


"""simple docstring"""







import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config
@pytest.fixture(scope='''session'''		)
def _A						(						):







    """simple docstring"""
    a					=10
    a					=datasets.Features(
        {
            '''tokens''': datasets.Sequence(datasets.Value('''string'''		)		),
            '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']		)		),
            '''answers''': datasets.Sequence(
                {
                    '''text''': datasets.Value('''string'''		),
                    '''answer_start''': datasets.Value('''int32'''		),
                }		),
            '''id''': datasets.Value('''int64'''		),
        }		)
    a					=datasets.Dataset.from_dict(
        {
            '''tokens''': [['''foo'''] * 5] * n,
            '''labels''': [[1] * 5] * n,
            '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
            '''id''': list(range(lowercase		)		),
        }	,   features=lowercase	,   )
    return dataset
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''file.arrow'''		)
    dataset.map(cache_file_name=lowercase		)
    return filename


# FILE_CONTENT + files


lowerCamelCase_       :       Union[str, Any]      						=  """\
    Text data.
    Second line of data."""
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt'''
    a					=FILE_CONTENT
    with open(lowercase	,   '''w'''		) as f:
        f.write(lowercase		)
    return filename
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    import bza

    a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.bz2'''
    a					=bytes(lowercase	,   '''utf-8'''		)
    with bza.open(lowercase	,   '''wb'''		) as f:
        f.write(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    import gzip

    a					=str(tmp_path_factory.mktemp('''data'''		) / '''file.txt.gz'''		)
    a					=bytes(lowercase	,   '''utf-8'''		)
    with gzip.open(lowercase	,   '''wb'''		) as f:
        f.write(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    if datasets.config.LZ4_AVAILABLE:
        import lza.frame

        a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.lz4'''
        a					=bytes(lowercase	,   '''utf-8'''		)
        with lza.frame.open(lowercase	,   '''wb'''		) as f:
            f.write(lowercase		)
        return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    if datasets.config.PY7ZR_AVAILABLE:
        import pyazr

        a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.7z'''
        with pyazr.SevenZipFile(lowercase	,   '''w'''		) as archive:
            archive.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
        return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    import tarfile

    a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.tar'''
    with tarfile.TarFile(lowercase	,   '''w'''		) as f:
        f.add(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    import lzma

    a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.xz'''
    a					=bytes(lowercase	,   '''utf-8'''		)
    with lzma.open(lowercase	,   '''wb'''		) as f:
        f.write(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    import zipfile

    a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    if datasets.config.ZSTANDARD_AVAILABLE:
        import zstandard as zstd

        a					=tmp_path_factory.mktemp('''data'''		) / '''file.txt.zst'''
        a					=bytes(lowercase	,   '''utf-8'''		)
        with zstd.open(lowercase	,   '''wb'''		) as f:
            f.write(lowercase		)
        return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''file.xml'''
    a					=textwrap.dedent(
        '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''		)
    with open(lowercase	,   '''w'''		) as f:
        f.write(lowercase		)
    return filename


lowerCamelCase_       :       Tuple      						=  [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
lowerCamelCase_       :       Dict      						=  [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
lowerCamelCase_       :       Tuple      						=  {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

lowerCamelCase_       :       Optional[Any]      						=  [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

lowerCamelCase_       :       Optional[int]      						=  [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]
@pytest.fixture(scope='''session'''		)
def _A						(						):







    """simple docstring"""
    return DATA_DICT_OF_LISTS
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=datasets.Dataset.from_dict(lowercase		)
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.arrow'''		)
    dataset.map(cache_file_name=lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.sqlite'''		)
    with contextlib.closing(sqlitea.connect(lowercase		)		) as con:
        a					=con.cursor()
        cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''		)
        for item in DATA:
            cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''	,   tuple(item.values()		)		)
        con.commit()
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.csv'''		)
    with open(lowercase	,   '''w'''	,   newline=''''''		) as f:
        a					=csv.DictWriter(lowercase	,   fieldnames=['''col_1''', '''col_2''', '''col_3''']		)
        writer.writeheader()
        for item in DATA:
            writer.writerow(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.csv'''		)
    with open(lowercase	,   '''w'''	,   newline=''''''		) as f:
        a					=csv.DictWriter(lowercase	,   fieldnames=['''col_1''', '''col_2''', '''col_3''']		)
        writer.writeheader()
        for item in DATA:
            writer.writerow(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    import bza

    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.bz2'''
    with open(lowercase	,   '''rb'''		) as f:
        a					=f.read()
    # data = bytes(FILE_CONTENT, "utf-8")
    with bza.open(lowercase	,   '''wb'''		) as f:
        f.write(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.csv.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(csv_path.replace('''.csv'''	,   '''.CSV'''		)		)		)
        f.write(lowercase	,   arcname=os.path.basename(csva_path.replace('''.csv'''	,   '''.CSV'''		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.csv.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.parquet'''		)
    a					=pa.schema(
        {
            '''col_1''': pa.string(),
            '''col_2''': pa.intaa(),
            '''col_3''': pa.floataa(),
        }		)
    with open(lowercase	,   '''wb'''		) as f:
        a					=pq.ParquetWriter(lowercase	,   schema=lowercase		)
        a					=pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(lowercase		)		)] for k in DATA[0]}	,   schema=lowercase		)
        writer.write_table(lowercase		)
        writer.close()
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.json'''		)
    a					={'''data''': DATA}
    with open(lowercase	,   '''w'''		) as f:
        json.dump(lowercase	,   lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.json'''		)
    a					={'''data''': DATA_DICT_OF_LISTS}
    with open(lowercase	,   '''w'''		) as f:
        json.dump(lowercase	,   lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in DATA:
            f.write(json.dumps(lowercase		) + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.jsonl'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in DATA:
            f.write(json.dumps(lowercase		) + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset_312.jsonl'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in DATA_312:
            f.write(json.dumps(lowercase		) + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset-str.jsonl'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in DATA_STR:
            f.write(json.dumps(lowercase		) + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    import gzip

    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.txt.gz'''		)
    with open(lowercase	,   '''rb'''		) as orig_file:
        with gzip.open(lowercase	,   '''wb'''		) as zipped_file:
            zipped_file.writelines(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    import gzip

    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.gz'''		)
    with open(lowercase	,   '''rb'''		) as orig_file:
        with gzip.open(lowercase	,   '''wb'''		) as zipped_file:
            zipped_file.writelines(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset_nested.jsonl.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.join('''nested'''	,   os.path.basename(lowercase		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.jsonl.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.jsonl.tar'''
    with tarfile.TarFile(lowercase	,   '''w'''		) as f:
        f.add(lowercase	,   arcname=os.path.basename(lowercase		)		)
        f.add(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset_nested.jsonl.tar'''
    with tarfile.TarFile(lowercase	,   '''w'''		) as f:
        f.add(lowercase	,   arcname=os.path.join('''nested'''	,   os.path.basename(lowercase		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=['''0''', '''1''', '''2''', '''3''']
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset.txt'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in data:
            f.write(item + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=['''0''', '''1''', '''2''', '''3''']
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset2.txt'''		)
    with open(lowercase	,   '''w'''		) as f:
        for item in data:
            f.write(item + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=['''0''', '''1''', '''2''', '''3''']
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.abc'''
    with open(lowercase	,   '''w'''		) as f:
        for item in data:
            f.write(item + '''\n'''		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.text.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset_with_dir.text.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
        f.write(lowercase	,   arcname=os.path.join('''main_dir'''	,   os.path.basename(lowercase		)		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.ext.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename('''unsupported.ext'''		)		)
        f.write(lowercase	,   arcname=os.path.basename('''unsupported_2.ext'''		)		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					='''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']		)
    a					=str(tmp_path_factory.mktemp('''data'''		) / '''dataset_with_unicode_new_lines.txt'''		)
    with open(lowercase	,   '''w'''	,   encoding='''utf-8'''		) as f:
        f.write(lowercase		)
    return path
@pytest.fixture(scope='''session'''		)
def _A						(						):







    """simple docstring"""
    return os.path.join('''tests'''	,   '''features'''	,   '''data'''	,   '''test_image_rgb.jpg'''		)
@pytest.fixture(scope='''session'''		)
def _A						(						):







    """simple docstring"""
    return os.path.join('''tests'''	,   '''features'''	,   '''data'''	,   '''test_audio_44100.wav'''		)
@pytest.fixture(scope='''session'''		)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data'''		) / '''dataset.img.zip'''
    with zipfile.ZipFile(lowercase	,   '''w'''		) as f:
        f.write(lowercase	,   arcname=os.path.basename(lowercase		)		)
        f.write(lowercase	,   arcname=os.path.basename(lowercase		).replace('''.jpg'''	,   '''2.jpg'''		)		)
    return path





@pytest.fixture(scope='''session'''		)
def _A						(						lowercase		):







    """simple docstring"""
    a					=tmp_path_factory.mktemp('''data_dir'''		)

    (data_dir / "subdir").mkdir()
    with open(data_dir / '''subdir''' / '''train.txt'''	,   '''w'''		) as f:
        f.write('''foo\n''' * 10		)
    with open(data_dir / '''subdir''' / '''test.txt'''	,   '''w'''		) as f:
        f.write('''bar\n''' * 10		)
    # hidden file
    with open(data_dir / '''subdir''' / '''.test.txt'''	,   '''w'''		) as f:
        f.write('''bar\n''' * 10		)

    # hidden directory
    (data_dir / ".subdir").mkdir()
    with open(data_dir / '''.subdir''' / '''train.txt'''	,   '''w'''		) as f:
        f.write('''foo\n''' * 10		)
    with open(data_dir / '''.subdir''' / '''test.txt'''	,   '''w'''		) as f:
        f.write('''bar\n''' * 10		)

    return data_dir



"""simple docstring"""







import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_rjieba
@require_tokenizers
class 				__A    (	_SCREAMING_SNAKE_CASE,   unittest.TestCase      ):






 """simple docstring"""




 __lowerCAmelCase			 =  RoFormerTokenizer
 __lowerCAmelCase			 =  RoFormerTokenizerFast
 __lowerCAmelCase			 =  True
 __lowerCAmelCase			 =  True


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					List[str]:
     super().setUp()


 def 				SCREAMING_SNAKE_CASE							(		self			,   **__A						) ->					Optional[int]:
     return self.tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''			,   **__A						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   **__A						) ->					List[Any]:
     return self.rust_tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''			,   **__A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Dict:
     a					='''永和服装饰品有限公司,今天天气非常好'''
     a					='''永和 服装 饰品 有限公司 , 今 天 天 气 非常 好'''
     return input_text, output_text


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Optional[int]:
     a					=self.get_tokenizer()
     a		,				a					=self.get_chinese_input_output_texts()
     a					=tokenizer.tokenize(__A						)

     self.assertListEqual(__A			,   output_text.split()						)

     a					=tokens + [tokenizer.unk_token]
     a					=[2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A						)			,   __A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Union[str, Any]:
     a					=self.get_rust_tokenizer()
     a		,				a					=self.get_chinese_input_output_texts()
     a					=tokenizer.tokenize(__A						)
     self.assertListEqual(__A			,   output_text.split()						)
     a					=tokens + [tokenizer.unk_token]
     a					=[2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A						)			,   __A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Tuple:
     pass


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Tuple:
     pass





 def 				SCREAMING_SNAKE_CASE							(		self						) ->					int:
     pass