


"""simple docstring"""







import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


lowerCamelCase_       :       Tuple      						=  logging.getLogger()






@unittest.skip("Temporarily disable the doc tests."      )
@require_torch
@require_tf
@slow
class 				__A    (	unittest.TestCase      ):






 """simple docstring"""




 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A = None			,   __A = None			,   __A = None			,   __A = True			,   ) ->					int:
     a					=[file for file in os.listdir(lowerCamelCase__						) if os.path.isfile(os.path.join(lowerCamelCase__			,   lowerCamelCase__						)						)]

     if identifier is not None:
         a					=[file for file in files if identifier in file]

     if n_identifier is not None:
         if isinstance(lowerCamelCase__			,   lowerCamelCase__						):
             for n_ in n_identifier:
                 a					=[file for file in files if n_ not in file]
         else:
             a					=[file for file in files if n_identifier not in file]

     a					=ignore_files or []
     ignore_files.append('''__init__.py'''						)
     a					=[file for file in files if file not in ignore_files]

     for file in files:
         # Open all files
         print('''Testing'''			,   lowerCamelCase__						)

         if only_modules:
             a					=file.split('''.'''						)[0]
             try:
                 a					=getattr(lowerCamelCase__			,   lowerCamelCase__						)
                 a					=doctest.DocTestSuite(lowerCamelCase__						)
                 a					=unittest.TextTestRunner().run(lowerCamelCase__						)
                 self.assertIs(len(result.failures						)			,   0						)
             except AttributeError:
                 logger.info(f'''{module_identifier} is not a module.'''						)
         else:
             a					=doctest.testfile(str('''..''' / directory / file						)			,   optionflags=doctest.ELLIPSIS						)
             self.assertIs(result.failed			,   0						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					str:
     a					=Path('''src/transformers'''						)
     a					='''modeling'''
     a					=[
         '''modeling_ctrl.py''',
         '''modeling_tf_ctrl.py''',
     ]
     self.analyze_directory(lowerCamelCase__			,   identifier=lowerCamelCase__			,   ignore_files=lowerCamelCase__						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Tuple:
     a					=Path('''src/transformers'''						)
     a					='''tokenization'''
     self.analyze_directory(lowerCamelCase__			,   identifier=lowerCamelCase__						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					List[Any]:
     a					=Path('''src/transformers'''						)
     a					='''configuration'''
     self.analyze_directory(lowerCamelCase__			,   identifier=lowerCamelCase__						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Optional[Any]:
     a					=Path('''src/transformers'''						)
     a					=['''configuration''', '''modeling''', '''tokenization''']
     self.analyze_directory(lowerCamelCase__			,   n_identifier=lowerCamelCase__						)





 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Dict:
     a					=Path('''docs/source'''						)
     a					=['''favicon.ico''']
     self.analyze_directory(lowerCamelCase__			,   ignore_files=lowerCamelCase__			,   only_modules=lowerCamelCase__						)



"""simple docstring"""







import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_rjieba
@require_tokenizers
class 				__A    (	_SCREAMING_SNAKE_CASE,   unittest.TestCase      ):






 """simple docstring"""




 __lowerCAmelCase			 =  RoFormerTokenizer
 __lowerCAmelCase			 =  RoFormerTokenizerFast
 __lowerCAmelCase			 =  True
 __lowerCAmelCase			 =  True


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					List[str]:
     super().setUp()


 def 				SCREAMING_SNAKE_CASE							(		self			,   **__A						) ->					Optional[int]:
     return self.tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''			,   **__A						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   **__A						) ->					List[Any]:
     return self.rust_tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''			,   **__A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Dict:
     a					='''永和服装饰品有限公司,今天天气非常好'''
     a					='''永和 服装 饰品 有限公司 , 今 天 天 气 非常 好'''
     return input_text, output_text


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Optional[int]:
     a					=self.get_tokenizer()
     a		,				a					=self.get_chinese_input_output_texts()
     a					=tokenizer.tokenize(__A						)

     self.assertListEqual(__A			,   output_text.split()						)

     a					=tokens + [tokenizer.unk_token]
     a					=[2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A						)			,   __A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Union[str, Any]:
     a					=self.get_rust_tokenizer()
     a		,				a					=self.get_chinese_input_output_texts()
     a					=tokenizer.tokenize(__A						)
     self.assertListEqual(__A			,   output_text.split()						)
     a					=tokens + [tokenizer.unk_token]
     a					=[2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A						)			,   __A						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Tuple:
     pass


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Tuple:
     pass





 def 				SCREAMING_SNAKE_CASE							(		self						) ->					int:
     pass