




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 						__lowercase		(a__					,		unittest.TestCase							):




      """simple docstring"""





      _snake_case												=						RoCBertTokenizer
      _snake_case												=						None
      _snake_case												=						False
      _snake_case												=						True
      _snake_case												=						filter_non_english

      def      UpperCAmelCase				(				self     )		->    int:
         super().setUp()

         snake_case				:    Optional[Any]           =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
         snake_case				:    Optional[int]           =	{}
         snake_case				:    int           =	{}
         for i, value in enumerate(_lowerCamelCase     ):
            snake_case				:    Tuple           =	i
            snake_case				:    Optional[int]           =	i
         snake_case				:    Tuple           =	os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES["""vocab_file"""]     )
         snake_case				:    Tuple           =	os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES["""word_shape_file"""]     )
         snake_case				:    int           =	os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
         with open(self.vocab_file    ,				"""w"""    ,				encoding="""utf-8"""     ) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
         with open(self.word_shape_file    ,				"""w"""    ,				encoding="""utf-8"""     ) as word_shape_writer:
            json.dump(_lowerCamelCase    ,				_lowerCamelCase    ,				ensure_ascii=_lowerCamelCase     )
         with open(self.word_pronunciation_file    ,				"""w"""    ,				encoding="""utf-8"""     ) as word_pronunciation_writer:
            json.dump(_lowerCamelCase    ,				_lowerCamelCase    ,				ensure_ascii=_lowerCamelCase     )

      def      UpperCAmelCase				(				self     )		->    Optional[Any]:
         snake_case				:    Optional[Any]           =	self.tokenizer_class(self.vocab_file    ,				self.word_shape_file    ,				self.word_pronunciation_file     )

         snake_case				:    List[str]           =	tokenizer.tokenize("""你好[SEP]你是谁"""     )
         self.assertListEqual(_lowerCamelCase    ,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase     )    ,				[5, 6, 2, 5, 7, 8]     )
         self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase     )    ,				[5, 6, 2, 5, 7, 8]     )
         self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase     )    ,				[5, 6, 2, 5, 7, 8]     )

      def      UpperCAmelCase				(				self     )		->    Tuple:
         snake_case				:    Optional[Any]           =	RoCBertBasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )    ,				["""ah""", """\u535A""", """\u63A8""", """zz"""]     )

      def      UpperCAmelCase				(				self     )		->    str:
         snake_case				:    str           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )    ,				["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )    ,				["""hello"""]     )

      def      UpperCAmelCase				(				self     )		->    Optional[Any]:
         snake_case				:    str           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase    ,				strip_accents=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )    ,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )    ,				["""h\u00E9llo"""]     )

      def      UpperCAmelCase				(				self     )		->    List[str]:
         snake_case				:    Union[str, Any]           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase    ,				strip_accents=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )    ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )    ,				["""hello"""]     )

      def      UpperCAmelCase				(				self     )		->    List[Any]:
         snake_case				:    str           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )    ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )    ,				["""hello"""]     )

      def      UpperCAmelCase				(				self     )		->    int:
         snake_case				:    Dict           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )    ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )

      def      UpperCAmelCase				(				self     )		->    Dict:
         snake_case				:    Tuple           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase    ,				strip_accents=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )    ,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )

      def      UpperCAmelCase				(				self     )		->    List[Any]:
         snake_case				:    Optional[int]           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase    ,				strip_accents=_lowerCamelCase     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )    ,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )

      def      UpperCAmelCase				(				self     )		->    Tuple:
         snake_case				:    Optional[int]           =	RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase    ,				never_split=["""[UNK]"""]     )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )    ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )

      def      UpperCAmelCase				(				self     )		->    Dict:
         snake_case				:    int           =	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         snake_case				:    Union[str, Any]           =	{}
         for i, token in enumerate(_lowerCamelCase     ):
            snake_case				:    Dict           =	i
         snake_case				:    List[str]           =	RoCBertWordpieceTokenizer(vocab=_lowerCamelCase    ,				unk_token="""[UNK]"""     )

         self.assertListEqual(tokenizer.tokenize(""""""     )    ,				[]     )

         self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )    ,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

         self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )    ,				["""[UNK]""", """runn""", """##ing"""]     )

      def      UpperCAmelCase				(				self     )		->    Optional[Any]:
         self.assertTrue(_is_whitespace(""" """     )     )
         self.assertTrue(_is_whitespace("""\t"""     )     )
         self.assertTrue(_is_whitespace("""\r"""     )     )
         self.assertTrue(_is_whitespace("""\n"""     )     )
         self.assertTrue(_is_whitespace("""\u00A0"""     )     )

         self.assertFalse(_is_whitespace("""A"""     )     )
         self.assertFalse(_is_whitespace("""-"""     )     )

      def      UpperCAmelCase				(				self     )		->    Dict:
         self.assertTrue(_is_control("""\u0005"""     )     )

         self.assertFalse(_is_control("""A"""     )     )
         self.assertFalse(_is_control(""" """     )     )
         self.assertFalse(_is_control("""\t"""     )     )
         self.assertFalse(_is_control("""\r"""     )     )

      def      UpperCAmelCase				(				self     )		->    Union[str, Any]:
         self.assertTrue(_is_punctuation("""-"""     )     )
         self.assertTrue(_is_punctuation("""$"""     )     )
         self.assertTrue(_is_punctuation("""`"""     )     )
         self.assertTrue(_is_punctuation("""."""     )     )

         self.assertFalse(_is_punctuation("""A"""     )     )
         self.assertFalse(_is_punctuation(""" """     )     )

      def      UpperCAmelCase				(				self     )		->    Optional[Any]:
         snake_case				:    Any           =	self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(_lowerCamelCase     ) for t in ["""Test""", """\xad""", """test"""]]    ,				[["""[UNK]"""], [], ["""[UNK]"""]]     )

         if self.test_rust_tokenizer:
            snake_case				:    List[Any]           =	self.get_rust_tokenizer()
            self.assertListEqual(
                [rust_tokenizer.tokenize(_lowerCamelCase     ) for t in ["""Test""", """\xad""", """test"""]]    ,				[["""[UNK]"""], [], ["""[UNK]"""]]     )

      def      UpperCAmelCase				(				self     )		->    Optional[Any]:







         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
               snake_case				:    List[Any]           =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,				**_lowerCamelCase     )

               snake_case				:    Any           =	f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
               snake_case				:    Dict           =	tokenizer_r.encode_plus(
                   _lowerCamelCase    ,				return_attention_mask=_lowerCamelCase    ,				return_token_type_ids=_lowerCamelCase    ,				return_offsets_mapping=_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase    ,				)

               snake_case				:    int           =	tokenizer_r.do_lower_case if hasattr(_lowerCamelCase    ,				"""do_lower_case"""     ) else False
               snake_case				:    int           =	(
                   [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), '''A'''),
                       ((1, 2), ''','''),
                       ((3, 5), '''na'''),
                       ((5, 6), '''##ï'''),
                       ((6, 8), '''##ve'''),
                       ((9, 1_5), tokenizer_r.mask_token),
                       ((1_6, 2_1), '''Allen'''),
                       ((2_1, 2_3), '''##NL'''),
                       ((2_3, 2_4), '''##P'''),
                       ((2_5, 3_3), '''sentence'''),
                       ((3_3, 3_4), '''.'''),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
                   if not do_lower_case
                   else [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), '''a'''),
                       ((1, 2), ''','''),
                       ((3, 8), '''naive'''),
                       ((9, 1_5), tokenizer_r.mask_token),
                       ((1_6, 2_1), '''allen'''),
                       ((2_1, 2_3), '''##nl'''),
                       ((2_3, 2_4), '''##p'''),
                       ((2_5, 3_3), '''sentence'''),
                       ((3_3, 3_4), '''.'''),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
               )

               self.assertEqual(
                   [e[1] for e in expected_results]    ,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
               self.assertEqual([e[0] for e in expected_results]    ,				tokens["""offset_mapping"""]     )

      def      UpperCAmelCase				(				self     )		->    str:
         snake_case				:    Tuple           =	['''的''', '''人''', '''有''']
         snake_case				:    Optional[int]           =	''''''.join(_lowerCamelCase     )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
               snake_case				:    List[str]           =	True
               snake_case				:    List[Any]           =	self.tokenizer_class.from_pretrained(_lowerCamelCase    ,				**_lowerCamelCase     )
               snake_case				:    Optional[Any]           =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,				**_lowerCamelCase     )

               snake_case				:    str           =	tokenizer_p.encode(_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )
               snake_case				:    Union[str, Any]           =	tokenizer_r.encode(_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )

               snake_case				:    str           =	tokenizer_r.convert_ids_to_tokens(_lowerCamelCase     )
               snake_case				:    Dict           =	tokenizer_p.convert_ids_to_tokens(_lowerCamelCase     )

               # it is expected that each Chinese character is not preceded by "##"
               self.assertListEqual(_lowerCamelCase    ,				_lowerCamelCase     )
               self.assertListEqual(_lowerCamelCase    ,				_lowerCamelCase     )

               snake_case				:    List[str]           =	False
               snake_case				:    List[Any]           =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase    ,				**_lowerCamelCase     )
               snake_case				:    int           =	self.tokenizer_class.from_pretrained(_lowerCamelCase    ,				**_lowerCamelCase     )

               snake_case				:    Any           =	tokenizer_r.encode(_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )
               snake_case				:    Union[str, Any]           =	tokenizer_p.encode(_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )

               snake_case				:    Optional[int]           =	tokenizer_r.convert_ids_to_tokens(_lowerCamelCase     )
               snake_case				:    List[Any]           =	tokenizer_p.convert_ids_to_tokens(_lowerCamelCase     )

               # it is expected that only the first Chinese character is not preceded by "##".
               snake_case				:    Optional[Any]           =	[
                   f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase     )
               ]
               self.assertListEqual(_lowerCamelCase    ,				_lowerCamelCase     )
               self.assertListEqual(_lowerCamelCase    ,				_lowerCamelCase     )

      @slow
      def      UpperCAmelCase				(				self     )		->    Dict:
         snake_case				:    Any           =	self.tokenizer_class(self.vocab_file    ,				self.word_shape_file    ,				self.word_pronunciation_file     )

         snake_case				:    Tuple           =	tokenizer.encode("""你好"""    ,				add_special_tokens=_lowerCamelCase     )
         snake_case				:    Dict           =	tokenizer.encode("""你是谁"""    ,				add_special_tokens=_lowerCamelCase     )

         snake_case				:    Dict           =	tokenizer.build_inputs_with_special_tokens(_lowerCamelCase     )
         snake_case				:    Optional[Any]           =	tokenizer.build_inputs_with_special_tokens(_lowerCamelCase    ,				_lowerCamelCase     )

         assert encoded_sentence == [1] + text + [2]
         assert encoded_pair == [1] + text + [2] + text_a + [2]

      def      UpperCAmelCase				(				self     )		->    List[Any]:
         snake_case				:    Optional[Any]           =	self.get_tokenizers(do_lower_case=_lowerCamelCase     )
         for tokenizer in tokenizers:
            with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
               snake_case				:    Dict           =	'''你好，你是谁'''
               snake_case				:    List[Any]           =	tokenizer.tokenize(_lowerCamelCase     )
               snake_case				:    Union[str, Any]           =	tokenizer.convert_tokens_to_ids(_lowerCamelCase     )
               snake_case				:    int           =	tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase     )
               snake_case				:    Any           =	tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase     )
               snake_case				:    Any           =	tokenizer.prepare_for_model(
                   _lowerCamelCase    ,				_lowerCamelCase    ,				_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )

               snake_case				:    Optional[int]           =	tokenizer.encode_plus(_lowerCamelCase    ,				add_special_tokens=_lowerCamelCase     )

               self.assertEqual(_lowerCamelCase    ,				_lowerCamelCase     )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase	:	str				=   {
    'configuration_roformer': ['ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RoFormerConfig', 'RoFormerOnnxConfig'],
    'tokenization_roformer': ['RoFormerTokenizer'],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase	:	List[Any]				=   ['RoFormerTokenizerFast']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase	:	Dict				=   [
           'ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
           'RoFormerForCausalLM',
           'RoFormerForMaskedLM',
           'RoFormerForMultipleChoice',
           'RoFormerForQuestionAnswering',
           'RoFormerForSequenceClassification',
           'RoFormerForTokenClassification',
           'RoFormerLayer',
           'RoFormerModel',
           'RoFormerPreTrainedModel',
           'load_tf_weights_in_roformer',
       ]


try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase	:	Any				=   [
           'TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
           'TFRoFormerForCausalLM',
           'TFRoFormerForMaskedLM',
           'TFRoFormerForMultipleChoice',
           'TFRoFormerForQuestionAnswering',
           'TFRoFormerForSequenceClassification',
           'TFRoFormerForTokenClassification',
           'TFRoFormerLayer',
           'TFRoFormerModel',
           'TFRoFormerPreTrainedModel',
       ]


try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase	:	Union[str, Any]				=   [
           'FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
           'FlaxRoFormerForMaskedLM',
           'FlaxRoFormerForMultipleChoice',
           'FlaxRoFormerForQuestionAnswering',
           'FlaxRoFormerForSequenceClassification',
           'FlaxRoFormerForTokenClassification',
           'FlaxRoFormerModel',
           'FlaxRoFormerPreTrainedModel',
       ]


if TYPE_CHECKING:
       from .configuration_roformer import ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RoFormerConfig, RoFormerOnnxConfig
       from .tokenization_roformer import RoFormerTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_roformer_fast import RoFormerTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_roformer import (
                  ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  RoFormerForCausalLM,
                  RoFormerForMaskedLM,
                  RoFormerForMultipleChoice,
                  RoFormerForQuestionAnswering,
                  RoFormerForSequenceClassification,
                  RoFormerForTokenClassification,
                  RoFormerLayer,
                  RoFormerModel,
                  RoFormerPreTrainedModel,
                  load_tf_weights_in_roformer,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_roformer import (
                  TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFRoFormerForCausalLM,
                  TFRoFormerForMaskedLM,
                  TFRoFormerForMultipleChoice,
                  TFRoFormerForQuestionAnswering,
                  TFRoFormerForSequenceClassification,
                  TFRoFormerForTokenClassification,
                  TFRoFormerLayer,
                  TFRoFormerModel,
                  TFRoFormerPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_roformer import (
                  FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  FlaxRoFormerForMaskedLM,
                  FlaxRoFormerForMultipleChoice,
                  FlaxRoFormerForQuestionAnswering,
                  FlaxRoFormerForSequenceClassification,
                  FlaxRoFormerForTokenClassification,
                  FlaxRoFormerModel,
                  FlaxRoFormerPreTrainedModel,
              )


else:
       import sys

       lowerCamelCase	:	Optional[int]				=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





