'''simple docstring'''




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    a__				(       _SCREAMING_SNAKE_CASE							,       unittest.TestCase			):
     lowerCamelCase  :					List[str]							=RoCBertTokenizer
     lowerCamelCase  :					Dict							=None
     lowerCamelCase  :					str							=False
     lowerCamelCase  :					Optional[int]							=True
     lowerCamelCase  :					str							=filter_non_english






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         super().setUp()

         __lowerCamelCase          =  ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
         __lowerCamelCase          =  {}
         __lowerCamelCase          =  {}
         for i, value in enumerate(a   ):
             __lowerCamelCase          =  i
             __lowerCamelCase          =  i
         __lowerCamelCase          =  os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES['''vocab_file''']   )
         __lowerCamelCase          =  os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES['''word_shape_file''']   )
         __lowerCamelCase          =  os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES['''word_pronunciation_file''']   )
         with open(self.vocab_file  ,    '''w'''  ,    encoding='''utf-8'''   ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
         with open(self.word_shape_file  ,    '''w'''  ,    encoding='''utf-8'''   ) as word_shape_writer:
             json.dump(a  ,    a  ,    ensure_ascii=a   )
         with open(self.word_pronunciation_file  ,    '''w'''  ,    encoding='''utf-8'''   ) as word_pronunciation_writer:
             json.dump(a  ,    a  ,    ensure_ascii=a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.tokenizer_class(self.vocab_file  ,    self.word_shape_file  ,    self.word_pronunciation_file   )

         __lowerCamelCase          =  tokenizer.tokenize('''你好[SEP]你是谁'''   )
         self.assertListEqual(a  ,    ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']   )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )  ,    [5, 6, 2, 5, 7, 8]   )
         self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(a   )  ,    [5, 6, 2, 5, 7, 8]   )
         self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(a   )  ,    [5, 6, 2, 5, 7, 8]   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )  ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )  ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''h\u00E9llo''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )  ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple   ):





         """simple docstring"""



         __lowerCamelCase          =  RoCBertBasicTokenizer(do_lower_case=a  ,    never_split=['''[UNK]''']   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )  ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

         __lowerCamelCase          =  {}
         for i, token in enumerate(a   ):
             __lowerCamelCase          =  i
         __lowerCamelCase          =  RoCBertWordpieceTokenizer(vocab=a  ,    unk_token='''[UNK]'''   )

         self.assertListEqual(tokenizer.tokenize(''''''   )  ,    []   )

         self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )  ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

         self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )  ,    ['''[UNK]''', '''runn''', '''##ing''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         self.assertTrue(_is_whitespace(''' '''   )   )
         self.assertTrue(_is_whitespace('''\t'''   )   )
         self.assertTrue(_is_whitespace('''\r'''   )   )
         self.assertTrue(_is_whitespace('''\n'''   )   )
         self.assertTrue(_is_whitespace('''\u00A0'''   )   )

         self.assertFalse(_is_whitespace('''A'''   )   )
         self.assertFalse(_is_whitespace('''-'''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple   ):





         """simple docstring"""



         self.assertTrue(_is_control('''\u0005'''   )   )

         self.assertFalse(_is_control('''A'''   )   )
         self.assertFalse(_is_control(''' '''   )   )
         self.assertFalse(_is_control('''\t'''   )   )
         self.assertFalse(_is_control('''\r'''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         self.assertTrue(_is_punctuation('''-'''   )   )
         self.assertTrue(_is_punctuation('''$'''   )   )
         self.assertTrue(_is_punctuation('''`'''   )   )
         self.assertTrue(_is_punctuation('''.'''   )   )

         self.assertFalse(_is_punctuation('''A'''   )   )
         self.assertFalse(_is_punctuation(''' '''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]  ,    [['''[UNK]'''], [], ['''[UNK]''']]   )

         if self.test_rust_tokenizer:
             __lowerCamelCase          =  self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]  ,    [['''[UNK]'''], [], ['''[UNK]''']]   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                 __lowerCamelCase          =  tokenizer_r.encode_plus(
                     a  ,    return_attention_mask=a  ,    return_token_type_ids=a  ,    return_offsets_mapping=a  ,    add_special_tokens=a  ,    )

                 __lowerCamelCase          =  tokenizer_r.do_lower_case if hasattr(a  ,    '''do_lower_case'''   ) else False
                 __lowerCamelCase          =  (
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), """A"""),
                         ((1, 2), ""","""),
                         ((3, 5), """na"""),
                         ((5, 6), """##ï"""),
                         ((6, 8), """##ve"""),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), """Allen"""),
                         ((21, 23), """##NL"""),
                         ((23, 24), """##P"""),
                         ((25, 33), """sentence"""),
                         ((33, 34), """."""),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), """a"""),
                         ((1, 2), ""","""),
                         ((3, 8), """naive"""),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), """allen"""),
                         ((21, 23), """##nl"""),
                         ((23, 24), """##p"""),
                         ((25, 33), """sentence"""),
                         ((33, 34), """."""),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]  ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
                 self.assertEqual([e[0] for e in expected_results]  ,    tokens['''offset_mapping''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  ["""的""", """人""", """有"""]
         __lowerCamelCase          =  """""".join(a   )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                 __lowerCamelCase          =  True
                 __lowerCamelCase          =  self.tokenizer_class.from_pretrained(a  ,    **a   )
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  tokenizer_p.encode(a  ,    add_special_tokens=a   )
                 __lowerCamelCase          =  tokenizer_r.encode(a  ,    add_special_tokens=a   )

                 __lowerCamelCase          =  tokenizer_r.convert_ids_to_tokens(a   )
                 __lowerCamelCase          =  tokenizer_p.convert_ids_to_tokens(a   )

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(a  ,    a   )
                 self.assertListEqual(a  ,    a   )

                 __lowerCamelCase          =  False
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )
                 __lowerCamelCase          =  self.tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  tokenizer_r.encode(a  ,    add_special_tokens=a   )
                 __lowerCamelCase          =  tokenizer_p.encode(a  ,    add_special_tokens=a   )

                 __lowerCamelCase          =  tokenizer_r.convert_ids_to_tokens(a   )
                 __lowerCamelCase          =  tokenizer_p.convert_ids_to_tokens(a   )

                 # it is expected that only the first Chinese character is not preceded by "##".
                 __lowerCamelCase          =  [
                     f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a   )
                 ]
                 self.assertListEqual(a  ,    a   )
                 self.assertListEqual(a  ,    a   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.tokenizer_class(self.vocab_file  ,    self.word_shape_file  ,    self.word_pronunciation_file   )

         __lowerCamelCase          =  tokenizer.encode('''你好'''  ,    add_special_tokens=a   )
         __lowerCamelCase          =  tokenizer.encode('''你是谁'''  ,    add_special_tokens=a   )

         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a   )
         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a  ,    a   )

         assert encoded_sentence == [1] + text + [2]
         assert encoded_pair == [1] + text + [2] + text_a + [2]






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  self.get_tokenizers(do_lower_case=a   )
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                 __lowerCamelCase          =  """你好，你是谁"""
                 __lowerCamelCase          =  tokenizer.tokenize(a   )
                 __lowerCamelCase          =  tokenizer.convert_tokens_to_ids(a   )
                 __lowerCamelCase          =  tokenizer.convert_tokens_to_shape_ids(a   )
                 __lowerCamelCase          =  tokenizer.convert_tokens_to_pronunciation_ids(a   )
                 __lowerCamelCase          =  tokenizer.prepare_for_model(
                     a  ,    a  ,    a  ,    add_special_tokens=a   )

                 __lowerCamelCase          =  tokenizer.encode_plus(a  ,    add_special_tokens=a   )

                 self.assertEqual(a  ,    a   )

'''simple docstring'''




def __lowerCAmelCase			(							UpperCamelCase__    )     ->   str:
    return "".join(chr(ord(UpperCamelCase__    ) - 32    ) if '''a''' <= char <= '''z''' else char for char in word    )


if __name__ == "__main__":
  from doctest import testmod

  testmod()
