





"""simple docstring"""
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase_       =		get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
@require_tokenizers
class  _SCREAMING_SNAKE_CASE(       A   ,     unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_       :				Union[str, Any]     						=  XLMRobertaTokenizer
       SCREAMING_SNAKE_CASE_       :				Optional[Any]     						=  XLMRobertaTokenizerFast
       SCREAMING_SNAKE_CASE_       :				List[str]     						=  True
       SCREAMING_SNAKE_CASE_       :				List[Any]     						=  True




       def    _UpperCamelCase					(						self							)      ->				int:





             """simple docstring"""






             super().setUp()

             # We have a SentencePiece fixture for testing
             __SCREAMING_SNAKE_CASE       :int		    =  XLMRobertaTokenizer(SCREAMING_SNAKE_CASE__		,keep_accents=SCREAMING_SNAKE_CASE__							)
             tokenizer.save_pretrained(self.tmpdirname							)




       def    _UpperCamelCase					(						self							)      ->				int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :int		    =  '''<pad>'''
             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__							)		,SCREAMING_SNAKE_CASE__							)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__							)		,SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[str]		    =  list(self.get_tokenizer().get_vocab().keys()							)

             self.assertEqual(vocab_keys[0]		,'''<s>'''							)
             self.assertEqual(vocab_keys[1]		,'''<pad>'''							)
             self.assertEqual(vocab_keys[-1]		,'''<mask>'''							)
             self.assertEqual(len(SCREAMING_SNAKE_CASE__							)		,10_02							)




       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             self.assertEqual(self.get_tokenizer().vocab_size		,10_02							)




       def    _UpperCamelCase					(						self							)      ->				List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[Any]		    =  XLMRobertaTokenizer(SCREAMING_SNAKE_CASE__		,keep_accents=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE       :List[Any]		    =  tokenizer.tokenize('''This is a test'''							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)		,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,)

             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
             self.assertListEqual(
                 SCREAMING_SNAKE_CASE__		,[
                     SPIECE_UNDERLINE + '''I''',
                     SPIECE_UNDERLINE + '''was''',
                     SPIECE_UNDERLINE + '''b''',
                     '''or''',
                     '''n''',
                     SPIECE_UNDERLINE + '''in''',
                     SPIECE_UNDERLINE + '''''',
                     '''9''',
                     '''2''',
                     '''0''',
                     '''0''',
                     '''0''',
                     ''',''',
                     SPIECE_UNDERLINE + '''and''',
                     SPIECE_UNDERLINE + '''this''',
                     SPIECE_UNDERLINE + '''is''',
                     SPIECE_UNDERLINE + '''f''',
                     '''al''',
                     '''s''',
                     '''é''',
                     '''.''',
                 ]		,)
             __SCREAMING_SNAKE_CASE       :int		    =  tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(
                 SCREAMING_SNAKE_CASE__		,[
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                     #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
                 ]		,)

             __SCREAMING_SNAKE_CASE       :List[Any]		    =  tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(
                 SCREAMING_SNAKE_CASE__		,[
                     SPIECE_UNDERLINE + '''I''',
                     SPIECE_UNDERLINE + '''was''',
                     SPIECE_UNDERLINE + '''b''',
                     '''or''',
                     '''n''',
                     SPIECE_UNDERLINE + '''in''',
                     SPIECE_UNDERLINE + '''''',
                     '''<unk>''',
                     '''2''',
                     '''0''',
                     '''0''',
                     '''0''',
                     ''',''',
                     SPIECE_UNDERLINE + '''and''',
                     SPIECE_UNDERLINE + '''this''',
                     SPIECE_UNDERLINE + '''is''',
                     SPIECE_UNDERLINE + '''f''',
                     '''al''',
                     '''s''',
                     '''<unk>''',
                     '''.''',
                 ]		,)




       def    _UpperCamelCase					(						self							)      ->				int:





             """simple docstring"""






             if not self.test_slow_tokenizer:
                   # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                   return

             __SCREAMING_SNAKE_CASE       :List[str]		    =  (self.rust_tokenizer_class, '''hf-internal-testing/tiny-xlm-roberta''', {})
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
                         __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)

                         __SCREAMING_SNAKE_CASE       :str		    =  tempfile.mkdtemp()

                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :int		    =  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Checks it save with the same files + the tokenizer.json file for the fast one
                         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
                         self.assertSequenceEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

                         # Checks everything loads correctly in the same way
                         __SCREAMING_SNAKE_CASE       :Tuple		    =  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :Dict		    =  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)							)
                               # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                               # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                         shutil.rmtree(SCREAMING_SNAKE_CASE__							)

                         # Save tokenizer rust, legacy_format=True
                         __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  tempfile.mkdtemp()

                         __SCREAMING_SNAKE_CASE       :List[str]		    =  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__		,legacy_format=SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :List[str]		    =  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Checks it save with the same files
                         self.assertSequenceEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

                         # Checks everything loads correctly in the same way
                         __SCREAMING_SNAKE_CASE       :Tuple		    =  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :List[Any]		    =  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)							)

                         shutil.rmtree(SCREAMING_SNAKE_CASE__							)

                         # Save tokenizer rust, legacy_format=False
                         __SCREAMING_SNAKE_CASE       :List[Any]		    =  tempfile.mkdtemp()

                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__		,legacy_format=SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Checks it saved the tokenizer.json file
                         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

                         # Checks everything loads correctly in the same way
                         __SCREAMING_SNAKE_CASE       :Tuple		    =  tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__							)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)							)

                         shutil.rmtree(SCREAMING_SNAKE_CASE__							)




       @cached_property
       def    _UpperCamelCase					(						self							)      ->				Dict:





             """simple docstring"""






             return XLMRobertaTokenizer.from_pretrained('''xlm-roberta-base'''							)




       def    _UpperCamelCase					(						self							)      ->				List[Any]:





             """simple docstring"""






             with tempfile.NamedTemporaryFile() as f:
                   shutil.copyfile(SCREAMING_SNAKE_CASE__		,f.name							)
                   __SCREAMING_SNAKE_CASE       :List[Any]		    =  XLMRobertaTokenizer(f.name		,keep_accents=SCREAMING_SNAKE_CASE__							)
                   __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  pickle.dumps(SCREAMING_SNAKE_CASE__							)
             pickle.loads(SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				str:





             """simple docstring"""






             if not self.test_rust_tokenizer:
                   return

             __SCREAMING_SNAKE_CASE       :Any		    =  self.get_tokenizer()
             __SCREAMING_SNAKE_CASE       :Dict		    =  self.get_rust_tokenizer()

             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  '''I was born in 92000, and this is falsé.'''

             __SCREAMING_SNAKE_CASE       :Dict		    =  tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE       :Tuple		    =  rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE       :List[str]		    =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  self.get_rust_tokenizer()
             __SCREAMING_SNAKE_CASE       :int		    =  tokenizer.encode(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE       :int		    =  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)




       @slow
       def    _UpperCamelCase					(						self							)      ->				Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[int]		    =  '''Hello World!'''
             __SCREAMING_SNAKE_CASE       :List[str]		    =  [0, 3_53_78, 66_61, 38, 2]
             # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
             # xlmr.eval()
             # xlmr.encode(symbols)

             self.assertListEqual(SCREAMING_SNAKE_CASE__		,self.big_tokenizer.encode(SCREAMING_SNAKE_CASE__							)							)




       @slow
       def    _UpperCamelCase					(						self							)      ->				Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :int		    =  (
                 '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
                 ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
             )
             __SCREAMING_SNAKE_CASE       :int		    =  [
                 0,
                 32_93,
                 83,
                 10,
                 45_52,
                 49_89,
                 79_86,
                 6_78,
                 10,
                 59_15,
                 1_11,
                 17_94_59,
                 12_48_50,
                 4,
                 60_44,
                 2_37,
                 12,
                 6,
                 5,
                 6,
                 4,
                 67_80,
                 7_05,
                 15,
                 13_88,
                 44,
                 3_78,
                 1_01_14,
                 7_11,
                 1_52,
                 20,
                 6,
                 5,
                 2_23_76,
                 6_42,
                 12_21,
                 1_51_90,
                 3_41_53,
                 4_50,
                 56_08,
                 9_59,
                 11_19,
                 5_77_02,
                 1_36,
                 1_86,
                 47,
                 10_98,
                 2_93_67,
                 47,
                 # 4426, # What fairseq tokenizes from "<unk>": "_<"
                 # 3678, # What fairseq tokenizes from "<unk>": "unk"
                 # 2740, # What fairseq tokenizes from "<unk>": ">"
                 3,  # What we tokenize from "<unk>": "<unk>"
                 6,  # Residue from the tokenization: an extra sentencepiece underline
                 4,
                 60_44,
                 2_37,
                 62_84,
                 5_09_01,
                 5_28,
                 31,
                 90,
                 34,
                 9_27,
                 2,
             ]
             # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
             # xlmr.eval()
             # xlmr.encode(symbols)

             self.assertListEqual(SCREAMING_SNAKE_CASE__		,self.big_tokenizer.encode(SCREAMING_SNAKE_CASE__							)							)




       @slow
       def    _UpperCamelCase					(						self							)      ->				Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Dict		    =  {'''input_ids''': [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=SCREAMING_SNAKE_CASE__		,model_name='''xlm-roberta-base'''		,revision='''d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'''		,)






"""simple docstring"""
def 					__lowerCamelCase       (							a_					:		int = 10			,       a_					:		int = 22							)      ->				int:
      __SCREAMING_SNAKE_CASE       :Optional[int]		    =  range(1			,       a_							)
      __SCREAMING_SNAKE_CASE       :List[Any]		    =  range(1			,       a_							)
      return sum(
          1 for power in powers for base in bases if len(str(base**power							)							) == power							)


if __name__ == "__main__":
  print(f'{solution(1_0, 2_2) = }')