





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_						       =	get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_torch_available():
      from transformers.models.mbart.modeling_mbart import shift_tokens_right

SCREAMING_SNAKE_CASE_						       =	2_5_0_0_0_4
SCREAMING_SNAKE_CASE_						       =	2_5_0_0_2_0

@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__			(				lowercase__							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     List[str]		     =      MBartaaTokenizer
     __snake_case   :     str		     =      MBartaaTokenizerFast
     __snake_case   :     Tuple		     =      True
     __snake_case   :     List[Any]		     =      True






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		str:




           '''simple docstring'''


           super().setUp()

           # We have a SentencePiece fixture for testing
           SCREAMING_SNAKE_CASE		=						MBartaaTokenizer(_a						,src_lang="""en_XX"""						,tgt_lang="""ro_RO"""						,keep_accents=_a	)
           tokenizer.save_pretrained(self.tmpdirname	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						'<s>'
           SCREAMING_SNAKE_CASE		=						0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a	)						,_a	)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a	)						,_a	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						list(self.get_tokenizer().get_vocab().keys()	)

           self.assertEqual(vocab_keys[0]						,"""<s>"""	)
           self.assertEqual(vocab_keys[1]						,"""<pad>"""	)
           self.assertEqual(vocab_keys[-1]						,"""<mask>"""	)
           self.assertEqual(len(_a	)						,1054	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[Any]:




           '''simple docstring'''


           self.assertEqual(self.get_tokenizer().vocab_size						,1054	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						MBartaaTokenizer(_a						,src_lang="""en_XX"""						,tgt_lang="""ro_RO"""						,keep_accents=_a	)

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize("""This is a test"""	)
           self.assertListEqual(_a						,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(_a	)						,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,)

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
           self.assertListEqual(
               _a						,[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]						,)
           SCREAMING_SNAKE_CASE		=						tokenizer.convert_tokens_to_ids(_a	)
           self.assertListEqual(
               _a						,[
                   value + tokenizer.fairseq_offset
                   for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
               ]						,)

           SCREAMING_SNAKE_CASE		=						tokenizer.convert_ids_to_tokens(_a	)
           self.assertListEqual(
               _a						,[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]						,)






     @slow
     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]	)					->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						{'input_ids': [[250004, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [250004, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [250004, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=_a						,model_name="""facebook/mbart-large-50"""						,revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		Optional[Any]:




           '''simple docstring'''


           if not self.test_slow_tokenizer:
                 # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                 return

           SCREAMING_SNAKE_CASE		=						(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-mbart50', {})
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(_a						,**_a	)
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(_a						,**_a	)

                       SCREAMING_SNAKE_CASE		=						tempfile.mkdtemp()

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.save_pretrained(_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.save_pretrained(_a	)

                       # Checks it save with the same files + the tokenizer.json file for the fast one
                       self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files	)	)
                       SCREAMING_SNAKE_CASE		=						tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f	)
                       self.assertSequenceEqual(_a						,_a	)

                       # Checks everything loads correctly in the same way
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.from_pretrained(_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.from_pretrained(_a	)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_a						,_a	)	)
                             # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                             # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                       shutil.rmtree(_a	)

                       # Save tokenizer rust, legacy_format=True
                       SCREAMING_SNAKE_CASE		=						tempfile.mkdtemp()

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.save_pretrained(_a						,legacy_format=_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.save_pretrained(_a	)

                       # Checks it save with the same files
                       self.assertSequenceEqual(_a						,_a	)

                       # Checks everything loads correctly in the same way
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.from_pretrained(_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.from_pretrained(_a	)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_a						,_a	)	)

                       shutil.rmtree(_a	)

                       # Save tokenizer rust, legacy_format=False
                       SCREAMING_SNAKE_CASE		=						tempfile.mkdtemp()

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.save_pretrained(_a						,legacy_format=_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.save_pretrained(_a	)

                       # Checks it saved the tokenizer.json file
                       self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files	)	)

                       # Checks everything loads correctly in the same way
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.from_pretrained(_a	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.from_pretrained(_a	)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_a						,_a	)	)

                       shutil.rmtree(_a	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__			(				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Any		     =      '''facebook/mbart-large-50-one-to-many-mmt'''
     __snake_case   :     Dict		     =      [
         ''' UN Chief Says There Is No Military Solution in Syria''',
         ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
     ]
     __snake_case   :     Union[str, Any]		     =      [
         '''Şeful ONU declară că nu există o soluţie militară în Siria''',
         '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
         ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
         ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
     ]
     __snake_case   :     Any		     =      [EN_CODE, 8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2]






     @classmethod
     def     SCREAMING_SNAKE_CASE__  (							cls						:   Optional[int]	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						MBartaaTokenizer.from_pretrained(
               cls.checkpoint_name						,src_lang="""en_XX"""						,tgt_lang="""ro_RO"""	)
           SCREAMING_SNAKE_CASE		=						1
           return cls






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Tuple:




           '''simple docstring'''


           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]						,250001	)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]						,250004	)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]						,250020	)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]						,250038	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
           self.assertListEqual(self.expected_src_tokens						,_a	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Optional[int]:




           '''simple docstring'''


           self.assertIn(_a						,self.tokenizer.all_special_ids	)
           SCREAMING_SNAKE_CASE		=						[RO_CODE, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2]
           SCREAMING_SNAKE_CASE		=						self.tokenizer.decode(_a						,skip_special_tokens=_a	)
           SCREAMING_SNAKE_CASE		=						self.tokenizer.decode(generated_ids[1:]						,skip_special_tokens=_a	)
           self.assertEqual(_a						,_a	)
           self.assertNotIn(self.tokenizer.eos_token						,_a	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						['this is gunna be a long sentence ' * 20]
           assert isinstance(src_text[0]						,_a	)
           SCREAMING_SNAKE_CASE		=						10
           SCREAMING_SNAKE_CASE		=						self.tokenizer(_a						,max_length=_a						,truncation=_a	).input_ids[0]
           self.assertEqual(ids[0]						,_a	)
           self.assertEqual(ids[-1]						,2	)
           self.assertEqual(len(_a	)						,_a	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]	)					->		str:




           '''simple docstring'''


           self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]	)						,[250053, 250001]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						tempfile.mkdtemp()
           SCREAMING_SNAKE_CASE		=						self.tokenizer.fairseq_tokens_to_ids
           self.tokenizer.save_pretrained(_a	)
           SCREAMING_SNAKE_CASE		=						MBartaaTokenizer.from_pretrained(_a	)
           self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,_a	)






     @require_torch
     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer(self.src_text						,text_target=self.tgt_text						,padding=_a						,return_tensors="""pt"""	)
           SCREAMING_SNAKE_CASE		=						shift_tokens_right(batch["""labels"""]						,self.tokenizer.pad_token_id	)

           # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
           assert batch.input_ids[1][0] == EN_CODE
           assert batch.input_ids[1][-1] == 2
           assert batch.labels[1][0] == RO_CODE
           assert batch.labels[1][-1] == 2
           assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]






     @require_torch
     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer(
               self.src_text						,text_target=self.tgt_text						,padding=_a						,truncation=_a						,max_length=len(self.expected_src_tokens	)						,return_tensors="""pt"""						,)
           SCREAMING_SNAKE_CASE		=						shift_tokens_right(batch["""labels"""]						,self.tokenizer.pad_token_id	)

           self.assertIsInstance(_a						,_a	)

           self.assertEqual((2, 14)						,batch.input_ids.shape	)
           self.assertEqual((2, 14)						,batch.attention_mask.shape	)
           SCREAMING_SNAKE_CASE		=						batch.input_ids.tolist()[0]
           self.assertListEqual(self.expected_src_tokens						,_a	)
           self.assertEqual(2						,batch.decoder_input_ids[0, 0]	)  # decoder_start_token_id
           # Test that special tokens are reset
           self.assertEqual(self.tokenizer.prefix_tokens						,[EN_CODE]	)
           self.assertEqual(self.tokenizer.suffix_tokens						,[self.tokenizer.eos_token_id]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer(self.src_text						,padding=_a						,truncation=_a						,max_length=3						,return_tensors="""pt"""	)
           SCREAMING_SNAKE_CASE		=						self.tokenizer(
               text_target=self.tgt_text						,padding=_a						,truncation=_a						,max_length=10						,return_tensors="""pt"""	)
           SCREAMING_SNAKE_CASE		=						targets['input_ids']
           SCREAMING_SNAKE_CASE		=						shift_tokens_right(_a						,self.tokenizer.pad_token_id	)

           self.assertEqual(batch.input_ids.shape[1]						,3	)
           self.assertEqual(batch.decoder_input_ids.shape[1]						,10	)






     @require_torch
     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer._build_translation_inputs(
               """A test"""						,return_tensors="""pt"""						,src_lang="""en_XX"""						,tgt_lang="""ar_AR"""	)

           self.assertEqual(
               nested_simplify(_a	)						,{
                   # en_XX, A, test, EOS
                   """input_ids""": [[250004, 62, 3034, 2]],
                   """attention_mask""": [[1, 1, 1, 1]],
                   # ar_AR
                   """forced_bos_token_id""": 250001,
               }						,)









def        __lowercase       (				_SCREAMING_SNAKE_CASE = 1			,						_SCREAMING_SNAKE_CASE = 10_00     )			->       int:


      '''simple docstring'''


      SCREAMING_SNAKE_CASE		=						1
      SCREAMING_SNAKE_CASE		=						0

      for divide_by_number in range(_SCREAMING_SNAKE_CASE			,						digit + 1     ):
            SCREAMING_SNAKE_CASE		=						[]
            SCREAMING_SNAKE_CASE		=						numerator
            for _ in range(1			,						digit + 1     ):
                  if now_divide in has_been_divided:
                        if longest_list_length < len(_SCREAMING_SNAKE_CASE     ):
                              SCREAMING_SNAKE_CASE		=						len(_SCREAMING_SNAKE_CASE     )
                              SCREAMING_SNAKE_CASE		=						divide_by_number
                  else:
                        has_been_divided.append(_SCREAMING_SNAKE_CASE     )
                        SCREAMING_SNAKE_CASE		=						now_divide * 10 % divide_by_number

      return the_digit


# Tests
if __name__ == "__main__":
      import doctest

      doctest.testmod()


