





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase				:	Optional[int]      						=  get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase				:	str      						=  2_5_6_0_4_7
lowercase				:	Tuple      						=  2_5_6_1_4_5





@require_sentencepiece
@require_tokenizers
class 			A__   (							__UpperCAmelCase			,  unittest.TestCase     ):




   """simple docstring"""



   __A       :    str	  =  NllbTokenizer
   __A       :    Optional[int]	  =  NllbTokenizerFast
   __A       :    Optional[int]	  =  True
   __A       :    str	  =  True
   __A       :    Tuple	  =  {}

   def 							__lowercase		( self)      ->						List[str]:







          '''simple docstring'''

          super().setUp()

          # We have a SentencePiece fixture for testing
          a__						:     List[Any]			=    NllbTokenizer(lowercase ,  keep_accents=lowercase)
          tokenizer.save_pretrained(self.tmpdirname)

   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     int			=    NllbTokenizer(lowercase ,  keep_accents=lowercase)

          a__						:     List[str]			=    tokenizer.tokenize('This is a test')
          self.assertListEqual(lowercase ,  ['▁This', '▁is', '▁a', '▁t', 'est'])

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(lowercase) ,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,  )

          a__						:     Tuple			=    tokenizer.tokenize('I was born in 92000, and this is falsé.')
          self.assertListEqual(
              lowercase ,  [
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ] ,  )
          a__						:     Any			=    tokenizer.convert_tokens_to_ids(lowercase)
          self.assertListEqual(
              lowercase ,  [
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ] ,  )

          a__						:     int			=    tokenizer.convert_ids_to_tokens(lowercase)
          self.assertListEqual(
              lowercase ,  [
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '<unk>',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '<unk>',
                  '.',
              ] ,  )

   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          a__						:     Optional[int]			=    (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                        a__						:     Optional[int]			=    self.rust_tokenizer_class.from_pretrained(lowercase ,  **lowercase)
                        a__						:     Tuple			=    self.tokenizer_class.from_pretrained(lowercase ,  **lowercase)

                        a__						:     List[Any]			=    tempfile.mkdtemp()

                        a__						:     int			=    tokenizer_r.save_pretrained(lowercase)
                        a__						:     Any			=    tokenizer_p.save_pretrained(lowercase)

                        # Checks it save with the same files + the tokenizer.json file for the fast one
                        self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))
                        a__						:     List[str]			=    tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f)
                        self.assertSequenceEqual(lowercase ,  lowercase)

                        # Checks everything loads correctly in the same way
                        a__						:     Any			=    tokenizer_r.from_pretrained(lowercase)
                        a__						:     Optional[int]			=    tokenizer_p.from_pretrained(lowercase)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(lowercase ,  lowercase))

                        shutil.rmtree(lowercase)

                        # Save tokenizer rust, legacy_format=True
                        a__						:     Optional[int]			=    tempfile.mkdtemp()

                        a__						:     Tuple			=    tokenizer_r.save_pretrained(lowercase ,  legacy_format=lowercase)
                        a__						:     Tuple			=    tokenizer_p.save_pretrained(lowercase)

                        # Checks it save with the same files
                        self.assertSequenceEqual(lowercase ,  lowercase)

                        # Checks everything loads correctly in the same way
                        a__						:     Dict			=    tokenizer_r.from_pretrained(lowercase)
                        a__						:     Optional[int]			=    tokenizer_p.from_pretrained(lowercase)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(lowercase ,  lowercase))

                        shutil.rmtree(lowercase)

                        # Save tokenizer rust, legacy_format=False
                        a__						:     int			=    tempfile.mkdtemp()

                        a__						:     Any			=    tokenizer_r.save_pretrained(lowercase ,  legacy_format=lowercase)
                        a__						:     Union[str, Any]			=    tokenizer_p.save_pretrained(lowercase)

                        # Checks it saved the tokenizer.json file
                        self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))

                        # Checks everything loads correctly in the same way
                        a__						:     List[str]			=    tokenizer_r.from_pretrained(lowercase)
                        a__						:     Dict			=    tokenizer_p.from_pretrained(lowercase)

                        # Check special tokens are set accordingly on Rust and Python
                        for key in tokenizer_pp.special_tokens_map:
                               self.assertTrue(hasattr(lowercase ,  lowercase))

                        shutil.rmtree(lowercase)

   @require_torch
   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          if not self.test_seqaseq:
                 return

          a__						:     List[str]			=    self.get_tokenizers()
          for tokenizer in tokenizers:
                 with self.subTest(F'{tokenizer.__class__.__name__}'):
                        # Longer text that will definitely require truncation.
                        a__						:     Optional[int]			=    [
                            ' UN Chief Says There Is No Military Solution in Syria',
                            ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                            ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                            ' will only worsen the violence and misery for millions of people.',
                        ]
                        a__						:     Tuple			=    [
                            'Şeful ONU declară că nu există o soluţie militară în Siria',
                            'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                            ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                            ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                        ]
                        try:
                               a__						:     Union[str, Any]			=    tokenizer.prepare_seqaseq_batch(
                                   src_texts=lowercase ,  tgt_texts=lowercase ,  max_length=3 ,  max_target_length=10 ,  return_tensors='pt' ,  src_lang='eng_Latn' ,  tgt_lang='ron_Latn' ,  )
                        except NotImplementedError:
                               return
                        self.assertEqual(batch.input_ids.shape[1] ,  3)
                        self.assertEqual(batch.labels.shape[1] ,  10)
                        # max_target_length will default to max_length if not specified
                        a__						:     Optional[Any]			=    tokenizer.prepare_seqaseq_batch(
                            lowercase ,  tgt_texts=lowercase ,  max_length=3 ,  return_tensors='pt')
                        self.assertEqual(batch.input_ids.shape[1] ,  3)
                        self.assertEqual(batch.labels.shape[1] ,  3)

                        a__						:     Optional[int]			=    tokenizer.prepare_seqaseq_batch(
                            src_texts=lowercase ,  max_length=3 ,  max_target_length=10 ,  return_tensors='pt')
                        self.assertEqual(batch_encoder_only.input_ids.shape[1] ,  3)
                        self.assertEqual(batch_encoder_only.attention_mask.shape[1] ,  3)
                        self.assertNotIn('decoder_input_ids' ,  lowercase)

   @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.')
   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          pass

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                        a__						:     int			=    [AddedToken('<special>' ,  lstrip=lowercase)]

                        a__						:     Dict			=    self.rust_tokenizer_class.from_pretrained(
                            lowercase ,  additional_special_tokens=lowercase ,  **lowercase)
                        a__						:     Union[str, Any]			=    tokenizer_r.encode('Hey this is a <special> token')

                        a__						:     Optional[int]			=    tokenizer_r.encode('<special>' ,  add_special_tokens=lowercase)[0]

                        self.assertTrue(special_token_id in r_output)

                        if self.test_slow_tokenizer:
                               a__						:     Optional[int]			=    self.rust_tokenizer_class.from_pretrained(
                                   lowercase ,  additional_special_tokens=lowercase ,  **lowercase ,  )
                               a__						:     Any			=    self.tokenizer_class.from_pretrained(
                                   lowercase ,  additional_special_tokens=lowercase ,  **lowercase)

                               a__						:     Dict			=    tokenizer_p.encode('Hey this is a <special> token')

                               a__						:     Any			=    tokenizer_cr.encode('Hey this is a <special> token')

                               self.assertEqual(lowercase ,  lowercase)
                               self.assertEqual(lowercase ,  lowercase)
                               self.assertTrue(special_token_id in p_output)
                               self.assertTrue(special_token_id in cr_output)





@require_torch
@require_sentencepiece
@require_tokenizers
class 			A__   (							unittest.TestCase     ):




   """simple docstring"""



   __A       :    Any	  =  '''facebook/nllb-200-distilled-600M'''
   __A       :    Optional[int]	  =  [
       ''' UN Chief Says There Is No Military Solution in Syria''',
       ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
   ]
   __A       :    List[Any]	  =  [
       '''Şeful ONU declară că nu există o soluţie militară în Siria''',
       '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
       ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
       ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
   ]
   __A       :    Union[str, Any]	  =  [
       2_5_6_0_4_7,
       1_6_2_9_7,
       1_3_4_4_0_8,
       8_1_6_5,
       2_4_8_0_6_6,
       1_4_7_3_4,
       9_5_0,
       1_1_3_5,
       1_0_5_7_2_1,
       3_5_7_3,
       8_3,
       2_7_3_5_2,
       1_0_8,
       4_9_4_8_6,
       2,
   ]

   @classmethod
   def 							__lowercase		( cls)      ->						Tuple:







          '''simple docstring'''

          a__						:     NllbTokenizer			=    NllbTokenizer.from_pretrained(
              cls.checkpoint_name ,  src_lang='eng_Latn' ,  tgt_lang='ron_Latn')
          a__						:     List[str]			=    1
          return cls

   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab'] ,  25_6001)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn'] ,  25_6002)
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn'] ,  25_6057)

   def 							__lowercase		( self)      ->						int:







          '''simple docstring'''

          a__						:     str			=    self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
          self.assertListEqual(self.expected_src_tokens ,  lowercase)

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          self.assertIn(lowercase ,  self.tokenizer.all_special_ids)
          # fmt: off
          a__						:     List[Any]			=    [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
          # fmt: on

          a__						:     List[str]			=    self.tokenizer.decode(lowercase ,  skip_special_tokens=lowercase)
          a__						:     List[Any]			=    self.tokenizer.decode(generated_ids[1:] ,  skip_special_tokens=lowercase)
          self.assertEqual(lowercase ,  lowercase)
          self.assertNotIn(self.tokenizer.eos_token ,  lowercase)

   def 							__lowercase		( self)      ->						Dict:







          '''simple docstring'''

          a__						:     int			=    ['this is gunna be a long sentence ' * 20]
          assert isinstance(src_text[0] ,  lowercase)
          a__						:     List[str]			=    10
          a__						:     int			=    self.tokenizer(lowercase ,  max_length=lowercase ,  truncation=lowercase).input_ids[0]
          self.assertEqual(ids[-1] ,  2)
          self.assertEqual(ids[0] ,  lowercase)
          self.assertEqual(len(lowercase) ,  lowercase)

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']) ,  [25_6203, 3])

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     Any			=    tempfile.mkdtemp()
          a__						:     str			=    self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(lowercase)
          a__						:     Optional[Any]			=    NllbTokenizer.from_pretrained(lowercase)
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,  lowercase)

   @require_torch
   def 							__lowercase		( self)      ->						List[str]:







          '''simple docstring'''

          a__						:     List[str]			=    self.tokenizer(
              self.src_text ,  text_target=self.tgt_text ,  padding=lowercase ,  truncation=lowercase ,  max_length=len(self.expected_src_tokens) ,  return_tensors='pt' ,  )
          a__						:     Any			=    shift_tokens_right(
              batch['labels'] ,  self.tokenizer.pad_token_id ,  self.tokenizer.lang_code_to_id['ron_Latn'])

          self.assertIsInstance(lowercase ,  lowercase)

          self.assertEqual((2, 15) ,  batch.input_ids.shape)
          self.assertEqual((2, 15) ,  batch.attention_mask.shape)
          a__						:     Optional[Any]			=    batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens ,  lowercase)
          self.assertEqual(lowercase ,  batch.decoder_input_ids[0, 0])  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens ,  [EN_CODE])
          self.assertEqual(self.tokenizer.suffix_tokens ,  [self.tokenizer.eos_token_id])

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    self.tokenizer(self.src_text ,  padding=lowercase ,  truncation=lowercase ,  max_length=3 ,  return_tensors='pt')
          a__						:     Optional[int]			=    self.tokenizer(
              text_target=self.tgt_text ,  padding=lowercase ,  truncation=lowercase ,  max_length=10 ,  return_tensors='pt')
          a__						:     Dict			=    targets['input_ids']
          a__						:     str			=    shift_tokens_right(
              lowercase ,  self.tokenizer.pad_token_id ,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang] ,  )

          self.assertEqual(batch.input_ids.shape[1] ,  3)
          self.assertEqual(batch.decoder_input_ids.shape[1] ,  10)

   @require_torch
   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          a__						:     Optional[int]			=    self.tokenizer._build_translation_inputs(
              'A test' ,  return_tensors='pt' ,  src_lang='eng_Latn' ,  tgt_lang='fra_Latn')

          self.assertEqual(
              nested_simplify(lowercase) ,  {
                  # A, test, EOS, en_XX
                  'input_ids': [[25_6047, 70, 7356, 2]],
                  'attention_mask': [[1, 1, 1, 1]],
                  # ar_AR
                  'forced_bos_token_id': 25_6057,
              } ,  )

   @require_torch
   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          a__						:     Optional[int]			=    True
          a__						:     Union[str, Any]			=    self.tokenizer(
              'UN Chief says there is no military solution in Syria' ,  src_lang='eng_Latn' ,  tgt_lang='fra_Latn')
          self.assertEqual(
              inputs.input_ids ,  [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047])

          a__						:     str			=    False
          a__						:     List[str]			=    self.tokenizer(
              'UN Chief says there is no military solution in Syria' ,  src_lang='eng_Latn' ,  tgt_lang='fra_Latn')
          self.assertEqual(
              inputs.input_ids ,  [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2])









import enum
import shutil
import sys


lowercase						, lowercase				:	List[Any]      						=  shutil.get_terminal_size()

lowercase				:	Union[str, Any]      						=  {"""UP""": """A""", """DOWN""": """B""", """RIGHT""": """C""", """LEFT""": """D"""}





class 			A__   (							enum.Enum     ):




   """simple docstring"""



   __A       :    List[str]	  =  0
   __A       :    str	  =  1



def A_     (		A__		,				A__=""    )  ->		int:
       sys.stdout.write(str(A__    ) + end    )
       sys.stdout.flush()



def A_     (		A__		,				A__		,				A__=""    )  ->		int:
       forceWrite(F'\u001b[{color}m{content}\u001b[0m'		,				A__    )



def A_     (		)  ->		Any:
       forceWrite('\r'    )



def A_     (		A__		,				A__    )  ->		List[str]:
       forceWrite(F'\033[{num_lines}{CURSOR_TO_CHAR[direction.upper()]}'    )



def A_     (		)  ->		Any:
       forceWrite(' ' * TERMINAL_WIDTH    )
       reset_cursor()



def A_     (		)  ->		Any:
       reset_cursor()
       forceWrite('-' * TERMINAL_WIDTH    )


