





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case           =			get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_snake_case           =			25_6047
_snake_case           =			25_6145





@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase							(       lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      NllbTokenizer
  __lowerCamelCase		=      NllbTokenizerFast
  __lowerCamelCase		=      True
  __lowerCamelCase		=      True
  __lowerCamelCase		=      {}







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__			       =  NllbTokenizer(_lowercase   ,							keep_accents=_lowercase )
     tokenizer.save_pretrained(self.tmpdirname )







  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  NllbTokenizer(_lowercase   ,							keep_accents=_lowercase )

     lowercase__			       =  tokenizer.tokenize("This is a test" )
     self.assertListEqual(_lowercase   ,							["▁This", "▁is", "▁a", "▁t", "est"] )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_lowercase )   ,							[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]   ,							)

     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "é",
             ".",
         ]   ,							)
     lowercase__			       =  tokenizer.convert_tokens_to_ids(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[
             value + tokenizer.fairseq_offset
             for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
         ]   ,							)

     lowercase__			       =  tokenizer.convert_ids_to_tokens(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "<unk>",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "<unk>",
             ".",
         ]   ,							)







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
           lowercase__			       =  self.rust_tokenizer_class.from_pretrained(_lowercase   ,							**_lowercase )
           lowercase__			       =  self.tokenizer_class.from_pretrained(_lowercase   ,							**_lowercase )

           lowercase__			       =  tempfile.mkdtemp()

           lowercase__			       =  tokenizer_r.save_pretrained(_lowercase )
           lowercase__			       =  tokenizer_p.save_pretrained(_lowercase )

           # Checks it save with the same files + the tokenizer.json file for the fast one
           self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )
           lowercase__			       =  tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f )
           self.assertSequenceEqual(_lowercase   ,							_lowercase )

           # Checks everything loads correctly in the same way
           lowercase__			       =  tokenizer_r.from_pretrained(_lowercase )
           lowercase__			       =  tokenizer_p.from_pretrained(_lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(_lowercase   ,							_lowercase ) )

           shutil.rmtree(_lowercase )

           # Save tokenizer rust, legacy_format=True
           lowercase__			       =  tempfile.mkdtemp()

           lowercase__			       =  tokenizer_r.save_pretrained(_lowercase   ,							legacy_format=_lowercase )
           lowercase__			       =  tokenizer_p.save_pretrained(_lowercase )

           # Checks it save with the same files
           self.assertSequenceEqual(_lowercase   ,							_lowercase )

           # Checks everything loads correctly in the same way
           lowercase__			       =  tokenizer_r.from_pretrained(_lowercase )
           lowercase__			       =  tokenizer_p.from_pretrained(_lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(_lowercase   ,							_lowercase ) )

           shutil.rmtree(_lowercase )

           # Save tokenizer rust, legacy_format=False
           lowercase__			       =  tempfile.mkdtemp()

           lowercase__			       =  tokenizer_r.save_pretrained(_lowercase   ,							legacy_format=_lowercase )
           lowercase__			       =  tokenizer_p.save_pretrained(_lowercase )

           # Checks it saved the tokenizer.json file
           self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )

           # Checks everything loads correctly in the same way
           lowercase__			       =  tokenizer_r.from_pretrained(_lowercase )
           lowercase__			       =  tokenizer_p.from_pretrained(_lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(_lowercase   ,							_lowercase ) )

           shutil.rmtree(_lowercase )







  @require_torch
  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     if not self.test_seqaseq:
        return

     lowercase__			       =  self.get_tokenizers()
     for tokenizer in tokenizers:
        with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           # Longer text that will definitely require truncation.
           lowercase__			       =  [
               " UN Chief Says There Is No Military Solution in Syria",
               " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
               " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
               " will only worsen the violence and misery for millions of people.",
           ]
           lowercase__			       =  [
               "Şeful ONU declară că nu există o soluţie militară în Siria",
               "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
               " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
               " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
           ]
           try:
              lowercase__			       =  tokenizer.prepare_seqaseq_batch(
                  src_texts=_lowercase   ,							tgt_texts=_lowercase   ,							max_length=3   ,							max_target_length=10   ,							return_tensors="pt"   ,							src_lang="eng_Latn"   ,							tgt_lang="ron_Latn"   ,							)
           except NotImplementedError:
              return
           self.assertEqual(batch.input_ids.shape[1]   ,							3 )
           self.assertEqual(batch.labels.shape[1]   ,							10 )
           # max_target_length will default to max_length if not specified
           lowercase__			       =  tokenizer.prepare_seqaseq_batch(
               _lowercase   ,							tgt_texts=_lowercase   ,							max_length=3   ,							return_tensors="pt" )
           self.assertEqual(batch.input_ids.shape[1]   ,							3 )
           self.assertEqual(batch.labels.shape[1]   ,							3 )

           lowercase__			       =  tokenizer.prepare_seqaseq_batch(
               src_texts=_lowercase   ,							max_length=3   ,							max_target_length=10   ,							return_tensors="pt" )
           self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,							3 )
           self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,							3 )
           self.assertNotIn("decoder_input_ids"   ,							_lowercase )







  @unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece." )
  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     pass







  def   UpperCAmelCase					(      self  :Tuple ):






     '''simple docstring'''





     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
           lowercase__			       =  [AddedToken("<special>"   ,							lstrip=_lowercase )]

           lowercase__			       =  self.rust_tokenizer_class.from_pretrained(
               _lowercase   ,							additional_special_tokens=_lowercase   ,							**_lowercase )
           lowercase__			       =  tokenizer_r.encode("Hey this is a <special> token" )

           lowercase__			       =  tokenizer_r.encode("<special>"   ,							add_special_tokens=_lowercase )[0]

           self.assertTrue(special_token_id in r_output )

           if self.test_slow_tokenizer:
              lowercase__			       =  self.rust_tokenizer_class.from_pretrained(
                  _lowercase   ,							additional_special_tokens=_lowercase   ,							**_lowercase   ,							)
              lowercase__			       =  self.tokenizer_class.from_pretrained(
                  _lowercase   ,							additional_special_tokens=_lowercase   ,							**_lowercase )

              lowercase__			       =  tokenizer_p.encode("Hey this is a <special> token" )

              lowercase__			       =  tokenizer_cr.encode("Hey this is a <special> token" )

              self.assertEqual(_lowercase   ,							_lowercase )
              self.assertEqual(_lowercase   ,							_lowercase )
              self.assertTrue(special_token_id in p_output )
              self.assertTrue(special_token_id in cr_output )





@require_torch
@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase							(       unittest.TestCase ):
  __lowerCamelCase		=      'facebook/nllb-200-distilled-600M'
  __lowerCamelCase		=      [
      ' UN Chief Says There Is No Military Solution in Syria',
      ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
  ]
  __lowerCamelCase		=      [
      'Şeful ONU declară că nu există o soluţie militară în Siria',
      'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
      ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
      ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
  ]
  __lowerCamelCase		=      [
      256_047,
      16_297,
      134_408,
      8_165,
      248_066,
      14_734,
      950,
      1_135,
      105_721,
      3_573,
      83,
      27_352,
      108,
      49_486,
      2,
  ]







  @classmethod
  def   UpperCAmelCase					(      cls  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  NllbTokenizer.from_pretrained(
         cls.checkpoint_name   ,							src_lang="eng_Latn"   ,							tgt_lang="ron_Latn" )
     lowercase__			       =  1
     return cls







  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]   ,							25_60_01 )
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]   ,							25_60_02 )
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]   ,							25_60_57 )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
     self.assertListEqual(self.expected_src_tokens   ,							_lowercase )







  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     self.assertIn(_lowercase   ,							self.tokenizer.all_special_ids )
     # fmt: off
     lowercase__			       =  [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
     # fmt: on

     lowercase__			       =  self.tokenizer.decode(_lowercase   ,							skip_special_tokens=_lowercase )
     lowercase__			       =  self.tokenizer.decode(generated_ids[1:]   ,							skip_special_tokens=_lowercase )
     self.assertEqual(_lowercase   ,							_lowercase )
     self.assertNotIn(self.tokenizer.eos_token   ,							_lowercase )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     lowercase__			       =  ["this is gunna be a long sentence " * 20]
     assert isinstance(src_text[0]   ,							_lowercase )
     lowercase__			       =  10
     lowercase__			       =  self.tokenizer(_lowercase   ,							max_length=_lowercase   ,							truncation=_lowercase ).input_ids[0]
     self.assertEqual(ids[-1]   ,							2 )
     self.assertEqual(ids[0]   ,							_lowercase )
     self.assertEqual(len(_lowercase )   ,							_lowercase )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"] )   ,							[25_62_03, 3] )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  tempfile.mkdtemp()
     lowercase__			       =  self.tokenizer.fairseq_tokens_to_ids
     self.tokenizer.save_pretrained(_lowercase )
     lowercase__			       =  NllbTokenizer.from_pretrained(_lowercase )
     self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,							_lowercase )







  @require_torch
  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  self.tokenizer(
         self.src_text   ,							text_target=self.tgt_text   ,							padding=_lowercase   ,							truncation=_lowercase   ,							max_length=len(self.expected_src_tokens )   ,							return_tensors="pt"   ,							)
     lowercase__			       =  shift_tokens_right(
         batch["labels"]   ,							self.tokenizer.pad_token_id   ,							self.tokenizer.lang_code_to_id["ron_Latn"] )

     self.assertIsInstance(_lowercase   ,							_lowercase )

     self.assertEqual((2, 15)   ,							batch.input_ids.shape )
     self.assertEqual((2, 15)   ,							batch.attention_mask.shape )
     lowercase__			       =  batch.input_ids.tolist()[0]
     self.assertListEqual(self.expected_src_tokens   ,							_lowercase )
     self.assertEqual(_lowercase   ,							batch.decoder_input_ids[0, 0] )  # EOS
     # Test that special tokens are reset
     self.assertEqual(self.tokenizer.prefix_tokens   ,							[EN_CODE] )
     self.assertEqual(self.tokenizer.suffix_tokens   ,							[self.tokenizer.eos_token_id] )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  self.tokenizer(self.src_text   ,							padding=_lowercase   ,							truncation=_lowercase   ,							max_length=3   ,							return_tensors="pt" )
     lowercase__			       =  self.tokenizer(
         text_target=self.tgt_text   ,							padding=_lowercase   ,							truncation=_lowercase   ,							max_length=10   ,							return_tensors="pt" )
     lowercase__			       =  targets["input_ids"]
     lowercase__			       =  shift_tokens_right(
         _lowercase   ,							self.tokenizer.pad_token_id   ,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,							)

     self.assertEqual(batch.input_ids.shape[1]   ,							3 )
     self.assertEqual(batch.decoder_input_ids.shape[1]   ,							10 )







  @require_torch
  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  self.tokenizer._build_translation_inputs(
         "A test"   ,							return_tensors="pt"   ,							src_lang="eng_Latn"   ,							tgt_lang="fra_Latn" )

     self.assertEqual(
         nested_simplify(_lowercase )   ,							{
             # A, test, EOS, en_XX
             "input_ids": [[25_60_47, 70, 73_56, 2]],
             "attention_mask": [[1, 1, 1, 1]],
             # ar_AR
             "forced_bos_token_id": 25_60_57,
         }   ,							)







  @require_torch
  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  True
     lowercase__			       =  self.tokenizer(
         "UN Chief says there is no military solution in Syria"   ,							src_lang="eng_Latn"   ,							tgt_lang="fra_Latn" )
     self.assertEqual(
         inputs.input_ids   ,							[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47] )

     lowercase__			       =  False
     lowercase__			       =  self.tokenizer(
         "UN Chief says there is no military solution in Syria"   ,							src_lang="eng_Latn"   ,							tgt_lang="fra_Latn" )
     self.assertEqual(
         inputs.input_ids   ,							[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2] )













from itertools import count


def _A       (   __magic_name__ = 50	):
   lowercase__			       =  [1] * min_block_length

   for n in count(__magic_name__	):
      fill_count_functions.append(1	)

      for block_length in range(__magic_name__  , n + 1	):
         for block_start in range(n - block_length	):
            fill_count_functions[n] += fill_count_functions[
                n - block_start - block_length - 1
            ]

         fill_count_functions[n] += 1

      if fill_count_functions[n] > 100_0000:
         break

   return n


if __name__ == "__main__":
  print(F"""{solution() = }""")






