





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_           =      get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

snake_case_           =      256047
snake_case_           =      256145

@require_sentencepiece
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase						,				unittest.TestCase  ):
      A_						:    Optional[int]							=   NllbTokenizer
      A_						:    Union[str, Any]							=   NllbTokenizerFast
      A_						:    List[str]							=   True
      A_						:    Dict							=   True
      A_						:    Dict							=   {}


      def        a			(self	:				str				):





         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __snake_case								=       NllbTokenizer(a__    ,      keep_accents=a__				)
         tokenizer.save_pretrained(self.tmpdirname				)


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       NllbTokenizer(a__    ,      keep_accents=a__				)

         __snake_case								=       tokenizer.tokenize('''This is a test'''				)
         self.assertListEqual(a__    ,      ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(a__				)    ,      [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,      )

         __snake_case								=       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
         self.assertListEqual(
             a__    ,      [
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ]    ,      )
         __snake_case								=       tokenizer.convert_tokens_to_ids(a__				)
         self.assertListEqual(
             a__    ,      [
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]    ,      )

         __snake_case								=       tokenizer.convert_ids_to_tokens(a__				)
         self.assertListEqual(
             a__    ,      [
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ]    ,      )


      def        a			(self	:				str				):





         """simple docstring"""



         __snake_case								=       (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
               __snake_case								=       self.rust_tokenizer_class.from_pretrained(a__    ,      **a__				)
               __snake_case								=       self.tokenizer_class.from_pretrained(a__    ,      **a__				)

               __snake_case								=       tempfile.mkdtemp()

               __snake_case								=       tokenizer_r.save_pretrained(a__				)
               __snake_case								=       tokenizer_p.save_pretrained(a__				)

               # Checks it save with the same files + the tokenizer.json file for the fast one
               self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)
               __snake_case								=       tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f				)
               self.assertSequenceEqual(a__    ,      a__				)

               # Checks everything loads correctly in the same way
               __snake_case								=       tokenizer_r.from_pretrained(a__				)
               __snake_case								=       tokenizer_p.from_pretrained(a__				)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(a__    ,      a__				)				)

               shutil.rmtree(a__				)

               # Save tokenizer rust, legacy_format=True
               __snake_case								=       tempfile.mkdtemp()

               __snake_case								=       tokenizer_r.save_pretrained(a__    ,      legacy_format=a__				)
               __snake_case								=       tokenizer_p.save_pretrained(a__				)

               # Checks it save with the same files
               self.assertSequenceEqual(a__    ,      a__				)

               # Checks everything loads correctly in the same way
               __snake_case								=       tokenizer_r.from_pretrained(a__				)
               __snake_case								=       tokenizer_p.from_pretrained(a__				)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(a__    ,      a__				)				)

               shutil.rmtree(a__				)

               # Save tokenizer rust, legacy_format=False
               __snake_case								=       tempfile.mkdtemp()

               __snake_case								=       tokenizer_r.save_pretrained(a__    ,      legacy_format=a__				)
               __snake_case								=       tokenizer_p.save_pretrained(a__				)

               # Checks it saved the tokenizer.json file
               self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)

               # Checks everything loads correctly in the same way
               __snake_case								=       tokenizer_r.from_pretrained(a__				)
               __snake_case								=       tokenizer_p.from_pretrained(a__				)

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(a__    ,      a__				)				)

               shutil.rmtree(a__				)


      @require_torch
      def        a			(self	:				Any				):





         """simple docstring"""



         if not self.test_seqaseq:
            return

         __snake_case								=       self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
               # Longer text that will definitely require truncation.
               __snake_case								=       [
                   ''' UN Chief Says There Is No Military Solution in Syria''',
                   ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
                   ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
                   ''' will only worsen the violence and misery for millions of people.''',
               ]
               __snake_case								=       [
                   '''Şeful ONU declară că nu există o soluţie militară în Siria''',
                   '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
                   ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
                   ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
               ]
               try:
                  __snake_case								=       tokenizer.prepare_seqaseq_batch(
                      src_texts=a__    ,      tgt_texts=a__    ,      max_length=3    ,      max_target_length=10    ,      return_tensors='''pt'''    ,      src_lang='''eng_Latn'''    ,      tgt_lang='''ron_Latn'''    ,      )
               except NotImplementedError:
                  return
               self.assertEqual(batch.input_ids.shape[1]    ,      3				)
               self.assertEqual(batch.labels.shape[1]    ,      10				)
               # max_target_length will default to max_length if not specified
               __snake_case								=       tokenizer.prepare_seqaseq_batch(
                   a__    ,      tgt_texts=a__    ,      max_length=3    ,      return_tensors='''pt'''				)
               self.assertEqual(batch.input_ids.shape[1]    ,      3				)
               self.assertEqual(batch.labels.shape[1]    ,      3				)

               __snake_case								=       tokenizer.prepare_seqaseq_batch(
                   src_texts=a__    ,      max_length=3    ,      max_target_length=10    ,      return_tensors='''pt'''				)
               self.assertEqual(batch_encoder_only.input_ids.shape[1]    ,      3				)
               self.assertEqual(batch_encoder_only.attention_mask.shape[1]    ,      3				)
               self.assertNotIn('''decoder_input_ids'''    ,      a__				)


      @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''				)
      def        a			(self	:				Tuple				):





         """simple docstring"""



         pass


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
               __snake_case								=       [AddedToken('''<special>'''    ,      lstrip=a__				)]

               __snake_case								=       self.rust_tokenizer_class.from_pretrained(
                   a__    ,      additional_special_tokens=a__    ,      **a__				)
               __snake_case								=       tokenizer_r.encode('''Hey this is a <special> token'''				)

               __snake_case								=       tokenizer_r.encode('''<special>'''    ,      add_special_tokens=a__				)[0]

               self.assertTrue(special_token_id in r_output				)

               if self.test_slow_tokenizer:
                  __snake_case								=       self.rust_tokenizer_class.from_pretrained(
                      a__    ,      additional_special_tokens=a__    ,      **a__    ,      )
                  __snake_case								=       self.tokenizer_class.from_pretrained(
                      a__    ,      additional_special_tokens=a__    ,      **a__				)

                  __snake_case								=       tokenizer_p.encode('''Hey this is a <special> token'''				)

                  __snake_case								=       tokenizer_cr.encode('''Hey this is a <special> token'''				)

                  self.assertEqual(a__    ,      a__				)
                  self.assertEqual(a__    ,      a__				)
                  self.assertTrue(special_token_id in p_output				)
                  self.assertTrue(special_token_id in cr_output				)






@require_torch
@require_sentencepiece
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__    (	unittest.TestCase  ):
      A_						:    Any							=   'facebook/nllb-200-distilled-600M'
      A_						:    Union[str, Any]							=   [
          ' UN Chief Says There Is No Military Solution in Syria',
          ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
      ]
      A_						:    List[Any]							=   [
          'Şeful ONU declară că nu există o soluţie militară în Siria',
          'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
          ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
          ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
      ]
      A_						:    List[Any]							=   [
          256_047,
          16_297,
          134_408,
          8_165,
          248_066,
          14_734,
          950,
          1_135,
          105_721,
          3_573,
          83,
          27_352,
          108,
          49_486,
          2,
      ]


      @classmethod
      def        a			(cls	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       NllbTokenizer.from_pretrained(
             cls.checkpoint_name    ,      src_lang='''eng_Latn'''    ,      tgt_lang='''ron_Latn'''				)
         __snake_case								=       1
         return cls


      def        a			(self	:				int				):





         """simple docstring"""



         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']    ,      25_6001				)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']    ,      25_6002				)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']    ,      25_6057				)


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
         self.assertListEqual(self.expected_src_tokens    ,      a__				)


      def        a			(self	:				Tuple				):





         """simple docstring"""



         self.assertIn(a__    ,      self.tokenizer.all_special_ids				)
         # fmt: off
         __snake_case								=       [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
         # fmt: on

         __snake_case								=       self.tokenizer.decode(a__    ,      skip_special_tokens=a__				)
         __snake_case								=       self.tokenizer.decode(generated_ids[1:]    ,      skip_special_tokens=a__				)
         self.assertEqual(a__    ,      a__				)
         self.assertNotIn(self.tokenizer.eos_token    ,      a__				)


      def        a			(self	:				Any				):





         """simple docstring"""



         __snake_case								=       ['''this is gunna be a long sentence ''' * 20]
         assert isinstance(src_text[0]    ,      a__				)
         __snake_case								=       10
         __snake_case								=       self.tokenizer(a__    ,      max_length=a__    ,      truncation=a__				).input_ids[0]
         self.assertEqual(ids[-1]    ,      2				)
         self.assertEqual(ids[0]    ,      a__				)
         self.assertEqual(len(a__				)    ,      a__				)


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']				)    ,      [25_6203, 3]				)


      def        a			(self	:				int				):





         """simple docstring"""



         __snake_case								=       tempfile.mkdtemp()
         __snake_case								=       self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(a__				)
         __snake_case								=       NllbTokenizer.from_pretrained(a__				)
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,      a__				)


      @require_torch
      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       self.tokenizer(
             self.src_text    ,      text_target=self.tgt_text    ,      padding=a__    ,      truncation=a__    ,      max_length=len(self.expected_src_tokens				)    ,      return_tensors='''pt'''    ,      )
         __snake_case								=       shift_tokens_right(
             batch['''labels''']    ,      self.tokenizer.pad_token_id    ,      self.tokenizer.lang_code_to_id['''ron_Latn''']				)

         self.assertIsInstance(a__    ,      a__				)

         self.assertEqual((2, 15)    ,      batch.input_ids.shape				)
         self.assertEqual((2, 15)    ,      batch.attention_mask.shape				)
         __snake_case								=       batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens    ,      a__				)
         self.assertEqual(a__    ,      batch.decoder_input_ids[0, 0]				)  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens    ,      [EN_CODE]				)
         self.assertEqual(self.tokenizer.suffix_tokens    ,      [self.tokenizer.eos_token_id]				)


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       self.tokenizer(self.src_text    ,      padding=a__    ,      truncation=a__    ,      max_length=3    ,      return_tensors='''pt'''				)
         __snake_case								=       self.tokenizer(
             text_target=self.tgt_text    ,      padding=a__    ,      truncation=a__    ,      max_length=10    ,      return_tensors='''pt'''				)
         __snake_case								=       targets['''input_ids''']
         __snake_case								=       shift_tokens_right(
             a__    ,      self.tokenizer.pad_token_id    ,      decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]    ,      )

         self.assertEqual(batch.input_ids.shape[1]    ,      3				)
         self.assertEqual(batch.decoder_input_ids.shape[1]    ,      10				)


      @require_torch
      def        a			(self	:				Any				):





         """simple docstring"""



         __snake_case								=       self.tokenizer._build_translation_inputs(
             '''A test'''    ,      return_tensors='''pt'''    ,      src_lang='''eng_Latn'''    ,      tgt_lang='''fra_Latn'''				)

         self.assertEqual(
             nested_simplify(a__				)    ,      {
                 # A, test, EOS, en_XX
                 '''input_ids''': [[25_6047, 70, 7356, 2]],
                 '''attention_mask''': [[1, 1, 1, 1]],
                 # ar_AR
                 '''forced_bos_token_id''': 25_6057,
             }    ,      )


      @require_torch
      def        a			(self	:				Any				):





         """simple docstring"""



         __snake_case								=       True
         __snake_case								=       self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''    ,      src_lang='''eng_Latn'''    ,      tgt_lang='''fra_Latn'''				)
         self.assertEqual(
             inputs.input_ids    ,      [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]				)

         __snake_case								=       False
         __snake_case								=       self.tokenizer(
             '''UN Chief says there is no military solution in Syria'''    ,      src_lang='''eng_Latn'''    ,      tgt_lang='''fra_Latn'''				)
         self.assertEqual(
             inputs.input_ids    ,      [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]				)








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_           =      logging.get_logger(__name__)

snake_case_           =      {
    'asapp/sew-d-tiny-100k': 'https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    str							=   'sew-d'


      def __init__(self	:				str    ,      a__	:				List[Any]=32    ,      a__	:				Optional[int]=768    ,      a__	:				Union[str, Any]=12    ,      a__	:				Tuple=12    ,      a__	:				int=3072    ,      a__	:				List[Any]=2    ,      a__	:				Dict=512    ,      a__	:				Dict=256    ,      a__	:				List[str]=True    ,      a__	:				List[str]=True    ,      a__	:				str=("p2c", "c2p")    ,      a__	:				str="layer_norm"    ,      a__	:				str="gelu_python"    ,      a__	:				List[Any]=0.1    ,      a__	:				Dict=0.1    ,      a__	:				Optional[Any]=0.1    ,      a__	:				Union[str, Any]=0.0    ,      a__	:				Dict=0.1    ,      a__	:				int=0.0_2    ,      a__	:				str=1E-7    ,      a__	:				Union[str, Any]=1E-5    ,      a__	:				Union[str, Any]="group"    ,      a__	:				Optional[Any]="gelu"    ,      a__	:				List[Any]=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)    ,      a__	:				Union[str, Any]=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)    ,      a__	:				Optional[int]=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)    ,      a__	:				List[str]=False    ,      a__	:				Optional[Any]=128    ,      a__	:				Union[str, Any]=16    ,      a__	:				Optional[Any]=True    ,      a__	:				List[Any]=0.0_5    ,      a__	:				Tuple=10    ,      a__	:				str=2    ,      a__	:				int=0.0    ,      a__	:				Optional[Any]=10    ,      a__	:				List[Any]=0    ,      a__	:				Optional[int]="mean"    ,      a__	:				List[str]=False    ,      a__	:				Tuple=False    ,      a__	:				Optional[int]=256    ,      a__	:				str=0    ,      a__	:				List[Any]=1    ,      a__	:				int=2    ,      **a__	:				List[str]    ,      ):





         """simple docstring"""



         super().__init__(**a__    ,      pad_token_id=a__    ,      bos_token_id=a__    ,      eos_token_id=a__				)
         __snake_case								=       hidden_size
         __snake_case								=       feat_extract_norm
         __snake_case								=       feat_extract_activation
         __snake_case								=       list(a__				)
         __snake_case								=       list(a__				)
         __snake_case								=       list(a__				)
         __snake_case								=       conv_bias
         __snake_case								=       num_conv_pos_embeddings
         __snake_case								=       num_conv_pos_embedding_groups
         __snake_case								=       len(self.conv_dim				)
         __snake_case								=       num_hidden_layers
         __snake_case								=       intermediate_size
         __snake_case								=       squeeze_factor
         __snake_case								=       max_position_embeddings
         __snake_case								=       position_buckets
         __snake_case								=       share_att_key
         __snake_case								=       relative_attention
         __snake_case								=       norm_rel_ebd
         __snake_case								=       list(a__				)
         __snake_case								=       hidden_act
         __snake_case								=       num_attention_heads
         __snake_case								=       hidden_dropout
         __snake_case								=       attention_dropout
         __snake_case								=       activation_dropout
         __snake_case								=       feat_proj_dropout
         __snake_case								=       final_dropout
         __snake_case								=       layer_norm_eps
         __snake_case								=       feature_layer_norm_eps
         __snake_case								=       initializer_range
         __snake_case								=       vocab_size

         if (
             (len(self.conv_stride				) != self.num_feat_extract_layers)
             or (len(self.conv_kernel				) != self.num_feat_extract_layers)
             or (len(self.conv_dim				) != self.num_feat_extract_layers)
         ):
            raise ValueError(
                '''Configuration for convolutional layers is incorrect.'''
                '''It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,'''
                f"""but is `len(config.conv_dim) = {len(self.conv_dim				)}`, `len(config.conv_stride)"""
                f"""= {len(self.conv_stride				)}`, `len(config.conv_kernel) = {len(self.conv_kernel				)}`."""				)

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __snake_case								=       apply_spec_augment
         __snake_case								=       mask_time_prob
         __snake_case								=       mask_time_length
         __snake_case								=       mask_time_min_masks
         __snake_case								=       mask_feature_prob
         __snake_case								=       mask_feature_length
         __snake_case								=       mask_feature_min_masks

         # ctc loss
         __snake_case								=       ctc_loss_reduction
         __snake_case								=       ctc_zero_infinity

         # sequence classification
         __snake_case								=       use_weighted_layer_sum
         __snake_case								=       classifier_proj_size


      @property
      def        a			(self	:				List[Any]				):





         """simple docstring"""



         return functools.reduce(operator.mul    ,      self.conv_stride    ,      1				)

