

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase		:       Union[str, Any]						    =	get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
     from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_UpperCAmelCase		:       Tuple						    =	25_6047
_UpperCAmelCase		:       Optional[Any]						    =	25_6145







@require_sentencepiece
@require_tokenizers
class 			lowercase    (	__snake_case     ,  unittest.TestCase      ):
    __SCREAMING_SNAKE_CASE :			Tuple	      =      NllbTokenizer
    __SCREAMING_SNAKE_CASE :			Any	      =      NllbTokenizerFast
    __SCREAMING_SNAKE_CASE :			Optional[Any]	      =      True
    __SCREAMING_SNAKE_CASE :			Tuple	      =      True
    __SCREAMING_SNAKE_CASE :			List[Any]	      =      {}




    def    a							( self	):
           super().setUp()

           # We have a SentencePiece fixture for testing
           snake_case_          =   NllbTokenizer(lowerCamelCase_				,       keep_accents=lowerCamelCase_	)
           tokenizer.save_pretrained(self.tmpdirname	)




    def    a							( self	):
           snake_case_          =   NllbTokenizer(lowerCamelCase_				,       keep_accents=lowerCamelCase_	)

           snake_case_          =   tokenizer.tokenize('This is a test'	)
           self.assertListEqual(lowerCamelCase_				,       ['▁This', '▁is', '▁a', '▁t', 'est']	)

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(lowerCamelCase_	)				,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,       )

           snake_case_          =   tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
           self.assertListEqual(
               lowerCamelCase_				,       [
                   SPIECE_UNDERLINE + 'I',
                   SPIECE_UNDERLINE + 'was',
                   SPIECE_UNDERLINE + 'b',
                   'or',
                   'n',
                   SPIECE_UNDERLINE + 'in',
                   SPIECE_UNDERLINE + '',
                   '9',
                   '2',
                   '0',
                   '0',
                   '0',
                   ',',
                   SPIECE_UNDERLINE + 'and',
                   SPIECE_UNDERLINE + 'this',
                   SPIECE_UNDERLINE + 'is',
                   SPIECE_UNDERLINE + 'f',
                   'al',
                   's',
                   'é',
                   '.',
               ]				,       )
           snake_case_          =   tokenizer.convert_tokens_to_ids(lowerCamelCase_	)
           self.assertListEqual(
               lowerCamelCase_				,       [
                   value + tokenizer.fairseq_offset
                   for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
               ]				,       )

           snake_case_          =   tokenizer.convert_ids_to_tokens(lowerCamelCase_	)
           self.assertListEqual(
               lowerCamelCase_				,       [
                   SPIECE_UNDERLINE + 'I',
                   SPIECE_UNDERLINE + 'was',
                   SPIECE_UNDERLINE + 'b',
                   'or',
                   'n',
                   SPIECE_UNDERLINE + 'in',
                   SPIECE_UNDERLINE + '',
                   '<unk>',
                   '2',
                   '0',
                   '0',
                   '0',
                   ',',
                   SPIECE_UNDERLINE + 'and',
                   SPIECE_UNDERLINE + 'this',
                   SPIECE_UNDERLINE + 'is',
                   SPIECE_UNDERLINE + 'f',
                   'al',
                   's',
                   '<unk>',
                   '.',
               ]				,       )




    def    a							( self	):
           snake_case_          =   (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                         snake_case_          =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase_				,       **lowerCamelCase_	)
                         snake_case_          =   self.tokenizer_class.from_pretrained(lowerCamelCase_				,       **lowerCamelCase_	)

                         snake_case_          =   tempfile.mkdtemp()

                         snake_case_          =   tokenizer_r.save_pretrained(lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.save_pretrained(lowerCamelCase_	)

                         # Checks it save with the same files + the tokenizer.json file for the fast one
                         self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files	)	)
                         snake_case_          =   tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f	)
                         self.assertSequenceEqual(lowerCamelCase_				,       lowerCamelCase_	)

                         # Checks everything loads correctly in the same way
                         snake_case_          =   tokenizer_r.from_pretrained(lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.from_pretrained(lowerCamelCase_	)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                                self.assertTrue(hasattr(lowerCamelCase_				,       lowerCamelCase_	)	)

                         shutil.rmtree(lowerCamelCase_	)

                         # Save tokenizer rust, legacy_format=True
                         snake_case_          =   tempfile.mkdtemp()

                         snake_case_          =   tokenizer_r.save_pretrained(lowerCamelCase_				,       legacy_format=lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.save_pretrained(lowerCamelCase_	)

                         # Checks it save with the same files
                         self.assertSequenceEqual(lowerCamelCase_				,       lowerCamelCase_	)

                         # Checks everything loads correctly in the same way
                         snake_case_          =   tokenizer_r.from_pretrained(lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.from_pretrained(lowerCamelCase_	)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                                self.assertTrue(hasattr(lowerCamelCase_				,       lowerCamelCase_	)	)

                         shutil.rmtree(lowerCamelCase_	)

                         # Save tokenizer rust, legacy_format=False
                         snake_case_          =   tempfile.mkdtemp()

                         snake_case_          =   tokenizer_r.save_pretrained(lowerCamelCase_				,       legacy_format=lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.save_pretrained(lowerCamelCase_	)

                         # Checks it saved the tokenizer.json file
                         self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files	)	)

                         # Checks everything loads correctly in the same way
                         snake_case_          =   tokenizer_r.from_pretrained(lowerCamelCase_	)
                         snake_case_          =   tokenizer_p.from_pretrained(lowerCamelCase_	)

                         # Check special tokens are set accordingly on Rust and Python
                         for key in tokenizer_pp.special_tokens_map:
                                self.assertTrue(hasattr(lowerCamelCase_				,       lowerCamelCase_	)	)

                         shutil.rmtree(lowerCamelCase_	)




    @require_torch
    def    a							( self	):
           if not self.test_seqaseq:
                  return

           snake_case_          =   self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(F'''{tokenizer.__class__.__name__}'''	):
                         # Longer text that will definitely require truncation.
                         snake_case_          =   [
                             """ UN Chief Says There Is No Military Solution in Syria""",
                             """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                             """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                             """ will only worsen the violence and misery for millions of people.""",
                         ]
                         snake_case_          =   [
                             """Şeful ONU declară că nu există o soluţie militară în Siria""",
                             """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                             """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                             """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                         ]
                         try:
                                snake_case_          =   tokenizer.prepare_seqaseq_batch(
                                    src_texts=lowerCamelCase_				,       tgt_texts=lowerCamelCase_				,       max_length=3				,       max_target_length=10				,       return_tensors='pt'				,       src_lang='eng_Latn'				,       tgt_lang='ron_Latn'				,       )
                         except NotImplementedError:
                                return
                         self.assertEqual(batch.input_ids.shape[1]				,       3	)
                         self.assertEqual(batch.labels.shape[1]				,       10	)
                         # max_target_length will default to max_length if not specified
                         snake_case_          =   tokenizer.prepare_seqaseq_batch(
                             lowerCamelCase_				,       tgt_texts=lowerCamelCase_				,       max_length=3				,       return_tensors='pt'	)
                         self.assertEqual(batch.input_ids.shape[1]				,       3	)
                         self.assertEqual(batch.labels.shape[1]				,       3	)

                         snake_case_          =   tokenizer.prepare_seqaseq_batch(
                             src_texts=lowerCamelCase_				,       max_length=3				,       max_target_length=10				,       return_tensors='pt'	)
                         self.assertEqual(batch_encoder_only.input_ids.shape[1]				,       3	)
                         self.assertEqual(batch_encoder_only.attention_mask.shape[1]				,       3	)
                         self.assertNotIn('decoder_input_ids'				,       lowerCamelCase_	)




    @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'	)
    def    a							( self	):
           pass




    def    a							( self	):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                         snake_case_          =   [AddedToken('<special>'				,       lstrip=lowerCamelCase_	)]

                         snake_case_          =   self.rust_tokenizer_class.from_pretrained(
                             lowerCamelCase_				,       additional_special_tokens=lowerCamelCase_				,       **lowerCamelCase_	)
                         snake_case_          =   tokenizer_r.encode('Hey this is a <special> token'	)

                         snake_case_          =   tokenizer_r.encode('<special>'				,       add_special_tokens=lowerCamelCase_	)[0]

                         self.assertTrue(special_token_id in r_output	)

                         if self.test_slow_tokenizer:
                                snake_case_          =   self.rust_tokenizer_class.from_pretrained(
                                    lowerCamelCase_				,       additional_special_tokens=lowerCamelCase_				,       **lowerCamelCase_				,       )
                                snake_case_          =   self.tokenizer_class.from_pretrained(
                                    lowerCamelCase_				,       additional_special_tokens=lowerCamelCase_				,       **lowerCamelCase_	)

                                snake_case_          =   tokenizer_p.encode('Hey this is a <special> token'	)

                                snake_case_          =   tokenizer_cr.encode('Hey this is a <special> token'	)

                                self.assertEqual(lowerCamelCase_				,       lowerCamelCase_	)
                                self.assertEqual(lowerCamelCase_				,       lowerCamelCase_	)
                                self.assertTrue(special_token_id in p_output	)
                                self.assertTrue(special_token_id in cr_output	)







@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowercase    (	unittest.TestCase      ):
    __SCREAMING_SNAKE_CASE :			Tuple	      =      "facebook/nllb-200-distilled-600M"
    __SCREAMING_SNAKE_CASE :			Tuple	      =      [
        " UN Chief Says There Is No Military Solution in Syria",
        " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
    ]
    __SCREAMING_SNAKE_CASE :			List[Any]	      =      [
        "Şeful ONU declară că nu există o soluţie militară în Siria",
        "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
        " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
        " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
    ]
    __SCREAMING_SNAKE_CASE :			List[str]	      =      [
        256_047,
        16_297,
        134_408,
        8_165,
        248_066,
        14_734,
        950,
        1_135,
        105_721,
        3_573,
        83,
        27_352,
        108,
        49_486,
        2,
    ]




    @classmethod
    def    a							( cls	):
           snake_case_          =   NllbTokenizer.from_pretrained(
               cls.checkpoint_name				,       src_lang='eng_Latn'				,       tgt_lang='ron_Latn'	)
           snake_case_          =   1
           return cls




    def    a							( self	):
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']				,       25_6001	)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']				,       25_6002	)
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']				,       25_6057	)




    def    a							( self	):
           snake_case_          =   self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
           self.assertListEqual(self.expected_src_tokens				,       lowerCamelCase_	)




    def    a							( self	):
           self.assertIn(lowerCamelCase_				,       self.tokenizer.all_special_ids	)
           # fmt: off
           snake_case_          =   [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
           # fmt: on

           snake_case_          =   self.tokenizer.decode(lowerCamelCase_				,       skip_special_tokens=lowerCamelCase_	)
           snake_case_          =   self.tokenizer.decode(generated_ids[1:]				,       skip_special_tokens=lowerCamelCase_	)
           self.assertEqual(lowerCamelCase_				,       lowerCamelCase_	)
           self.assertNotIn(self.tokenizer.eos_token				,       lowerCamelCase_	)




    def    a							( self	):
           snake_case_          =   ["""this is gunna be a long sentence """ * 20]
           assert isinstance(src_text[0]				,       lowerCamelCase_	)
           snake_case_          =   10
           snake_case_          =   self.tokenizer(lowerCamelCase_				,       max_length=lowerCamelCase_				,       truncation=lowerCamelCase_	).input_ids[0]
           self.assertEqual(ids[-1]				,       2	)
           self.assertEqual(ids[0]				,       lowerCamelCase_	)
           self.assertEqual(len(lowerCamelCase_	)				,       lowerCamelCase_	)




    def    a							( self	):
           self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']	)				,       [25_6203, 3]	)




    def    a							( self	):
           snake_case_          =   tempfile.mkdtemp()
           snake_case_          =   self.tokenizer.fairseq_tokens_to_ids
           self.tokenizer.save_pretrained(lowerCamelCase_	)
           snake_case_          =   NllbTokenizer.from_pretrained(lowerCamelCase_	)
           self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,       lowerCamelCase_	)




    @require_torch
    def    a							( self	):
           snake_case_          =   self.tokenizer(
               self.src_text				,       text_target=self.tgt_text				,       padding=lowerCamelCase_				,       truncation=lowerCamelCase_				,       max_length=len(self.expected_src_tokens	)				,       return_tensors='pt'				,       )
           snake_case_          =   shift_tokens_right(
               batch['labels']				,       self.tokenizer.pad_token_id				,       self.tokenizer.lang_code_to_id['ron_Latn']	)

           self.assertIsInstance(lowerCamelCase_				,       lowerCamelCase_	)

           self.assertEqual((2, 15)				,       batch.input_ids.shape	)
           self.assertEqual((2, 15)				,       batch.attention_mask.shape	)
           snake_case_          =   batch.input_ids.tolist()[0]
           self.assertListEqual(self.expected_src_tokens				,       lowerCamelCase_	)
           self.assertEqual(lowerCamelCase_				,       batch.decoder_input_ids[0, 0]	)  # EOS
           # Test that special tokens are reset
           self.assertEqual(self.tokenizer.prefix_tokens				,       [EN_CODE]	)
           self.assertEqual(self.tokenizer.suffix_tokens				,       [self.tokenizer.eos_token_id]	)




    def    a							( self	):
           snake_case_          =   self.tokenizer(self.src_text				,       padding=lowerCamelCase_				,       truncation=lowerCamelCase_				,       max_length=3				,       return_tensors='pt'	)
           snake_case_          =   self.tokenizer(
               text_target=self.tgt_text				,       padding=lowerCamelCase_				,       truncation=lowerCamelCase_				,       max_length=10				,       return_tensors='pt'	)
           snake_case_          =   targets["""input_ids"""]
           snake_case_          =   shift_tokens_right(
               lowerCamelCase_				,       self.tokenizer.pad_token_id				,       decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]				,       )

           self.assertEqual(batch.input_ids.shape[1]				,       3	)
           self.assertEqual(batch.decoder_input_ids.shape[1]				,       10	)




    @require_torch
    def    a							( self	):
           snake_case_          =   self.tokenizer._build_translation_inputs(
               'A test'				,       return_tensors='pt'				,       src_lang='eng_Latn'				,       tgt_lang='fra_Latn'	)

           self.assertEqual(
               nested_simplify(lowerCamelCase_	)				,       {
                   # A, test, EOS, en_XX
                   'input_ids': [[25_6047, 70, 7356, 2]],
                   'attention_mask': [[1, 1, 1, 1]],
                   # ar_AR
                   'forced_bos_token_id': 25_6057,
               }				,       )




    @require_torch
    def    a							( self	):
           snake_case_          =   True
           snake_case_          =   self.tokenizer(
               'UN Chief says there is no military solution in Syria'				,       src_lang='eng_Latn'				,       tgt_lang='fra_Latn'	)
           self.assertEqual(
               inputs.input_ids				,       [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]	)

           snake_case_          =   False
           snake_case_          =   self.tokenizer(
               'UN Chief says there is no military solution in Syria'				,       src_lang='eng_Latn'				,       tgt_lang='fra_Latn'	)
           self.assertEqual(
               inputs.input_ids				,       [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]	)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_UpperCAmelCase		:       Dict						    =	{
    """configuration_blip""": [
        """BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """BlipConfig""",
        """BlipTextConfig""",
        """BlipVisionConfig""",
    ],
    """processing_blip""": ["""BlipProcessor"""],
}

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase		:       Optional[int]						    =	["""BlipImageProcessor"""]


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase		:       Any						    =	[
					    """BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """BlipModel""",
					    """BlipPreTrainedModel""",
					    """BlipForConditionalGeneration""",
					    """BlipForQuestionAnswering""",
					    """BlipVisionModel""",
					    """BlipTextModel""",
					    """BlipForImageTextRetrieval""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase		:       Any						    =	[
					    """TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFBlipModel""",
					    """TFBlipPreTrainedModel""",
					    """TFBlipForConditionalGeneration""",
					    """TFBlipForQuestionAnswering""",
					    """TFBlipVisionModel""",
					    """TFBlipTextModel""",
					    """TFBlipForImageTextRetrieval""",
					]

if TYPE_CHECKING:
					from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig
					from .processing_blip import BlipProcessor

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .image_processing_blip import BlipImageProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_blip import (
										    BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
										    BlipForConditionalGeneration,
										    BlipForImageTextRetrieval,
										    BlipForQuestionAnswering,
										    BlipModel,
										    BlipPreTrainedModel,
										    BlipTextModel,
										    BlipVisionModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_blip import (
										    TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFBlipForConditionalGeneration,
										    TFBlipForImageTextRetrieval,
										    TFBlipForQuestionAnswering,
										    TFBlipModel,
										    TFBlipPreTrainedModel,
										    TFBlipTextModel,
										    TFBlipVisionModel,
										)

else:
					import sys

					_UpperCAmelCase		:       Optional[int]						    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




