
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase_  :	Union[str, Any] 				=					get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
							from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCamelCase_  :	Union[str, Any] 				=					25_60_47
lowerCamelCase_  :	List[Any] 				=					25_61_45

@require_sentencepiece
@require_tokenizers
class     _UpperCamelCase			(   _A       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  List[str]    =	NllbTokenizer
			__UpperCamelCase :  int    =	NllbTokenizerFast
			__UpperCamelCase :  str    =	True
			__UpperCamelCase :  Dict    =	True
			__UpperCamelCase :  Union[str, Any]    =	{}

			def  lowerCAmelCase__		(    self	:    Any ):
							super().setUp()

							# We have a SentencePiece fixture for testing
							UpperCamelCase_:							Optional[int]		=  NllbTokenizer(__SCREAMING_SNAKE_CASE  ,		keep_accents=__SCREAMING_SNAKE_CASE )
							tokenizer.save_pretrained(self.tmpdirname )

			def  lowerCAmelCase__		(    self	:    Tuple ):
							UpperCamelCase_:							Tuple		=  NllbTokenizer(__SCREAMING_SNAKE_CASE  ,		keep_accents=__SCREAMING_SNAKE_CASE )

							UpperCamelCase_:							Optional[int]		=  tokenizer.tokenize("""This is a test""" )
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE )  ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]  ,		)

							UpperCamelCase_:							Optional[int]		=  tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
							self.assertListEqual(
							    __SCREAMING_SNAKE_CASE  ,		[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """9""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """é""",
							        """.""",
							    ]  ,		)
							UpperCamelCase_:							Tuple		=  tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE )
							self.assertListEqual(
							    __SCREAMING_SNAKE_CASE  ,		[
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]  ,		)

							UpperCamelCase_:							Any		=  tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE )
							self.assertListEqual(
							    __SCREAMING_SNAKE_CASE  ,		[
							        SPIECE_UNDERLINE + """I""",
							        SPIECE_UNDERLINE + """was""",
							        SPIECE_UNDERLINE + """b""",
							        """or""",
							        """n""",
							        SPIECE_UNDERLINE + """in""",
							        SPIECE_UNDERLINE + """""",
							        """<unk>""",
							        """2""",
							        """0""",
							        """0""",
							        """0""",
							        """,""",
							        SPIECE_UNDERLINE + """and""",
							        SPIECE_UNDERLINE + """this""",
							        SPIECE_UNDERLINE + """is""",
							        SPIECE_UNDERLINE + """f""",
							        """al""",
							        """s""",
							        """<unk>""",
							        """.""",
							    ]  ,		)

			def  lowerCAmelCase__		(    self	:    str ):
							UpperCamelCase_:							Tuple		=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							List[Any]		=  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,		**__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							Optional[int]		=  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,		**__SCREAMING_SNAKE_CASE )

															UpperCamelCase_:							Optional[Any]		=  tempfile.mkdtemp()

															UpperCamelCase_:							Any		=  tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							Union[str, Any]		=  tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE )

															# Checks it save with the same files + the tokenizer.json file for the fast one
															self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files ) )
															UpperCamelCase_:							List[Any]		=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f )
															self.assertSequenceEqual(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )

															# Checks everything loads correctly in the same way
															UpperCamelCase_:							Tuple		=  tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							Dict		=  tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE )

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE ) )

															shutil.rmtree(__SCREAMING_SNAKE_CASE )

															# Save tokenizer rust, legacy_format=True
															UpperCamelCase_:							Tuple		=  tempfile.mkdtemp()

															UpperCamelCase_:							Optional[Any]		=  tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE  ,		legacy_format=__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							str		=  tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE )

															# Checks it save with the same files
															self.assertSequenceEqual(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )

															# Checks everything loads correctly in the same way
															UpperCamelCase_:							str		=  tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							List[str]		=  tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE )

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE ) )

															shutil.rmtree(__SCREAMING_SNAKE_CASE )

															# Save tokenizer rust, legacy_format=False
															UpperCamelCase_:							List[Any]		=  tempfile.mkdtemp()

															UpperCamelCase_:							Tuple		=  tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE  ,		legacy_format=__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							List[Any]		=  tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE )

															# Checks it saved the tokenizer.json file
															self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files ) )

															# Checks everything loads correctly in the same way
															UpperCamelCase_:							List[str]		=  tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							List[str]		=  tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE )

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE ) )

															shutil.rmtree(__SCREAMING_SNAKE_CASE )

			@require_torch
			def  lowerCAmelCase__		(    self	:    str ):
							if not self.test_seqaseq:
											return

							UpperCamelCase_:							Optional[int]		=  self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
															# Longer text that will definitely require truncation.
															UpperCamelCase_:							Dict		=  [
															    """ UN Chief Says There Is No Military Solution in Syria""",
															    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
															    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
															    """ will only worsen the violence and misery for millions of people.""",
															]
															UpperCamelCase_:							Tuple		=  [
															    """Şeful ONU declară că nu există o soluţie militară în Siria""",
															    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
															    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
															    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
															]
															try:
																			UpperCamelCase_:							List[Any]		=  tokenizer.prepare_seqaseq_batch(
																			    src_texts=__SCREAMING_SNAKE_CASE  ,		tgt_texts=__SCREAMING_SNAKE_CASE  ,		max_length=3  ,		max_target_length=10  ,		return_tensors="""pt"""  ,		src_lang="""eng_Latn"""  ,		tgt_lang="""ron_Latn"""  ,		)
															except NotImplementedError:
																			return
															self.assertEqual(batch.input_ids.shape[1]  ,		3 )
															self.assertEqual(batch.labels.shape[1]  ,		10 )
															# max_target_length will default to max_length if not specified
															UpperCamelCase_:							int		=  tokenizer.prepare_seqaseq_batch(
															    __SCREAMING_SNAKE_CASE  ,		tgt_texts=__SCREAMING_SNAKE_CASE  ,		max_length=3  ,		return_tensors="""pt""" )
															self.assertEqual(batch.input_ids.shape[1]  ,		3 )
															self.assertEqual(batch.labels.shape[1]  ,		3 )

															UpperCamelCase_:							List[str]		=  tokenizer.prepare_seqaseq_batch(
															    src_texts=__SCREAMING_SNAKE_CASE  ,		max_length=3  ,		max_target_length=10  ,		return_tensors="""pt""" )
															self.assertEqual(batch_encoder_only.input_ids.shape[1]  ,		3 )
															self.assertEqual(batch_encoder_only.attention_mask.shape[1]  ,		3 )
															self.assertNotIn("""decoder_input_ids"""  ,		__SCREAMING_SNAKE_CASE )

			@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece.""" )
			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							pass




			def  lowerCAmelCase__		(    self	:    List[Any] ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							Dict		=  [AddedToken("""<special>"""  ,		lstrip=__SCREAMING_SNAKE_CASE )]

															UpperCamelCase_:							Any		=  self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,		additional_special_tokens=__SCREAMING_SNAKE_CASE  ,		**__SCREAMING_SNAKE_CASE )
															UpperCamelCase_:							Optional[int]		=  tokenizer_r.encode("""Hey this is a <special> token""" )

															UpperCamelCase_:							Optional[int]		=  tokenizer_r.encode("""<special>"""  ,		add_special_tokens=__SCREAMING_SNAKE_CASE )[0]

															self.assertTrue(special_token_id in r_output )

															if self.test_slow_tokenizer:
																			UpperCamelCase_:							Optional[Any]		=  self.rust_tokenizer_class.from_pretrained(
																			    __SCREAMING_SNAKE_CASE  ,		additional_special_tokens=__SCREAMING_SNAKE_CASE  ,		**__SCREAMING_SNAKE_CASE  ,		)
																			UpperCamelCase_:							int		=  self.tokenizer_class.from_pretrained(
																			    __SCREAMING_SNAKE_CASE  ,		additional_special_tokens=__SCREAMING_SNAKE_CASE  ,		**__SCREAMING_SNAKE_CASE )

																			UpperCamelCase_:							Optional[Any]		=  tokenizer_p.encode("""Hey this is a <special> token""" )

																			UpperCamelCase_:							Dict		=  tokenizer_cr.encode("""Hey this is a <special> token""" )

																			self.assertEqual(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )
																			self.assertEqual(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )
																			self.assertTrue(special_token_id in p_output )
																			self.assertTrue(special_token_id in cr_output )




@require_torch
@require_sentencepiece
@require_tokenizers
class     _UpperCamelCase			(   unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  Union[str, Any]    =	"""facebook/nllb-200-distilled-600M"""
			__UpperCamelCase :  str    =	[
			    """ UN Chief Says There Is No Military Solution in Syria""",
			    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
			]
			__UpperCamelCase :  List[str]    =	[
			    """Şeful ONU declară că nu există o soluţie militară în Siria""",
			    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
			    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
			    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
			]
			__UpperCamelCase :  Optional[int]    =	[
			    256_047,
			    16_297,
			    134_408,
			    8_165,
			    248_066,
			    14_734,
			    950,
			    1_135,
			    105_721,
			    3_573,
			    83,
			    27_352,
			    108,
			    49_486,
			    2,
			]

			@classmethod
			def  lowerCAmelCase__		(    cls	:    Optional[int] ):
							UpperCamelCase_:							Optional[Any]		=  NllbTokenizer.from_pretrained(
							    cls.checkpoint_name  ,		src_lang="""eng_Latn"""  ,		tgt_lang="""ron_Latn""" )
							UpperCamelCase_:							Any		=  1
							return cls

			def  lowerCAmelCase__		(    self	:    List[Any] ):
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]  ,		25_6001 )
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]  ,		25_6002 )
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]  ,		25_6057 )

			def  lowerCAmelCase__		(    self	:    str ):
							UpperCamelCase_:							str		=  self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
							self.assertListEqual(self.expected_src_tokens  ,		__SCREAMING_SNAKE_CASE )

			def  lowerCAmelCase__		(    self	:    str ):
							self.assertIn(__SCREAMING_SNAKE_CASE  ,		self.tokenizer.all_special_ids )
							# fmt: off
							UpperCamelCase_:							List[str]		=  [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
							# fmt: on

							UpperCamelCase_:							int		=  self.tokenizer.decode(__SCREAMING_SNAKE_CASE  ,		skip_special_tokens=__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							Optional[Any]		=  self.tokenizer.decode(generated_ids[1:]  ,		skip_special_tokens=__SCREAMING_SNAKE_CASE )
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )
							self.assertNotIn(self.tokenizer.eos_token  ,		__SCREAMING_SNAKE_CASE )

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Optional[Any]		=  ["""this is gunna be a long sentence """ * 20]
							assert isinstance(src_text[0]  ,		__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							Tuple		=  10
							UpperCamelCase_:							int		=  self.tokenizer(__SCREAMING_SNAKE_CASE  ,		max_length=__SCREAMING_SNAKE_CASE  ,		truncation=__SCREAMING_SNAKE_CASE ).input_ids[0]
							self.assertEqual(ids[-1]  ,		2 )
							self.assertEqual(ids[0]  ,		__SCREAMING_SNAKE_CASE )
							self.assertEqual(len(__SCREAMING_SNAKE_CASE )  ,		__SCREAMING_SNAKE_CASE )

			def  lowerCAmelCase__		(    self	:    List[str] ):
							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""] )  ,		[25_6203, 3] )

			def  lowerCAmelCase__		(    self	:    int ):
							UpperCamelCase_:							Dict		=  tempfile.mkdtemp()
							UpperCamelCase_:							Union[str, Any]		=  self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							Optional[int]		=  NllbTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE )
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids  ,		__SCREAMING_SNAKE_CASE )

			@require_torch
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							List[Any]		=  self.tokenizer(
							    self.src_text  ,		text_target=self.tgt_text  ,		padding=__SCREAMING_SNAKE_CASE  ,		truncation=__SCREAMING_SNAKE_CASE  ,		max_length=len(self.expected_src_tokens )  ,		return_tensors="""pt"""  ,		)
							UpperCamelCase_:							Dict		=  shift_tokens_right(
							    batch["""labels"""]  ,		self.tokenizer.pad_token_id  ,		self.tokenizer.lang_code_to_id["""ron_Latn"""] )

							self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE )

							self.assertEqual((2, 15)  ,		batch.input_ids.shape )
							self.assertEqual((2, 15)  ,		batch.attention_mask.shape )
							UpperCamelCase_:							Dict		=  batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens  ,		__SCREAMING_SNAKE_CASE )
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,		batch.decoder_input_ids[0, 0] )  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens  ,		[EN_CODE] )
							self.assertEqual(self.tokenizer.suffix_tokens  ,		[self.tokenizer.eos_token_id] )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							Optional[int]		=  self.tokenizer(self.src_text  ,		padding=__SCREAMING_SNAKE_CASE  ,		truncation=__SCREAMING_SNAKE_CASE  ,		max_length=3  ,		return_tensors="""pt""" )
							UpperCamelCase_:							Tuple		=  self.tokenizer(
							    text_target=self.tgt_text  ,		padding=__SCREAMING_SNAKE_CASE  ,		truncation=__SCREAMING_SNAKE_CASE  ,		max_length=10  ,		return_tensors="""pt""" )
							UpperCamelCase_:							str		=  targets["""input_ids"""]
							UpperCamelCase_:							Union[str, Any]		=  shift_tokens_right(
							    __SCREAMING_SNAKE_CASE  ,		self.tokenizer.pad_token_id  ,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]  ,		)

							self.assertEqual(batch.input_ids.shape[1]  ,		3 )
							self.assertEqual(batch.decoder_input_ids.shape[1]  ,		10 )

			@require_torch
			def  lowerCAmelCase__		(    self	:    str ):
							UpperCamelCase_:							Optional[int]		=  self.tokenizer._build_translation_inputs(
							    """A test"""  ,		return_tensors="""pt"""  ,		src_lang="""eng_Latn"""  ,		tgt_lang="""fra_Latn""" )

							self.assertEqual(
							    nested_simplify(__SCREAMING_SNAKE_CASE )  ,		{
							        # A, test, EOS, en_XX
							        """input_ids""": [[25_6047, 70, 7356, 2]],
							        """attention_mask""": [[1, 1, 1, 1]],
							        # ar_AR
							        """forced_bos_token_id""": 25_6057,
							    }  ,		)




			@require_torch
			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							Dict		=  True
							UpperCamelCase_:							Tuple		=  self.tokenizer(
							    """UN Chief says there is no military solution in Syria"""  ,		src_lang="""eng_Latn"""  ,		tgt_lang="""fra_Latn""" )
							self.assertEqual(
							    inputs.input_ids  ,		[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047] )

							UpperCamelCase_:							List[str]		=  False
							UpperCamelCase_:							Dict		=  self.tokenizer(
							    """UN Chief says there is no military solution in Syria"""  ,		src_lang="""eng_Latn"""  ,		tgt_lang="""fra_Latn""" )
							self.assertEqual(
							    inputs.input_ids  ,		[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2] )








from typing import TYPE_CHECKING

from ...utils import _LazyModule


lowerCamelCase_  :	Dict 				=					{"""tokenization_byt5""": ["""ByT5Tokenizer"""]}


if TYPE_CHECKING:
							from .tokenization_byta import ByTaTokenizer
else:
							import sys

							lowerCamelCase_  :	Any 				=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






