


'''simple docstring'''
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case											=					get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
		from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__snake_case											=					256047
__snake_case											=					256145
@require_sentencepiece
@require_tokenizers
class        lowercase     (							A__     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    NllbTokenizer
		_a   			=    NllbTokenizerFast
		_a   			=    True
		_a   			=    True
		_a   			=    {}




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							super().setUp()

							# We have a SentencePiece fixture for testing
							UpperCamelCase__       :str							 =			NllbTokenizer(UpperCamelCase_   ,       keep_accents=UpperCamelCase_							)
							tokenizer.save_pretrained(self.tmpdirname							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[Any]							 =			NllbTokenizer(UpperCamelCase_   ,       keep_accents=UpperCamelCase_							)

							UpperCamelCase__       :int							 =			tokenizer.tokenize('''This is a test'''							)
							self.assertListEqual(UpperCamelCase_   ,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCamelCase_							)   ,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,       )

							UpperCamelCase__       :List[str]							 =			tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''é''',
							        '''.''',
							    ]   ,       )
							UpperCamelCase__       :Tuple							 =			tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]   ,       )

							UpperCamelCase__       :Dict							 =			tokenizer.convert_ids_to_tokens(UpperCamelCase_							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''<unk>''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''<unk>''',
							        '''.''',
							    ]   ,       )




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__       :List[str]							 =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase_   ,       **UpperCamelCase_							)
																	UpperCamelCase__       :List[str]							 =			self.tokenizer_class.from_pretrained(UpperCamelCase_   ,       **UpperCamelCase_							)

																	UpperCamelCase__       :Dict							 =			tempfile.mkdtemp()

																	UpperCamelCase__       :str							 =			tokenizer_r.save_pretrained(UpperCamelCase_							)
																	UpperCamelCase__       :int							 =			tokenizer_p.save_pretrained(UpperCamelCase_							)

																	# Checks it save with the same files + the tokenizer.json file for the fast one
																	self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
																	UpperCamelCase__       :Optional[Any]							 =			tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
																	self.assertSequenceEqual(UpperCamelCase_   ,       UpperCamelCase_							)

																	# Checks everything loads correctly in the same way
																	UpperCamelCase__       :List[Any]							 =			tokenizer_r.from_pretrained(UpperCamelCase_							)
																	UpperCamelCase__       :Union[str, Any]							 =			tokenizer_p.from_pretrained(UpperCamelCase_							)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(UpperCamelCase_   ,       UpperCamelCase_							)							)

																	shutil.rmtree(UpperCamelCase_							)

																	# Save tokenizer rust, legacy_format=True
																	UpperCamelCase__       :Dict							 =			tempfile.mkdtemp()

																	UpperCamelCase__       :Union[str, Any]							 =			tokenizer_r.save_pretrained(UpperCamelCase_   ,       legacy_format=UpperCamelCase_							)
																	UpperCamelCase__       :Any							 =			tokenizer_p.save_pretrained(UpperCamelCase_							)

																	# Checks it save with the same files
																	self.assertSequenceEqual(UpperCamelCase_   ,       UpperCamelCase_							)

																	# Checks everything loads correctly in the same way
																	UpperCamelCase__       :Tuple							 =			tokenizer_r.from_pretrained(UpperCamelCase_							)
																	UpperCamelCase__       :str							 =			tokenizer_p.from_pretrained(UpperCamelCase_							)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(UpperCamelCase_   ,       UpperCamelCase_							)							)

																	shutil.rmtree(UpperCamelCase_							)

																	# Save tokenizer rust, legacy_format=False
																	UpperCamelCase__       :List[Any]							 =			tempfile.mkdtemp()

																	UpperCamelCase__       :Dict							 =			tokenizer_r.save_pretrained(UpperCamelCase_   ,       legacy_format=UpperCamelCase_							)
																	UpperCamelCase__       :List[str]							 =			tokenizer_p.save_pretrained(UpperCamelCase_							)

																	# Checks it saved the tokenizer.json file
																	self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

																	# Checks everything loads correctly in the same way
																	UpperCamelCase__       :int							 =			tokenizer_r.from_pretrained(UpperCamelCase_							)
																	UpperCamelCase__       :Dict							 =			tokenizer_p.from_pretrained(UpperCamelCase_							)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(UpperCamelCase_   ,       UpperCamelCase_							)							)

																	shutil.rmtree(UpperCamelCase_							)




		@require_torch
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							if not self.test_seqaseq:
												return

							UpperCamelCase__       :int							 =			self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	# Longer text that will definitely require truncation.
																	UpperCamelCase__       :List[Any]							 =			[
																	    ''' UN Chief Says There Is No Military Solution in Syria''',
																	    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
																	    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
																	    ''' will only worsen the violence and misery for millions of people.''',
																	]
																	UpperCamelCase__       :str							 =			[
																	    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
																	    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
																	    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
																	    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
																	]
																	try:
																						UpperCamelCase__       :Dict							 =			tokenizer.prepare_seqaseq_batch(
																						    src_texts=UpperCamelCase_   ,       tgt_texts=UpperCamelCase_   ,       max_length=3   ,       max_target_length=10   ,       return_tensors='''pt'''   ,       src_lang='''eng_Latn'''   ,       tgt_lang='''ron_Latn'''   ,       )
																	except NotImplementedError:
																						return
																	self.assertEqual(batch.input_ids.shape[1]   ,       3							)
																	self.assertEqual(batch.labels.shape[1]   ,       10							)
																	# max_target_length will default to max_length if not specified
																	UpperCamelCase__       :List[str]							 =			tokenizer.prepare_seqaseq_batch(
																	    UpperCamelCase_   ,       tgt_texts=UpperCamelCase_   ,       max_length=3   ,       return_tensors='''pt'''							)
																	self.assertEqual(batch.input_ids.shape[1]   ,       3							)
																	self.assertEqual(batch.labels.shape[1]   ,       3							)

																	UpperCamelCase__       :List[str]							 =			tokenizer.prepare_seqaseq_batch(
																	    src_texts=UpperCamelCase_   ,       max_length=3   ,       max_target_length=10   ,       return_tensors='''pt'''							)
																	self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,       3							)
																	self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,       3							)
																	self.assertNotIn('''decoder_input_ids'''   ,       UpperCamelCase_							)




		@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__       :Optional[Any]							 =			[AddedToken('''<special>'''   ,       lstrip=UpperCamelCase_							)]

																	UpperCamelCase__       :Union[str, Any]							 =			self.rust_tokenizer_class.from_pretrained(
																	    UpperCamelCase_   ,       additional_special_tokens=UpperCamelCase_   ,       **UpperCamelCase_							)
																	UpperCamelCase__       :str							 =			tokenizer_r.encode('''Hey this is a <special> token'''							)

																	UpperCamelCase__       :int							 =			tokenizer_r.encode('''<special>'''   ,       add_special_tokens=UpperCamelCase_							)[0]

																	self.assertTrue(special_token_id in r_output							)

																	if self.test_slow_tokenizer:
																						UpperCamelCase__       :int							 =			self.rust_tokenizer_class.from_pretrained(
																						    UpperCamelCase_   ,       additional_special_tokens=UpperCamelCase_   ,       **UpperCamelCase_   ,       )
																						UpperCamelCase__       :Tuple							 =			self.tokenizer_class.from_pretrained(
																						    UpperCamelCase_   ,       additional_special_tokens=UpperCamelCase_   ,       **UpperCamelCase_							)

																						UpperCamelCase__       :Union[str, Any]							 =			tokenizer_p.encode('''Hey this is a <special> token'''							)

																						UpperCamelCase__       :Any							 =			tokenizer_cr.encode('''Hey this is a <special> token'''							)

																						self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
																						self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
																						self.assertTrue(special_token_id in p_output							)
																						self.assertTrue(special_token_id in cr_output							)



@require_torch
@require_sentencepiece
@require_tokenizers
class        lowercase     (							unittest.TestCase ):

		"""simple docstring"""

		_a   			=    'facebook/nllb-200-distilled-600M'
		_a   			=    [
		    ' UN Chief Says There Is No Military Solution in Syria',
		    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
		]
		_a   			=    [
		    'Şeful ONU declară că nu există o soluţie militară în Siria',
		    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
		    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
		    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
		]
		_a   			=    [
		    25_60_47,
		    1_62_97,
		    13_44_08,
		    81_65,
		    24_80_66,
		    1_47_34,
		    9_50,
		    11_35,
		    10_57_21,
		    35_73,
		    83,
		    2_73_52,
		    1_08,
		    4_94_86,
		    2,
		]




		@classmethod
		def    lowerCAmelCase__    (		cls							):

							'''simple docstring'''


							UpperCamelCase__       :NllbTokenizer							 =			NllbTokenizer.from_pretrained(
							    cls.checkpoint_name   ,       src_lang='''eng_Latn'''   ,       tgt_lang='''ron_Latn'''							)
							UpperCamelCase__       :str							 =			1
							return cls




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']   ,       256001							)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']   ,       256002							)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']   ,       256057							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
							self.assertListEqual(self.expected_src_tokens   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertIn(UpperCamelCase_   ,       self.tokenizer.all_special_ids							)
							# fmt: off
							UpperCamelCase__       :Optional[Any]							 =			[RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
							# fmt: on

							UpperCamelCase__       :Dict							 =			self.tokenizer.decode(UpperCamelCase_   ,       skip_special_tokens=UpperCamelCase_							)
							UpperCamelCase__       :Optional[int]							 =			self.tokenizer.decode(generated_ids[1:]   ,       skip_special_tokens=UpperCamelCase_							)
							self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
							self.assertNotIn(self.tokenizer.eos_token   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			['''this is gunna be a long sentence ''' * 20]
							assert isinstance(src_text[0]   ,       UpperCamelCase_							)
							UpperCamelCase__       :Tuple							 =			10
							UpperCamelCase__       :List[Any]							 =			self.tokenizer(UpperCamelCase_   ,       max_length=UpperCamelCase_   ,       truncation=UpperCamelCase_							).input_ids[0]
							self.assertEqual(ids[-1]   ,       2							)
							self.assertEqual(ids[0]   ,       UpperCamelCase_							)
							self.assertEqual(len(UpperCamelCase_							)   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']							)   ,       [256203, 3]							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[str]							 =			tempfile.mkdtemp()
							UpperCamelCase__       :Optional[int]							 =			self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(UpperCamelCase_							)
							UpperCamelCase__       :Optional[int]							 =			NllbTokenizer.from_pretrained(UpperCamelCase_							)
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,       UpperCamelCase_							)




		@require_torch
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			self.tokenizer(
							    self.src_text   ,       text_target=self.tgt_text   ,       padding=UpperCamelCase_   ,       truncation=UpperCamelCase_   ,       max_length=len(self.expected_src_tokens							)   ,       return_tensors='''pt'''   ,       )
							UpperCamelCase__       :List[str]							 =			shift_tokens_right(
							    batch['''labels''']   ,       self.tokenizer.pad_token_id   ,       self.tokenizer.lang_code_to_id['''ron_Latn''']							)

							self.assertIsInstance(UpperCamelCase_   ,       UpperCamelCase_							)

							self.assertEqual((2, 15)   ,       batch.input_ids.shape							)
							self.assertEqual((2, 15)   ,       batch.attention_mask.shape							)
							UpperCamelCase__       :Tuple							 =			batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens   ,       UpperCamelCase_							)
							self.assertEqual(UpperCamelCase_   ,       batch.decoder_input_ids[0, 0]							)  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens   ,       [EN_CODE]							)
							self.assertEqual(self.tokenizer.suffix_tokens   ,       [self.tokenizer.eos_token_id]							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			self.tokenizer(self.src_text   ,       padding=UpperCamelCase_   ,       truncation=UpperCamelCase_   ,       max_length=3   ,       return_tensors='''pt'''							)
							UpperCamelCase__       :Dict							 =			self.tokenizer(
							    text_target=self.tgt_text   ,       padding=UpperCamelCase_   ,       truncation=UpperCamelCase_   ,       max_length=10   ,       return_tensors='''pt'''							)
							UpperCamelCase__       :List[str]							 =			targets['''input_ids''']
							UpperCamelCase__       :str							 =			shift_tokens_right(
							    UpperCamelCase_   ,       self.tokenizer.pad_token_id   ,       decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,       )

							self.assertEqual(batch.input_ids.shape[1]   ,       3							)
							self.assertEqual(batch.decoder_input_ids.shape[1]   ,       10							)




		@require_torch
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			self.tokenizer._build_translation_inputs(
							    '''A test'''   ,       return_tensors='''pt'''   ,       src_lang='''eng_Latn'''   ,       tgt_lang='''fra_Latn'''							)

							self.assertEqual(
							    nested_simplify(UpperCamelCase_							)   ,       {
							        # A, test, EOS, en_XX
							        '''input_ids''': [[256047, 70, 7356, 2]],
							        '''attention_mask''': [[1, 1, 1, 1]],
							        # ar_AR
							        '''forced_bos_token_id''': 256057,
							    }   ,       )




		@require_torch
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			True
							UpperCamelCase__       :str							 =			self.tokenizer(
							    '''UN Chief says there is no military solution in Syria'''   ,       src_lang='''eng_Latn'''   ,       tgt_lang='''fra_Latn'''							)
							self.assertEqual(
							    inputs.input_ids   ,       [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]							)

							UpperCamelCase__       :Union[str, Any]							 =			False
							UpperCamelCase__       :Union[str, Any]							 =			self.tokenizer(
							    '''UN Chief says there is no military solution in Syria'''   ,       src_lang='''eng_Latn'''   ,       tgt_lang='''fra_Latn'''							)
							self.assertEqual(
							    inputs.input_ids   ,       [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]							)



'''simple docstring'''
from __future__ import annotations

from typing import Any


def       a				(      __a       )  ->     None:



					'''simple docstring'''






					create_state_space_tree(__a			, []			, 0       )


def       a				(      __a			, __a			, __a       )  ->     None:



					'''simple docstring'''






					if index == len(__a       ):
										print(__a       )
										return

					create_state_space_tree(__a			, __a			, index + 1       )
					current_subsequence.append(sequence[index]       )
					create_state_space_tree(__a			, __a			, index + 1       )
					current_subsequence.pop()


if __name__ == "__main__":
		__snake_case											=					[3, 1, 2, 4]
		generate_all_subsequences(seq)

		seq.clear()
		seq.extend(['''A''', '''B''', '''C'''])
		generate_all_subsequences(seq)