


'''simple docstring'''
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case											=					get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
		import sentencepiece as sp


__snake_case											=					5
__snake_case											=					10
@require_sentencepiece
@require_tokenizers
class        lowercase     (							A__     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    SpeechaTextTokenizer
		_a   			=    False
		_a   			=    True




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__       :Any							 =			sp.SentencePieceProcessor()
							spm_model.Load(UpperCamelCase_							)
							UpperCamelCase__       :int							 =			['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

							vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(UpperCamelCase_							)							)]
							UpperCamelCase__       :Dict							 =			dict(zip(UpperCamelCase_   ,       range(len(UpperCamelCase_							)							)							)							)

							UpperCamelCase__       :Union[str, Any]							 =			Path(self.tmpdirname							)
							save_json(UpperCamelCase_   ,       save_dir / VOCAB_FILES_NAMES['''vocab_file''']							)
							if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
												copyfile(UpperCamelCase_   ,       save_dir / VOCAB_FILES_NAMES['''spm_file''']							)

							UpperCamelCase__       :Union[str, Any]							 =			SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
							tokenizer.save_pretrained(self.tmpdirname							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			'''<pad>'''
							UpperCamelCase__       :List[Any]							 =			1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_							)   ,       UpperCamelCase_							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_							)   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[str]							 =			list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]   ,       '''<s>'''							)
							self.assertEqual(vocab_keys[1]   ,       '''<pad>'''							)
							self.assertEqual(vocab_keys[-1]   ,       '''j'''							)
							self.assertEqual(len(UpperCamelCase_							)   ,       1001							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertEqual(self.get_tokenizer().vocab_size   ,       1001							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

							UpperCamelCase__       :Tuple							 =			tokenizer.tokenize('''This is a test'''							)
							self.assertListEqual(UpperCamelCase_   ,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCamelCase_							)   ,       [289, 50, 14, 174, 386]   ,       )

							UpperCamelCase__       :Optional[Any]							 =			tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']   ,       )
							UpperCamelCase__       :Union[str, Any]							 =			tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
							self.assertListEqual(UpperCamelCase_   ,       [12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]							)

							UpperCamelCase__       :Optional[Any]							 =			tokenizer.convert_ids_to_tokens(UpperCamelCase_							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']   ,       )




		@slow
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			{'''input_ids''': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=UpperCamelCase_   ,       model_name='''facebook/s2t-small-mustc-en-de-st'''   ,       revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''   ,       )



@require_sentencepiece
class        lowercase     (							unittest.TestCase ):

		"""simple docstring"""

		_a   			=    'valhalla/s2t_mustc_multilinguial_medium'

		_a   			=    'C\'est trop cool'
		_a   			=    'Esto es genial'




		@classmethod
		def    lowerCAmelCase__    (		cls							):

							'''simple docstring'''


							UpperCamelCase__       :SpeechaTextTokenizer							 =			SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
							return cls




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']   ,       4							)
							self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']   ,       6							)
							self.assertEqual(self.tokenizer.lang_code_to_id['''it''']   ,       9							)
							self.assertEqual(self.tokenizer.lang_code_to_id['''de''']   ,       11							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertEqual(self.tokenizer.vocab_size   ,       10000							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertIn(UpperCamelCase_   ,       self.tokenizer.all_special_ids							)
							UpperCamelCase__       :str							 =			[ES_CODE, 4, 1601, 47, 7647, 2]
							UpperCamelCase__       :Optional[Any]							 =			self.tokenizer.decode(UpperCamelCase_   ,       skip_special_tokens=UpperCamelCase_							)
							UpperCamelCase__       :Union[str, Any]							 =			self.tokenizer.decode(generated_ids[1:]   ,       skip_special_tokens=UpperCamelCase_							)
							self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
							self.assertNotIn(self.tokenizer.eos_token   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :str							 =			'''fr'''
							UpperCamelCase__       :List[str]							 =			self.tokenizer(self.french_text							).input_ids
							self.assertEqual(encoded[0]   ,       UpperCamelCase_							)
							self.assertEqual(encoded[-1]   ,       self.tokenizer.eos_token_id							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			'''fr'''
							self.assertListEqual(self.tokenizer.prefix_tokens   ,       [FR_CODE]							)

							UpperCamelCase__       :Optional[int]							 =			'''es'''
							self.assertListEqual(self.tokenizer.prefix_tokens   ,       [ES_CODE]							)



'''simple docstring'''
import argparse
from collections import defaultdict

import yaml


__snake_case											=					'''docs/source/en/_toctree.yml'''


def       a				(      __a       )  ->     int:



					'''simple docstring'''






					UpperCamelCase__       :int							 =			defaultdict(__a       )
					UpperCamelCase__       :int							 =			[]
					UpperCamelCase__       :int							 =			[]
					for doc in doc_list:
										if "local" in doc:
															counts[doc["local"]] += 1

										if doc["title"].lower() == "overview":
															overview_doc.append({'''local''': doc['''local'''], '''title''': doc['''title''']}       )
										else:
															new_doc_list.append(__a       )

					UpperCamelCase__       :Union[str, Any]							 =			new_doc_list
					UpperCamelCase__       :Tuple							 =			[key for key, value in counts.items() if value > 1]

					UpperCamelCase__       :Union[str, Any]							 =			[]
					for duplicate_key in duplicates:
										UpperCamelCase__       :Dict							 =			list({doc['''title'''] for doc in doc_list if doc['''local'''] == duplicate_key}       )
										if len(__a       ) > 1:
															raise ValueError(
															    f'''{duplicate_key} is present several times in the documentation table of content at '''
															    '''`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '''
															    '''others.'''       )
										# Only add this once
										new_doc.append({'''local''': duplicate_key, '''title''': titles[0]}       )

					# Add none duplicate-keys
					new_doc.extend([doc for doc in doc_list if '''local''' not in counts or counts[doc['''local''']] == 1]       )
					UpperCamelCase__       :Union[str, Any]							 =			sorted(__a			, key=lambda __a       : s["title"].lower()       )

					# "overview" gets special treatment and is always first
					if len(__a       ) > 1:
										raise ValueError('''{doc_list} has two \'overview\' docs which is not allowed.'''       )

					overview_doc.extend(__a       )

					# Sort
					return overview_doc


def       a				(      __a=False       )  ->     Any:



					'''simple docstring'''






					with open(__a			, encoding='''utf-8'''       ) as f:
										UpperCamelCase__       :Any							 =			yaml.safe_load(f.read()       )

					# Get to the API doc
					UpperCamelCase__       :str							 =			0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					UpperCamelCase__       :str							 =			content[api_idx]['''sections''']

					# Then to the model doc
					UpperCamelCase__       :Optional[Any]							 =			0
					while api_doc[scheduler_idx]["title"] != "Schedulers":
										scheduler_idx += 1

					UpperCamelCase__       :List[Any]							 =			api_doc[scheduler_idx]['''sections''']
					UpperCamelCase__       :Union[str, Any]							 =			clean_doc_toc(__a       )

					UpperCamelCase__       :List[Any]							 =			False
					if new_scheduler_doc != scheduler_doc:
										UpperCamelCase__       :Optional[int]							 =			True
										if overwrite:
															UpperCamelCase__       :Dict							 =			new_scheduler_doc

					if diff:
										if overwrite:
															UpperCamelCase__       :Any							 =			api_doc
															with open(__a			, '''w'''			, encoding='''utf-8'''       ) as f:
																				f.write(yaml.dump(__a			, allow_unicode=__a       )       )
										else:
															raise ValueError(
															    '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''       )


def       a				(      __a=False       )  ->     Optional[Any]:



					'''simple docstring'''






					with open(__a			, encoding='''utf-8'''       ) as f:
										UpperCamelCase__       :str							 =			yaml.safe_load(f.read()       )

					# Get to the API doc
					UpperCamelCase__       :Optional[Any]							 =			0
					while content[api_idx]["title"] != "API":
										api_idx += 1
					UpperCamelCase__       :Any							 =			content[api_idx]['''sections''']

					# Then to the model doc
					UpperCamelCase__       :str							 =			0
					while api_doc[pipeline_idx]["title"] != "Pipelines":
										pipeline_idx += 1

					UpperCamelCase__       :Any							 =			False
					UpperCamelCase__       :Union[str, Any]							 =			api_doc[pipeline_idx]['''sections''']
					UpperCamelCase__       :Tuple							 =			[]

					# sort sub pipeline docs
					for pipeline_doc in pipeline_docs:
										if "section" in pipeline_doc:
															UpperCamelCase__       :Dict							 =			pipeline_doc['''section''']
															UpperCamelCase__       :Optional[Any]							 =			clean_doc_toc(__a       )
															if overwrite:
																				UpperCamelCase__       :Optional[int]							 =			new_sub_pipeline_doc
										new_pipeline_docs.append(__a       )

					# sort overall pipeline doc
					UpperCamelCase__       :Optional[Any]							 =			clean_doc_toc(__a       )

					if new_pipeline_docs != pipeline_docs:
										UpperCamelCase__       :int							 =			True
										if overwrite:
															UpperCamelCase__       :Union[str, Any]							 =			new_pipeline_docs

					if diff:
										if overwrite:
															UpperCamelCase__       :Dict							 =			api_doc
															with open(__a			, '''w'''			, encoding='''utf-8'''       ) as f:
																				f.write(yaml.dump(__a			, allow_unicode=__a       )       )
										else:
															raise ValueError(
															    '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''       )


if __name__ == "__main__":
		__snake_case											=					argparse.ArgumentParser()
		parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
		__snake_case											=					parser.parse_args()

		check_scheduler_doc(args.fix_and_overwrite)
		check_pipeline_doc(args.fix_and_overwrite)