


'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case											=					get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')
@require_sentencepiece
@require_tokenizers
class        lowercase     (							A__     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    SpeechTaTokenizer
		_a   			=    False
		_a   			=    True




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							super().setUp()

							# We have a SentencePiece fixture for testing
							UpperCamelCase__       :str							 =			SpeechTaTokenizer(UpperCamelCase_							)

							UpperCamelCase__       :Optional[int]							 =			AddedToken('''<mask>'''   ,       lstrip=UpperCamelCase_   ,       rstrip=UpperCamelCase_							)
							UpperCamelCase__       :Any							 =			mask_token
							tokenizer.add_special_tokens({'''mask_token''': mask_token}							)
							tokenizer.add_tokens(['''<ctc_blank>''']							)

							tokenizer.save_pretrained(self.tmpdirname							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			'''this is a test'''
							UpperCamelCase__       :Any							 =			'''this is a test'''
							return input_text, output_text




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_=False   ,       UpperCamelCase_=20   ,       UpperCamelCase_=5							):

							'''simple docstring'''


							UpperCamelCase__ ,    UpperCamelCase__       :int							 =			self.get_input_output_texts(UpperCamelCase_							)
							UpperCamelCase__       :Dict							 =			tokenizer.encode(UpperCamelCase_   ,       add_special_tokens=UpperCamelCase_							)
							UpperCamelCase__       :Any							 =			tokenizer.decode(UpperCamelCase_   ,       clean_up_tokenization_spaces=UpperCamelCase_							)
							return text, ids




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			'''<pad>'''
							UpperCamelCase__       :Any							 =			1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_							)   ,       UpperCamelCase_							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_							)   ,       UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]   ,       '''<s>'''							)
							self.assertEqual(vocab_keys[1]   ,       '''<pad>'''							)
							self.assertEqual(vocab_keys[-4]   ,       '''œ'''							)
							self.assertEqual(vocab_keys[-2]   ,       '''<mask>'''							)
							self.assertEqual(vocab_keys[-1]   ,       '''<ctc_blank>'''							)
							self.assertEqual(len(UpperCamelCase_							)   ,       81							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.assertEqual(self.get_tokenizer().vocab_size   ,       79							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[Any]							 =			self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__       :List[str]							 =			tokenizer.vocab_size
																	UpperCamelCase__       :Dict							 =			len(UpperCamelCase_							)

																	self.assertNotEqual(UpperCamelCase_   ,       0							)

																	# We usually have added tokens from the start in tests because our vocab fixtures are
																	# smaller than the original vocabs - let's not assert this
																	# self.assertEqual(vocab_size, all_size)

																	UpperCamelCase__       :Union[str, Any]							 =			['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
																	UpperCamelCase__       :Optional[Any]							 =			tokenizer.add_tokens(UpperCamelCase_							)
																	UpperCamelCase__       :List[Any]							 =			tokenizer.vocab_size
																	UpperCamelCase__       :int							 =			len(UpperCamelCase_							)

																	self.assertNotEqual(UpperCamelCase_   ,       0							)
																	self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
																	self.assertEqual(UpperCamelCase_   ,       len(UpperCamelCase_							)							)
																	self.assertEqual(UpperCamelCase_   ,       all_size + len(UpperCamelCase_							)							)

																	UpperCamelCase__       :List[str]							 =			tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''   ,       add_special_tokens=UpperCamelCase_							)

																	self.assertGreaterEqual(len(UpperCamelCase_							)   ,       4							)
																	self.assertGreater(tokens[0]   ,       tokenizer.vocab_size - 1							)
																	self.assertGreater(tokens[-3]   ,       tokenizer.vocab_size - 1							)

																	UpperCamelCase__       :Dict							 =			{'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
																	UpperCamelCase__       :List[str]							 =			tokenizer.add_special_tokens(UpperCamelCase_							)
																	UpperCamelCase__       :Optional[Any]							 =			tokenizer.vocab_size
																	UpperCamelCase__       :str							 =			len(UpperCamelCase_							)

																	self.assertNotEqual(UpperCamelCase_   ,       0							)
																	self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)
																	self.assertEqual(UpperCamelCase_   ,       len(UpperCamelCase_							)							)
																	self.assertEqual(UpperCamelCase_   ,       all_size_a + len(UpperCamelCase_							)							)

																	UpperCamelCase__       :Dict							 =			tokenizer.encode(
																	    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''   ,       add_special_tokens=UpperCamelCase_							)

																	self.assertGreaterEqual(len(UpperCamelCase_							)   ,       6							)
																	self.assertGreater(tokens[0]   ,       tokenizer.vocab_size - 1							)
																	self.assertGreater(tokens[0]   ,       tokens[1]							)
																	self.assertGreater(tokens[-3]   ,       tokenizer.vocab_size - 1							)
																	self.assertGreater(tokens[-3]   ,       tokens[-4]							)
																	self.assertEqual(tokens[0]   ,       tokenizer.eos_token_id							)
																	self.assertEqual(tokens[-3]   ,       tokenizer.pad_token_id							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			self.get_tokenizer()

							UpperCamelCase__       :Any							 =			tokenizer.tokenize('''This is a test'''							)
							# fmt: off
							self.assertListEqual(UpperCamelCase_   ,       [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']							)
							# fmt: on

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCamelCase_							)   ,       [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]   ,       )

							UpperCamelCase__       :List[Any]							 =			tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']							)

							UpperCamelCase__       :Dict							 =			tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
							# fmt: off
							self.assertListEqual(UpperCamelCase_   ,       [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]							)
							# fmt: on

							UpperCamelCase__       :int							 =			tokenizer.convert_ids_to_tokens(UpperCamelCase_							)
							self.assertListEqual(
							    UpperCamelCase_   ,       [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']							)




		@slow
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			[
							    '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
							    '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
							    '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
							    '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
							    '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
							    '''conditioning on both left and right context in all layers.''',
							    '''The quick brown fox jumps over the lazy dog.''',
							]

							# fmt: off
							UpperCamelCase__       :Union[str, Any]							 =			{
							    '''input_ids''': [
							        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
							        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							    ],
							    '''attention_mask''': [
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    ]
							}
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=UpperCamelCase_   ,       model_name='''microsoft/speecht5_asr'''   ,       revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''   ,       sequences=UpperCamelCase_   ,       )



'''simple docstring'''
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__snake_case											=					logging.get_logger(__name__)

__snake_case											=					{
    '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/config.json''',
    '''distilbert-base-uncased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/config.json''',
    '''distilbert-base-cased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json''',
    '''distilbert-base-multilingual-cased''': (
        '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json'''
    ),
    '''distilbert-base-uncased-finetuned-sst-2-english''': (
        '''https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json'''
    ),
}
class        lowercase     (							A__ ):

		"""simple docstring"""

		_a   			=    'distilbert'
		_a   			=    {
		    'hidden_size': 'dim',
		    'num_attention_heads': 'n_heads',
		    'num_hidden_layers': 'n_layers',
		}




		def __init__(		self   ,       UpperCamelCase_=30522   ,       UpperCamelCase_=512   ,       UpperCamelCase_=False   ,       UpperCamelCase_=6   ,       UpperCamelCase_=12   ,       UpperCamelCase_=768   ,       UpperCamelCase_=4 * 768   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_="gelu"   ,       UpperCamelCase_=0.02   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_=0.2   ,       UpperCamelCase_=0   ,       **UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			vocab_size
							UpperCamelCase__       :Dict							 =			max_position_embeddings
							UpperCamelCase__       :str							 =			sinusoidal_pos_embds
							UpperCamelCase__       :Any							 =			n_layers
							UpperCamelCase__       :str							 =			n_heads
							UpperCamelCase__       :Tuple							 =			dim
							UpperCamelCase__       :str							 =			hidden_dim
							UpperCamelCase__       :Dict							 =			dropout
							UpperCamelCase__       :int							 =			attention_dropout
							UpperCamelCase__       :Optional[Any]							 =			activation
							UpperCamelCase__       :Optional[int]							 =			initializer_range
							UpperCamelCase__       :Union[str, Any]							 =			qa_dropout
							UpperCamelCase__       :Dict							 =			seq_classif_dropout
							super().__init__(**UpperCamelCase_   ,       pad_token_id=UpperCamelCase_							)



class        lowercase     (							A__ ):

		"""simple docstring"""

		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							if self.task == "multiple-choice":
												UpperCamelCase__       :str							 =			{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
							else:
												UpperCamelCase__       :str							 =			{0: '''batch''', 1: '''sequence'''}
							return OrderedDict(
							    [
							        ('''input_ids''', dynamic_axis),
							        ('''attention_mask''', dynamic_axis),
							    ]							)