




"""simple docstring"""



import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE    =	get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")




@require_sentencepiece
@require_tokenizers
class      UpperCAmelCase_  (						A_,       unittest.TestCase							):
    lowercase__						 =				SpeechTaTokenizer
    lowercase__						 =				False
    lowercase__						 =				True




    def 							__magic_name__							(   self						:	Dict				)				->							Optional[int]:




     '''simple docstring'''

     super().setUp()

     # We have a SentencePiece fixture for testing
     A__	=    SpeechTaTokenizer(snake_case_				)

     A__	=    AddedToken("<mask>"     ,			lstrip=snake_case_     ,			rstrip=snake_case_				)
     A__	=    mask_token
     tokenizer.add_special_tokens({"mask_token": mask_token}				)
     tokenizer.add_tokens(["<ctc_blank>"]				)

     tokenizer.save_pretrained(self.tmpdirname				)




    def 							__magic_name__							(   self						:	Optional[Any]     ,			snake_case_						:	List[str]				)				->							Tuple:




     '''simple docstring'''

     A__	=    "this is a test"
     A__	=    "this is a test"
     return input_text, output_text




    def 							__magic_name__							(   self						:	Dict     ,			snake_case_						:	Dict     ,			snake_case_						:	Any=False     ,			snake_case_						:	int=20     ,			snake_case_						:	Dict=5				)				->							Optional[int]:




     '''simple docstring'''

     A__,				A__	=    self.get_input_output_texts(snake_case_				)
     A__	=    tokenizer.encode(snake_case_     ,			add_special_tokens=snake_case_				)
     A__	=    tokenizer.decode(snake_case_     ,			clean_up_tokenization_spaces=snake_case_				)
     return text, ids




    def 							__magic_name__							(   self						:	Dict				)				->							List[str]:




     '''simple docstring'''

     A__	=    "<pad>"
     A__	=    1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_				)     ,			snake_case_				)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_				)     ,			snake_case_				)




    def 							__magic_name__							(   self						:	Dict				)				->							Dict:




     '''simple docstring'''

     A__	=    list(self.get_tokenizer().get_vocab().keys()				)

     self.assertEqual(vocab_keys[0]     ,			"<s>"				)
     self.assertEqual(vocab_keys[1]     ,			"<pad>"				)
     self.assertEqual(vocab_keys[-4]     ,			"œ"				)
     self.assertEqual(vocab_keys[-2]     ,			"<mask>"				)
     self.assertEqual(vocab_keys[-1]     ,			"<ctc_blank>"				)
     self.assertEqual(len(snake_case_				)     ,			81				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Dict:




     '''simple docstring'''

     self.assertEqual(self.get_tokenizer().vocab_size     ,			79				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Any:




     '''simple docstring'''

     A__	=    self.get_tokenizers(do_lower_case=snake_case_				)
     for tokenizer in tokenizers:
      with self.subTest(F"""{tokenizer.__class__.__name__}"""				):
       A__	=    tokenizer.vocab_size
       A__	=    len(snake_case_				)

       self.assertNotEqual(snake_case_     ,			0				)

       # We usually have added tokens from the start in tests because our vocab fixtures are
       # smaller than the original vocabs - let's not assert this
       # self.assertEqual(vocab_size, all_size)

       A__	=    ["aaaaa bbbbbb", "cccccccccdddddddd"]
       A__	=    tokenizer.add_tokens(snake_case_				)
       A__	=    tokenizer.vocab_size
       A__	=    len(snake_case_				)

       self.assertNotEqual(snake_case_     ,			0				)
       self.assertEqual(snake_case_     ,			snake_case_				)
       self.assertEqual(snake_case_     ,			len(snake_case_				)				)
       self.assertEqual(snake_case_     ,			all_size + len(snake_case_				)				)

       A__	=    tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l"     ,			add_special_tokens=snake_case_				)

       self.assertGreaterEqual(len(snake_case_				)     ,			4				)
       self.assertGreater(tokens[0]     ,			tokenizer.vocab_size - 1				)
       self.assertGreater(tokens[-3]     ,			tokenizer.vocab_size - 1				)

       A__	=    {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
       A__	=    tokenizer.add_special_tokens(snake_case_				)
       A__	=    tokenizer.vocab_size
       A__	=    len(snake_case_				)

       self.assertNotEqual(snake_case_     ,			0				)
       self.assertEqual(snake_case_     ,			snake_case_				)
       self.assertEqual(snake_case_     ,			len(snake_case_				)				)
       self.assertEqual(snake_case_     ,			all_size_a + len(snake_case_				)				)

       A__	=    tokenizer.encode(
           ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"     ,			add_special_tokens=snake_case_				)

       self.assertGreaterEqual(len(snake_case_				)     ,			6				)
       self.assertGreater(tokens[0]     ,			tokenizer.vocab_size - 1				)
       self.assertGreater(tokens[0]     ,			tokens[1]				)
       self.assertGreater(tokens[-3]     ,			tokenizer.vocab_size - 1				)
       self.assertGreater(tokens[-3]     ,			tokens[-4]				)
       self.assertEqual(tokens[0]     ,			tokenizer.eos_token_id				)
       self.assertEqual(tokens[-3]     ,			tokenizer.pad_token_id				)




    def 							__magic_name__							(   self						:	Tuple				)				->							int:




     '''simple docstring'''

     pass




    def 							__magic_name__							(   self						:	int				)				->							Union[str, Any]:




     '''simple docstring'''

     pass




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    self.get_tokenizer()

     A__	=    tokenizer.tokenize("This is a test"				)
     # fmt: off
     self.assertListEqual(snake_case_     ,			[SPIECE_UNDERLINE, "T", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "a", SPIECE_UNDERLINE, "t", "e", "s", "t"]				)
     # fmt: on

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(snake_case_				)     ,			[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]     ,			)

     A__	=    tokenizer.tokenize("I was born in 92000, and this is falsé."				)
     self.assertListEqual(
         snake_case_     ,			[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "92000", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]				)

     A__	=    tokenizer.convert_tokens_to_ids(snake_case_				)
     # fmt: off
     self.assertListEqual(snake_case_     ,			[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]				)
     # fmt: on

     A__	=    tokenizer.convert_ids_to_tokens(snake_case_				)
     self.assertListEqual(
         snake_case_     ,			[SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "<unk>", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]				)




    @slow
    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							str:




     '''simple docstring'''

     A__	=    [
         "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
         "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
         "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
         "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
         "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
         "conditioning on both left and right context in all layers.",
         "The quick brown fox jumps over the lazy dog.",
     ]

     # fmt: off
     A__	=    {
         "input_ids": [
             [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
             [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         ],
         "attention_mask": [
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         ]
     }
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=snake_case_     ,			model_name="microsoft/speecht5_asr"     ,			revision="c5ef64c71905caeccde0e4462ef3f9077224c524"     ,			sequences=snake_case_     ,			)











"""simple docstring"""



from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class      UpperCAmelCase_  (						A_							):
    lowercase__						 =				['''image_processor''', '''tokenizer''']
    lowercase__						 =				'''AutoImageProcessor'''
    lowercase__						 =				'''AutoTokenizer'''




    def __init__(   self						:	str     ,			snake_case_						:	Dict     ,			snake_case_						:	List[str]				)				->							str:




     '''simple docstring'''

     super().__init__(snake_case_     ,			snake_case_				)
     A__	=    self.image_processor




    def __call__(   self						:	int     ,			snake_case_						:	Any=None     ,			snake_case_						:	Any=None     ,			snake_case_						:	Union[str, Any]=None     ,			**snake_case_						:	Optional[int]				)				->							Optional[int]:




     '''simple docstring'''

     if text is None and images is None:
      raise ValueError("You have to specify either text or images. Both cannot be none."				)

     if text is not None:
      A__	=    self.tokenizer(snake_case_     ,			return_tensors=snake_case_     ,			**snake_case_				)

     if images is not None:
      A__	=    self.image_processor(snake_case_     ,			return_tensors=snake_case_     ,			**snake_case_				)

     if text is not None and images is not None:
      A__	=    image_features.pixel_values
      return encoding
     elif text is not None:
      return encoding
     else:
      return BatchEncoding(data=dict(**snake_case_				)     ,			tensor_type=snake_case_				)




    def 							__magic_name__							(   self						:	Optional[int]     ,			*snake_case_						:	Union[str, Any]     ,			**snake_case_						:	List[Any]				)				->							int:




     '''simple docstring'''

     return self.tokenizer.batch_decode(*snake_case_     ,			**snake_case_				)




    def 							__magic_name__							(   self						:	List[str]     ,			*snake_case_						:	List[str]     ,			**snake_case_						:	Optional[int]				)				->							Tuple:




     '''simple docstring'''

     return self.tokenizer.decode(*snake_case_     ,			**snake_case_				)




    @property
    def 							__magic_name__							(   self						:	List[Any]				)				->							List[Any]:




     '''simple docstring'''

     return ["input_ids", "attention_mask", "pixel_values"]





