




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			lowerCamelCase__			:       List[Any]         =							"pt"
elif is_tf_available():
			lowerCamelCase__			:       Tuple         =							"tf"
else:
			lowerCamelCase__			:       List[str]         =							"jax"





class 	lowerCamelCase_							(				__lowerCAmelCase	,       unittest.TestCase ):


	'''simple docstring'''







	lowercase_       =   ByTaTokenizer
	lowercase_       =   False



	def 				lowerCAmelCase_  (							self  :		List[str]						):
							super().setUp()
							SCREAMING_SNAKE_CASE_     						=      ByTaTokenizer()
							tokenizer.save_pretrained(self.tmpdirname						)



	@cached_property
	def 				lowerCAmelCase_  (							self  :		Union[str, Any]						):
							return ByTaTokenizer.from_pretrained('google/byt5-small'						)



	def 				lowerCAmelCase_  (							self  :		List[Any]    ,    **_lowerCAmelCase  :		int						):
							return self.tokenizer_class.from_pretrained(self.tmpdirname    ,    **lowerCamelCase__						)



	def 				lowerCAmelCase_  (							self  :		Dict    ,    _lowerCAmelCase  :		Dict    ,    _lowerCAmelCase  :		Optional[Any]=False    ,    _lowerCAmelCase  :		str=20    ,    _lowerCAmelCase  :		Optional[int]=5						):
							SCREAMING_SNAKE_CASE_     						=      []
							for i in range(len(lowerCamelCase__						)						):
													try:
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.decode([i]    ,    clean_up_tokenization_spaces=lowerCamelCase__						)
													except UnicodeDecodeError:
																			pass
													toks.append((i, tok)						)

							SCREAMING_SNAKE_CASE_     						=      list(filter(lambda _lowerCAmelCase						: re.match(R'^[ a-zA-Z]+$'    ,    t[1]						)    ,    lowerCamelCase__						)						)
							SCREAMING_SNAKE_CASE_     						=      list(filter(lambda _lowerCAmelCase						: [t[0]] == tokenizer.encode(t[1]    ,    add_special_tokens=lowerCamelCase__						)    ,    lowerCamelCase__						)						)
							if max_length is not None and len(lowerCamelCase__						) > max_length:
													SCREAMING_SNAKE_CASE_     						=      toks[:max_length]
							if min_length is not None and len(lowerCamelCase__						) < min_length and len(lowerCamelCase__						) > 0:
													while len(lowerCamelCase__						) < min_length:
																			SCREAMING_SNAKE_CASE_     						=      toks + toks
																			# toks_str = [t[1] for t in toks]
							SCREAMING_SNAKE_CASE_     						=      [t[0] for t in toks]

							# Ensure consistency
							SCREAMING_SNAKE_CASE_     						=      tokenizer.decode(lowerCamelCase__    ,    clean_up_tokenization_spaces=lowerCamelCase__						)
							if " " not in output_txt and len(lowerCamelCase__						) > 1:
													SCREAMING_SNAKE_CASE_     						=      (
													    tokenizer.decode([toks_ids[0]]    ,    clean_up_tokenization_spaces=lowerCamelCase__						)
													    + ''' '''
													    + tokenizer.decode(toks_ids[1:]    ,    clean_up_tokenization_spaces=lowerCamelCase__						)
													)
							if with_prefix_space:
													SCREAMING_SNAKE_CASE_     						=      ''' ''' + output_txt
							SCREAMING_SNAKE_CASE_     						=      tokenizer.encode(lowerCamelCase__    ,    add_special_tokens=lowerCamelCase__						)
							return output_txt, output_ids



	def 				lowerCAmelCase_  (							self  :		Dict						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']						)
							SCREAMING_SNAKE_CASE_     						=      tokenizer(['hi', 'I went to the gym', '']						)
							self.assertListEqual(batch_with_eos_added['input_ids']    ,    batch_without_eos_added['input_ids']						)



	def 				lowerCAmelCase_  (							self  :		str						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      '''Unicode €.'''
							SCREAMING_SNAKE_CASE_     						=      tokenizer(lowerCamelCase__						)
							SCREAMING_SNAKE_CASE_     						=      [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
							self.assertEqual(encoded['input_ids']    ,    lowerCamelCase__						)

							# decoding
							SCREAMING_SNAKE_CASE_     						=      tokenizer.decode(lowerCamelCase__						)
							self.assertEqual(lowerCamelCase__    ,    'Unicode €.</s>'						)

							SCREAMING_SNAKE_CASE_     						=      tokenizer('e è é ê ë'						)
							SCREAMING_SNAKE_CASE_     						=      [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
							self.assertEqual(encoded['input_ids']    ,    lowerCamelCase__						)
							# decoding
							SCREAMING_SNAKE_CASE_     						=      tokenizer.decode(lowerCamelCase__						)
							self.assertEqual(lowerCamelCase__    ,    'e è é ê ë</s>'						)

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'						)						)    ,    'e è é ê ë</s>'						)



	def 				lowerCAmelCase_  (							self  :		str						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							# fmt: off
							SCREAMING_SNAKE_CASE_     						=      [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
							# fmt: on
							SCREAMING_SNAKE_CASE_     						=      tokenizer(lowerCamelCase__    ,    padding=lowerCamelCase__    ,    return_tensors=lowerCamelCase__						)
							self.assertIsInstance(lowerCamelCase__    ,    lowerCamelCase__						)

							if FRAMEWORK != "jax":
													SCREAMING_SNAKE_CASE_     						=      list(batch.input_ids.numpy()[0]						)
							else:
													SCREAMING_SNAKE_CASE_     						=      list(batch.input_ids.tolist()[0]						)

							self.assertListEqual(lowerCamelCase__    ,    lowerCamelCase__						)

							self.assertEqual((2, 37)    ,    batch.input_ids.shape						)
							self.assertEqual((2, 37)    ,    batch.attention_mask.shape						)



	def 				lowerCAmelCase_  (							self  :		Dict						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							SCREAMING_SNAKE_CASE_     						=      tokenizer(lowerCamelCase__    ,    padding=lowerCamelCase__    ,    return_tensors=lowerCamelCase__						)
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn('input_ids'    ,    lowerCamelCase__						)
							self.assertIn('attention_mask'    ,    lowerCamelCase__						)
							self.assertNotIn('decoder_input_ids'    ,    lowerCamelCase__						)
							self.assertNotIn('decoder_attention_mask'    ,    lowerCamelCase__						)



	def 				lowerCAmelCase_  (							self  :		Optional[int]						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      [
							    '''Summary of the text.''',
							    '''Another summary.''',
							]
							SCREAMING_SNAKE_CASE_     						=      tokenizer(
							    text_target=lowerCamelCase__    ,    max_length=32    ,    padding='max_length'    ,    truncation=lowerCamelCase__    ,    return_tensors=lowerCamelCase__						)
							self.assertEqual(32    ,    targets['input_ids'].shape[1]						)



	def 				lowerCAmelCase_  (							self  :		List[Any]						):
							SCREAMING_SNAKE_CASE_     						=      self.ta_base_tokenizer
							SCREAMING_SNAKE_CASE_     						=      ['''A long paragraph for summarization. </s>''']
							SCREAMING_SNAKE_CASE_     						=      ['''Summary of the text. </s>''']
							# fmt: off
							SCREAMING_SNAKE_CASE_     						=      [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
							SCREAMING_SNAKE_CASE_     						=      [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
							# fmt: on

							SCREAMING_SNAKE_CASE_     						=      tokenizer(lowerCamelCase__    ,    text_target=lowerCamelCase__						)

							self.assertEqual(lowerCamelCase__    ,    batch['input_ids'][0]						)
							self.assertEqual(lowerCamelCase__    ,    batch['labels'][0]						)



	def 				lowerCAmelCase_  (							self  :		Dict						):
							SCREAMING_SNAKE_CASE_     						=      self.get_tokenizers()
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"						):
																			self.assertNotEqual(tokenizer.model_max_length    ,    42						)

																			# Now let's start the test
							SCREAMING_SNAKE_CASE_     						=      self.get_tokenizers()
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"						):
																			# Isolate this from the other tests because we save additional tokens/etc
																			SCREAMING_SNAKE_CASE_     						=      tempfile.mkdtemp()

																			SCREAMING_SNAKE_CASE_     						=      ''' He is very happy, UNwant\u00E9d,running'''
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.encode(lowerCamelCase__    ,    add_special_tokens=lowerCamelCase__						)
																			tokenizer.save_pretrained(lowerCamelCase__						)

																			SCREAMING_SNAKE_CASE_     						=      tokenizer.__class__.from_pretrained(lowerCamelCase__						)
																			SCREAMING_SNAKE_CASE_     						=      after_tokenizer.encode(lowerCamelCase__    ,    add_special_tokens=lowerCamelCase__						)
																			self.assertListEqual(lowerCamelCase__    ,    lowerCamelCase__						)

																			shutil.rmtree(lowerCamelCase__						)

							SCREAMING_SNAKE_CASE_     						=      self.get_tokenizers(model_max_length=42						)
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"						):
																			# Isolate this from the other tests because we save additional tokens/etc
																			SCREAMING_SNAKE_CASE_     						=      tempfile.mkdtemp()

																			SCREAMING_SNAKE_CASE_     						=      ''' He is very happy, UNwant\u00E9d,running'''
																			tokenizer.add_tokens(['bim', 'bambam']						)
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.additional_special_tokens
																			additional_special_tokens.append('new_additional_special_token'						)
																			tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}						)
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.encode(lowerCamelCase__    ,    add_special_tokens=lowerCamelCase__						)
																			tokenizer.save_pretrained(lowerCamelCase__						)

																			SCREAMING_SNAKE_CASE_     						=      tokenizer.__class__.from_pretrained(lowerCamelCase__						)
																			SCREAMING_SNAKE_CASE_     						=      after_tokenizer.encode(lowerCamelCase__    ,    add_special_tokens=lowerCamelCase__						)
																			self.assertListEqual(lowerCamelCase__    ,    lowerCamelCase__						)
																			self.assertIn('new_additional_special_token'    ,    after_tokenizer.additional_special_tokens						)
																			self.assertEqual(after_tokenizer.model_max_length    ,    42						)

																			SCREAMING_SNAKE_CASE_     						=      tokenizer.__class__.from_pretrained(lowerCamelCase__    ,    model_max_length=43						)
																			self.assertEqual(tokenizer.model_max_length    ,    43						)

																			shutil.rmtree(lowerCamelCase__						)



	def 				lowerCAmelCase_  (							self  :		Tuple						):
							SCREAMING_SNAKE_CASE_     						=      []
							if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

							if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer_utils.save_pretrained(lowerCamelCase__						)

																			with open(os.path.join(lowerCamelCase__    ,    'special_tokens_map.json'						)    ,    encoding='utf-8'						) as json_file:
																									SCREAMING_SNAKE_CASE_     						=      json.load(lowerCamelCase__						)

																			with open(os.path.join(lowerCamelCase__    ,    'tokenizer_config.json'						)    ,    encoding='utf-8'						) as json_file:
																									SCREAMING_SNAKE_CASE_     						=      json.load(lowerCamelCase__						)

																			SCREAMING_SNAKE_CASE_     						=      [F"<extra_id_{i}>" for i in range(125						)]

																			SCREAMING_SNAKE_CASE_     						=      added_tokens_extra_ids + [
																			    '''an_additional_special_token'''
																			]
																			SCREAMING_SNAKE_CASE_     						=      added_tokens_extra_ids + [
																			    '''an_additional_special_token'''
																			]

																			with open(os.path.join(lowerCamelCase__    ,    'special_tokens_map.json'						)    ,    'w'    ,    encoding='utf-8'						) as outfile:
																									json.dump(lowerCamelCase__    ,    lowerCamelCase__						)
																			with open(os.path.join(lowerCamelCase__    ,    'tokenizer_config.json'						)    ,    'w'    ,    encoding='utf-8'						) as outfile:
																									json.dump(lowerCamelCase__    ,    lowerCamelCase__						)

																			# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																			# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																			# "special_tokens_map.json" files
																			SCREAMING_SNAKE_CASE_     						=      tokenizer_class.from_pretrained(
																			    lowerCamelCase__    ,    )
																			self.assertIn(
																			    'an_additional_special_token'    ,    tokenizer_without_change_in_init.additional_special_tokens						)
																			# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																			self.assertEqual(
																			    ['an_additional_special_token']    ,    tokenizer_without_change_in_init.convert_ids_to_tokens(
																			        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']						)						)    ,    )

																			# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																			SCREAMING_SNAKE_CASE_     						=      added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'    ,    lstrip=lowerCamelCase__						)]
																			SCREAMING_SNAKE_CASE_     						=      tokenizer_class.from_pretrained(
																			    lowerCamelCase__    ,    additional_special_tokens=lowerCamelCase__    ,    )

																			self.assertIn('a_new_additional_special_token'    ,    tokenizer.additional_special_tokens						)
																			self.assertEqual(
																			    ['a_new_additional_special_token']    ,    tokenizer.convert_ids_to_tokens(
																			        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']						)						)    ,    )



	def 				lowerCAmelCase_  (							self  :		List[str]						):
							SCREAMING_SNAKE_CASE_     						=      []
							if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

							if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer_utils.save_pretrained(lowerCamelCase__						)

																			SCREAMING_SNAKE_CASE_     						=      tokenizer_class.from_pretrained(lowerCamelCase__						)

																			self.assertTrue(tokenizer.decode([255]						) == ''						)



	def 				lowerCAmelCase_  (							self  :		Optional[int]						):
							pass



	def 				lowerCAmelCase_  (							self  :		int						):
							pass



	def 				lowerCAmelCase_  (							self  :		Optional[Any]						):
							pass



	def 				lowerCAmelCase_  (							self  :		Optional[Any]						):
							pass



	def 				lowerCAmelCase_  (							self  :		List[Any]						):
							SCREAMING_SNAKE_CASE_     						=      self.get_tokenizers(fast=lowerCamelCase__    ,    do_lower_case=lowerCamelCase__						)
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"						):
																			SCREAMING_SNAKE_CASE_     						=      ['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.convert_tokens_to_string(lowerCamelCase__						)

																			self.assertIsInstance(lowerCamelCase__    ,    lowerCamelCase__						)




	def 				lowerCAmelCase_  (							self  :		int						):
							SCREAMING_SNAKE_CASE_     						=      self.get_tokenizers()
							for tokenizer in tokenizers:
													with self.subTest(F"{tokenizer.__class__.__name__}"						):
																			SCREAMING_SNAKE_CASE_     						=      [
																			    '''bos_token''',
																			    '''eos_token''',
																			    '''unk_token''',
																			    '''sep_token''',
																			    '''pad_token''',
																			    '''cls_token''',
																			    '''mask_token''',
																			]

																			SCREAMING_SNAKE_CASE_     						=      0
																			SCREAMING_SNAKE_CASE_     						=      tokenizer.convert_ids_to_tokens(
																			    lowerCamelCase__    ,    skip_special_tokens=lowerCamelCase__						)

																			for attr in attributes_list:
																									setattr(lowerCamelCase__    ,    attr + '_id'    ,    lowerCamelCase__						)
																									self.assertEqual(getattr(lowerCamelCase__    ,    lowerCamelCase__						)    ,    lowerCamelCase__						)
																									self.assertEqual(getattr(lowerCamelCase__    ,    attr + '_id'						)    ,    lowerCamelCase__						)

																									setattr(lowerCamelCase__    ,    attr + '_id'    ,    lowerCamelCase__						)
																									self.assertEqual(getattr(lowerCamelCase__    ,    lowerCamelCase__						)    ,    lowerCamelCase__						)
																									self.assertEqual(getattr(lowerCamelCase__    ,    attr + '_id'						)    ,    lowerCamelCase__						)

																			setattr(lowerCamelCase__    ,    'additional_special_tokens_ids'    ,    []						)
																			self.assertListEqual(getattr(lowerCamelCase__    ,    'additional_special_tokens'						)    ,    []						)
																			self.assertListEqual(getattr(lowerCamelCase__    ,    'additional_special_tokens_ids'						)    ,    []						)

																			setattr(lowerCamelCase__    ,    'additional_special_tokens_ids'    ,    [token_id_to_test_setters]						)
																			self.assertListEqual(getattr(lowerCamelCase__    ,    'additional_special_tokens'						)    ,    [token_to_test_setters]						)
																			self.assertListEqual(getattr(lowerCamelCase__    ,    'additional_special_tokens_ids'						)    ,    [token_id_to_test_setters]						)





def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[Any]      ,      __UpperCAmelCase						:     Union[str, Any]							) ->			List[str]:
      SCREAMING_SNAKE_CASE_     						=      0
      while b > 0:
            if b & 1:
                  res += a

            a += a
            b >>= 1

      return res

def  UpperCAmelCase_							(				__UpperCAmelCase						:     str      ,      __UpperCAmelCase						:     int      ,      __UpperCAmelCase						:     Tuple							) ->			int:
      SCREAMING_SNAKE_CASE_     						=      0
      while b > 0:
            if b & 1:
                  SCREAMING_SNAKE_CASE_     						=      ((res % c) + (a % c)) % c

            a += a
            b >>= 1

      return res