


'''simple docstring'''
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		__snake_case											=					'''pt'''
elif is_tf_available():
		__snake_case											=					'''tf'''
else:
		__snake_case											=					'''jax'''
class        lowercase     (							__lowerCamelCase     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    ByTaTokenizer
		_a   			=    False




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							super().setUp()
							UpperCamelCase__       :List[Any]							 =			ByTaTokenizer()
							tokenizer.save_pretrained(self.tmpdirname							)




		@cached_property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return ByTaTokenizer.from_pretrained('''google/byt5-small'''							)




		def    lowerCAmelCase__    (		self   ,       **UpperCamelCase_							):

							'''simple docstring'''


							return self.tokenizer_class.from_pretrained(self.tmpdirname   ,       **__lowercase							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_=False   ,       UpperCamelCase_=20   ,       UpperCamelCase_=5							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			[]
							for i in range(len(__lowercase							)							):
												try:
																	UpperCamelCase__       :Tuple							 =			tokenizer.decode([i]   ,       clean_up_tokenization_spaces=__lowercase							)
												except UnicodeDecodeError:
																	pass
												toks.append((i, tok)							)

							UpperCamelCase__       :str							 =			list(filter(lambda UpperCamelCase_							: re.match(r'''^[ a-zA-Z]+$'''   ,       t[1]							)   ,       __lowercase							)							)
							UpperCamelCase__       :Tuple							 =			list(filter(lambda UpperCamelCase_							: [t[0]] == tokenizer.encode(t[1]   ,       add_special_tokens=__lowercase							)   ,       __lowercase							)							)
							if max_length is not None and len(__lowercase							) > max_length:
												UpperCamelCase__       :List[str]							 =			toks[:max_length]
							if min_length is not None and len(__lowercase							) < min_length and len(__lowercase							) > 0:
												while len(__lowercase							) < min_length:
																	UpperCamelCase__       :Optional[Any]							 =			toks + toks
        # toks_str = [t[1] for t in toks]
							UpperCamelCase__       :List[Any]							 =			[t[0] for t in toks]

							# Ensure consistency
							UpperCamelCase__       :Dict							 =			tokenizer.decode(__lowercase   ,       clean_up_tokenization_spaces=__lowercase							)
							if " " not in output_txt and len(__lowercase							) > 1:
												UpperCamelCase__       :Optional[int]							 =			(
												    tokenizer.decode([toks_ids[0]]   ,       clean_up_tokenization_spaces=__lowercase							)
												    + ''' '''
												    + tokenizer.decode(toks_ids[1:]   ,       clean_up_tokenization_spaces=__lowercase							)
												)
							if with_prefix_space:
												UpperCamelCase__       :Dict							 =			''' ''' + output_txt
							UpperCamelCase__       :Union[str, Any]							 =			tokenizer.encode(__lowercase   ,       add_special_tokens=__lowercase							)
							return output_txt, output_ids




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[str]							 =			self.ta_base_tokenizer
							UpperCamelCase__       :List[Any]							 =			tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>''']							)
							UpperCamelCase__       :Optional[Any]							 =			tokenizer(['''hi''', '''I went to the gym''', '''''']							)
							self.assertListEqual(batch_with_eos_added['''input_ids''']   ,       batch_without_eos_added['''input_ids''']							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			self.ta_base_tokenizer
							UpperCamelCase__       :Any							 =			'''Unicode €.'''
							UpperCamelCase__       :Any							 =			tokenizer(__lowercase							)
							UpperCamelCase__       :Dict							 =			[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
							self.assertEqual(encoded['''input_ids''']   ,       __lowercase							)

							# decoding
							UpperCamelCase__       :List[Any]							 =			tokenizer.decode(__lowercase							)
							self.assertEqual(__lowercase   ,       '''Unicode €.</s>'''							)

							UpperCamelCase__       :Any							 =			tokenizer('''e è é ê ë'''							)
							UpperCamelCase__       :int							 =			[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
							self.assertEqual(encoded['''input_ids''']   ,       __lowercase							)
							# decoding
							UpperCamelCase__       :str							 =			tokenizer.decode(__lowercase							)
							self.assertEqual(__lowercase   ,       '''e è é ê ë</s>'''							)

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''							)							)   ,       '''e è é ê ë</s>'''							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			self.ta_base_tokenizer
							UpperCamelCase__       :Any							 =			['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							# fmt: off
							UpperCamelCase__       :Dict							 =			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
							# fmt: on
							UpperCamelCase__       :Optional[int]							 =			tokenizer(__lowercase   ,       padding=__lowercase   ,       return_tensors=__lowercase							)
							self.assertIsInstance(__lowercase   ,       __lowercase							)

							if FRAMEWORK != "jax":
												UpperCamelCase__       :Optional[Any]							 =			list(batch.input_ids.numpy()[0]							)
							else:
												UpperCamelCase__       :Optional[Any]							 =			list(batch.input_ids.tolist()[0]							)

							self.assertListEqual(__lowercase   ,       __lowercase							)

							self.assertEqual((2, 37)   ,       batch.input_ids.shape							)
							self.assertEqual((2, 37)   ,       batch.attention_mask.shape							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			self.ta_base_tokenizer
							UpperCamelCase__       :int							 =			['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
							UpperCamelCase__       :Tuple							 =			tokenizer(__lowercase   ,       padding=__lowercase   ,       return_tensors=__lowercase							)
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn('''input_ids'''   ,       __lowercase							)
							self.assertIn('''attention_mask'''   ,       __lowercase							)
							self.assertNotIn('''decoder_input_ids'''   ,       __lowercase							)
							self.assertNotIn('''decoder_attention_mask'''   ,       __lowercase							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			self.ta_base_tokenizer
							UpperCamelCase__       :Optional[int]							 =			[
							    '''Summary of the text.''',
							    '''Another summary.''',
							]
							UpperCamelCase__       :str							 =			tokenizer(
							    text_target=__lowercase   ,       max_length=32   ,       padding='''max_length'''   ,       truncation=__lowercase   ,       return_tensors=__lowercase							)
							self.assertEqual(32   ,       targets['''input_ids'''].shape[1]							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[Any]							 =			self.ta_base_tokenizer
							UpperCamelCase__       :Optional[Any]							 =			['''A long paragraph for summarization. </s>''']
							UpperCamelCase__       :Dict							 =			['''Summary of the text. </s>''']
							# fmt: off
							UpperCamelCase__       :str							 =			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
							UpperCamelCase__       :Optional[Any]							 =			[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
							# fmt: on

							UpperCamelCase__       :Union[str, Any]							 =			tokenizer(__lowercase   ,       text_target=__lowercase							)

							self.assertEqual(__lowercase   ,       batch['''input_ids'''][0]							)
							self.assertEqual(__lowercase   ,       batch['''labels'''][0]							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	self.assertNotEqual(tokenizer.model_max_length   ,       42							)

        # Now let's start the test
							UpperCamelCase__       :Union[str, Any]							 =			self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	# Isolate this from the other tests because we save additional tokens/etc
																	UpperCamelCase__       :Optional[Any]							 =			tempfile.mkdtemp()

																	UpperCamelCase__       :Any							 =			''' He is very happy, UNwant\u00E9d,running'''
																	UpperCamelCase__       :Optional[int]							 =			tokenizer.encode(__lowercase   ,       add_special_tokens=__lowercase							)
																	tokenizer.save_pretrained(__lowercase							)

																	UpperCamelCase__       :int							 =			tokenizer.__class__.from_pretrained(__lowercase							)
																	UpperCamelCase__       :List[str]							 =			after_tokenizer.encode(__lowercase   ,       add_special_tokens=__lowercase							)
																	self.assertListEqual(__lowercase   ,       __lowercase							)

																	shutil.rmtree(__lowercase							)

							UpperCamelCase__       :int							 =			self.get_tokenizers(model_max_length=42							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	# Isolate this from the other tests because we save additional tokens/etc
																	UpperCamelCase__       :int							 =			tempfile.mkdtemp()

																	UpperCamelCase__       :List[Any]							 =			''' He is very happy, UNwant\u00E9d,running'''
																	tokenizer.add_tokens(['''bim''', '''bambam''']							)
																	UpperCamelCase__       :Optional[Any]							 =			tokenizer.additional_special_tokens
																	additional_special_tokens.append('''new_additional_special_token'''							)
																	tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}							)
																	UpperCamelCase__       :Union[str, Any]							 =			tokenizer.encode(__lowercase   ,       add_special_tokens=__lowercase							)
																	tokenizer.save_pretrained(__lowercase							)

																	UpperCamelCase__       :Optional[Any]							 =			tokenizer.__class__.from_pretrained(__lowercase							)
																	UpperCamelCase__       :Tuple							 =			after_tokenizer.encode(__lowercase   ,       add_special_tokens=__lowercase							)
																	self.assertListEqual(__lowercase   ,       __lowercase							)
																	self.assertIn('''new_additional_special_token'''   ,       after_tokenizer.additional_special_tokens							)
																	self.assertEqual(after_tokenizer.model_max_length   ,       42							)

																	UpperCamelCase__       :Any							 =			tokenizer.__class__.from_pretrained(__lowercase   ,       model_max_length=43							)
																	self.assertEqual(tokenizer.model_max_length   ,       43							)

																	shutil.rmtree(__lowercase							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :str							 =			[]
							if self.test_slow_tokenizer:
												tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

							if self.test_rust_tokenizer:
												tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
												with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(__lowercase							)

																	with open(os.path.join(__lowercase   ,       '''special_tokens_map.json'''							)   ,       encoding='''utf-8'''							) as json_file:
																						UpperCamelCase__       :Union[str, Any]							 =			json.load(__lowercase							)

																	with open(os.path.join(__lowercase   ,       '''tokenizer_config.json'''							)   ,       encoding='''utf-8'''							) as json_file:
																						UpperCamelCase__       :Dict							 =			json.load(__lowercase							)

																	UpperCamelCase__       :str							 =			[F'''<extra_id_{i}>''' for i in range(125							)]

																	UpperCamelCase__       :Optional[Any]							 =			added_tokens_extra_ids + [
																	    '''an_additional_special_token'''
																	]
																	UpperCamelCase__       :str							 =			added_tokens_extra_ids + [
																	    '''an_additional_special_token'''
																	]

																	with open(os.path.join(__lowercase   ,       '''special_tokens_map.json'''							)   ,       '''w'''   ,       encoding='''utf-8'''							) as outfile:
																						json.dump(__lowercase   ,       __lowercase							)
																	with open(os.path.join(__lowercase   ,       '''tokenizer_config.json'''							)   ,       '''w'''   ,       encoding='''utf-8'''							) as outfile:
																						json.dump(__lowercase   ,       __lowercase							)

																	# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																	# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																	# "special_tokens_map.json" files
																	UpperCamelCase__       :List[str]							 =			tokenizer_class.from_pretrained(
																	    __lowercase   ,       )
																	self.assertIn(
																	    '''an_additional_special_token'''   ,       tokenizer_without_change_in_init.additional_special_tokens							)
																	# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																	self.assertEqual(
																	    ['''an_additional_special_token''']   ,       tokenizer_without_change_in_init.convert_ids_to_tokens(
																	        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']							)							)   ,       )

																	# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																	UpperCamelCase__       :Tuple							 =			added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''   ,       lstrip=__lowercase							)]
																	UpperCamelCase__       :List[str]							 =			tokenizer_class.from_pretrained(
																	    __lowercase   ,       additional_special_tokens=__lowercase   ,       )

																	self.assertIn('''a_new_additional_special_token'''   ,       tokenizer.additional_special_tokens							)
																	self.assertEqual(
																	    ['''a_new_additional_special_token''']   ,       tokenizer.convert_ids_to_tokens(
																	        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']							)							)   ,       )




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			[]
							if self.test_slow_tokenizer:
												tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

							if self.test_rust_tokenizer:
												tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
												with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(__lowercase							)

																	UpperCamelCase__       :Dict							 =			tokenizer_class.from_pretrained(__lowercase							)

																	self.assertTrue(tokenizer.decode([255]							) == ''''''							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			self.get_tokenizers(fast=__lowercase   ,       do_lower_case=__lowercase							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__       :Optional[Any]							 =			['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
																	UpperCamelCase__       :int							 =			tokenizer.convert_tokens_to_string(__lowercase							)

																	self.assertIsInstance(__lowercase   ,       __lowercase							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__       :str							 =			[
																	    '''bos_token''',
																	    '''eos_token''',
																	    '''unk_token''',
																	    '''sep_token''',
																	    '''pad_token''',
																	    '''cls_token''',
																	    '''mask_token''',
																	]

																	UpperCamelCase__       :List[Any]							 =			0
																	UpperCamelCase__       :Union[str, Any]							 =			tokenizer.convert_ids_to_tokens(
																	    __lowercase   ,       skip_special_tokens=__lowercase							)

																	for attr in attributes_list:
																						setattr(__lowercase   ,       attr + '''_id'''   ,       __lowercase							)
																						self.assertEqual(getattr(__lowercase   ,       __lowercase							)   ,       __lowercase							)
																						self.assertEqual(getattr(__lowercase   ,       attr + '''_id'''							)   ,       __lowercase							)

																						setattr(__lowercase   ,       attr + '''_id'''   ,       __lowercase							)
																						self.assertEqual(getattr(__lowercase   ,       __lowercase							)   ,       __lowercase							)
																						self.assertEqual(getattr(__lowercase   ,       attr + '''_id'''							)   ,       __lowercase							)

																	setattr(__lowercase   ,       '''additional_special_tokens_ids'''   ,       []							)
																	self.assertListEqual(getattr(__lowercase   ,       '''additional_special_tokens'''							)   ,       []							)
																	self.assertListEqual(getattr(__lowercase   ,       '''additional_special_tokens_ids'''							)   ,       []							)

																	setattr(__lowercase   ,       '''additional_special_tokens_ids'''   ,       [token_id_to_test_setters]							)
																	self.assertListEqual(getattr(__lowercase   ,       '''additional_special_tokens'''							)   ,       [token_to_test_setters]							)
																	self.assertListEqual(getattr(__lowercase   ,       '''additional_special_tokens_ids'''							)   ,       [token_id_to_test_setters]							)






'''simple docstring'''
from math import ceil


def       a				(      __a			, __a       )  ->     Any:



					'''simple docstring'''






					UpperCamelCase__       :str							 =			list(range(0			, __a       )       )

					UpperCamelCase__       :Optional[int]							 =			[item for sublist in list(device_map.values()       ) for item in sublist]

					# Duplicate check
					UpperCamelCase__       :Optional[int]							 =			[]
					for i in device_map_blocks:
										if device_map_blocks.count(__a       ) > 1 and i not in duplicate_blocks:
															duplicate_blocks.append(__a       )
    # Missing blocks
					UpperCamelCase__       :List[str]							 =			[i for i in blocks if i not in device_map_blocks]
					UpperCamelCase__       :Optional[Any]							 =			[i for i in device_map_blocks if i not in blocks]

					if len(__a       ) != 0:
										raise ValueError(
										    '''Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device.'''
										    ''' These attention blocks were specified more than once: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''There are attention blocks for this model that are not specified in the device_map. Add these attention '''
										    '''blocks to a device on the device_map: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''The device_map contains more attention blocks than this model has. Remove these from the device_map:'''
										    + str(__a       )       )







def       a				(      __a			, __a       )  ->     Tuple:



					'''simple docstring'''






					UpperCamelCase__       :Optional[Any]							 =			list(range(__a       )       )
					UpperCamelCase__       :Any							 =			int(ceil(n_layers / len(__a       )       )       )
					UpperCamelCase__       :List[Any]							 =			[layers[i : i + n_blocks] for i in range(0			, __a			, __a       )]

					return dict(zip(__a			, __a       )       )