




"""simple docstring"""


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  _A													=      """pt"""
elif is_tf_available():
  _A													=      """tf"""
else:
  _A													=      """jax"""


class _lowerCamelCase    (					a_       ,	unittest.TestCase						):
   _lowerCamelCase    :List[str]     				=     ByTaTokenizer
   _lowerCamelCase    :Dict     				=     False






   def 	_lowerCAmelCase   (		self       :		int   )			->						Dict:

         """simple docstring"""
         super().setUp()
         lowerCAmelCase__      :			str   =	ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname   )






   @cached_property
   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						List[str]:

         """simple docstring"""
         return ByTaTokenizer.from_pretrained("""google/byt5-small"""   )






   def 	_lowerCAmelCase   (		self       :		Dict   ,     **UpperCamelCase       :		int   )			->						ByTaTokenizer:

         """simple docstring"""
         return self.tokenizer_class.from_pretrained(self.tmpdirname   ,     **UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   ,     UpperCamelCase       :		List[Any]   ,     UpperCamelCase       :		Tuple=False   ,     UpperCamelCase       :		Optional[Any]=20   ,     UpperCamelCase       :		Union[str, Any]=5   )			->						Tuple[str, list]:

         """simple docstring"""
         # XXX The default common tokenizer tests assume that every ID is decodable on its own.
         # This assumption is invalid for ByT5 because single bytes might not be
         # valid utf-8 (byte 128 for instance).
         # Here we're overriding the smallest possible method to provide
         # a clean sequence without making the same assumption.

         lowerCAmelCase__      :			str   =	[]
         for i in range(len(UpperCamelCase   )   ):
               try:
                     lowerCAmelCase__      :			Optional[int]   =	tokenizer.decode([i]   ,     clean_up_tokenization_spaces=UpperCamelCase   )
               except UnicodeDecodeError:
                     pass
               toks.append((i, tok)   )

         lowerCAmelCase__      :			str   =	list(filter(lambda UpperCamelCase   : re.match(r"""^[ a-zA-Z]+$"""   ,     t[1]   )   ,     UpperCamelCase   )   )
         lowerCAmelCase__      :			List[Any]   =	list(filter(lambda UpperCamelCase   : [t[0]] == tokenizer.encode(t[1]   ,     add_special_tokens=UpperCamelCase   )   ,     UpperCamelCase   )   )
         if max_length is not None and len(UpperCamelCase   ) > max_length:
               lowerCAmelCase__      :			Any   =	toks[:max_length]
         if min_length is not None and len(UpperCamelCase   ) < min_length and len(UpperCamelCase   ) > 0:
               while len(UpperCamelCase   ) < min_length:
                     lowerCAmelCase__      :			str   =	toks + toks
        # toks_str = [t[1] for t in toks]
         lowerCAmelCase__      :			Optional[Any]   =	[t[0] for t in toks]

         # Ensure consistency
         lowerCAmelCase__      :			Optional[Any]   =	tokenizer.decode(UpperCamelCase   ,     clean_up_tokenization_spaces=UpperCamelCase   )
         if " " not in output_txt and len(UpperCamelCase   ) > 1:
               lowerCAmelCase__      :			List[Any]   =	(
                   tokenizer.decode([toks_ids[0]]   ,     clean_up_tokenization_spaces=UpperCamelCase   )
                   + """ """
                   + tokenizer.decode(toks_ids[1:]   ,     clean_up_tokenization_spaces=UpperCamelCase   )
               )
         if with_prefix_space:
               lowerCAmelCase__      :			str   =	""" """ + output_txt
         lowerCAmelCase__      :			Optional[Any]   =	tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
         return output_txt, output_ids






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Optional[int]:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			Tuple   =	tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]   )
         lowerCAmelCase__      :			List[Any]   =	tokenizer(["""hi""", """I went to the gym""", """"""]   )
         self.assertListEqual(batch_with_eos_added["""input_ids"""]   ,     batch_without_eos_added["""input_ids"""]   )






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Union[str, Any]:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			List[str]   =	"""Unicode €."""
         lowerCAmelCase__      :			List[str]   =	tokenizer(UpperCamelCase   )
         lowerCAmelCase__      :			Any   =	[88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
         self.assertEqual(encoded["""input_ids"""]   ,     UpperCamelCase   )

         # decoding
         lowerCAmelCase__      :			Optional[Any]   =	tokenizer.decode(UpperCamelCase   )
         self.assertEqual(UpperCamelCase   ,     """Unicode €.</s>"""   )

         lowerCAmelCase__      :			Union[str, Any]   =	tokenizer("""e è é ê ë"""   )
         lowerCAmelCase__      :			int   =	[1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
         self.assertEqual(encoded["""input_ids"""]   ,     UpperCamelCase   )
         # decoding
         lowerCAmelCase__      :			Optional[int]   =	tokenizer.decode(UpperCamelCase   )
         self.assertEqual(UpperCamelCase   ,     """e è é ê ë</s>"""   )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""   )   )   ,     """e è é ê ë</s>"""   )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Union[str, Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			Any   =	["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         # fmt: off
         lowerCAmelCase__      :			Union[str, Any]   =	[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
         # fmt: on
         lowerCAmelCase__      :			Tuple   =	tokenizer(UpperCamelCase   ,     padding=UpperCamelCase   ,     return_tensors=UpperCamelCase   )
         self.assertIsInstance(UpperCamelCase   ,     UpperCamelCase   )

         if FRAMEWORK != "jax":
               lowerCAmelCase__      :			Union[str, Any]   =	list(batch.input_ids.numpy()[0]   )
         else:
               lowerCAmelCase__      :			List[str]   =	list(batch.input_ids.tolist()[0]   )

         self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )

         self.assertEqual((2, 37)   ,     batch.input_ids.shape   )
         self.assertEqual((2, 37)   ,     batch.attention_mask.shape   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						Union[str, Any]:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			int   =	["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         lowerCAmelCase__      :			Optional[Any]   =	tokenizer(UpperCamelCase   ,     padding=UpperCamelCase   ,     return_tensors=UpperCamelCase   )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn("""input_ids"""   ,     UpperCamelCase   )
         self.assertIn("""attention_mask"""   ,     UpperCamelCase   )
         self.assertNotIn("""decoder_input_ids"""   ,     UpperCamelCase   )
         self.assertNotIn("""decoder_attention_mask"""   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						str:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			Dict   =	[
             """Summary of the text.""",
             """Another summary.""",
         ]
         lowerCAmelCase__      :			Dict   =	tokenizer(
             text_target=UpperCamelCase   ,     max_length=32   ,     padding="""max_length"""   ,     truncation=UpperCamelCase   ,     return_tensors=UpperCamelCase   )
         self.assertEqual(32   ,     targets["""input_ids"""].shape[1]   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						Any:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	self.ta_base_tokenizer
         lowerCAmelCase__      :			Optional[int]   =	["""A long paragraph for summarization. </s>"""]
         lowerCAmelCase__      :			Union[str, Any]   =	["""Summary of the text. </s>"""]
         # fmt: off
         lowerCAmelCase__      :			List[Any]   =	[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
         lowerCAmelCase__      :			Optional[Any]   =	[86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
         # fmt: on

         lowerCAmelCase__      :			Any   =	tokenizer(UpperCamelCase   ,     text_target=UpperCamelCase   )

         self.assertEqual(UpperCamelCase   ,     batch["""input_ids"""][0]   )
         self.assertEqual(UpperCamelCase   ,     batch["""labels"""][0]   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						Optional[int]:

         """simple docstring"""
         # safety check on max_len default value so we are sure the test works
         lowerCAmelCase__      :			int   =	self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     self.assertNotEqual(tokenizer.model_max_length   ,     42   )

        # Now let's start the test
         lowerCAmelCase__      :			int   =	self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     lowerCAmelCase__      :			int   =	tempfile.mkdtemp()

                     lowerCAmelCase__      :			str   =	""" He is very happy, UNwant\u00E9d,running"""
                     lowerCAmelCase__      :			Union[str, Any]   =	tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
                     tokenizer.save_pretrained(UpperCamelCase   )

                     lowerCAmelCase__      :			Any   =	tokenizer.__class__.from_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Optional[Any]   =	after_tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
                     self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )

                     shutil.rmtree(UpperCamelCase   )

         lowerCAmelCase__      :			Any   =	self.get_tokenizers(model_max_length=42   )
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     lowerCAmelCase__      :			Tuple   =	tempfile.mkdtemp()

                     lowerCAmelCase__      :			Optional[Any]   =	""" He is very happy, UNwant\u00E9d,running"""
                     tokenizer.add_tokens(["""bim""", """bambam"""]   )
                     lowerCAmelCase__      :			Union[str, Any]   =	tokenizer.additional_special_tokens
                     additional_special_tokens.append("""new_additional_special_token"""   )
                     tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}   )
                     lowerCAmelCase__      :			List[Any]   =	tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
                     tokenizer.save_pretrained(UpperCamelCase   )

                     lowerCAmelCase__      :			Tuple   =	tokenizer.__class__.from_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Union[str, Any]   =	after_tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
                     self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )
                     self.assertIn("""new_additional_special_token"""   ,     after_tokenizer.additional_special_tokens   )
                     self.assertEqual(after_tokenizer.model_max_length   ,     42   )

                     lowerCAmelCase__      :			Optional[int]   =	tokenizer.__class__.from_pretrained(UpperCamelCase   ,     model_max_length=43   )
                     self.assertEqual(tokenizer.model_max_length   ,     43   )

                     shutil.rmtree(UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						List[str]:

         """simple docstring"""
         lowerCAmelCase__      :			Dict   =	[]
         if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

         if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(UpperCamelCase   )

                     with open(os.path.join(UpperCamelCase   ,     """special_tokens_map.json"""   )   ,     encoding="""utf-8"""   ) as json_file:
                           lowerCAmelCase__      :			Optional[int]   =	json.load(UpperCamelCase   )

                     with open(os.path.join(UpperCamelCase   ,     """tokenizer_config.json"""   )   ,     encoding="""utf-8"""   ) as json_file:
                           lowerCAmelCase__      :			int   =	json.load(UpperCamelCase   )

                     lowerCAmelCase__      :			Any   =	[f"""<extra_id_{i}>""" for i in range(1_25   )]

                     lowerCAmelCase__      :			Union[str, Any]   =	added_tokens_extra_ids + [
                         """an_additional_special_token"""
                     ]
                     lowerCAmelCase__      :			List[str]   =	added_tokens_extra_ids + [
                         """an_additional_special_token"""
                     ]

                     with open(os.path.join(UpperCamelCase   ,     """special_tokens_map.json"""   )   ,     """w"""   ,     encoding="""utf-8"""   ) as outfile:
                           json.dump(UpperCamelCase   ,     UpperCamelCase   )
                     with open(os.path.join(UpperCamelCase   ,     """tokenizer_config.json"""   )   ,     """w"""   ,     encoding="""utf-8"""   ) as outfile:
                           json.dump(UpperCamelCase   ,     UpperCamelCase   )

                     # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                     # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                     # "special_tokens_map.json" files
                     lowerCAmelCase__      :			int   =	tokenizer_class.from_pretrained(
                         UpperCamelCase   ,     )
                     self.assertIn(
                         """an_additional_special_token"""   ,     tokenizer_without_change_in_init.additional_special_tokens   )
                     # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                     self.assertEqual(
                         ["""an_additional_special_token"""]   ,     tokenizer_without_change_in_init.convert_ids_to_tokens(
                             tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]   )   )   ,     )

                     # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                     lowerCAmelCase__      :			List[str]   =	added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""   ,     lstrip=UpperCamelCase   )]
                     lowerCAmelCase__      :			Union[str, Any]   =	tokenizer_class.from_pretrained(
                         UpperCamelCase   ,     additional_special_tokens=UpperCamelCase   ,     )

                     self.assertIn("""a_new_additional_special_token"""   ,     tokenizer.additional_special_tokens   )
                     self.assertEqual(
                         ["""a_new_additional_special_token"""]   ,     tokenizer.convert_ids_to_tokens(
                             tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]   )   )   ,     )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						Dict:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	[]
         if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

         if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(UpperCamelCase   )

                     lowerCAmelCase__      :			Optional[Any]   =	tokenizer_class.from_pretrained(UpperCamelCase   )

                     self.assertTrue(tokenizer.decode([2_55]   ) == """"""   )






   def 	_lowerCAmelCase   (		self       :		str   )			->						str:

         """simple docstring"""
         pass






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Optional[int]:

         """simple docstring"""
         pass






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Optional[Any]:

         """simple docstring"""
         pass






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Dict:

         """simple docstring"""
         pass






   def 	_lowerCAmelCase   (		self       :		int   )			->						Tuple:

         """simple docstring"""
         # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
         # and special added tokens as tokens
         lowerCAmelCase__      :			Optional[Any]   =	self.get_tokenizers(fast=UpperCamelCase   ,     do_lower_case=UpperCamelCase   )
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     lowerCAmelCase__      :			Tuple   =	["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                     lowerCAmelCase__      :			Dict   =	tokenizer.convert_tokens_to_string(UpperCamelCase   )

                     self.assertIsInstance(UpperCamelCase   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Tuple:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     lowerCAmelCase__      :			List[str]   =	[
                         """bos_token""",
                         """eos_token""",
                         """unk_token""",
                         """sep_token""",
                         """pad_token""",
                         """cls_token""",
                         """mask_token""",
                     ]

                     lowerCAmelCase__      :			str   =	0
                     lowerCAmelCase__      :			str   =	tokenizer.convert_ids_to_tokens(
                         UpperCamelCase   ,     skip_special_tokens=UpperCamelCase   )

                     for attr in attributes_list:
                           setattr(UpperCamelCase   ,     attr + """_id"""   ,     UpperCamelCase   )
                           self.assertEqual(getattr(UpperCamelCase   ,     UpperCamelCase   )   ,     UpperCamelCase   )
                           self.assertEqual(getattr(UpperCamelCase   ,     attr + """_id"""   )   ,     UpperCamelCase   )

                           setattr(UpperCamelCase   ,     attr + """_id"""   ,     UpperCamelCase   )
                           self.assertEqual(getattr(UpperCamelCase   ,     UpperCamelCase   )   ,     UpperCamelCase   )
                           self.assertEqual(getattr(UpperCamelCase   ,     attr + """_id"""   )   ,     UpperCamelCase   )

                     setattr(UpperCamelCase   ,     """additional_special_tokens_ids"""   ,     []   )
                     self.assertListEqual(getattr(UpperCamelCase   ,     """additional_special_tokens"""   )   ,     []   )
                     self.assertListEqual(getattr(UpperCamelCase   ,     """additional_special_tokens_ids"""   )   ,     []   )

                     setattr(UpperCamelCase   ,     """additional_special_tokens_ids"""   ,     [token_id_to_test_setters]   )
                     self.assertListEqual(getattr(UpperCamelCase   ,     """additional_special_tokens"""   )   ,     [token_to_test_setters]   )
                     self.assertListEqual(getattr(UpperCamelCase   ,     """additional_special_tokens_ids"""   )   ,     [token_id_to_test_setters]   )











"""simple docstring"""


# This is the module that test_patching.py uses to test patch_submodule()

import os  # noqa: this is just for tests
import os as renamed_os  # noqa: this is just for tests
from os import path  # noqa: this is just for tests
from os import path as renamed_path  # noqa: this is just for tests
from os.path import join  # noqa: this is just for tests
from os.path import join as renamed_join  # noqa: this is just for tests


_A													=      open  # noqa: we just need to have a builtin inside this module to test it properly





