


"""simple docstring"""







import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 			UpperCamelCase_     (			UpperCamelCase      ,  unittest.TestCase):

	"""simple docstring"""





	snake_case__    :  Union[str, Any]    =      BertTokenizer
	snake_case__    :  int    =      BertTokenizerFast
	snake_case__    :  List[str]    =      True
	snake_case__    :  int    =      True
	snake_case__    :  str    =      filter_non_english




	def   UpperCAmelCase_				(			self      :    Optional[Any]       )	->     str:
						super().setUp()

						__SCREAMING_SNAKE_CASE						       =    [
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "[PAD]",
						    "[MASK]",
						    "want",
						    "##want",
						    "##ed",
						    "wa",
						    "un",
						    "runn",
						    "##ing",
						    ",",
						    "low",
						    "lowest",
						]
						__SCREAMING_SNAKE_CASE						       =    os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["vocab_file"]       )
						with open(self.vocab_file      ,      "w"      ,      encoding="utf-8"       ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    Union[str, Any]       )	->     int:
						__SCREAMING_SNAKE_CASE						       =    "UNwant\u00E9d,running"
						__SCREAMING_SNAKE_CASE						       =    "unwanted, running"
						return input_text, output_text




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     Tuple:
						__SCREAMING_SNAKE_CASE						       =    self.tokenizer_class(self.vocab_file       )

						__SCREAMING_SNAKE_CASE						       =    tokenizer.tokenize("UNwant\u00E9d,running"       )
						self.assertListEqual(UpperCAmelCase__      ,      ["un", "##want", "##ed", ",", "runn", "##ing"]       )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__       )      ,      [9, 6, 7, 1_2, 1_0, 1_1]       )




	def   UpperCAmelCase_				(			self      :    Tuple       )	->     int:
						if not self.test_rust_tokenizer:
											return

						__SCREAMING_SNAKE_CASE						       =    self.get_tokenizer()
						__SCREAMING_SNAKE_CASE						       =    self.get_rust_tokenizer()

						__SCREAMING_SNAKE_CASE						       =    "UNwant\u00E9d,running"

						__SCREAMING_SNAKE_CASE						       =    tokenizer.tokenize(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.tokenize(UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    self.get_rust_tokenizer()
						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.encode(UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

						# With lower casing
						__SCREAMING_SNAKE_CASE						       =    self.get_tokenizer(do_lower_case=UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    "UNwant\u00E9d,running"

						__SCREAMING_SNAKE_CASE						       =    tokenizer.tokenize(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.tokenize(UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    self.get_rust_tokenizer()
						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    rust_tokenizer.encode(UpperCAmelCase__       )
						self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )      ,      ["ah", "\u535A", "\u63A8", "zz"]       )




	def   UpperCAmelCase_				(			self      :    List[str]       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )      ,      ["hello", "!", "how", "are", "you", "?"]       )
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )      ,      ["hello"]       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     int:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__      ,      strip_accents=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )      ,      ["hällo", "!", "how", "are", "you", "?"]       )
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )      ,      ["h\u00E9llo"]       )




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__      ,      strip_accents=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )      ,      ["hallo", "!", "how", "are", "you", "?"]       )
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )      ,      ["hello"]       )




	def   UpperCAmelCase_				(			self      :    str       )	->     Tuple:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )      ,      ["hallo", "!", "how", "are", "you", "?"]       )
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )      ,      ["hello"]       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )      ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]       )




	def   UpperCAmelCase_				(			self      :    Dict       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__      ,      strip_accents=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )      ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]       )




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     List[str]:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__      ,      strip_accents=UpperCAmelCase__       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )      ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]       )




	def   UpperCAmelCase_				(			self      :    Any       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer(do_lower_case=UpperCAmelCase__      ,      never_split=["[UNK]"]       )

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )      ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     List[str]:
						__SCREAMING_SNAKE_CASE						       =    BasicTokenizer()
						__SCREAMING_SNAKE_CASE						       =    "a\n'll !!to?'d of, can't."
						__SCREAMING_SNAKE_CASE						       =    ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
						self.assertListEqual(tokenizer.tokenize(UpperCAmelCase__       )      ,      UpperCAmelCase__       )




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__SCREAMING_SNAKE_CASE						       =    {}
						for i, token in enumerate(UpperCAmelCase__       ):
											__SCREAMING_SNAKE_CASE						       =    i
						__SCREAMING_SNAKE_CASE						       =    WordpieceTokenizer(vocab=UpperCAmelCase__      ,      unk_token="[UNK]"       )

						self.assertListEqual(tokenizer.tokenize(""       )      ,      []       )

						self.assertListEqual(tokenizer.tokenize("unwanted running"       )      ,      ["un", "##want", "##ed", "runn", "##ing"]       )

						self.assertListEqual(tokenizer.tokenize("unwantedX running"       )      ,      ["[UNK]", "runn", "##ing"]       )




	def   UpperCAmelCase_				(			self      :    str       )	->     int:
						self.assertTrue(_is_whitespace(" "       )       )
						self.assertTrue(_is_whitespace("\t"       )       )
						self.assertTrue(_is_whitespace("\r"       )       )
						self.assertTrue(_is_whitespace("\n"       )       )
						self.assertTrue(_is_whitespace("\u00A0"       )       )

						self.assertFalse(_is_whitespace("A"       )       )
						self.assertFalse(_is_whitespace("-"       )       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     Optional[Any]:
						self.assertTrue(_is_control("\u0005"       )       )

						self.assertFalse(_is_control("A"       )       )
						self.assertFalse(_is_control(" "       )       )
						self.assertFalse(_is_control("\t"       )       )
						self.assertFalse(_is_control("\r"       )       )




	def   UpperCAmelCase_				(			self      :    Tuple       )	->     Tuple:
						self.assertTrue(_is_punctuation("-"       )       )
						self.assertTrue(_is_punctuation("$"       )       )
						self.assertTrue(_is_punctuation("`"       )       )
						self.assertTrue(_is_punctuation("."       )       )

						self.assertFalse(_is_punctuation("A"       )       )
						self.assertFalse(_is_punctuation(" "       )       )




	def   UpperCAmelCase_				(			self      :    Any       )	->     Tuple:
						__SCREAMING_SNAKE_CASE						       =    self.get_tokenizer()
						__SCREAMING_SNAKE_CASE						       =    self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__       ) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]       )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(UpperCAmelCase__       ) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]       )




	@slow
	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     List[Any]:
						__SCREAMING_SNAKE_CASE						       =    self.tokenizer_class.from_pretrained("bert-base-uncased"       )

						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode("sequence builders"      ,      add_special_tokens=UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    tokenizer.encode("multi-sequence build"      ,      add_special_tokens=UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__      ,      UpperCAmelCase__       )

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




	def   UpperCAmelCase_				(			self      :    List[str]       )	->     Optional[int]:
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																__SCREAMING_SNAKE_CASE						       =    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,      **UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.encode_plus(
																    UpperCAmelCase__      ,      return_attention_mask=UpperCAmelCase__      ,      return_token_type_ids=UpperCAmelCase__      ,      return_offsets_mapping=UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__      ,      )

																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__      ,      "do_lower_case"       ) else False
																__SCREAMING_SNAKE_CASE						       =    (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "A"),
																        ((1, 2), ","),
																        ((3, 5), "na"),
																        ((5, 6), "##ï"),
																        ((6, 8), "##ve"),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), "Allen"),
																        ((2_1, 2_3), "##NL"),
																        ((2_3, 2_4), "##P"),
																        ((2_5, 3_3), "sentence"),
																        ((3_3, 3_4), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "a"),
																        ((1, 2), ","),
																        ((3, 8), "naive"),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), "allen"),
																        ((2_1, 2_3), "##nl"),
																        ((2_3, 2_4), "##p"),
																        ((2_5, 3_3), "sentence"),
																        ((3_3, 3_4), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]      ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
																self.assertEqual([e[0] for e in expected_results]      ,      tokens["offset_mapping"]       )




	def   UpperCAmelCase_				(			self      :    Dict       )	->     Any:
						__SCREAMING_SNAKE_CASE						       =    ["的", "人", "有"]
						__SCREAMING_SNAKE_CASE						       =    "".join(UpperCAmelCase__       )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,      **UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,      **UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    tokenizer_p.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__       )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )
																self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    False
																__SCREAMING_SNAKE_CASE						       =    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,      **UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,      **UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    tokenizer_p.encode(UpperCAmelCase__      ,      add_special_tokens=UpperCAmelCase__       )

																__SCREAMING_SNAKE_CASE						       =    tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__       )
																__SCREAMING_SNAKE_CASE						       =    tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__       )

																# it is expected that only the first Chinese character is not preceded by "##".
																__SCREAMING_SNAKE_CASE						       =    [
																    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__       )
																]
																self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )
																self.assertListEqual(UpperCAmelCase__      ,      UpperCAmelCase__       )




"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


if is_sentencepiece_available():
		from ..ta.tokenization_ta import TaTokenizer
else:
		from ...utils.dummy_sentencepiece_objects import TaTokenizer

a__    :							Dict            =					TaTokenizer

if is_tokenizers_available():
		from ..ta.tokenization_ta_fast import TaTokenizerFast
else:
		from ...utils.dummy_tokenizers_objects import TaTokenizerFast

a__    :							Any            =					TaTokenizerFast

a__    :							Tuple            =					{'''configuration_mt5''': ['''MT5Config''', '''MT5OnnxConfig''']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a__    :							int            =					[
		    '''MT5EncoderModel''',
		    '''MT5ForConditionalGeneration''',
		    '''MT5ForQuestionAnswering''',
		    '''MT5Model''',
		    '''MT5PreTrainedModel''',
		    '''MT5Stack''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a__    :							str            =					['''TFMT5EncoderModel''', '''TFMT5ForConditionalGeneration''', '''TFMT5Model''']

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a__    :							Union[str, Any]            =					['''FlaxMT5EncoderModel''', '''FlaxMT5ForConditionalGeneration''', '''FlaxMT5Model''']


if TYPE_CHECKING:
		from .configuration_mta import MTaConfig, MTaOnnxConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mta import (
				    MTaEncoderModel,
				    MTaForConditionalGeneration,
				    MTaForQuestionAnswering,
				    MTaModel,
				    MTaPreTrainedModel,
				    MTaStack,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_mta import TFMTaEncoderModel, TFMTaForConditionalGeneration, TFMTaModel

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_mta import FlaxMTaEncoderModel, FlaxMTaForConditionalGeneration, FlaxMTaModel

else:
		import sys

		a__    :							str            =					_LazyModule(
		    __name__,
		    globals()['''__file__'''],
		    _import_structure,
		    extra_objects={'''MT5Tokenizer''': MTaTokenizer, '''MT5TokenizerFast''': MTaTokenizerFast},
		    module_spec=__spec__,
		)
