



"""simple docstring"""




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 			_snake_case     (   __lowerCAmelCase      ,		unittest.TestCase):
				UpperCamelCase__			:   List[str]       						=BertTokenizer
				UpperCamelCase__			:   List[str]       						=BertTokenizerFast
				UpperCamelCase__			:   Union[str, Any]       						=True
				UpperCamelCase__			:   Dict       						=True
				UpperCamelCase__			:   str       						=filter_non_english




				def 							A__ (  self   :  int  ):
							super().setUp()

							lowercase__   					=			[
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							lowercase__   					=			os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["vocab_file"]  )
							with open(self.vocab_file,   "w",   encoding="utf-8"  ) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]  )  )




				def 							A__ (  self   :  str,   __lowercase   :  List[str]  ):
							lowercase__   					=			"UNwant\u00E9d,running"
							lowercase__   					=			"unwanted, running"
							return input_text, output_text




				def 							A__ (  self   :  List[Any]  ):
							lowercase__   					=			self.tokenizer_class(self.vocab_file  )

							lowercase__   					=			tokenizer.tokenize("UNwant\u00E9d,running"  )
							self.assertListEqual(lowerCAmelCase_,   ["un", "##want", "##ed", ",", "runn", "##ing"]  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_  ),   [9, 6, 7, 12, 10, 11]  )




				def 							A__ (  self   :  Dict  ):
							if not self.test_rust_tokenizer:
										return

							lowercase__   					=			self.get_tokenizer()
							lowercase__   					=			self.get_rust_tokenizer()

							lowercase__   					=			"UNwant\u00E9d,running"

							lowercase__   					=			tokenizer.tokenize(lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.tokenize(lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

							lowercase__   					=			tokenizer.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

							lowercase__   					=			self.get_rust_tokenizer()
							lowercase__   					=			tokenizer.encode(lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.encode(lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

							# With lower casing
							lowercase__   					=			self.get_tokenizer(do_lower_case=lowerCAmelCase_  )
							lowercase__   					=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_  )

							lowercase__   					=			"UNwant\u00E9d,running"

							lowercase__   					=			tokenizer.tokenize(lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.tokenize(lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

							lowercase__   					=			tokenizer.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

							lowercase__   					=			self.get_rust_tokenizer()
							lowercase__   					=			tokenizer.encode(lowerCAmelCase_  )
							lowercase__   					=			rust_tokenizer.encode(lowerCAmelCase_  )
							self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )




				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"  ),   ["ah", "\u535A", "\u63A8", "zz"]  )




				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  ),   ["hello", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  ),   ["hello"]  )




				def 							A__ (  self   :  int  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_,   strip_accents=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  ),   ["hällo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  ),   ["h\u00E9llo"]  )




				def 							A__ (  self   :  List[str]  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_,   strip_accents=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  ),   ["hallo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  ),   ["hello"]  )




				def 							A__ (  self   :  str  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  ),   ["hallo", "!", "how", "are", "you", "?"]  )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  ),   ["hello"]  )




				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  ),   ["HeLLo", "!", "how", "Are", "yoU", "?"]  )




				def 							A__ (  self   :  int  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_,   strip_accents=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  ),   ["HäLLo", "!", "how", "Are", "yoU", "?"]  )




				def 							A__ (  self   :  List[Any]  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_,   strip_accents=lowerCAmelCase_  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  ),   ["HaLLo", "!", "how", "Are", "yoU", "?"]  )




				def 							A__ (  self   :  Tuple  ):
							lowercase__   					=			BasicTokenizer(do_lower_case=lowerCAmelCase_,   never_split=["[UNK]"]  )

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"  ),   ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]  )




				def 							A__ (  self   :  List[Any]  ):
							lowercase__   					=			BasicTokenizer()
							lowercase__   					=			"a\n'll !!to?'d of, can't."
							lowercase__   					=			["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
							self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_  ),   lowerCAmelCase_  )




				def 							A__ (  self   :  Union[str, Any]  ):
							lowercase__   					=			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							lowercase__   					=			{}
							for i, token in enumerate(lowerCAmelCase_  ):
										lowercase__   					=			i
							lowercase__   					=			WordpieceTokenizer(vocab=lowerCAmelCase_,   unk_token="[UNK]"  )

							self.assertListEqual(tokenizer.tokenize(""  ),   []  )

							self.assertListEqual(tokenizer.tokenize("unwanted running"  ),   ["un", "##want", "##ed", "runn", "##ing"]  )

							self.assertListEqual(tokenizer.tokenize("unwantedX running"  ),   ["[UNK]", "runn", "##ing"]  )




				def 							A__ (  self   :  int  ):
							self.assertTrue(_is_whitespace(" "  )  )
							self.assertTrue(_is_whitespace("\t"  )  )
							self.assertTrue(_is_whitespace("\r"  )  )
							self.assertTrue(_is_whitespace("\n"  )  )
							self.assertTrue(_is_whitespace("\u00A0"  )  )

							self.assertFalse(_is_whitespace("A"  )  )
							self.assertFalse(_is_whitespace("-"  )  )




				def 							A__ (  self   :  Dict  ):
							self.assertTrue(_is_control("\u0005"  )  )

							self.assertFalse(_is_control("A"  )  )
							self.assertFalse(_is_control(" "  )  )
							self.assertFalse(_is_control("\t"  )  )
							self.assertFalse(_is_control("\r"  )  )




				def 							A__ (  self   :  List[str]  ):
							self.assertTrue(_is_punctuation("-"  )  )
							self.assertTrue(_is_punctuation("$"  )  )
							self.assertTrue(_is_punctuation("`"  )  )
							self.assertTrue(_is_punctuation("."  )  )

							self.assertFalse(_is_punctuation("A"  )  )
							self.assertFalse(_is_punctuation(" "  )  )




				def 							A__ (  self   :  Dict  ):
							lowercase__   					=			self.get_tokenizer()
							lowercase__   					=			self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_  ) for t in ["Test", "\xad", "test"]],   [["[UNK]"], [], ["[UNK]"]]  )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(lowerCAmelCase_  ) for t in ["Test", "\xad", "test"]],   [["[UNK]"], [], ["[UNK]"]]  )




				@slow
				def 							A__ (  self   :  List[Any]  ):
							lowercase__   					=			self.tokenizer_class.from_pretrained("bert-base-uncased"  )

							lowercase__   					=			tokenizer.encode("sequence builders",   add_special_tokens=lowerCAmelCase_  )
							lowercase__   					=			tokenizer.encode("multi-sequence build",   add_special_tokens=lowerCAmelCase_  )

							lowercase__   					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_  )
							lowercase__   					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_,   lowerCAmelCase_  )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]




				def 							A__ (  self   :  Any  ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
													lowercase__   					=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_,   **lowerCAmelCase_  )

													lowercase__   					=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													lowercase__   					=			tokenizer_r.encode_plus(
													    lowerCAmelCase_,   return_attention_mask=lowerCAmelCase_,   return_token_type_ids=lowerCAmelCase_,   return_offsets_mapping=lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_,   )

													lowercase__   					=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_,   "do_lower_case"  ) else False
													lowercase__   					=			(
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results],   tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]  )  )
													self.assertEqual([e[0] for e in expected_results],   tokens["offset_mapping"]  )




				def 							A__ (  self   :  Optional[Any]  ):
							lowercase__   					=			["的", "人", "有"]
							lowercase__   					=			"".join(lowerCAmelCase_  )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
													lowercase__   					=			True
													lowercase__   					=			self.tokenizer_class.from_pretrained(lowerCAmelCase_,   **lowerCAmelCase_  )
													lowercase__   					=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_,   **lowerCAmelCase_  )

													lowercase__   					=			tokenizer_p.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
													lowercase__   					=			tokenizer_r.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )

													lowercase__   					=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_  )
													lowercase__   					=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_  )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )
													self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )

													lowercase__   					=			False
													lowercase__   					=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_,   **lowerCAmelCase_  )
													lowercase__   					=			self.tokenizer_class.from_pretrained(lowerCAmelCase_,   **lowerCAmelCase_  )

													lowercase__   					=			tokenizer_r.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )
													lowercase__   					=			tokenizer_p.encode(lowerCAmelCase_,   add_special_tokens=lowerCAmelCase_  )

													lowercase__   					=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_  )
													lowercase__   					=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_  )

													# it is expected that only the first Chinese character is not preceded by "##".
													lowercase__   					=			[
													    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_  )
													]
													self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )
													self.assertListEqual(lowerCAmelCase_,   lowerCAmelCase_  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_						=       logging.get_logger(__name__)

lowercase_						=       {
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}
class 			_snake_case     (   lowercase__):
				UpperCamelCase__			:   Optional[Any]       						="""transfo-xl"""
				UpperCamelCase__			:   Dict       						=["""mems"""]
				UpperCamelCase__			:   Optional[int]       						={
				    """n_token""": """vocab_size""",
				    """hidden_size""": """d_model""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}




				def __init__(  self   :  Optional[Any],   __lowercase   :  Optional[Any]=26_7735,   __lowercase   :  int=[2_0000, 4_0000, 20_0000],   __lowercase   :  Union[str, Any]=1024,   __lowercase   :  Tuple=1024,   __lowercase   :  Tuple=16,   __lowercase   :  Optional[Any]=64,   __lowercase   :  str=4096,   __lowercase   :  Optional[int]=4,   __lowercase   :  Union[str, Any]=False,   __lowercase   :  Union[str, Any]=18,   __lowercase   :  List[str]=1600,   __lowercase   :  List[Any]=1000,   __lowercase   :  Union[str, Any]=True,   __lowercase   :  Tuple=True,   __lowercase   :  Optional[Any]=0,   __lowercase   :  List[str]=-1,   __lowercase   :  int=True,   __lowercase   :  Dict=0.1,   __lowercase   :  Union[str, Any]=0.0,   __lowercase   :  str=True,   __lowercase   :  Optional[Any]="normal",   __lowercase   :  str=0.01,   __lowercase   :  Tuple=0.01,   __lowercase   :  List[Any]=0.02,   __lowercase   :  Any=1e-5,   __lowercase   :  Union[str, Any]=0,   **__lowercase   :  Union[str, Any],   ):
							lowercase__   					=			vocab_size
							lowercase__   					=			[]
							self.cutoffs.extend(__lowercase  )
							if proj_share_all_but_first:
										lowercase__   					=			[False] + [True] * len(self.cutoffs  )
							else:
										lowercase__   					=			[False] + [False] * len(self.cutoffs  )
							lowercase__   					=			d_model
							lowercase__   					=			d_embed
							lowercase__   					=			d_head
							lowercase__   					=			d_inner
							lowercase__   					=			div_val
							lowercase__   					=			pre_lnorm
							lowercase__   					=			n_layer
							lowercase__   					=			n_head
							lowercase__   					=			mem_len
							lowercase__   					=			same_length
							lowercase__   					=			attn_type
							lowercase__   					=			clamp_len
							lowercase__   					=			sample_softmax
							lowercase__   					=			adaptive
							lowercase__   					=			dropout
							lowercase__   					=			dropatt
							lowercase__   					=			untie_r
							lowercase__   					=			init
							lowercase__   					=			init_range
							lowercase__   					=			proj_init_std
							lowercase__   					=			init_std
							lowercase__   					=			layer_norm_epsilon
							super().__init__(eos_token_id=__lowercase,   **__lowercase  )




				@property
				def 							A__ (  self   :  Optional[Any]  ):
							# Message copied from Transformer-XL documentation
							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )
							return -1




				@max_position_embeddings.setter
				def 							A__ (  self   :  List[str],   __lowercase   :  Union[str, Any]  ):
							# Message copied from Transformer-XL documentation
							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )




