






"""simple docstring"""



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class        lowercase__      (							snake_case__,       unittest.TestCase       ):
			_UpperCAmelCase		:Optional[Any]             =			BertTokenizer
			_UpperCAmelCase		:Dict             =			BertTokenizerFast
			_UpperCAmelCase		:str             =			True
			_UpperCAmelCase		:Union[str, Any]             =			True
			_UpperCAmelCase		:Any             =			filter_non_english


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										super().setUp()

										lowerCamelCase_	:							Any									=[
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										lowerCamelCase_	:							str									=os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["vocab_file"] )
										with open(self.vocab_file       ,	"w"       ,	encoding="utf-8" ) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  Any ):
										lowerCamelCase_	:							List[Any]									="UNwant\u00E9d,running"
										lowerCamelCase_	:							Union[str, Any]									="unwanted, running"
										return input_text, output_text


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										lowerCamelCase_	:							int									=self.tokenizer_class(self.vocab_file )

										lowerCamelCase_	:							Optional[int]									=tokenizer.tokenize("UNwant\u00E9d,running" )
										self.assertListEqual(snake_case__       ,	["un", "##want", "##ed", ",", "runn", "##ing"] )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )       ,	[9, 6, 7, 12, 10, 11] )


			def      UpperCAmelCase__						(				self					:  Optional[Any] ):
										if not self.test_rust_tokenizer:
																	return

										lowerCamelCase_	:							List[Any]									=self.get_tokenizer()
										lowerCamelCase_	:							Optional[int]									=self.get_rust_tokenizer()

										lowerCamelCase_	:							int									="UNwant\u00E9d,running"

										lowerCamelCase_	:							Union[str, Any]									=tokenizer.tokenize(snake_case__ )
										lowerCamelCase_	:							List[Any]									=rust_tokenizer.tokenize(snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )

										lowerCamelCase_	:							Optional[Any]									=tokenizer.encode(snake_case__       ,	add_special_tokens=snake_case__ )
										lowerCamelCase_	:							Dict									=rust_tokenizer.encode(snake_case__       ,	add_special_tokens=snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )

										lowerCamelCase_	:							Optional[int]									=self.get_rust_tokenizer()
										lowerCamelCase_	:							Dict									=tokenizer.encode(snake_case__ )
										lowerCamelCase_	:							Union[str, Any]									=rust_tokenizer.encode(snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )

										# With lower casing
										lowerCamelCase_	:							str									=self.get_tokenizer(do_lower_case=snake_case__ )
										lowerCamelCase_	:							Union[str, Any]									=self.get_rust_tokenizer(do_lower_case=snake_case__ )

										lowerCamelCase_	:							List[Any]									="UNwant\u00E9d,running"

										lowerCamelCase_	:							Optional[Any]									=tokenizer.tokenize(snake_case__ )
										lowerCamelCase_	:							int									=rust_tokenizer.tokenize(snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )

										lowerCamelCase_	:							Union[str, Any]									=tokenizer.encode(snake_case__       ,	add_special_tokens=snake_case__ )
										lowerCamelCase_	:							List[str]									=rust_tokenizer.encode(snake_case__       ,	add_special_tokens=snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )

										lowerCamelCase_	:							List[Any]									=self.get_rust_tokenizer()
										lowerCamelCase_	:							str									=tokenizer.encode(snake_case__ )
										lowerCamelCase_	:							Dict									=rust_tokenizer.encode(snake_case__ )
										self.assertListEqual(snake_case__       ,	snake_case__ )


			def      UpperCAmelCase__						(				self					:  List[Any] ):
										lowerCamelCase_	:							Union[str, Any]									=BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz" )       ,	["ah", "\u535A", "\u63A8", "zz"] )


			def      UpperCAmelCase__						(				self					:  Optional[Any] ):
										lowerCamelCase_	:							Optional[int]									=BasicTokenizer(do_lower_case=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )       ,	["hello", "!", "how", "are", "you", "?"] )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,	["hello"] )


			def      UpperCAmelCase__						(				self					:  int ):
										lowerCamelCase_	:							Dict									=BasicTokenizer(do_lower_case=snake_case__       ,	strip_accents=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,	["hällo", "!", "how", "are", "you", "?"] )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,	["h\u00E9llo"] )


			def      UpperCAmelCase__						(				self					:  List[Any] ):
										lowerCamelCase_	:							Any									=BasicTokenizer(do_lower_case=snake_case__       ,	strip_accents=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,	["hallo", "!", "how", "are", "you", "?"] )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,	["hello"] )


			def      UpperCAmelCase__						(				self					:  List[Any] ):
										lowerCamelCase_	:							Optional[Any]									=BasicTokenizer(do_lower_case=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,	["hallo", "!", "how", "are", "you", "?"] )
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )       ,	["hello"] )


			def      UpperCAmelCase__						(				self					:  List[str] ):
										lowerCamelCase_	:							Tuple									=BasicTokenizer(do_lower_case=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )       ,	["HeLLo", "!", "how", "Are", "yoU", "?"] )


			def      UpperCAmelCase__						(				self					:  List[Any] ):
										lowerCamelCase_	:							Union[str, Any]									=BasicTokenizer(do_lower_case=snake_case__       ,	strip_accents=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,	["HäLLo", "!", "how", "Are", "yoU", "?"] )


			def      UpperCAmelCase__						(				self					:  Tuple ):
										lowerCamelCase_	:							List[Any]									=BasicTokenizer(do_lower_case=snake_case__       ,	strip_accents=snake_case__ )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )       ,	["HaLLo", "!", "how", "Are", "yoU", "?"] )


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										lowerCamelCase_	:							List[str]									=BasicTokenizer(do_lower_case=snake_case__       ,	never_split=["[UNK]"] )

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]" )       ,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] )


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										lowerCamelCase_	:							Dict									=BasicTokenizer()
										lowerCamelCase_	:							int									="a\n'll !!to?'d of, can't."
										lowerCamelCase_	:							Tuple									=["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
										self.assertListEqual(tokenizer.tokenize(snake_case__ )       ,	snake_case__ )


			def      UpperCAmelCase__						(				self					:  Optional[Any] ):
										lowerCamelCase_	:							Dict									=["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										lowerCamelCase_	:							List[str]									={}
										for i, token in enumerate(snake_case__ ):
																	lowerCamelCase_	:							Dict									=i
										lowerCamelCase_	:							List[Any]									=WordpieceTokenizer(vocab=snake_case__       ,	unk_token="[UNK]" )

										self.assertListEqual(tokenizer.tokenize("" )       ,	[] )

										self.assertListEqual(tokenizer.tokenize("unwanted running" )       ,	["un", "##want", "##ed", "runn", "##ing"] )

										self.assertListEqual(tokenizer.tokenize("unwantedX running" )       ,	["[UNK]", "runn", "##ing"] )


			def      UpperCAmelCase__						(				self					:  List[Any] ):
										self.assertTrue(_is_whitespace(" " ) )
										self.assertTrue(_is_whitespace("\t" ) )
										self.assertTrue(_is_whitespace("\r" ) )
										self.assertTrue(_is_whitespace("\n" ) )
										self.assertTrue(_is_whitespace("\u00A0" ) )

										self.assertFalse(_is_whitespace("A" ) )
										self.assertFalse(_is_whitespace("-" ) )


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										self.assertTrue(_is_control("\u0005" ) )

										self.assertFalse(_is_control("A" ) )
										self.assertFalse(_is_control(" " ) )
										self.assertFalse(_is_control("\t" ) )
										self.assertFalse(_is_control("\r" ) )


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										self.assertTrue(_is_punctuation("-" ) )
										self.assertTrue(_is_punctuation("$" ) )
										self.assertTrue(_is_punctuation("`" ) )
										self.assertTrue(_is_punctuation("." ) )

										self.assertFalse(_is_punctuation("A" ) )
										self.assertFalse(_is_punctuation(" " ) )


			def      UpperCAmelCase__						(				self					:  Tuple ):
										lowerCamelCase_	:							Any									=self.get_tokenizer()
										lowerCamelCase_	:							Optional[int]									=self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(snake_case__ ) for t in ["Test", "\xad", "test"]]       ,	[["[UNK]"], [], ["[UNK]"]] )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(snake_case__ ) for t in ["Test", "\xad", "test"]]       ,	[["[UNK]"], [], ["[UNK]"]] )


			@slow
			def      UpperCAmelCase__						(				self					:  List[Any] ):
										lowerCamelCase_	:							str									=self.tokenizer_class.from_pretrained("bert-base-uncased" )

										lowerCamelCase_	:							List[str]									=tokenizer.encode("sequence builders"       ,	add_special_tokens=snake_case__ )
										lowerCamelCase_	:							List[Any]									=tokenizer.encode("multi-sequence build"       ,	add_special_tokens=snake_case__ )

										lowerCamelCase_	:							int									=tokenizer.build_inputs_with_special_tokens(snake_case__ )
										lowerCamelCase_	:							Dict									=tokenizer.build_inputs_with_special_tokens(snake_case__       ,	snake_case__ )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]


			def      UpperCAmelCase__						(				self					:  List[str] ):
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																								lowerCamelCase_	:							List[str]									=self.rust_tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__ )

																								lowerCamelCase_	:							List[Any]									=F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																								lowerCamelCase_	:							int									=tokenizer_r.encode_plus(
																								    snake_case__       ,	return_attention_mask=snake_case__       ,	return_token_type_ids=snake_case__       ,	return_offsets_mapping=snake_case__       ,	add_special_tokens=snake_case__       ,	)

																								lowerCamelCase_	:							List[str]									=tokenizer_r.do_lower_case if hasattr(snake_case__       ,	"do_lower_case" ) else False
																								lowerCamelCase_	:							Optional[int]									=(
																								    [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "A"),
																								        ((1, 2), ","),
																								        ((3, 5), "na"),
																								        ((5, 6), "##ï"),
																								        ((6, 8), "##ve"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "Allen"),
																								        ((21, 23), "##NL"),
																								        ((23, 24), "##P"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								    if not do_lower_case
																								    else [
																								        ((0, 0), tokenizer_r.cls_token),
																								        ((0, 1), "a"),
																								        ((1, 2), ","),
																								        ((3, 8), "naive"),
																								        ((9, 15), tokenizer_r.mask_token),
																								        ((16, 21), "allen"),
																								        ((21, 23), "##nl"),
																								        ((23, 24), "##p"),
																								        ((25, 33), "sentence"),
																								        ((33, 34), "."),
																								        ((0, 0), tokenizer_r.sep_token),
																								    ]
																								)

																								self.assertEqual(
																								    [e[1] for e in expected_results]       ,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"] ) )
																								self.assertEqual([e[0] for e in expected_results]       ,	tokens["offset_mapping"] )


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										lowerCamelCase_	:							str									=["的", "人", "有"]
										lowerCamelCase_	:							str									="".join(snake_case__ )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																								lowerCamelCase_	:							Optional[int]									=True
																								lowerCamelCase_	:							Any									=self.tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__ )
																								lowerCamelCase_	:							Optional[Any]									=self.rust_tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__ )

																								lowerCamelCase_	:							Optional[Any]									=tokenizer_p.encode(snake_case__       ,	add_special_tokens=snake_case__ )
																								lowerCamelCase_	:							List[Any]									=tokenizer_r.encode(snake_case__       ,	add_special_tokens=snake_case__ )

																								lowerCamelCase_	:							Union[str, Any]									=tokenizer_r.convert_ids_to_tokens(snake_case__ )
																								lowerCamelCase_	:							List[str]									=tokenizer_p.convert_ids_to_tokens(snake_case__ )

																								# it is expected that each Chinese character is not preceded by "##"
																								self.assertListEqual(snake_case__       ,	snake_case__ )
																								self.assertListEqual(snake_case__       ,	snake_case__ )

																								lowerCamelCase_	:							List[str]									=False
																								lowerCamelCase_	:							Tuple									=self.rust_tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__ )
																								lowerCamelCase_	:							Union[str, Any]									=self.tokenizer_class.from_pretrained(snake_case__       ,	**snake_case__ )

																								lowerCamelCase_	:							List[Any]									=tokenizer_r.encode(snake_case__       ,	add_special_tokens=snake_case__ )
																								lowerCamelCase_	:							Tuple									=tokenizer_p.encode(snake_case__       ,	add_special_tokens=snake_case__ )

																								lowerCamelCase_	:							int									=tokenizer_r.convert_ids_to_tokens(snake_case__ )
																								lowerCamelCase_	:							List[str]									=tokenizer_p.convert_ids_to_tokens(snake_case__ )

																								# it is expected that only the first Chinese character is not preceded by "##".
																								lowerCamelCase_	:							Tuple									=[
																								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__ )
																								]
																								self.assertListEqual(snake_case__       ,	snake_case__ )
																								self.assertListEqual(snake_case__       ,	snake_case__ )











"""simple docstring"""



def 			_snake_case    (					lowerCamelCase__     :   int  )    ->     int:

							if not isinstance(lowerCamelCase__    ,						lowerCamelCase__  ):
														raise TypeError("only integers accepted as input"  )
							else:
														lowerCamelCase_	:							str									=str(abs(lowerCamelCase__  )  )
														lowerCamelCase_	:							Tuple									=[list(lowerCamelCase__  ) for char in range(len(lowerCamelCase__  )  )]
														for index in range(len(lowerCamelCase__  )  ):
																					num_transpositions[index].pop(lowerCamelCase__  )
														return max(
														    int("".join(list(lowerCamelCase__  )  )  ) for transposition in num_transpositions  )


if __name__ == "__main__":
						__import__('doctest').testmod()



