


'''simple docstring'''






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       Optional[int]     						=				MobileBertTokenizer
							lowercase__      :       Union[str, Any]     						=				MobileBertTokenizerFast
							lowercase__      :       Any     						=				True
							lowercase__      :       int     						=				True
							lowercase__      :       int     						=				filter_non_english
							lowercase__      :       Optional[int]     						=				"google/mobilebert-uncased"







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														super().setUp()

														lowerCAmelCase__	       =     [
														    '''[UNK]''',
														    '''[CLS]''',
														    '''[SEP]''',
														    '''[PAD]''',
														    '''[MASK]''',
														    '''want''',
														    '''##want''',
														    '''##ed''',
														    '''wa''',
														    '''un''',
														    '''runn''',
														    '''##ing''',
														    ''',''',
														    '''low''',
														    '''lowest''',
														]
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as vocab_writer:
																					vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

														lowerCAmelCase__	       =     [
														    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
														    for tokenizer_def in self.tokenizers_list
														]







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							Any:
														lowerCAmelCase__	       =     '''UNwant\u00E9d,running'''
														lowerCAmelCase__	       =     '''unwanted, running'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														lowerCAmelCase__	       =     self.tokenizer_class(self.vocab_file       )

														lowerCAmelCase__	       =     tokenizer.tokenize('''UNwant\u00E9d,running'''       )
														self.assertListEqual(lowerCamelCase_ ,   ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   [9, 6, 7, 12, 10, 11]       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														if not self.test_rust_tokenizer:
																					return

														lowerCAmelCase__	       =     self.get_tokenizer()
														lowerCAmelCase__	       =     self.get_rust_tokenizer()

														lowerCAmelCase__	       =     '''UNwant\u00E9d,running'''

														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     self.get_rust_tokenizer()
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# With lower casing
														lowerCAmelCase__	       =     self.get_tokenizer(do_lower_case=lowerCamelCase_       )
														lowerCAmelCase__	       =     self.get_rust_tokenizer(do_lower_case=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''UNwant\u00E9d,running'''

														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     self.get_rust_tokenizer()
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														lowerCAmelCase__	       =     BasicTokenizer()

														self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       ) ,   ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       ) ,   ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
														self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,   ['''hello''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							int:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,   strip_accents=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,   ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
														self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,   ['''h\u00E9llo''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,   strip_accents=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
														self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,   ['''hello''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
														self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,   ['''hello''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       ) ,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,   strip_accents=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,   ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,   strip_accents=lowerCamelCase_       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,   ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														lowerCAmelCase__	       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,   never_split=['''[UNK]''']       )

														self.assertListEqual(
														    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       ) ,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

														lowerCAmelCase__	       =     {}
														for i, token in enumerate(lowerCamelCase_       ):
																					lowerCAmelCase__	       =     i
														lowerCAmelCase__	       =     WordpieceTokenizer(vocab=lowerCamelCase_ ,   unk_token='''[UNK]'''       )

														self.assertListEqual(tokenizer.tokenize(''''''       ) ,   []       )

														self.assertListEqual(tokenizer.tokenize('''unwanted running'''       ) ,   ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

														self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       ) ,   ['''[UNK]''', '''runn''', '''##ing''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														self.assertTrue(_is_whitespace(''' '''       )       )
														self.assertTrue(_is_whitespace('''\t'''       )       )
														self.assertTrue(_is_whitespace('''\r'''       )       )
														self.assertTrue(_is_whitespace('''\n'''       )       )
														self.assertTrue(_is_whitespace('''\u00A0'''       )       )

														self.assertFalse(_is_whitespace('''A'''       )       )
														self.assertFalse(_is_whitespace('''-'''       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														self.assertTrue(_is_control('''\u0005'''       )       )

														self.assertFalse(_is_control('''A'''       )       )
														self.assertFalse(_is_control(''' '''       )       )
														self.assertFalse(_is_control('''\t'''       )       )
														self.assertFalse(_is_control('''\r'''       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														self.assertTrue(_is_punctuation('''-'''       )       )
														self.assertTrue(_is_punctuation('''$'''       )       )
														self.assertTrue(_is_punctuation('''`'''       )       )
														self.assertTrue(_is_punctuation('''.'''       )       )

														self.assertFalse(_is_punctuation('''A'''       )       )
														self.assertFalse(_is_punctuation(''' '''       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     self.get_tokenizer()
														lowerCAmelCase__	       =     self.get_rust_tokenizer()

														# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
														self.assertListEqual([tokenizer.tokenize(lowerCamelCase_       ) for t in ['''Test''', '''\xad''', '''test''']] ,   [['''[UNK]'''], [], ['''[UNK]''']]       )

														self.assertListEqual(
														    [rust_tokenizer.tokenize(lowerCamelCase_       ) for t in ['''Test''', '''\xad''', '''test''']] ,   [['''[UNK]'''], [], ['''[UNK]''']]       )







							@slow
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''       )

														lowerCAmelCase__	       =     tokenizer.encode('''sequence builders''' ,   add_special_tokens=lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer.encode('''multi-sequence build''' ,   add_special_tokens=lowerCamelCase_       )

														lowerCAmelCase__	       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_ ,   lowerCamelCase_       )

														assert encoded_sentence == [1_01] + text + [1_02]
														assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																												lowerCAmelCase__	       =     tokenizer_r.encode_plus(
																												    lowerCamelCase_ ,   return_attention_mask=lowerCamelCase_ ,   return_token_type_ids=lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   )

																												lowerCAmelCase__	       =     tokenizer_r.do_lower_case if hasattr(lowerCamelCase_ ,   '''do_lower_case'''       ) else False
																												lowerCAmelCase__	       =     (
																												    [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), '''A'''),
																												        ((1, 2), ''','''),
																												        ((3, 5), '''na'''),
																												        ((5, 6), '''##ï'''),
																												        ((6, 8), '''##ve'''),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), '''Allen'''),
																												        ((21, 23), '''##NL'''),
																												        ((23, 24), '''##P'''),
																												        ((25, 33), '''sentence'''),
																												        ((33, 34), '''.'''),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												    if not do_lower_case
																												    else [
																												        ((0, 0), tokenizer_r.cls_token),
																												        ((0, 1), '''a'''),
																												        ((1, 2), ''','''),
																												        ((3, 8), '''naive'''),
																												        ((9, 15), tokenizer_r.mask_token),
																												        ((16, 21), '''allen'''),
																												        ((21, 23), '''##nl'''),
																												        ((23, 24), '''##p'''),
																												        ((25, 33), '''sentence'''),
																												        ((33, 34), '''.'''),
																												        ((0, 0), tokenizer_r.sep_token),
																												    ]
																												)

																												self.assertEqual(
																												    [e[1] for e in expected_results] ,   tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
																												self.assertEqual([e[0] for e in expected_results] ,   tokens['''offset_mapping''']       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														lowerCAmelCase__	       =     ['''的''', '''人''', '''有''']
														lowerCAmelCase__	       =     ''''''.join(lowerCamelCase_       )
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     True
																												lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     tokenizer_p.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )

																												lowerCAmelCase__	       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_       )

																												# it is expected that each Chinese character is not preceded by "##"
																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )
																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												lowerCAmelCase__	       =     False
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )
																												lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     tokenizer_r.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_p.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )

																												lowerCAmelCase__	       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_       )

																												# it is expected that only the first Chinese character is not preceded by "##".
																												lowerCAmelCase__	       =     [
																												    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase_       )
																												]
																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )
																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )



'''simple docstring'''






def  _snake_case       (					A = 10			,   A = 22	)						->   int:
							lowerCAmelCase__	       =     range(1			,   A	)
							lowerCAmelCase__	       =     range(1			,   A	)
							return sum(
							    1 for power in powers for base in bases if len(str(base**power	)	) == power	)


if __name__ == "__main__":
		print(f"""{solution(10, 22) = }""")