




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     _UpperCamelCase			(   __lowercase       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  Dict    =	MobileBertTokenizer
			__UpperCamelCase :  Dict    =	MobileBertTokenizerFast
			__UpperCamelCase :  List[str]    =	True
			__UpperCamelCase :  Optional[Any]    =	True
			__UpperCamelCase :  List[Any]    =	filter_non_english
			__UpperCamelCase :  Optional[Any]    =	"""google/mobilebert-uncased"""

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							super().setUp()

							UpperCamelCase_:							Union[str, Any]		=  [
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """[PAD]""",
							    """[MASK]""",
							    """want""",
							    """##want""",
							    """##ed""",
							    """wa""",
							    """un""",
							    """runn""",
							    """##ing""",
							    """,""",
							    """low""",
							    """lowest""",
							]
							UpperCamelCase_:							List[str]		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""vocab_file"""] )
							with open(self.vocab_file  ,		"""w"""  ,		encoding="""utf-8""" ) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )

							UpperCamelCase_:							int		=  [
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]

			def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Tuple ):
							UpperCamelCase_:							Tuple		=  """UNwant\u00E9d,running"""
							UpperCamelCase_:							Optional[int]		=  """unwanted, running"""
							return input_text, output_text

			def  lowerCAmelCase__		(    self	:    Tuple ):
							UpperCamelCase_:							Optional[int]		=  self.tokenizer_class(self.vocab_file )

							UpperCamelCase_:							Optional[Any]		=  tokenizer.tokenize("""UNwant\u00E9d,running""" )
							self.assertListEqual(_a  ,		["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""] )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_a )  ,		[9, 6, 7, 12, 10, 11] )

			def  lowerCAmelCase__		(    self	:    Dict ):
							if not self.test_rust_tokenizer:
											return

							UpperCamelCase_:							Union[str, Any]		=  self.get_tokenizer()
							UpperCamelCase_:							int		=  self.get_rust_tokenizer()

							UpperCamelCase_:							List[Any]		=  """UNwant\u00E9d,running"""

							UpperCamelCase_:							int		=  tokenizer.tokenize(_a )
							UpperCamelCase_:							List[Any]		=  rust_tokenizer.tokenize(_a )
							self.assertListEqual(_a  ,		_a )

							UpperCamelCase_:							str		=  tokenizer.encode(_a  ,		add_special_tokens=_a )
							UpperCamelCase_:							List[str]		=  rust_tokenizer.encode(_a  ,		add_special_tokens=_a )
							self.assertListEqual(_a  ,		_a )

							UpperCamelCase_:							List[Any]		=  self.get_rust_tokenizer()
							UpperCamelCase_:							List[str]		=  tokenizer.encode(_a )
							UpperCamelCase_:							Optional[int]		=  rust_tokenizer.encode(_a )
							self.assertListEqual(_a  ,		_a )

							# With lower casing
							UpperCamelCase_:							int		=  self.get_tokenizer(do_lower_case=_a )
							UpperCamelCase_:							Dict		=  self.get_rust_tokenizer(do_lower_case=_a )

							UpperCamelCase_:							Optional[Any]		=  """UNwant\u00E9d,running"""

							UpperCamelCase_:							List[Any]		=  tokenizer.tokenize(_a )
							UpperCamelCase_:							Tuple		=  rust_tokenizer.tokenize(_a )
							self.assertListEqual(_a  ,		_a )

							UpperCamelCase_:							Tuple		=  tokenizer.encode(_a  ,		add_special_tokens=_a )
							UpperCamelCase_:							Optional[int]		=  rust_tokenizer.encode(_a  ,		add_special_tokens=_a )
							self.assertListEqual(_a  ,		_a )

							UpperCamelCase_:							Optional[Any]		=  self.get_rust_tokenizer()
							UpperCamelCase_:							int		=  tokenizer.encode(_a )
							UpperCamelCase_:							str		=  rust_tokenizer.encode(_a )
							self.assertListEqual(_a  ,		_a )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							List[str]		=  BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )  ,		["""ah""", """\u535A""", """\u63A8""", """zz"""] )

			def  lowerCAmelCase__		(    self	:    str ):
							UpperCamelCase_:							Optional[Any]		=  BasicTokenizer(do_lower_case=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )  ,		["""hello""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							Optional[Any]		=  BasicTokenizer(do_lower_case=_a  ,		strip_accents=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""h\u00E9llo"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							Dict		=  BasicTokenizer(do_lower_case=_a  ,		strip_accents=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							int		=  BasicTokenizer(do_lower_case=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    List[str] ):
							UpperCamelCase_:							Union[str, Any]		=  BasicTokenizer(do_lower_case=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )  ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							Any		=  BasicTokenizer(do_lower_case=_a  ,		strip_accents=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							List[Any]		=  BasicTokenizer(do_lower_case=_a  ,		strip_accents=_a )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Tuple ):
							UpperCamelCase_:							Optional[Any]		=  BasicTokenizer(do_lower_case=_a  ,		never_split=["""[UNK]"""] )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )  ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )

			def  lowerCAmelCase__		(    self	:    List[str] ):
							UpperCamelCase_:							List[Any]		=  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

							UpperCamelCase_:							Dict		=  {}
							for i, token in enumerate(_a ):
											UpperCamelCase_:							Dict		=  i
							UpperCamelCase_:							List[str]		=  WordpieceTokenizer(vocab=_a  ,		unk_token="""[UNK]""" )

							self.assertListEqual(tokenizer.tokenize("""""" )  ,		[] )

							self.assertListEqual(tokenizer.tokenize("""unwanted running""" )  ,		["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

							self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )  ,		["""[UNK]""", """runn""", """##ing"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							self.assertTrue(_is_whitespace(""" """ ) )
							self.assertTrue(_is_whitespace("""\t""" ) )
							self.assertTrue(_is_whitespace("""\r""" ) )
							self.assertTrue(_is_whitespace("""\n""" ) )
							self.assertTrue(_is_whitespace("""\u00A0""" ) )

							self.assertFalse(_is_whitespace("""A""" ) )
							self.assertFalse(_is_whitespace("""-""" ) )

			def  lowerCAmelCase__		(    self	:    str ):
							self.assertTrue(_is_control("""\u0005""" ) )

							self.assertFalse(_is_control("""A""" ) )
							self.assertFalse(_is_control(""" """ ) )
							self.assertFalse(_is_control("""\t""" ) )
							self.assertFalse(_is_control("""\r""" ) )

			def  lowerCAmelCase__		(    self	:    int ):
							self.assertTrue(_is_punctuation("""-""" ) )
							self.assertTrue(_is_punctuation("""$""" ) )
							self.assertTrue(_is_punctuation("""`""" ) )
							self.assertTrue(_is_punctuation(""".""" ) )

							self.assertFalse(_is_punctuation("""A""" ) )
							self.assertFalse(_is_punctuation(""" """ ) )

			def  lowerCAmelCase__		(    self	:    Tuple ):
							UpperCamelCase_:							str		=  self.get_tokenizer()
							UpperCamelCase_:							Dict		=  self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_a ) for t in ["""Test""", """\xad""", """test"""]]  ,		[["""[UNK]"""], [], ["""[UNK]"""]] )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(_a ) for t in ["""Test""", """\xad""", """test"""]]  ,		[["""[UNK]"""], [], ["""[UNK]"""]] )

			@slow
			def  lowerCAmelCase__		(    self	:    List[Any] ):
							UpperCamelCase_:							Optional[int]		=  self.tokenizer_class.from_pretrained("""google/mobilebert-uncased""" )

							UpperCamelCase_:							List[str]		=  tokenizer.encode("""sequence builders"""  ,		add_special_tokens=_a )
							UpperCamelCase_:							Dict		=  tokenizer.encode("""multi-sequence build"""  ,		add_special_tokens=_a )

							UpperCamelCase_:							int		=  tokenizer.build_inputs_with_special_tokens(_a )
							UpperCamelCase_:							List[str]		=  tokenizer.build_inputs_with_special_tokens(_a  ,		_a )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]

			def  lowerCAmelCase__		(    self	:    str ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							Optional[int]		=  self.rust_tokenizer_class.from_pretrained(_a  ,		**_a )

															UpperCamelCase_:							Dict		=  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
															UpperCamelCase_:							Dict		=  tokenizer_r.encode_plus(
															    _a  ,		return_attention_mask=_a  ,		return_token_type_ids=_a  ,		return_offsets_mapping=_a  ,		add_special_tokens=_a  ,		)

															UpperCamelCase_:							Any		=  tokenizer_r.do_lower_case if hasattr(_a  ,		"""do_lower_case""" ) else False
															UpperCamelCase_:							List[Any]		=  (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """A"""),
															        ((1, 2), ""","""),
															        ((3, 5), """na"""),
															        ((5, 6), """##ï"""),
															        ((6, 8), """##ve"""),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), """Allen"""),
															        ((21, 23), """##NL"""),
															        ((23, 24), """##P"""),
															        ((25, 33), """sentence"""),
															        ((33, 34), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """a"""),
															        ((1, 2), ""","""),
															        ((3, 8), """naive"""),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), """allen"""),
															        ((21, 23), """##nl"""),
															        ((23, 24), """##p"""),
															        ((25, 33), """sentence"""),
															        ((33, 34), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]  ,		tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
															self.assertEqual([e[0] for e in expected_results]  ,		tokens["""offset_mapping"""] )




			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							int		=  ["""的""", """人""", """有"""]
							UpperCamelCase_:							Tuple		=  """""".join(_a )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							Tuple		=  True
															UpperCamelCase_:							Any		=  self.tokenizer_class.from_pretrained(_a  ,		**_a )
															UpperCamelCase_:							int		=  self.rust_tokenizer_class.from_pretrained(_a  ,		**_a )

															UpperCamelCase_:							List[Any]		=  tokenizer_p.encode(_a  ,		add_special_tokens=_a )
															UpperCamelCase_:							str		=  tokenizer_r.encode(_a  ,		add_special_tokens=_a )

															UpperCamelCase_:							Tuple		=  tokenizer_r.convert_ids_to_tokens(_a )
															UpperCamelCase_:							List[Any]		=  tokenizer_p.convert_ids_to_tokens(_a )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(_a  ,		_a )
															self.assertListEqual(_a  ,		_a )

															UpperCamelCase_:							int		=  False
															UpperCamelCase_:							Any		=  self.rust_tokenizer_class.from_pretrained(_a  ,		**_a )
															UpperCamelCase_:							Dict		=  self.tokenizer_class.from_pretrained(_a  ,		**_a )

															UpperCamelCase_:							int		=  tokenizer_r.encode(_a  ,		add_special_tokens=_a )
															UpperCamelCase_:							str		=  tokenizer_p.encode(_a  ,		add_special_tokens=_a )

															UpperCamelCase_:							Tuple		=  tokenizer_r.convert_ids_to_tokens(_a )
															UpperCamelCase_:							Any		=  tokenizer_p.convert_ids_to_tokens(_a )

															# it is expected that only the first Chinese character is not preceded by "##".
															UpperCamelCase_:							List[Any]		=  [
															    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a )
															]
															self.assertListEqual(_a  ,		_a )
															self.assertListEqual(_a  ,		_a )








import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class     _UpperCamelCase			(   _A       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  Tuple    =	CTRLTokenizer
			__UpperCamelCase :  int    =	False
			__UpperCamelCase :  List[str]    =	False

			def  lowerCAmelCase__		(    self	:    List[str] ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							UpperCamelCase_:							int		=  ["""adapt""", """re@@""", """a@@""", """apt""", """c@@""", """t""", """<unk>"""]
							UpperCamelCase_:							int		=  dict(zip(snake_case_  ,		range(len(snake_case_ ) ) ) )
							UpperCamelCase_:							Union[str, Any]		=  ["""#version: 0.2""", """a p""", """ap t</w>""", """r e""", """a d""", """ad apt</w>""", """"""]
							UpperCamelCase_:							Tuple		=  {"""unk_token""": """<unk>"""}

							UpperCamelCase_:							Optional[Any]		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""vocab_file"""] )
							UpperCamelCase_:							Dict		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""merges_file"""] )
							with open(self.vocab_file  ,		"""w"""  ,		encoding="""utf-8""" ) as fp:
											fp.write(json.dumps(snake_case_ ) + """\n""" )
							with open(self.merges_file  ,		"""w"""  ,		encoding="""utf-8""" ) as fp:
											fp.write("""\n""".join(snake_case_ ) )

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		**snake_case_	:    int ):
							kwargs.update(self.special_tokens_map )
							return CTRLTokenizer.from_pretrained(self.tmpdirname  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		snake_case_	:    List[str] ):
							UpperCamelCase_:							Dict		=  """adapt react readapt apt"""
							UpperCamelCase_:							List[str]		=  """adapt react readapt apt"""
							return input_text, output_text




			def  lowerCAmelCase__		(    self	:    List[str] ):
							UpperCamelCase_:							str		=  CTRLTokenizer(self.vocab_file  ,		self.merges_file  ,		**self.special_tokens_map )
							UpperCamelCase_:							List[Any]		=  """adapt react readapt apt"""
							UpperCamelCase_:							Optional[int]		=  """adapt re@@ a@@ c@@ t re@@ adapt apt""".split()
							UpperCamelCase_:							int		=  tokenizer.tokenize(snake_case_ )
							self.assertListEqual(snake_case_  ,		snake_case_ )

							UpperCamelCase_:							List[Any]		=  tokens + [tokenizer.unk_token]

							UpperCamelCase_:							Union[str, Any]		=  [0, 1, 2, 4, 5, 1, 0, 3, 6]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_ )  ,		snake_case_ )






