





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     _UpperCamelCase			(   a__       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  List[Any]    =	RoCBertTokenizer
			__UpperCamelCase :  List[str]    =	None
			__UpperCamelCase :  Optional[int]    =	False
			__UpperCamelCase :  int    =	True
			__UpperCamelCase :  List[Any]    =	filter_non_english

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							super().setUp()

							UpperCamelCase_:							Optional[Any]		=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase_:							Optional[int]		=  {}
							UpperCamelCase_:							int		=  {}
							for i, value in enumerate(_lowerCamelCase ):
											UpperCamelCase_:							Tuple		=  i
											UpperCamelCase_:							Optional[int]		=  i
							UpperCamelCase_:							Tuple		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""vocab_file"""] )
							UpperCamelCase_:							Tuple		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""word_shape_file"""] )
							UpperCamelCase_:							int		=  os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES["""word_pronunciation_file"""] )
							with open(self.vocab_file  ,		"""w"""  ,		encoding="""utf-8""" ) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )
							with open(self.word_shape_file  ,		"""w"""  ,		encoding="""utf-8""" ) as word_shape_writer:
											json.dump(_lowerCamelCase  ,		_lowerCamelCase  ,		ensure_ascii=_lowerCamelCase )
							with open(self.word_pronunciation_file  ,		"""w"""  ,		encoding="""utf-8""" ) as word_pronunciation_writer:
											json.dump(_lowerCamelCase  ,		_lowerCamelCase  ,		ensure_ascii=_lowerCamelCase )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							Optional[Any]		=  self.tokenizer_class(self.vocab_file  ,		self.word_shape_file  ,		self.word_pronunciation_file )

							UpperCamelCase_:							List[str]		=  tokenizer.tokenize("""你好[SEP]你是谁""" )
							self.assertListEqual(_lowerCamelCase  ,		["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""] )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase )  ,		[5, 6, 2, 5, 7, 8] )
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase )  ,		[5, 6, 2, 5, 7, 8] )
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase )  ,		[5, 6, 2, 5, 7, 8] )

			def  lowerCAmelCase__		(    self	:    List[str] ):
							UpperCamelCase_:							Optional[Any]		=  RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )  ,		["""ah""", """\u535A""", """\u63A8""", """zz"""] )

			def  lowerCAmelCase__		(    self	:    Dict ):
							UpperCamelCase_:							str		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )  ,		["""hello""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							str		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,		strip_accents=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""h\u00E9llo"""] )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							Union[str, Any]		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,		strip_accents=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							str		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
							self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )  ,		["""hello"""] )

			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							Dict		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )  ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							Tuple		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,		strip_accents=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							Optional[int]		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,		strip_accents=_lowerCamelCase )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )  ,		["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Optional[int]		=  RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,		never_split=["""[UNK]"""] )

							self.assertListEqual(
							    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )  ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							int		=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase_:							Union[str, Any]		=  {}
							for i, token in enumerate(_lowerCamelCase ):
											UpperCamelCase_:							Dict		=  i
							UpperCamelCase_:							List[str]		=  RoCBertWordpieceTokenizer(vocab=_lowerCamelCase  ,		unk_token="""[UNK]""" )

							self.assertListEqual(tokenizer.tokenize("""""" )  ,		[] )

							self.assertListEqual(tokenizer.tokenize("""unwanted running""" )  ,		["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

							self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )  ,		["""[UNK]""", """runn""", """##ing"""] )

			def  lowerCAmelCase__		(    self	:    int ):
							self.assertTrue(_is_whitespace(""" """ ) )
							self.assertTrue(_is_whitespace("""\t""" ) )
							self.assertTrue(_is_whitespace("""\r""" ) )
							self.assertTrue(_is_whitespace("""\n""" ) )
							self.assertTrue(_is_whitespace("""\u00A0""" ) )

							self.assertFalse(_is_whitespace("""A""" ) )
							self.assertFalse(_is_whitespace("""-""" ) )

			def  lowerCAmelCase__		(    self	:    Tuple ):
							self.assertTrue(_is_control("""\u0005""" ) )

							self.assertFalse(_is_control("""A""" ) )
							self.assertFalse(_is_control(""" """ ) )
							self.assertFalse(_is_control("""\t""" ) )
							self.assertFalse(_is_control("""\r""" ) )

			def  lowerCAmelCase__		(    self	:    Tuple ):
							self.assertTrue(_is_punctuation("""-""" ) )
							self.assertTrue(_is_punctuation("""$""" ) )
							self.assertTrue(_is_punctuation("""`""" ) )
							self.assertTrue(_is_punctuation(""".""" ) )

							self.assertFalse(_is_punctuation("""A""" ) )
							self.assertFalse(_is_punctuation(""" """ ) )

			def  lowerCAmelCase__		(    self	:    int ):
							UpperCamelCase_:							Any		=  self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_lowerCamelCase ) for t in ["""Test""", """\xad""", """test"""]]  ,		[["""[UNK]"""], [], ["""[UNK]"""]] )

							if self.test_rust_tokenizer:
											UpperCamelCase_:							List[Any]		=  self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(_lowerCamelCase ) for t in ["""Test""", """\xad""", """test"""]]  ,		[["""[UNK]"""], [], ["""[UNK]"""]] )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):







							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							List[Any]		=  self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,		**_lowerCamelCase )

															UpperCamelCase_:							Any		=  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
															UpperCamelCase_:							Dict		=  tokenizer_r.encode_plus(
															    _lowerCamelCase  ,		return_attention_mask=_lowerCamelCase  ,		return_token_type_ids=_lowerCamelCase  ,		return_offsets_mapping=_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase  ,		)

															UpperCamelCase_:							int		=  tokenizer_r.do_lower_case if hasattr(_lowerCamelCase  ,		"""do_lower_case""" ) else False
															UpperCamelCase_:							int		=  (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''A'''),
															        ((1, 2), ''','''),
															        ((3, 5), '''na'''),
															        ((5, 6), '''##ï'''),
															        ((6, 8), '''##ve'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''Allen'''),
															        ((21, 23), '''##NL'''),
															        ((23, 24), '''##P'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''a'''),
															        ((1, 2), ''','''),
															        ((3, 8), '''naive'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''allen'''),
															        ((21, 23), '''##nl'''),
															        ((23, 24), '''##p'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]  ,		tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
															self.assertEqual([e[0] for e in expected_results]  ,		tokens["""offset_mapping"""] )

			def  lowerCAmelCase__		(    self	:    int ):
							UpperCamelCase_:							Tuple		=  ['''的''', '''人''', '''有''']
							UpperCamelCase_:							Optional[int]		=  ''''''.join(_lowerCamelCase )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
															UpperCamelCase_:							List[str]		=  True
															UpperCamelCase_:							List[Any]		=  self.tokenizer_class.from_pretrained(_lowerCamelCase  ,		**_lowerCamelCase )
															UpperCamelCase_:							Optional[Any]		=  self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,		**_lowerCamelCase )

															UpperCamelCase_:							str		=  tokenizer_p.encode(_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )
															UpperCamelCase_:							Union[str, Any]		=  tokenizer_r.encode(_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )

															UpperCamelCase_:							str		=  tokenizer_r.convert_ids_to_tokens(_lowerCamelCase )
															UpperCamelCase_:							Dict		=  tokenizer_p.convert_ids_to_tokens(_lowerCamelCase )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(_lowerCamelCase  ,		_lowerCamelCase )
															self.assertListEqual(_lowerCamelCase  ,		_lowerCamelCase )

															UpperCamelCase_:							List[str]		=  False
															UpperCamelCase_:							List[Any]		=  self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,		**_lowerCamelCase )
															UpperCamelCase_:							int		=  self.tokenizer_class.from_pretrained(_lowerCamelCase  ,		**_lowerCamelCase )

															UpperCamelCase_:							Any		=  tokenizer_r.encode(_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )
															UpperCamelCase_:							Union[str, Any]		=  tokenizer_p.encode(_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )

															UpperCamelCase_:							Optional[int]		=  tokenizer_r.convert_ids_to_tokens(_lowerCamelCase )
															UpperCamelCase_:							List[Any]		=  tokenizer_p.convert_ids_to_tokens(_lowerCamelCase )

															# it is expected that only the first Chinese character is not preceded by "##".
															UpperCamelCase_:							Optional[Any]		=  [
															    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase )
															]
															self.assertListEqual(_lowerCamelCase  ,		_lowerCamelCase )
															self.assertListEqual(_lowerCamelCase  ,		_lowerCamelCase )

			@slow
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							Any		=  self.tokenizer_class(self.vocab_file  ,		self.word_shape_file  ,		self.word_pronunciation_file )

							UpperCamelCase_:							Tuple		=  tokenizer.encode("""你好"""  ,		add_special_tokens=_lowerCamelCase )
							UpperCamelCase_:							Dict		=  tokenizer.encode("""你是谁"""  ,		add_special_tokens=_lowerCamelCase )

							UpperCamelCase_:							Dict		=  tokenizer.build_inputs_with_special_tokens(_lowerCamelCase )
							UpperCamelCase_:							Optional[Any]		=  tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,		_lowerCamelCase )

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Optional[Any]		=  self.get_tokenizers(do_lower_case=_lowerCamelCase )
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
															UpperCamelCase_:							Dict		=  '''你好，你是谁'''
															UpperCamelCase_:							List[Any]		=  tokenizer.tokenize(_lowerCamelCase )
															UpperCamelCase_:							Union[str, Any]		=  tokenizer.convert_tokens_to_ids(_lowerCamelCase )
															UpperCamelCase_:							int		=  tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase )
															UpperCamelCase_:							Any		=  tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase )
															UpperCamelCase_:							Any		=  tokenizer.prepare_for_model(
															    _lowerCamelCase  ,		_lowerCamelCase  ,		_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )

															UpperCamelCase_:							Optional[int]		=  tokenizer.encode_plus(_lowerCamelCase  ,		add_special_tokens=_lowerCamelCase )

															self.assertEqual(_lowerCamelCase  ,		_lowerCamelCase )








import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_rjieba
@require_tokenizers
class     _UpperCamelCase			(   _A       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  Union[str, Any]    =	RoFormerTokenizer
			__UpperCamelCase :  Union[str, Any]    =	RoFormerTokenizerFast
			__UpperCamelCase :  Optional[Any]    =	True
			__UpperCamelCase :  Tuple    =	True

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							super().setUp()

			def  lowerCAmelCase__		(    self	:    List[str]  ,		**snake_case_	:    Union[str, Any] ):
							return self.tokenizer_class.from_pretrained("""junnyu/roformer_chinese_base"""  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Dict  ,		**snake_case_	:    Union[str, Any] ):
							return self.rust_tokenizer_class.from_pretrained("""junnyu/roformer_chinese_base"""  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_:							int		=  """永和服装饰品有限公司,今天天气非常好"""
							UpperCamelCase_:							int		=  """永和 服装 饰品 有限公司 , 今 天 天 气 非常 好"""
							return input_text, output_text

			def  lowerCAmelCase__		(    self	:    Dict ):
							UpperCamelCase_:							Dict		=  self.get_tokenizer()
							UpperCamelCase_,				UpperCamelCase_:							Optional[int]		=  self.get_chinese_input_output_texts()
							UpperCamelCase_:							Optional[int]		=  tokenizer.tokenize(snake_case_ )

							self.assertListEqual(snake_case_  ,		output_text.split() )

							UpperCamelCase_:							Dict		=  tokens + [tokenizer.unk_token]
							UpperCamelCase_:							int		=  [2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_ )  ,		snake_case_ )

			def  lowerCAmelCase__		(    self	:    Dict ):
							UpperCamelCase_:							Optional[int]		=  self.get_rust_tokenizer()
							UpperCamelCase_,				UpperCamelCase_:							Any		=  self.get_chinese_input_output_texts()
							UpperCamelCase_:							int		=  tokenizer.tokenize(snake_case_ )
							self.assertListEqual(snake_case_  ,		output_text.split() )
							UpperCamelCase_:							Union[str, Any]		=  tokens + [tokenizer.unk_token]
							UpperCamelCase_:							Optional[Any]		=  [2_2943, 2_1332, 3_4431, 4_5904, 117, 306, 1231, 1231, 2653, 3_3994, 1266, 100]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_ )  ,		snake_case_ )

			def  lowerCAmelCase__		(    self	:    Dict ):
							pass

			def  lowerCAmelCase__		(    self	:    Tuple ):
							pass




			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							pass






