





'''simple docstring'''



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 							__lowercase    ( lowerCAmelCase__	, unittest.TestCase  ):







	'''simple docstring'''

	a      :			Union[str, Any]     =    RoCBertTokenizer
	a      :			int     =    None
	a      :			Tuple     =    False
	a      :			Optional[int]     =    True
	a      :			Dict     =    filter_non_english


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				super().setUp()

				__lowercase			  =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
				__lowercase			  =							{}
				__lowercase			  =							{}
				for i, value in enumerate(_lowerCamelCase	):
							__lowercase			  =							i
							__lowercase			  =							i
				__lowercase			  =							os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''vocab_file''']	)
				__lowercase			  =							os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''word_shape_file''']	)
				__lowercase			  =							os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES['''word_pronunciation_file''']	)
				with open(self.vocab_file       ,'''w'''       ,encoding='''utf-8'''	) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
				with open(self.word_shape_file       ,'''w'''       ,encoding='''utf-8'''	) as word_shape_writer:
							json.dump(_lowerCamelCase       ,_lowerCamelCase       ,ensure_ascii=_lowerCamelCase	)
				with open(self.word_pronunciation_file       ,'''w'''       ,encoding='''utf-8'''	) as word_pronunciation_writer:
							json.dump(_lowerCamelCase       ,_lowerCamelCase       ,ensure_ascii=_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class(self.vocab_file       ,self.word_shape_file       ,self.word_pronunciation_file	)

				__lowercase			  =							tokenizer.tokenize('''你好[SEP]你是谁'''	)
				self.assertListEqual(_lowerCamelCase       ,['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']	)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase	)       ,[5, 6, 2, 5, 7, 8]	)
				self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase	)       ,[5, 6, 2, 5, 7, 8]	)
				self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase	)       ,[5, 6, 2, 5, 7, 8]	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)       ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)       ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,['''hello''']	)


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       ,strip_accents=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,['''h\u00E9llo''']	)


	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       ,strip_accents=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,['''hello''']	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,['''hello''']	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)       ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)


	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       ,strip_accents=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       ,strip_accents=_lowerCamelCase	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase       ,never_split=['''[UNK]''']	)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)       ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				__lowercase			  =							{}
				for i, token in enumerate(_lowerCamelCase	):
							__lowercase			  =							i
				__lowercase			  =							RoCBertWordpieceTokenizer(vocab=_lowerCamelCase       ,unk_token='''[UNK]'''	)

				self.assertListEqual(tokenizer.tokenize(''''''	)       ,[]	)

				self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)       ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

				self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)       ,['''[UNK]''', '''runn''', '''##ing''']	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				self.assertTrue(_is_whitespace(''' '''	)	)
				self.assertTrue(_is_whitespace('''\t'''	)	)
				self.assertTrue(_is_whitespace('''\r'''	)	)
				self.assertTrue(_is_whitespace('''\n'''	)	)
				self.assertTrue(_is_whitespace('''\u00A0'''	)	)

				self.assertFalse(_is_whitespace('''A'''	)	)
				self.assertFalse(_is_whitespace('''-'''	)	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				self.assertTrue(_is_control('''\u0005'''	)	)

				self.assertFalse(_is_control('''A'''	)	)
				self.assertFalse(_is_control(''' '''	)	)
				self.assertFalse(_is_control('''\t'''	)	)
				self.assertFalse(_is_control('''\r'''	)	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				self.assertTrue(_is_punctuation('''-'''	)	)
				self.assertTrue(_is_punctuation('''$'''	)	)
				self.assertTrue(_is_punctuation('''`'''	)	)
				self.assertTrue(_is_punctuation('''.'''	)	)

				self.assertFalse(_is_punctuation('''A'''	)	)
				self.assertFalse(_is_punctuation(''' '''	)	)


	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]       ,[['''[UNK]'''], [], ['''[UNK]''']]	)

				if self.test_rust_tokenizer:
							__lowercase			  =							self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(_lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]       ,[['''[UNK]'''], [], ['''[UNK]''']]	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
										__lowercase			  =							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase       ,**_lowerCamelCase	)

										__lowercase			  =							f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowercase			  =							tokenizer_r.encode_plus(
										    _lowerCamelCase       ,return_attention_mask=_lowerCamelCase       ,return_token_type_ids=_lowerCamelCase       ,return_offsets_mapping=_lowerCamelCase       ,add_special_tokens=_lowerCamelCase       ,)

										__lowercase			  =							tokenizer_r.do_lower_case if hasattr(_lowerCamelCase       ,'''do_lower_case'''	) else False
										__lowercase			  =							(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''A'''),
										        ((1, 2), ''','''),
										        ((3, 5), '''na'''),
										        ((5, 6), '''##ï'''),
										        ((6, 8), '''##ve'''),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), '''Allen'''),
										        ((21, 23), '''##NL'''),
										        ((23, 24), '''##P'''),
										        ((25, 33), '''sentence'''),
										        ((33, 34), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), '''a'''),
										        ((1, 2), ''','''),
										        ((3, 8), '''naive'''),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), '''allen'''),
										        ((21, 23), '''##nl'''),
										        ((23, 24), '''##p'''),
										        ((25, 33), '''sentence'''),
										        ((33, 34), '''.'''),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]       ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
										self.assertEqual([e[0] for e in expected_results]       ,tokens['''offset_mapping''']	)


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				__lowercase			  =							['''的''', '''人''', '''有''']
				__lowercase			  =							''''''.join(_lowerCamelCase	)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
										__lowercase			  =							True
										__lowercase			  =							self.tokenizer_class.from_pretrained(_lowerCamelCase       ,**_lowerCamelCase	)
										__lowercase			  =							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase       ,**_lowerCamelCase	)

										__lowercase			  =							tokenizer_p.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer_r.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer_r.convert_ids_to_tokens(_lowerCamelCase	)
										__lowercase			  =							tokenizer_p.convert_ids_to_tokens(_lowerCamelCase	)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

										__lowercase			  =							False
										__lowercase			  =							self.rust_tokenizer_class.from_pretrained(_lowerCamelCase       ,**_lowerCamelCase	)
										__lowercase			  =							self.tokenizer_class.from_pretrained(_lowerCamelCase       ,**_lowerCamelCase	)

										__lowercase			  =							tokenizer_r.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer_p.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer_r.convert_ids_to_tokens(_lowerCamelCase	)
										__lowercase			  =							tokenizer_p.convert_ids_to_tokens(_lowerCamelCase	)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowercase			  =							[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase	)
										]
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)


	@slow
	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class(self.vocab_file       ,self.word_shape_file       ,self.word_pronunciation_file	)

				__lowercase			  =							tokenizer.encode('''你好'''       ,add_special_tokens=_lowerCamelCase	)
				__lowercase			  =							tokenizer.encode('''你是谁'''       ,add_special_tokens=_lowerCamelCase	)

				__lowercase			  =							tokenizer.build_inputs_with_special_tokens(_lowerCamelCase	)
				__lowercase			  =							tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       ,_lowerCamelCase	)

				assert encoded_sentence == [1] + text + [2]
				assert encoded_pair == [1] + text + [2] + text_a + [2]


	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							'''你好，你是谁'''
										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)
										__lowercase			  =							tokenizer.convert_tokens_to_ids(_lowerCamelCase	)
										__lowercase			  =							tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase	)
										__lowercase			  =							tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase	)
										__lowercase			  =							tokenizer.prepare_for_model(
										    _lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer.encode_plus(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										self.assertEqual(_lowerCamelCase       ,_lowerCamelCase	)












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Dict     =    "open-llama"


	def __init__(self       ,_lowerCamelCase=100000       ,_lowerCamelCase=4096       ,_lowerCamelCase=11008       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase="silu"       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-6       ,_lowerCamelCase=True       ,_lowerCamelCase=0       ,_lowerCamelCase=1       ,_lowerCamelCase=2       ,_lowerCamelCase=False       ,_lowerCamelCase=True       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.1       ,_lowerCamelCase=True       ,_lowerCamelCase=True       ,_lowerCamelCase=None       ,**_lowerCamelCase       ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							intermediate_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							hidden_act
				__lowercase			  =							initializer_range
				__lowercase			  =							rms_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							kwargs.pop(
				    '''use_memorry_efficient_attention'''       ,_lowerCamelCase	)
				__lowercase			  =							hidden_dropout_prob
				__lowercase			  =							attention_dropout_prob
				__lowercase			  =							use_stable_embedding
				__lowercase			  =							shared_input_output_embedding
				__lowercase			  =							rope_scaling
				self._rope_scaling_validation()

				super().__init__(
				    pad_token_id=_lowerCamelCase       ,bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,tie_word_embeddings=_lowerCamelCase       ,**_lowerCamelCase       ,)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				if self.rope_scaling is None:
							return

				if not isinstance(self.rope_scaling       ,_lowerCamelCase	) or len(self.rope_scaling	) != 2:
							raise ValueError(
							    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
							    f"got {self.rope_scaling}"	)
				__lowercase			  =							self.rope_scaling.get('''type'''       ,_lowerCamelCase	)
				__lowercase			  =							self.rope_scaling.get('''factor'''       ,_lowerCamelCase	)
				if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
							raise ValueError(
							    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"	)
				if rope_scaling_factor is None or not isinstance(_lowerCamelCase       ,_lowerCamelCase	) or rope_scaling_factor <= 1.0:
							raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"	)





