

from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
		from transformers import (
		    TFAutoModel,
		    TFAutoModelForCausalLM,
		    TFAutoModelForMaskedLM,
		    TFAutoModelForPreTraining,
		    TFAutoModelForQuestionAnswering,
		    TFAutoModelForSeqaSeqLM,
		    TFAutoModelForSequenceClassification,
		    TFAutoModelForTableQuestionAnswering,
		    TFAutoModelForTokenClassification,
		    TFAutoModelWithLMHead,
		    TFBertForMaskedLM,
		    TFBertForPreTraining,
		    TFBertForQuestionAnswering,
		    TFBertForSequenceClassification,
		    TFBertModel,
		    TFFunnelBaseModel,
		    TFFunnelModel,
		    TFGPTaLMHeadModel,
		    TFRobertaForMaskedLM,
		    TFTaForConditionalGeneration,
		    TFTapasForQuestionAnswering,
		)
		from transformers.models.auto.modeling_tf_auto import (
		    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_MASKED_LM_MAPPING,
		    TF_MODEL_FOR_PRETRAINING_MAPPING,
		    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    TF_MODEL_MAPPING,
		)
		from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
		from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST





class  _a							(    _lowercase):
	_a  :  Optional[int]				   =							'''new-model'''


if is_tf_available():





		class  _a							(    _lowercase):
			_a  :  List[Any]				   =							NewModelConfig





@require_tf
class  _a							(    unittest.TestCase):




	@slow
	def 							UpperCAmelCase__( self						:		List[str]		)->	List[str]:
			lowerCAmelCase__       :      List[str]   					= '''bert-base-cased'''
			lowerCAmelCase__       :      List[Any]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      Any   					= TFAutoModel.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		int		)->	Optional[Any]:
			lowerCAmelCase__       :      Tuple   					= '''bert-base-cased'''
			lowerCAmelCase__       :      List[str]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      List[Any]   					= TFAutoModelForPreTraining.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		Optional[int]		)->	str:
			for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCAmelCase__       :      Dict   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Optional[Any]   					= TFAutoModelForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
					lowerCAmelCase__       :      Any   					= TFAutoModelForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		, output_loading_info=_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		Optional[int]		)->	List[Any]:
			for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCAmelCase__       :      Any   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Union[str, Any]   					= TFAutoModelWithLMHead.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		Union[str, Any]		)->	str:
			for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCAmelCase__       :      int   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Optional[Any]   					= TFAutoModelForMaskedLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
					lowerCAmelCase__       :      Tuple   					= TFAutoModelForMaskedLM.from_pretrained(_SCREAMING_SNAKE_CASE		, output_loading_info=_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		int		)->	Optional[Any]:
			for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCAmelCase__       :      Union[str, Any]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Union[str, Any]   					= TFAutoModelForSeqaSeqLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
					lowerCAmelCase__       :      int   					= TFAutoModelForSeqaSeqLM.from_pretrained(_SCREAMING_SNAKE_CASE		, output_loading_info=_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		Union[str, Any]		)->	Optional[Any]:
			# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			for model_name in ["bert-base-uncased"]:
					lowerCAmelCase__       :      List[Any]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      List[Any]   					= TFAutoModelForSequenceClassification.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	def 							UpperCAmelCase__( self						:		Union[str, Any]		)->	Tuple:
			# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			for model_name in ["bert-base-uncased"]:
					lowerCAmelCase__       :      Union[str, Any]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Optional[Any]   					= TFAutoModelForQuestionAnswering.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	@slow
	@require_tensorflow_probability
	def 							UpperCAmelCase__( self						:		Dict		)->	Any:
			for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
					lowerCAmelCase__       :      Optional[Any]   					= AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      List[Any]   					= TFAutoModelForTableQuestionAnswering.from_pretrained(_SCREAMING_SNAKE_CASE		)
					lowerCAmelCase__       :      Union[str, Any]   					= TFAutoModelForTableQuestionAnswering.from_pretrained(
					    _SCREAMING_SNAKE_CASE		, output_loading_info=_SCREAMING_SNAKE_CASE		)
					self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	def 							UpperCAmelCase__( self						:		int		)->	str:
			lowerCAmelCase__       :      Optional[Any]   					= TFAutoModelWithLMHead.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)
			self.assertEqual(model.num_parameters()		, 1_4410		)
			self.assertEqual(model.num_parameters(only_trainable=_SCREAMING_SNAKE_CASE		)		, 1_4410		)




	def 							UpperCAmelCase__( self						:		Any		)->	Tuple:
			lowerCAmelCase__       :      List[str]   					= TFAutoModelWithLMHead.from_pretrained(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)
			self.assertEqual(model.num_parameters()		, 1_4410		)
			self.assertEqual(model.num_parameters(only_trainable=_SCREAMING_SNAKE_CASE		)		, 1_4410		)




	def 							UpperCAmelCase__( self						:		Dict		)->	Optional[int]:
			# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
			lowerCAmelCase__       :      int   					= TFAutoModel.from_pretrained('''sgugger/funnel-random-tiny'''		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      str   					= copy.deepcopy(model.config		)
			lowerCAmelCase__       :      Union[str, Any]   					= ['''FunnelBaseModel''']
			lowerCAmelCase__       :      Dict   					= TFAutoModel.from_config(_SCREAMING_SNAKE_CASE		)
			self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			with tempfile.TemporaryDirectory() as tmp_dir:
					model.save_pretrained(_SCREAMING_SNAKE_CASE		)
					lowerCAmelCase__       :      int   					= TFAutoModel.from_pretrained(_SCREAMING_SNAKE_CASE		)
					self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)




	def 							UpperCAmelCase__( self						:		int		)->	Tuple:
			try:
					AutoConfig.register('''new-model'''		, _SCREAMING_SNAKE_CASE		)

					lowerCAmelCase__       :      Optional[int]   					= [
					    TFAutoModel,
					    TFAutoModelForCausalLM,
					    TFAutoModelForMaskedLM,
					    TFAutoModelForPreTraining,
					    TFAutoModelForQuestionAnswering,
					    TFAutoModelForSequenceClassification,
					    TFAutoModelForTokenClassification,
					]

					for auto_class in auto_classes:
							with self.subTest(auto_class.__name__		):
									# Wrong config class will raise an error
									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
											auto_class.register(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)
									auto_class.register(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
											auto_class.register(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

									# Now that the config is registered, it can be used as any other config with the auto-API
									lowerCAmelCase__       :      Tuple   					= BertModelTester(self		).get_config()
									lowerCAmelCase__       :      Optional[int]   					= NewModelConfig(**tiny_config.to_dict()		)
									lowerCAmelCase__       :      List[Any]   					= auto_class.from_config(_SCREAMING_SNAKE_CASE		)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(_SCREAMING_SNAKE_CASE		)
											lowerCAmelCase__       :      Any   					= auto_class.from_pretrained(_SCREAMING_SNAKE_CASE		)
											self.assertIsInstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			finally:
					if "new-model" in CONFIG_MAPPING._extra_content:
							del CONFIG_MAPPING._extra_content["new-model"]
					for mapping in (
					    TF_MODEL_MAPPING,
					    TF_MODEL_FOR_PRETRAINING_MAPPING,
					    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
					    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
					    TF_MODEL_FOR_MASKED_LM_MAPPING,
					):
							if NewModelConfig in mapping._extra_content:
									del mapping._extra_content[NewModelConfig]




	def 							UpperCAmelCase__( self						:		int		)->	List[str]:
			with self.assertRaisesRegex(
			    _SCREAMING_SNAKE_CASE		, '''bert-base is not a local folder and is not a valid model identifier'''		):
					lowerCAmelCase__       :      List[Any]   					= TFAutoModel.from_pretrained('''bert-base'''		)




	def 							UpperCAmelCase__( self						:		Optional[int]		)->	Union[str, Any]:
			with self.assertRaisesRegex(
			    _SCREAMING_SNAKE_CASE		, r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''		):
					lowerCAmelCase__       :      List[str]   					= TFAutoModel.from_pretrained(_SCREAMING_SNAKE_CASE		, revision='''aaaaaa'''		)




	def 							UpperCAmelCase__( self						:		Dict		)->	Tuple:
			with self.assertRaisesRegex(
			    _SCREAMING_SNAKE_CASE		, '''hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'''		, ):
					lowerCAmelCase__       :      Optional[int]   					= TFAutoModel.from_pretrained('''hf-internal-testing/config-no-model'''		)




	def 							UpperCAmelCase__( self						:		List[Any]		)->	Dict:
			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE		, '''Use `from_pt=True` to load this model'''		):
					lowerCAmelCase__       :      str   					= TFAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only'''		)




	def 							UpperCAmelCase__( self						:		int		)->	Any:
			# Make sure we have cached the model.
			lowerCAmelCase__       :      int   					= TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
			with RequestCounter() as counter:
					lowerCAmelCase__       :      int   					= TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
					self.assertEqual(counter.get_request_count		, 0		)
					self.assertEqual(counter.head_request_count		, 1		)
					self.assertEqual(counter.other_request_count		, 0		)

			# With a sharded checkpoint
			lowerCAmelCase__       :      Optional[Any]   					= TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''		)
			with RequestCounter() as counter:
					lowerCAmelCase__       :      Any   					= TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''		)
					self.assertEqual(counter.get_request_count		, 0		)
					self.assertEqual(counter.head_request_count		, 1		)
					self.assertEqual(counter.other_request_count		, 0		)








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase						=		[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase						=		{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}





class  _a							(    _lowercase):
	_a  :  Tuple				   =							VOCAB_FILES_NAMES
	_a  :  Dict				   =							PRETRAINED_VOCAB_FILES_MAP
	_a  :  Dict				   =							PRETRAINED_INIT_CONFIGURATION
	_a  :  Union[str, Any]				   =							FunnelTokenizer
	_a  :  int				   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a  :  int				   =							2




	def __init__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Any="<unk>"		, _SCREAMING_SNAKE_CASE						:		Dict="<sep>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"		, _SCREAMING_SNAKE_CASE						:		str="<cls>"		, _SCREAMING_SNAKE_CASE						:		List[str]="<mask>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<s>"		, _SCREAMING_SNAKE_CASE						:		Dict="</s>"		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Dict=True		, _SCREAMING_SNAKE_CASE						:		Tuple=None		, _SCREAMING_SNAKE_CASE						:		str="##"		, **_SCREAMING_SNAKE_CASE						:		List[str]		, )->	List[str]:
			super().__init__(
			    _SCREAMING_SNAKE_CASE		, tokenizer_file=_SCREAMING_SNAKE_CASE		, do_lower_case=_SCREAMING_SNAKE_CASE		, unk_token=_SCREAMING_SNAKE_CASE		, sep_token=_SCREAMING_SNAKE_CASE		, pad_token=_SCREAMING_SNAKE_CASE		, cls_token=_SCREAMING_SNAKE_CASE		, mask_token=_SCREAMING_SNAKE_CASE		, bos_token=_SCREAMING_SNAKE_CASE		, eos_token=_SCREAMING_SNAKE_CASE		, clean_text=_SCREAMING_SNAKE_CASE		, tokenize_chinese_chars=_SCREAMING_SNAKE_CASE		, strip_accents=_SCREAMING_SNAKE_CASE		, wordpieces_prefix=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		, )

			lowerCAmelCase__       :      Any   					= json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
			if (
			    normalizer_state.get('''lowercase'''		, _SCREAMING_SNAKE_CASE		) != do_lower_case
			    or normalizer_state.get('''strip_accents'''		, _SCREAMING_SNAKE_CASE		) != strip_accents
			    or normalizer_state.get('''handle_chinese_chars'''		, _SCREAMING_SNAKE_CASE		) != tokenize_chinese_chars
			):
					lowerCAmelCase__       :      int   					= getattr(_SCREAMING_SNAKE_CASE		, normalizer_state.pop('''type'''		)		)
					lowerCAmelCase__       :      Dict   					= do_lower_case
					lowerCAmelCase__       :      str   					= strip_accents
					lowerCAmelCase__       :      Dict   					= tokenize_chinese_chars
					lowerCAmelCase__       :      str   					= normalizer_class(**_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      List[Any]   					= do_lower_case




	def 							UpperCAmelCase__( self						:		Any		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=None		)->	Optional[int]:
			lowerCAmelCase__       :      Tuple   					= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output




	def 							UpperCAmelCase__( self						:		Optional[Any]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      str   					= [self.sep_token_id]
			lowerCAmelCase__       :      Optional[int]   					= [self.cls_token_id]
			if token_ids_a is None:
					return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
			return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




	def 							UpperCAmelCase__( self						:		str		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str] = None		)->	Tuple[str]:
			lowerCAmelCase__       :      Any   					= self._tokenizer.model.save(_SCREAMING_SNAKE_CASE		, name=_SCREAMING_SNAKE_CASE		)
			return tuple(_SCREAMING_SNAKE_CASE		)




