



"""simple docstring"""




import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel






class      UpperCAmelCase_  :



      def __init__(					self					:     Optional[Any]					,     A					:     Dict					,     A					:     int=9_9					,     A					:     List[Any]=1_3					,     A					:     int=7					,     A					:     List[str]=9					,     A					:     List[Any]=True					,     A					:     Any=True					,     A					:     List[str]=False					,     A					:     Dict=3_2					,     A					:     List[Any]=5					,     A					:     Union[str, Any]=4					,     A					:     str=3_7					,     A					:     List[Any]=8					,     A					:     Optional[Any]=0.1					,     A					:     List[str]=0.002					,     A					:     Any=1					,     A					:     List[Any]=0					,     A					:     int=0					,     A					:     str=None					,     A					:     Tuple=None					,     ):
           _UpperCAmelCase      :     Dict        =   parent
           _UpperCAmelCase      :     str        =   batch_size
           _UpperCAmelCase      :     List[str]        =   encoder_seq_length
           _UpperCAmelCase      :     int        =   decoder_seq_length
           # For common tests
           _UpperCAmelCase      :     List[str]        =   self.decoder_seq_length
           _UpperCAmelCase      :     str        =   is_training
           _UpperCAmelCase      :     Dict        =   use_attention_mask
           _UpperCAmelCase      :     Dict        =   use_labels
           _UpperCAmelCase      :     int        =   vocab_size
           _UpperCAmelCase      :     Dict        =   hidden_size
           _UpperCAmelCase      :     Optional[int]        =   num_hidden_layers
           _UpperCAmelCase      :     List[Any]        =   num_attention_heads
           _UpperCAmelCase      :     Dict        =   d_ff
           _UpperCAmelCase      :     Optional[int]        =   relative_attention_num_buckets
           _UpperCAmelCase      :     List[Any]        =   dropout_rate
           _UpperCAmelCase      :     Any        =   initializer_factor
           _UpperCAmelCase      :     int        =   eos_token_id
           _UpperCAmelCase      :     Optional[Any]        =   pad_token_id
           _UpperCAmelCase      :     List[Any]        =   decoder_start_token_id
           _UpperCAmelCase      :     List[Any]        =   None
           _UpperCAmelCase      :     Dict        =   decoder_layers



      def 						snake_case_ (					self					:     Optional[int]						):
           return TaConfig.from_pretrained("google/umt5-base"						)



      def 						snake_case_ (					self					:     Union[str, Any]					,     A					:     List[Any]					,     A					:     List[Any]					,     A					:     Optional[Any]					,     A					:     List[str]=None					,     A					:     Any=None					,     A					:     Optional[int]=None					,     A					:     Any=None					,     A					:     List[Any]=None					,     ):
           if attention_mask is None:
                _UpperCAmelCase      :     int        =   input_ids.ne(config.pad_token_id						)
           if decoder_attention_mask is None:
                _UpperCAmelCase      :     Optional[Any]        =   decoder_input_ids.ne(config.pad_token_id						)
           if head_mask is None:
                _UpperCAmelCase      :     Dict        =   torch.ones(config.num_hidden_layers					,     config.num_attention_heads					,     device=A						)
           if decoder_head_mask is None:
                _UpperCAmelCase      :     Tuple        =   torch.ones(config.num_decoder_layers					,     config.num_attention_heads					,     device=A						)
           if cross_attn_head_mask is None:
                _UpperCAmelCase      :     Dict        =   torch.ones(
                    config.num_decoder_layers					,     config.num_attention_heads					,     device=A						)
           return {
               "input_ids": input_ids,
               "decoder_input_ids": decoder_input_ids,
               "attention_mask": attention_mask,
               "decoder_attention_mask": decoder_attention_mask,
               "head_mask": head_mask,
               "decoder_head_mask": decoder_head_mask,
               "cross_attn_head_mask": cross_attn_head_mask,
           }



      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     Optional[Any]        =   ids_tensor([self.batch_size, self.encoder_seq_length]					,     self.vocab_size						)
           _UpperCAmelCase      :     str        =   ids_tensor([self.batch_size, self.decoder_seq_length]					,     self.vocab_size						)

           # we need to clamp the input ids here to avoid having pad token in between
           # this is because for NllbMoe the position_ids are prepared such that
           # all pad tokens have pos id = 2 and rest are between 2..seq_length
           # and the seq_length here is seq_length - num_pad_tokens
           # but when using past, there is no way of knowing if the past input ids had
           # pad tokens in them, which results in incorrect seq_lenth and which in turn results in
           # position_ids being off by num_pad_tokens in past input
           _UpperCAmelCase      :     Dict        =   input_ids.clamp(self.pad_token_id + 1						)
           _UpperCAmelCase      :     Dict        =   decoder_input_ids.clamp(self.pad_token_id + 1						)

           _UpperCAmelCase      :     Optional[int]        =   self.get_config()
           _UpperCAmelCase      :     Tuple        =   config.num_attention_heads
           _UpperCAmelCase      :     Union[str, Any]        =   self.prepare_inputs_dict(A					,     A					,     A						)
           return config, input_dict



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     int        =   self.prepare_config_and_inputs()
           return config, inputs_dict



      def 						snake_case_ (					self					:     Optional[int]						):
           return TaConfig(
               vocab_size=1_6_6					,     d_model=self.hidden_size					,     d_ff=self.d_ff					,     d_kv=self.hidden_size // self.num_attention_heads					,     num_layers=self.num_hidden_layers					,     num_decoder_layers=self.decoder_layers					,     num_heads=self.num_attention_heads					,     relative_attention_num_buckets=self.relative_attention_num_buckets					,     dropout_rate=self.dropout_rate					,     initializer_factor=self.initializer_factor					,     eos_token_id=self.eos_token_id					,     bos_token_id=self.pad_token_id					,     pad_token_id=self.pad_token_id					,     decoder_start_token_id=self.decoder_start_token_id					,     )



      def 						snake_case_ (					self					:     Optional[Any]						):
           return TaConfig(
               vocab_size=self.vocab_size					,     d_model=self.hidden_size					,     d_ff=self.d_ff					,     d_kv=self.hidden_size // self.num_attention_heads					,     num_layers=self.num_hidden_layers					,     num_decoder_layers=self.decoder_layers					,     num_heads=self.num_attention_heads					,     relative_attention_num_buckets=self.relative_attention_num_buckets					,     dropout_rate=self.dropout_rate					,     initializer_factor=self.initializer_factor					,     eos_token_id=self.eos_token_id					,     bos_token_id=self.pad_token_id					,     pad_token_id=self.pad_token_id					,     decoder_start_token_id=self.decoder_start_token_id					,     )



      def 						snake_case_ (					self					:     Any					,     A					:     Union[str, Any]					,     A					:     Dict					,     A					:     Optional[int]					,     A					:     List[Any]					,     A					:     Union[str, Any]					,     A					:     int					,     ):
           _UpperCAmelCase      :     Any        =   UMTaModel(config=A						)
           model.to(A						)
           model.eval()
           _UpperCAmelCase      :     Union[str, Any]        =   model(
               input_ids=A					,     decoder_input_ids=A					,     attention_mask=A					,     decoder_attention_mask=A					,     )
           _UpperCAmelCase      :     List[Any]        =   model(input_ids=A					,     decoder_input_ids=A						)
           _UpperCAmelCase      :     int        =   result.last_hidden_state
           _UpperCAmelCase      :     Dict        =   result.past_key_values
           _UpperCAmelCase      :     Tuple        =   result.encoder_last_hidden_state

           self.parent.assertEqual(encoder_output.size()					,     (self.batch_size, self.encoder_seq_length, self.hidden_size)						)
           self.parent.assertEqual(decoder_output.size()					,     (self.batch_size, self.decoder_seq_length, self.hidden_size)						)
           # There should be `num_layers` key value embeddings stored in decoder_past
           self.parent.assertEqual(len(A						)					,     config.num_layers						)
           # There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
           self.parent.assertEqual(len(decoder_past[0]						)					,     4						)



      def 						snake_case_ (					self					:     int					,     A					:     Optional[Any]					,     A					:     Tuple					,     A					:     List[Any]					,     A					:     int					,     A					:     int					,     A					:     Any					,     ):
           _UpperCAmelCase      :     List[Any]        =   UMTaModel(config=A						).get_decoder().to(A						).eval()
           # first forward pass
           _UpperCAmelCase      :     str        =   model(A					,     use_cache=A						)
           _UpperCAmelCase      :     Any        =   model(A						)
           _UpperCAmelCase      :     Any        =   model(A					,     use_cache=A						)

           self.parent.assertTrue(len(A						) == len(A						)						)
           self.parent.assertTrue(len(A						) == len(A						) + 1						)

           _UpperCAmelCase    ,	_UpperCAmelCase      :     str        =   outputs.to_tuple()

           # create hypothetical next token and extent to next_input_ids
           _UpperCAmelCase      :     Any        =   ids_tensor((self.batch_size, 1)					,     config.vocab_size						)

           # append to next input_ids and
           _UpperCAmelCase      :     Any        =   torch.cat([input_ids, next_tokens]					,     dim=-1						)

           _UpperCAmelCase      :     Dict        =   model(A						)["last_hidden_state"]
           _UpperCAmelCase      :     Optional[Any]        =   model(A					,     past_key_values=A						)["last_hidden_state"]

           # select random slice
           _UpperCAmelCase      :     Union[str, Any]        =   ids_tensor((1,)					,     output_from_past.shape[-1]						).item()
           _UpperCAmelCase      :     Tuple        =   output_from_no_past[:, -1, random_slice_idx].detach()
           _UpperCAmelCase      :     str        =   output_from_past[:, 0, random_slice_idx].detach()

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(A					,     A					,     atol=1e-3						)						)



      def 						snake_case_ (					self					:     List[str]					,     A					:     List[Any]					,     A					:     List[str]					,     ):
           _UpperCAmelCase      :     List[Any]        =   UMTaModel(config=A						).to(A						).half().eval()
           _UpperCAmelCase      :     Optional[Any]        =   model(**A						)["last_hidden_state"]
           self.parent.assertFalse(torch.isnan(A						).any().item()						)







@require_torch
class      UpperCAmelCase_  (    _UpperCamelCase	,      _UpperCamelCase	,      _UpperCamelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       str								=			(
          (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
      )
      __SCREAMING_SNAKE_CASE						:       str								=			(UMTaForConditionalGeneration,) if is_torch_available() else ()
      __SCREAMING_SNAKE_CASE						:       int								=			(
          {
              'conversational': UMTaForConditionalGeneration,
              'feature-extraction': UMTaModel,
              'summarization': UMTaForConditionalGeneration,
              'text2text-generation': UMTaForConditionalGeneration,
              'translation': UMTaForConditionalGeneration,
              'question-answering': UMTaForQuestionAnswering,
          }
          if is_torch_available()
          else {}
      )
      __SCREAMING_SNAKE_CASE						:       List[str]								=			True
      __SCREAMING_SNAKE_CASE						:       List[str]								=			False
      __SCREAMING_SNAKE_CASE						:       Tuple								=			False
      __SCREAMING_SNAKE_CASE						:       Optional[int]								=			True
      __SCREAMING_SNAKE_CASE						:       str								=			True
      # The small UMT5 model needs higher percentages for CPU/MP tests
      __SCREAMING_SNAKE_CASE						:       str								=			[0.8, 0.9]



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     Tuple        =   UMTaModelTester(self						)



      @unittest.skip("Test has a segmentation fault on torch 1.8.0"						)
      def 						snake_case_ (					self					:     int						):
           _UpperCAmelCase      :     Dict        =   self.model_tester.prepare_config_and_inputs()
           _UpperCAmelCase      :     Optional[Any]        =   UMTaModel(config_and_inputs[0]						).to(A						)
           with tempfile.TemporaryDirectory() as tmpdirname:
                torch.onnx.export(
                    A					,     (config_and_inputs[1], config_and_inputs[3], config_and_inputs[2])					,     f'{tmpdirname}/t5_test.onnx'					,     export_params=A					,     opset_version=9					,     input_names=["input_ids", "decoder_input_ids"]					,     )



      @unittest.skipIf(torch_device == "cpu"					,     "Cant do half precision"						)
      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     Union[str, Any]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model_fpaa_forward(*A						)



      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     Optional[int]        =   ["encoder_attentions", "decoder_attentions", "cross_attentions"]
           _UpperCAmelCase      :     str        =   self.model_tester.prepare_config_and_inputs()
           _UpperCAmelCase      :     Dict        =   config_and_inputs[0]
           _UpperCAmelCase      :     List[Any]        =   UMTaForConditionalGeneration(A						).eval()
           model.to(A						)

           _UpperCAmelCase      :     int        =   {
               "head_mask": torch.zeros(config.num_layers					,     config.num_heads					,     device=A						),
               "decoder_head_mask": torch.zeros(config.num_decoder_layers					,     config.num_heads					,     device=A						),
               "cross_attn_head_mask": torch.zeros(config.num_decoder_layers					,     config.num_heads					,     device=A						),
           }

           for attn_name, (name, mask) in zip(A					,     head_masking.items()						):
                _UpperCAmelCase      :     Any        =   {name: mask}
                # Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
                if name == "head_mask":
                     _UpperCAmelCase      :     int        =   torch.ones(
                         config.num_decoder_layers					,     config.num_heads					,     device=A						)

                _UpperCAmelCase      :     Any        =   model.generate(
                    config_and_inputs[1]["input_ids"]					,     num_beams=1					,     max_length=3					,     output_attentions=A					,     return_dict_in_generate=A					,     **A					,     )
                # We check the state of decoder_attentions and cross_attentions just from the last step
                _UpperCAmelCase      :     int        =   out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
                self.assertEqual(sum([w.sum().item() for w in attn_weights]						)					,     0.0						)



      @unittest.skip("Does not work on the tiny model as we keep hitting edge cases."						)
      def 						snake_case_ (					self					:     Dict						):
           pass







@require_torch
@require_sentencepiece
@require_tokenizers
class      UpperCAmelCase_  (    unittest.TestCase						):



      @slow
      @unittest.skip(
          "Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged"						)
      def 						snake_case_ (					self					:     int						):
           _UpperCAmelCase      :     Tuple        =   UMTaForConditionalGeneration.from_pretrained("google/umt5-small"					,     return_dict=A						).to(A						)
           _UpperCAmelCase      :     Optional[Any]        =   AutoTokenizer.from_pretrained("google/umt5-small"					,     use_fast=A					,     legacy=A						)
           _UpperCAmelCase      :     Optional[Any]        =   [
               "Bonjour monsieur <extra_id_0> bien <extra_id_1>.",
               "No se como puedo <extra_id_0>.",
               "This is the reason why we <extra_id_0> them.",
               "The <extra_id_0> walks in <extra_id_1>, seats",
               "A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.",
           ]
           _UpperCAmelCase      :     Optional[Any]        =   tokenizer(A					,     return_tensors="pt"					,     padding=A						).input_ids
           # fmt: off
           _UpperCAmelCase      :     Union[str, Any]        =   torch.tensor(
               [
                   [ 3_8_5_3_0, 2_1_0_7_0_3, 2_5_6_2_9_9, 1_4_1_0, 2_5_6_2_9_8, 2_7_4, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
                   [   8_2_6, 3_2_1, 6_7_1, 2_5_9_2_2, 2_5_6_2_9_9, 2_7_4, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
                   [  1_4_6_0, 3_3_9, 3_1_2, 1_9_0_1_4, 1_0_6_2_0, 7_5_8, 2_5_6_2_9_9, 2_3_5_5,2_7_4, 1, 0, 0, 0, 0, 0, 0,0, 0],
                   [   5_1_7, 2_5_6_2_9_9, 1_4_8_6_9, 2_8_1, 3_0_1, 2_5_6_2_9_8, 2_7_5, 1_1_9_9_8_3,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
                   [   3_2_0, 2_5_6_2_9_9, 1_4_8_6_9, 2_8_1, 2_2_3_4, 2_8_9, 2_2_7_5, 3_3_3,6_1_3_9_1, 2_8_9, 2_5_6_2_9_8, 5_4_3, 2_5_6_2_9_7, 1_6_8_7_1_4, 3_2_9, 2_5_6_2_9_6,2_7_4, 1],
               ]						)
           # fmt: on
           torch.testing.assert_allclose(A					,     A						)

           _UpperCAmelCase      :     List[Any]        =   model.generate(input_ids.to(A						)						)
           _UpperCAmelCase      :     List[Any]        =   [
               "<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>",
               "<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>",
               "<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>",
               "<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>",
               "<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>",
           ]
           _UpperCAmelCase      :     str        =   tokenizer.batch_decode(A						)
           self.assertEqual(A					,     A						)





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCAmelCase    : List[str]							=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase    : int							=							{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCAmelCase    : List[Any]							=							{
    "google/electra-small-generator": 5_12,
    "google/electra-base-generator": 5_12,
    "google/electra-large-generator": 5_12,
    "google/electra-small-discriminator": 5_12,
    "google/electra-base-discriminator": 5_12,
    "google/electra-large-discriminator": 5_12,
}

_lowerCAmelCase    : Optional[Any]							=							{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}






class      UpperCAmelCase_  (    _UpperCamelCase						):
      __SCREAMING_SNAKE_CASE						:       Optional[Any]								=			VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE						:       Any								=			PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_INIT_CONFIGURATION
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE						:       Tuple								=			ElectraTokenizer



      def __init__(					self					:     Dict					,     A					:     Dict=None					,     A					:     Optional[int]=None					,     A					:     Dict=True					,     A					:     Optional[Any]="[UNK]"					,     A					:     Any="[SEP]"					,     A					:     str="[PAD]"					,     A					:     Tuple="[CLS]"					,     A					:     Optional[Any]="[MASK]"					,     A					:     Any=True					,     A					:     Tuple=None					,     **A					:     Any					,     ):
           super().__init__(
               A					,     tokenizer_file=A					,     do_lower_case=A					,     unk_token=A					,     sep_token=A					,     pad_token=A					,     cls_token=A					,     mask_token=A					,     tokenize_chinese_chars=A					,     strip_accents=A					,     **A					,     )

           _UpperCAmelCase      :     List[str]        =   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
           if (
               normalizer_state.get("lowercase"					,     A						) != do_lower_case
               or normalizer_state.get("strip_accents"					,     A						) != strip_accents
               or normalizer_state.get("handle_chinese_chars"					,     A						) != tokenize_chinese_chars
           ):
                _UpperCAmelCase      :     Union[str, Any]        =   getattr(A					,     normalizer_state.pop("type"						)						)
                _UpperCAmelCase      :     Dict        =   do_lower_case
                _UpperCAmelCase      :     Optional[int]        =   strip_accents
                _UpperCAmelCase      :     Any        =   tokenize_chinese_chars
                _UpperCAmelCase      :     Optional[Any]        =   normalizer_class(**A						)

           _UpperCAmelCase      :     int        =   do_lower_case



      def 						snake_case_ (					self					:     Tuple					,     A					:     str					,     A					:     int=None						):
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

           return output



      def 						snake_case_ (					self					:     Any					,     A					:     List[int]					,     A					:     Optional[List[int]] = None						):
           _UpperCAmelCase      :     Any        =   [self.sep_token_id]
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id]
           if token_ids_a is None:
                return len(cls + token_ids_a + sep						) * [0]
           return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



      def 						snake_case_ (					self					:     Any					,     A					:     str					,     A					:     Optional[str] = None						):
           _UpperCAmelCase      :     List[Any]        =   self._tokenizer.model.save(A					,     name=A						)
           return tuple(A						)
