



"""simple docstring"""




import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
 from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor






class      UpperCAmelCase_  :



      def __init__(					self					:     Union[str, Any]					,     A					:     List[str]					,     A					:     List[str]=1_3					,     A					:     Dict=3_2					,     A					:     Dict=2					,     A					:     List[Any]=3					,     A					:     List[str]=1_6					,     A					:     Any=[1, 2, 1]					,     A					:     Optional[Any]=[2, 2, 4]					,     A					:     Dict=2					,     A					:     Optional[int]=2.0					,     A					:     List[str]=True					,     A					:     List[str]=0.0					,     A					:     str=0.0					,     A					:     Optional[Any]=0.1					,     A					:     Optional[int]="gelu"					,     A					:     Tuple=False					,     A					:     Optional[int]=True					,     A					:     Union[str, Any]=0.02					,     A					:     List[str]=1e-5					,     A					:     int=True					,     A					:     Union[str, Any]=None					,     A					:     List[str]=True					,     A					:     Any=1_0					,     A					:     Optional[Any]=8					,     ):
           _UpperCAmelCase      :     Any        =   parent
           _UpperCAmelCase      :     Any        =   batch_size
           _UpperCAmelCase      :     List[Any]        =   image_size
           _UpperCAmelCase      :     Union[str, Any]        =   patch_size
           _UpperCAmelCase      :     List[str]        =   num_channels
           _UpperCAmelCase      :     str        =   embed_dim
           _UpperCAmelCase      :     str        =   depths
           _UpperCAmelCase      :     int        =   num_heads
           _UpperCAmelCase      :     Any        =   window_size
           _UpperCAmelCase      :     List[Any]        =   mlp_ratio
           _UpperCAmelCase      :     Any        =   qkv_bias
           _UpperCAmelCase      :     List[Any]        =   hidden_dropout_prob
           _UpperCAmelCase      :     List[Any]        =   attention_probs_dropout_prob
           _UpperCAmelCase      :     str        =   drop_path_rate
           _UpperCAmelCase      :     Tuple        =   hidden_act
           _UpperCAmelCase      :     Any        =   use_absolute_embeddings
           _UpperCAmelCase      :     List[Any]        =   patch_norm
           _UpperCAmelCase      :     List[str]        =   layer_norm_eps
           _UpperCAmelCase      :     Any        =   initializer_range
           _UpperCAmelCase      :     str        =   is_training
           _UpperCAmelCase      :     str        =   scope
           _UpperCAmelCase      :     List[Any]        =   use_labels
           _UpperCAmelCase      :     List[str]        =   type_sequence_label_size
           _UpperCAmelCase      :     Tuple        =   encoder_stride



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     Optional[Any]        =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

           _UpperCAmelCase      :     Dict        =   None
           if self.use_labels:
                _UpperCAmelCase      :     List[Any]        =   ids_tensor([self.batch_size]					,     self.type_sequence_label_size						)

           _UpperCAmelCase      :     Optional[Any]        =   self.get_config()

           return config, pixel_values, labels



      def 						snake_case_ (					self					:     Optional[int]						):
           return SwinvaConfig(
               image_size=self.image_size					,     patch_size=self.patch_size					,     num_channels=self.num_channels					,     embed_dim=self.embed_dim					,     depths=self.depths					,     num_heads=self.num_heads					,     window_size=self.window_size					,     mlp_ratio=self.mlp_ratio					,     qkv_bias=self.qkv_bias					,     hidden_dropout_prob=self.hidden_dropout_prob					,     attention_probs_dropout_prob=self.attention_probs_dropout_prob					,     drop_path_rate=self.drop_path_rate					,     hidden_act=self.hidden_act					,     use_absolute_embeddings=self.use_absolute_embeddings					,     path_norm=self.patch_norm					,     layer_norm_eps=self.layer_norm_eps					,     initializer_range=self.initializer_range					,     encoder_stride=self.encoder_stride					,     )



      def 						snake_case_ (					self					:     List[str]					,     A					:     Optional[Any]					,     A					:     Tuple					,     A					:     Tuple						):
           _UpperCAmelCase      :     List[Any]        =   SwinvaModel(config=A						)
           model.to(A						)
           model.eval()
           _UpperCAmelCase      :     Optional[int]        =   model(A						)

           _UpperCAmelCase      :     Optional[Any]        =   ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths						) - 1))
           _UpperCAmelCase      :     Union[str, Any]        =   int(config.embed_dim * 2 ** (len(config.depths						) - 1)						)

           self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, expected_seq_len, expected_dim)						)



      def 						snake_case_ (					self					:     List[str]					,     A					:     Dict					,     A					:     Dict					,     A					:     Optional[Any]						):
           _UpperCAmelCase      :     Tuple        =   SwinvaForMaskedImageModeling(config=A						)
           model.to(A						)
           model.eval()
           _UpperCAmelCase      :     Optional[int]        =   model(A						)
           self.parent.assertEqual(
               result.logits.shape					,     (self.batch_size, self.num_channels, self.image_size, self.image_size)						)

           # test greyscale images
           _UpperCAmelCase      :     int        =   1
           _UpperCAmelCase      :     int        =   SwinvaForMaskedImageModeling(A						)
           model.to(A						)
           model.eval()

           _UpperCAmelCase      :     Union[str, Any]        =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
           _UpperCAmelCase      :     Dict        =   model(A						)
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, 1, self.image_size, self.image_size)						)



      def 						snake_case_ (					self					:     Tuple					,     A					:     List[Any]					,     A					:     Union[str, Any]					,     A					:     str						):
           _UpperCAmelCase      :     int        =   self.type_sequence_label_size
           _UpperCAmelCase      :     Dict        =   SwinvaForImageClassification(A						)
           model.to(A						)
           model.eval()
           _UpperCAmelCase      :     Optional[Any]        =   model(A					,     labels=A						)
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.type_sequence_label_size)						)



      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     List[str]        =   self.prepare_config_and_inputs()
           _UpperCAmelCase    ,	_UpperCAmelCase    ,	_UpperCAmelCase      :     List[str]        =   config_and_inputs
           _UpperCAmelCase      :     Any        =   {"pixel_values": pixel_values}
           return config, inputs_dict







@require_torch
class      UpperCAmelCase_  (    _UpperCamelCase	,      _UpperCamelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       int								=			(
          (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
      )
      __SCREAMING_SNAKE_CASE						:       List[Any]								=			(
          {'feature-extraction': SwinvaModel, 'image-classification': SwinvaForImageClassification}
          if is_torch_available()
          else {}
      )

      __SCREAMING_SNAKE_CASE						:       Dict								=			False
      __SCREAMING_SNAKE_CASE						:       Any								=			False
      __SCREAMING_SNAKE_CASE						:       str								=			False
      __SCREAMING_SNAKE_CASE						:       Dict								=			False



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     Dict        =   SwinvaModelTester(self						)
           _UpperCAmelCase      :     Tuple        =   ConfigTester(self					,     config_class=A					,     embed_dim=3_7						)



      def 						snake_case_ (					self					:     Any						):
           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()



      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     List[Any]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*A						)



      @unittest.skip(reason="Got `CUDA error: misaligned address` with PyTorch 2.0.0."						)
      def 						snake_case_ (					self					:     str						):
           pass



      @unittest.skip(reason="Swinv2 does not use inputs_embeds"						)
      def 						snake_case_ (					self					:     Union[str, Any]						):
           pass



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     Dict        =   self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     Union[str, Any]        =   model_class(A						)
                self.assertIsInstance(model.get_input_embeddings()					,     (nn.Module)						)
                _UpperCAmelCase      :     Tuple        =   model.get_output_embeddings()
                self.assertTrue(x is None or isinstance(A					,     nn.Linear						)						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     Optional[Any]        =   self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     Tuple        =   model_class(A						)
                _UpperCAmelCase      :     List[Any]        =   inspect.signature(model.forward						)
                # signature.parameters is an OrderedDict => so arg_names order is deterministic
                _UpperCAmelCase      :     Union[str, Any]        =   [*signature.parameters.keys()]

                _UpperCAmelCase      :     Tuple        =   ["pixel_values"]
                self.assertListEqual(arg_names[:1]					,     A						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     str        =   self.model_tester.prepare_config_and_inputs_for_common()
           _UpperCAmelCase      :     Optional[Any]        =   True

           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     Any        =   True
                _UpperCAmelCase      :     Tuple        =   False
                _UpperCAmelCase      :     Dict        =   True
                _UpperCAmelCase      :     Any        =   model_class(A						)
                model.to(A						)
                model.eval()
                with torch.no_grad():
                     _UpperCAmelCase      :     List[Any]        =   model(**self._prepare_for_class(A					,     A						)						)
                _UpperCAmelCase      :     Any        =   outputs.attentions
                _UpperCAmelCase      :     Optional[int]        =   len(self.model_tester.depths						)
                self.assertEqual(len(A						)					,     A						)

                # check that output_attentions also work using config
                del inputs_dict["output_attentions"]
                _UpperCAmelCase      :     List[Any]        =   True
                _UpperCAmelCase      :     Optional[int]        =   config.window_size**2
                _UpperCAmelCase      :     int        =   model_class(A						)
                model.to(A						)
                model.eval()
                with torch.no_grad():
                     _UpperCAmelCase      :     int        =   model(**self._prepare_for_class(A					,     A						)						)
                _UpperCAmelCase      :     Dict        =   outputs.attentions
                self.assertEqual(len(A						)					,     A						)

                self.assertListEqual(
                    list(attentions[0].shape[-3:]						)					,     [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,     )
                _UpperCAmelCase      :     Dict        =   len(A						)

                # Check attention is always last and order is fine
                _UpperCAmelCase      :     Dict        =   True
                _UpperCAmelCase      :     int        =   True
                _UpperCAmelCase      :     Tuple        =   model_class(A						)
                model.to(A						)
                model.eval()
                with torch.no_grad():
                     _UpperCAmelCase      :     Optional[int]        =   model(**self._prepare_for_class(A					,     A						)						)

                if hasattr(self.model_tester					,     "num_hidden_states_types"						):
                     _UpperCAmelCase      :     Union[str, Any]        =   self.model_tester.num_hidden_states_types
                else:
                     # also another +1 for reshaped_hidden_states
                     _UpperCAmelCase      :     int        =   2
                self.assertEqual(out_len + added_hidden_states					,     len(A						)						)

                _UpperCAmelCase      :     Optional[Any]        =   outputs.attentions

                self.assertEqual(len(A						)					,     A						)

                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]						)					,     [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,     )



      def 						snake_case_ (					self					:     str					,     A					:     List[str]					,     A					:     Tuple					,     A					:     int					,     A					:     int						):
           _UpperCAmelCase      :     Any        =   model_class(A						)
           model.to(A						)
           model.eval()

           with torch.no_grad():
                _UpperCAmelCase      :     Any        =   model(**self._prepare_for_class(A					,     A						)						)

           _UpperCAmelCase      :     Dict        =   outputs.hidden_states

           _UpperCAmelCase      :     Any        =   getattr(
               self.model_tester					,     "expected_num_hidden_layers"					,     len(self.model_tester.depths						) + 1						)
           self.assertEqual(len(A						)					,     A						)

           # Swinv2 has a different seq_length
           _UpperCAmelCase      :     Dict        =   (
               config.patch_size
               if isinstance(config.patch_size					,     collections.abc.Iterable						)
               else (config.patch_size, config.patch_size)
           )

           _UpperCAmelCase      :     Dict        =   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

           self.assertListEqual(
               list(hidden_states[0].shape[-2:]						)					,     [num_patches, self.model_tester.embed_dim]					,     )

           _UpperCAmelCase      :     Optional[int]        =   outputs.reshaped_hidden_states
           self.assertEqual(len(A						)					,     A						)

           _UpperCAmelCase    ,	_UpperCAmelCase    ,	_UpperCAmelCase    ,	_UpperCAmelCase      :     int        =   reshaped_hidden_states[0].shape
           _UpperCAmelCase      :     List[str]        =   (
               reshaped_hidden_states[0].view(A					,     A					,     height * width						).permute(0					,     2					,     1						)
           )
           self.assertListEqual(
               list(reshaped_hidden_states.shape[-2:]						)					,     [num_patches, self.model_tester.embed_dim]					,     )



      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     int        =   self.model_tester.prepare_config_and_inputs_for_common()

           _UpperCAmelCase      :     Dict        =   (
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size					,     collections.abc.Iterable						)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )

           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     int        =   True
                self.check_hidden_states_output(A					,     A					,     A					,     A						)

                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                _UpperCAmelCase      :     Optional[Any]        =   True

                self.check_hidden_states_output(A					,     A					,     A					,     A						)



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     str        =   self.model_tester.prepare_config_and_inputs_for_common()
           _UpperCAmelCase      :     int        =   3

           _UpperCAmelCase      :     List[Any]        =   (
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size					,     collections.abc.Iterable						)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )
           _UpperCAmelCase      :     int        =   (
               config.patch_size
               if isinstance(config.patch_size					,     collections.abc.Iterable						)
               else (config.patch_size, config.patch_size)
           )

           _UpperCAmelCase      :     int        =   image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
           _UpperCAmelCase      :     Union[str, Any]        =   image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     List[str]        =   True
                self.check_hidden_states_output(A					,     A					,     A					,     (padded_height, padded_width)						)

                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                _UpperCAmelCase      :     List[Any]        =   True
                self.check_hidden_states_output(A					,     A					,     A					,     (padded_height, padded_width)						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     Dict        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_image_modeling(*A						)



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     int        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_image_classification(*A						)



      @slow
      def 						snake_case_ (					self					:     Any						):
           for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _UpperCAmelCase      :     int        =   SwinvaModel.from_pretrained(A						)
                self.assertIsNotNone(A						)



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     Tuple        =   self.model_tester.prepare_config_and_inputs_for_common()

           _UpperCAmelCase      :     str        =   _config_zero_init(A						)
           for model_class in self.all_model_classes:
                _UpperCAmelCase      :     Union[str, Any]        =   model_class(config=A						)
                for name, param in model.named_parameters():
                     if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
                          self.assertIn(
                              ((param.data.mean() * 1e9).round() / 1e9).item()					,     [0.0, 1.0]					,     msg=f'Parameter {name} of model {model_class} seems not properly initialized'					,     )







@require_vision
@require_torch
class      UpperCAmelCase_  (    unittest.TestCase						):



      @cached_property
      def 						snake_case_ (					self					:     str						):
           return (
               AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256"						)
               if is_vision_available()
               else None
           )



      @slow
      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     Any        =   SwinvaForImageClassification.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256"						).to(
               A						)
           _UpperCAmelCase      :     Optional[int]        =   self.default_image_processor

           _UpperCAmelCase      :     Tuple        =   Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"						)
           _UpperCAmelCase      :     Union[str, Any]        =   image_processor(images=A					,     return_tensors="pt"						).to(A						)

           # forward pass
           with torch.no_grad():
                _UpperCAmelCase      :     List[Any]        =   model(**A						)

           # verify the logits
           _UpperCAmelCase      :     int        =   torch.Size((1, 1_0_0_0)						)
           self.assertEqual(outputs.logits.shape					,     A						)
           _UpperCAmelCase      :     List[Any]        =   torch.tensor([-0.3_947, -0.4_306, 0.0_026]						).to(A						)
           self.assertTrue(torch.allclose(outputs.logits[0, :3]					,     A					,     atol=1e-4						)						)





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCAmelCase    : List[str]							=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase    : int							=							{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCAmelCase    : List[Any]							=							{
    "google/electra-small-generator": 5_12,
    "google/electra-base-generator": 5_12,
    "google/electra-large-generator": 5_12,
    "google/electra-small-discriminator": 5_12,
    "google/electra-base-discriminator": 5_12,
    "google/electra-large-discriminator": 5_12,
}

_lowerCAmelCase    : Optional[Any]							=							{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}






class      UpperCAmelCase_  (    _UpperCamelCase						):
      __SCREAMING_SNAKE_CASE						:       Optional[Any]								=			VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE						:       Any								=			PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_INIT_CONFIGURATION
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE						:       Tuple								=			ElectraTokenizer



      def __init__(					self					:     Dict					,     A					:     Dict=None					,     A					:     Optional[int]=None					,     A					:     Dict=True					,     A					:     Optional[Any]="[UNK]"					,     A					:     Any="[SEP]"					,     A					:     str="[PAD]"					,     A					:     Tuple="[CLS]"					,     A					:     Optional[Any]="[MASK]"					,     A					:     Any=True					,     A					:     Tuple=None					,     **A					:     Any					,     ):
           super().__init__(
               A					,     tokenizer_file=A					,     do_lower_case=A					,     unk_token=A					,     sep_token=A					,     pad_token=A					,     cls_token=A					,     mask_token=A					,     tokenize_chinese_chars=A					,     strip_accents=A					,     **A					,     )

           _UpperCAmelCase      :     List[str]        =   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
           if (
               normalizer_state.get("lowercase"					,     A						) != do_lower_case
               or normalizer_state.get("strip_accents"					,     A						) != strip_accents
               or normalizer_state.get("handle_chinese_chars"					,     A						) != tokenize_chinese_chars
           ):
                _UpperCAmelCase      :     Union[str, Any]        =   getattr(A					,     normalizer_state.pop("type"						)						)
                _UpperCAmelCase      :     Dict        =   do_lower_case
                _UpperCAmelCase      :     Optional[int]        =   strip_accents
                _UpperCAmelCase      :     Any        =   tokenize_chinese_chars
                _UpperCAmelCase      :     Optional[Any]        =   normalizer_class(**A						)

           _UpperCAmelCase      :     int        =   do_lower_case



      def 						snake_case_ (					self					:     Tuple					,     A					:     str					,     A					:     int=None						):
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

           return output



      def 						snake_case_ (					self					:     Any					,     A					:     List[int]					,     A					:     Optional[List[int]] = None						):
           _UpperCAmelCase      :     Any        =   [self.sep_token_id]
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id]
           if token_ids_a is None:
                return len(cls + token_ids_a + sep						) * [0]
           return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



      def 						snake_case_ (					self					:     Any					,     A					:     str					,     A					:     Optional[str] = None						):
           _UpperCAmelCase      :     List[Any]        =   self._tokenizer.model.save(A					,     name=A						)
           return tuple(A						)
