
from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


lowerCamelCase_  :	List[Any] 				=					logging.get_logger(__name__)


# General docstring
lowerCamelCase_  :	Optional[int] 				=					'MobileNetV1Config'

# Base docstring
lowerCamelCase_  :	Optional[Any] 				=					'google/mobilenet_v1_1.0_224'
lowerCamelCase_  :	Any 				=					[1, 10_24, 7, 7]

# Image classification docstring
lowerCamelCase_  :	Dict 				=					'google/mobilenet_v1_1.0_224'
lowerCamelCase_  :	List[str] 				=					'tabby, tabby cat'


lowerCamelCase_  :	str 				=					[
    'google/mobilenet_v1_1.0_224',
    'google/mobilenet_v1_0.75_192',
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]







def     A__   (  lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase=None  )					->		Optional[int]:
    UpperCamelCase_:							List[str]		=  {}

    if isinstance(lowerCAmelCase__       ,      lowerCAmelCase__  ):
        UpperCamelCase_:							Optional[Any]		=  model.mobilenet_va
    else:
        UpperCamelCase_:							Optional[int]		=  model

    UpperCamelCase_:							Tuple		=  """MobilenetV1/Conv2d_0/"""
    UpperCamelCase_:							Optional[Any]		=  backbone.conv_stem.convolution.weight
    UpperCamelCase_:							Dict		=  backbone.conv_stem.normalization.bias
    UpperCamelCase_:							int		=  backbone.conv_stem.normalization.weight
    UpperCamelCase_:							Tuple		=  backbone.conv_stem.normalization.running_mean
    UpperCamelCase_:							Any		=  backbone.conv_stem.normalization.running_var

    for i in range(13  ):
        UpperCamelCase_:							Any		=  i + 1
        UpperCamelCase_:							Union[str, Any]		=  i * 2

        UpperCamelCase_:							str		=  backbone.layer[pt_index]
        UpperCamelCase_:							List[Any]		=  F'''MobilenetV1/Conv2d_{tf_index}_depthwise/'''
        UpperCamelCase_:							int		=  pointer.convolution.weight
        UpperCamelCase_:							Tuple		=  pointer.normalization.bias
        UpperCamelCase_:							Dict		=  pointer.normalization.weight
        UpperCamelCase_:							Optional[Any]		=  pointer.normalization.running_mean
        UpperCamelCase_:							Dict		=  pointer.normalization.running_var

        UpperCamelCase_:							List[Any]		=  backbone.layer[pt_index + 1]
        UpperCamelCase_:							int		=  F'''MobilenetV1/Conv2d_{tf_index}_pointwise/'''
        UpperCamelCase_:							Any		=  pointer.convolution.weight
        UpperCamelCase_:							str		=  pointer.normalization.bias
        UpperCamelCase_:							Any		=  pointer.normalization.weight
        UpperCamelCase_:							List[str]		=  pointer.normalization.running_mean
        UpperCamelCase_:							int		=  pointer.normalization.running_var

    if isinstance(lowerCAmelCase__       ,      lowerCAmelCase__  ):
        UpperCamelCase_:							Optional[int]		=  """MobilenetV1/Logits/Conv2d_1c_1x1/"""
        UpperCamelCase_:							str		=  model.classifier.weight
        UpperCamelCase_:							str		=  model.classifier.bias

    return tf_to_pt_map







def     A__   (  lowerCamelCase       ,      lowerCamelCase       ,      lowerCamelCase  )					->		Union[str, Any]:


    try:
        import numpy as np
        import tensorflow as tf
    except ImportError:
        logger.error(
            """Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see """
            """https://www.tensorflow.org/install/ for installation instructions."""  )
        raise

    # Load weights from TF model
    UpperCamelCase_:							List[str]		=  tf.train.list_variables(lowerCAmelCase__  )
    UpperCamelCase_:							Union[str, Any]		=  {}
    for name, shape in init_vars:
        logger.info(F'''Loading TF weight {name} with shape {shape}'''  )
        UpperCamelCase_:							Dict		=  tf.train.load_variable(lowerCAmelCase__       ,      lowerCAmelCase__  )
        UpperCamelCase_:							Dict		=  array

    # Build TF to PyTorch weights loading map
    UpperCamelCase_:							List[str]		=  _build_tf_to_pytorch_map(lowerCAmelCase__       ,      lowerCAmelCase__       ,      lowerCAmelCase__  )

    for name, pointer in tf_to_pt_map.items():
        logger.info(F'''Importing {name}'''  )
        if name not in tf_weights:
            logger.info(F'''{name} not in tf pre-trained weights, skipping'''  )
            continue

        UpperCamelCase_:							Dict		=  tf_weights[name]

        if "depthwise_weights" in name:
            logger.info("""Transposing depthwise"""  )
            UpperCamelCase_:							str		=  np.transpose(lowerCAmelCase__       ,      (2, 3, 0, 1)  )
        elif "weights" in name:
            logger.info("""Transposing"""  )
            if len(pointer.shape  ) == 2:  # copying into linear layer
                UpperCamelCase_:							Tuple		=  array.squeeze().transpose()
            else:
                UpperCamelCase_:							Union[str, Any]		=  np.transpose(lowerCAmelCase__       ,      (3, 2, 0, 1)  )

        if pointer.shape != array.shape:
            raise ValueError(F'''Pointer shape {pointer.shape} and array shape {array.shape} mismatched'''  )

        logger.info(F'''Initialize PyTorch weight {name} {array.shape}'''  )
        UpperCamelCase_:							Tuple		=  torch.from_numpy(lowerCAmelCase__  )

        tf_weights.pop(lowerCAmelCase__       ,      lowerCAmelCase__  )
        tf_weights.pop(name + """/RMSProp"""       ,      lowerCAmelCase__  )
        tf_weights.pop(name + """/RMSProp_1"""       ,      lowerCAmelCase__  )
        tf_weights.pop(name + """/ExponentialMovingAverage"""       ,      lowerCAmelCase__  )

    logger.info(F'''Weights not copied to PyTorch model: {", ".join(tf_weights.keys()  )}'''  )
    return model







def     A__   (  lowerCamelCase       ,      lowerCamelCase  )					->		List[Any]:
    UpperCamelCase_:							Tuple		=  features.shape[-2:]
    UpperCamelCase_:							str		=  conv_layer.stride
    UpperCamelCase_:							str		=  conv_layer.kernel_size

    if in_height % stride_height == 0:
        UpperCamelCase_:							List[Any]		=  max(kernel_height - stride_height       ,      0  )
    else:
        UpperCamelCase_:							List[str]		=  max(kernel_height - (in_height % stride_height)       ,      0  )

    if in_width % stride_width == 0:
        UpperCamelCase_:							List[str]		=  max(kernel_width - stride_width       ,      0  )
    else:
        UpperCamelCase_:							Any		=  max(kernel_width - (in_width % stride_width)       ,      0  )

    UpperCamelCase_:							Optional[Any]		=  pad_along_width // 2
    UpperCamelCase_:							List[str]		=  pad_along_width - pad_left
    UpperCamelCase_:							Optional[Any]		=  pad_along_height // 2
    UpperCamelCase_:							Dict		=  pad_along_height - pad_top

    UpperCamelCase_:							Tuple		=  (pad_left, pad_right, pad_top, pad_bottom)
    return nn.functional.pad(lowerCAmelCase__       ,      lowerCAmelCase__       ,      """constant"""       ,      0.0  )

class     _UpperCamelCase			(   nn.Module					):






   '''simple docstring'''

   def __init__(    self	:    Union[str, Any]  ,		snake_case_	:    List[Any]  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Dict  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Union[str, Any] = 1  ,		snake_case_	:    Optional[int] = 1  ,		snake_case_	:    int = False  ,		snake_case_	:    List[str] = True  ,		snake_case_	:    Union[str, Any] = True  ,		):
       super().__init__()
       UpperCamelCase_:							List[Any]		=  config

       if in_channels % groups != 0:
           raise ValueError(f'''Input channels ({in_channels}) are not divisible by {groups} groups.''' )
       if out_channels % groups != 0:
           raise ValueError(f'''Output channels ({out_channels}) are not divisible by {groups} groups.''' )

       UpperCamelCase_:							Optional[int]		=  0 if config.tf_padding else int((kernel_size - 1) / 2 )

       UpperCamelCase_:							Any		=  nn.Convad(
           in_channels=SCREAMING_SNAKE_CASE_  ,		out_channels=SCREAMING_SNAKE_CASE_  ,		kernel_size=SCREAMING_SNAKE_CASE_  ,		stride=SCREAMING_SNAKE_CASE_  ,		padding=SCREAMING_SNAKE_CASE_  ,		groups=SCREAMING_SNAKE_CASE_  ,		bias=SCREAMING_SNAKE_CASE_  ,		padding_mode="""zeros"""  ,		)

       if use_normalization:
           UpperCamelCase_:							List[Any]		=  nn.BatchNormad(
               num_features=SCREAMING_SNAKE_CASE_  ,		eps=config.layer_norm_eps  ,		momentum=0.9997  ,		affine=SCREAMING_SNAKE_CASE_  ,		track_running_stats=SCREAMING_SNAKE_CASE_  ,		)
       else:
           UpperCamelCase_:							Optional[Any]		=  None

       if use_activation:
           if isinstance(SCREAMING_SNAKE_CASE_  ,		SCREAMING_SNAKE_CASE_ ):
               UpperCamelCase_:							List[Any]		=  ACTaFN[use_activation]
           elif isinstance(config.hidden_act  ,		SCREAMING_SNAKE_CASE_ ):
               UpperCamelCase_:							Optional[Any]		=  ACTaFN[config.hidden_act]
           else:
               UpperCamelCase_:							Union[str, Any]		=  config.hidden_act
       else:
           UpperCamelCase_:							Optional[Any]		=  None




   def  lowerCAmelCase__		(    self	:    int  ,		snake_case_	:    List[Any] ):
       if self.config.tf_padding:
           UpperCamelCase_:							int		=  apply_tf_padding(SCREAMING_SNAKE_CASE_  ,		self.convolution )
       UpperCamelCase_:							Dict		=  self.convolution(SCREAMING_SNAKE_CASE_ )
       if self.normalization is not None:
           UpperCamelCase_:							Tuple		=  self.normalization(SCREAMING_SNAKE_CASE_ )
       if self.activation is not None:
           UpperCamelCase_:							Union[str, Any]		=  self.activation(SCREAMING_SNAKE_CASE_ )
       return features




class     _UpperCamelCase			(   _lowerCAmelCase					):






   '''simple docstring'''

   __UpperCamelCase :  str    =	MobileNetVaConfig
   __UpperCamelCase :  Dict    =	load_tf_weights_in_mobilenet_va
   __UpperCamelCase :  Tuple    =	"""mobilenet_v1"""
   __UpperCamelCase :  List[str]    =	"""pixel_values"""
   __UpperCamelCase :  Dict    =	False

   def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		snake_case_	:    List[Any] ):
       if isinstance(SCREAMING_SNAKE_CASE_  ,		(nn.Linear, nn.Convad) ):
           module.weight.data.normal_(mean=0.0  ,		std=self.config.initializer_range )
           if module.bias is not None:
               module.bias.data.zero_()
       elif isinstance(SCREAMING_SNAKE_CASE_  ,		nn.BatchNormad ):
           module.bias.data.zero_()
           module.weight.data.fill_(1.0 )




lowerCamelCase_  :	Union[str, Any] 				=					r'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

lowerCamelCase_  :	int 				=					r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`MobileNetV1ImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'

@add_start_docstrings(
    """The bare MobileNetV1 model outputting raw hidden-states without any specific head on top."""       ,    _lowerCAmelCase       ,    )
class     _UpperCamelCase			(   _lowerCAmelCase					):






   '''simple docstring'''

   def __init__(    self	:    Dict  ,		snake_case_	:    Dict  ,		snake_case_	:    Optional[Any] = True ):
       super().__init__(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase_:							Any		=  config

       UpperCamelCase_:							Optional[int]		=  32
       UpperCamelCase_:							Union[str, Any]		=  max(int(depth * config.depth_multiplier )  ,		config.min_depth )

       UpperCamelCase_:							Dict		=  MobileNetVaConvLayer(
           SCREAMING_SNAKE_CASE_  ,		in_channels=config.num_channels  ,		out_channels=SCREAMING_SNAKE_CASE_  ,		kernel_size=3  ,		stride=2  ,		)

       UpperCamelCase_:							int		=  [1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

       UpperCamelCase_:							Dict		=  nn.ModuleList()
       for i in range(13 ):
           UpperCamelCase_:							Any		=  out_channels

           if strides[i] == 2 or i == 0:
               depth *= 2
               UpperCamelCase_:							List[Any]		=  max(int(depth * config.depth_multiplier )  ,		config.min_depth )

           self.layer.append(
               MobileNetVaConvLayer(
                   SCREAMING_SNAKE_CASE_  ,		in_channels=SCREAMING_SNAKE_CASE_  ,		out_channels=SCREAMING_SNAKE_CASE_  ,		kernel_size=3  ,		stride=strides[i]  ,		groups=SCREAMING_SNAKE_CASE_  ,		) )

           self.layer.append(
               MobileNetVaConvLayer(
                   SCREAMING_SNAKE_CASE_  ,		in_channels=SCREAMING_SNAKE_CASE_  ,		out_channels=SCREAMING_SNAKE_CASE_  ,		kernel_size=1  ,		) )

       UpperCamelCase_:							Dict		=  nn.AdaptiveAvgPoolad((1, 1) ) if add_pooling_layer else None

       # Initialize weights and apply final processing
       self.post_init()

   def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    Dict ):
       raise NotImplementedError




   @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_ )
   @add_code_sample_docstrings(
       checkpoint=_CHECKPOINT_FOR_DOC  ,		output_type=SCREAMING_SNAKE_CASE_  ,		config_class=_CONFIG_FOR_DOC  ,		modality="""vision"""  ,		expected_output=_EXPECTED_OUTPUT_SHAPE  ,		)
   def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    int = None  ,		snake_case_	:    str = None  ,		snake_case_	:    List[Any] = None  ,		):
       UpperCamelCase_:							Union[str, Any]		=  (
           output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
       )
       UpperCamelCase_:							str		=  return_dict if return_dict is not None else self.config.use_return_dict

       if pixel_values is None:
           raise ValueError("""You have to specify pixel_values""" )

       UpperCamelCase_:							int		=  self.conv_stem(SCREAMING_SNAKE_CASE_ )

       UpperCamelCase_:							Union[str, Any]		=  () if output_hidden_states else None

       for i, layer_module in enumerate(self.layer ):
           UpperCamelCase_:							Any		=  layer_module(SCREAMING_SNAKE_CASE_ )

           if output_hidden_states:
               UpperCamelCase_:							int		=  all_hidden_states + (hidden_states,)

       UpperCamelCase_:							Optional[int]		=  hidden_states

       if self.pooler is not None:
           UpperCamelCase_:							Any		=  torch.flatten(self.pooler(SCREAMING_SNAKE_CASE_ )  ,		start_dim=1 )
       else:
           UpperCamelCase_:							Tuple		=  None

       if not return_dict:
           return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None )

       return BaseModelOutputWithPoolingAndNoAttention(
           last_hidden_state=SCREAMING_SNAKE_CASE_  ,		pooler_output=SCREAMING_SNAKE_CASE_  ,		hidden_states=SCREAMING_SNAKE_CASE_  ,		)




@add_start_docstrings(
    """
    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    """       ,    _lowerCAmelCase       ,    )
class     _UpperCamelCase			(   _lowerCAmelCase					):






   '''simple docstring'''

   def __init__(    self	:    List[str]  ,		snake_case_	:    Any ):
       super().__init__(SCREAMING_SNAKE_CASE_ )

       UpperCamelCase_:							Union[str, Any]		=  config.num_labels
       UpperCamelCase_:							Any		=  MobileNetVaModel(SCREAMING_SNAKE_CASE_ )

       UpperCamelCase_:							Optional[Any]		=  self.mobilenet_va.layer[-1].convolution.out_channels

       # Classifier head
       UpperCamelCase_:							str		=  nn.Dropout(config.classifier_dropout_prob  ,		inplace=SCREAMING_SNAKE_CASE_ )
       UpperCamelCase_:							int		=  nn.Linear(SCREAMING_SNAKE_CASE_  ,		config.num_labels ) if config.num_labels > 0 else nn.Identity()

       # Initialize weights and apply final processing
       self.post_init()




   @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_ )
   @add_code_sample_docstrings(
       checkpoint=_IMAGE_CLASS_CHECKPOINT  ,		output_type=SCREAMING_SNAKE_CASE_  ,		config_class=_CONFIG_FOR_DOC  ,		expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT  ,		)
   def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    Tuple = None  ,		snake_case_	:    Union[str, Any] = None  ,		snake_case_	:    Optional[int] = None  ,		snake_case_	:    int = None  ,		):
       UpperCamelCase_:							Tuple		=  return_dict if return_dict is not None else self.config.use_return_dict

       UpperCamelCase_:							Any		=  self.mobilenet_va(SCREAMING_SNAKE_CASE_  ,		output_hidden_states=SCREAMING_SNAKE_CASE_  ,		return_dict=SCREAMING_SNAKE_CASE_ )

       UpperCamelCase_:							Tuple		=  outputs.pooler_output if return_dict else outputs[1]

       UpperCamelCase_:							Dict		=  self.classifier(self.dropout(SCREAMING_SNAKE_CASE_ ) )

       UpperCamelCase_:							List[str]		=  None
       if labels is not None:
           if self.config.problem_type is None:
               if self.num_labels == 1:
                   UpperCamelCase_:							str		=  """regression"""
               elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                   UpperCamelCase_:							Optional[int]		=  """single_label_classification"""
               else:
                   UpperCamelCase_:							Any		=  """multi_label_classification"""

           if self.config.problem_type == "regression":
               UpperCamelCase_:							Optional[Any]		=  MSELoss()
               if self.num_labels == 1:
                   UpperCamelCase_:							List[str]		=  loss_fct(logits.squeeze()  ,		labels.squeeze() )
               else:
                   UpperCamelCase_:							int		=  loss_fct(SCREAMING_SNAKE_CASE_  ,		SCREAMING_SNAKE_CASE_ )
           elif self.config.problem_type == "single_label_classification":
               UpperCamelCase_:							List[str]		=  CrossEntropyLoss()
               UpperCamelCase_:							Optional[int]		=  loss_fct(logits.view(-1  ,		self.num_labels )  ,		labels.view(-1 ) )
           elif self.config.problem_type == "multi_label_classification":
               UpperCamelCase_:							List[str]		=  BCEWithLogitsLoss()
               UpperCamelCase_:							Tuple		=  loss_fct(SCREAMING_SNAKE_CASE_  ,		SCREAMING_SNAKE_CASE_ )

       if not return_dict:
           UpperCamelCase_:							int		=  (logits,) + outputs[2:]
           return ((loss,) + output) if loss is not None else output

       return ImageClassifierOutputWithNoAttention(
           loss=SCREAMING_SNAKE_CASE_  ,		logits=SCREAMING_SNAKE_CASE_  ,		hidden_states=outputs.hidden_states  ,		)








import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


lowerCamelCase_  :	Any 				=					logging.get_logger(__name__)


lowerCamelCase_  :	Optional[int] 				=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
lowerCamelCase_  :	str 				=					{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase_  :	Union[str, Any] 				=					{
    """RUCAIBox/mvp""": 10_24,
}

class     _UpperCamelCase			(   _A					):






			'''simple docstring'''

			__UpperCamelCase :  List[Any]    =	VOCAB_FILES_NAMES
			__UpperCamelCase :  int    =	PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase :  Any    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase :  Any    =	["""input_ids""", """attention_mask"""]
			__UpperCamelCase :  int    =	MvpTokenizer

			def __init__(    self	:    Union[str, Any]  ,		snake_case_	:    Any=None  ,		snake_case_	:    Any=None  ,		snake_case_	:    Union[str, Any]=None  ,		snake_case_	:    Tuple="replace"  ,		snake_case_	:    Dict="<s>"  ,		snake_case_	:    Dict="</s>"  ,		snake_case_	:    Tuple="</s>"  ,		snake_case_	:    int="<s>"  ,		snake_case_	:    Tuple="<unk>"  ,		snake_case_	:    Optional[int]="<pad>"  ,		snake_case_	:    Any="<mask>"  ,		snake_case_	:    Union[str, Any]=False  ,		snake_case_	:    Optional[int]=True  ,		**snake_case_	:    Union[str, Any]  ,		):
							super().__init__(
							    snake_case_  ,		snake_case_  ,		tokenizer_file=snake_case_  ,		errors=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		sep_token=snake_case_  ,		cls_token=snake_case_  ,		unk_token=snake_case_  ,		pad_token=snake_case_  ,		mask_token=snake_case_  ,		add_prefix_space=snake_case_  ,		trim_offsets=snake_case_  ,		**snake_case_  ,		)

							UpperCamelCase_:							Any		=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
							if pre_tok_state.get("""add_prefix_space"""  ,		snake_case_ ) != add_prefix_space:
											UpperCamelCase_:							str		=  getattr(snake_case_  ,		pre_tok_state.pop("""type""" ) )
											UpperCamelCase_:							int		=  add_prefix_space
											UpperCamelCase_:							int		=  pre_tok_class(**snake_case_ )

							UpperCamelCase_:							int		=  add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							UpperCamelCase_:							Tuple		=  """post_processor"""
							UpperCamelCase_:							Optional[int]		=  getattr(self.backend_tokenizer  ,		snake_case_  ,		snake_case_ )
							if tokenizer_component_instance:
											UpperCamelCase_:							Optional[int]		=  json.loads(tokenizer_component_instance.__getstate__() )

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
															UpperCamelCase_:							Union[str, Any]		=  tuple(state["""sep"""] )
											if "cls" in state:
															UpperCamelCase_:							List[Any]		=  tuple(state["""cls"""] )

											UpperCamelCase_:							List[str]		=  False

											if state.get("""add_prefix_space"""  ,		snake_case_ ) != add_prefix_space:
															UpperCamelCase_:							str		=  add_prefix_space
															UpperCamelCase_:							Union[str, Any]		=  True

											if state.get("""trim_offsets"""  ,		snake_case_ ) != trim_offsets:
															UpperCamelCase_:							Optional[int]		=  trim_offsets
															UpperCamelCase_:							Any		=  True

											if changes_to_apply:
															UpperCamelCase_:							Optional[int]		=  getattr(snake_case_  ,		state.pop("""type""" ) )
															UpperCamelCase_:							Union[str, Any]		=  component_class(**snake_case_ )
															setattr(self.backend_tokenizer  ,		snake_case_  ,		snake_case_ )

			@property
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							if self._mask_token is None:
											if self.verbose:
															logger.error("""Using mask_token, but it is not set yet.""" )
											return None
							return str(self._mask_token )

			@mask_token.setter
			def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		snake_case_	:    Dict ):
							UpperCamelCase_:							List[str]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else value
							UpperCamelCase_:							Optional[int]		=  value

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		*snake_case_	:    Dict  ,		**snake_case_	:    Optional[int] ):
							UpperCamelCase_:							Optional[int]		=  kwargs.get("""is_split_into_words"""  ,		snake_case_ )

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs.""" )

							return super()._batch_encode_plus(*snake_case_  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		*snake_case_	:    Dict  ,		**snake_case_	:    Tuple ):
							UpperCamelCase_:							Any		=  kwargs.get("""is_split_into_words"""  ,		snake_case_ )

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs.""" )

							return super()._encode_plus(*snake_case_  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    str  ,		snake_case_	:    Optional[str] = None ):
							UpperCamelCase_:							Optional[int]		=  self._tokenizer.model.save(snake_case_  ,		name=snake_case_ )
							return tuple(snake_case_ )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    Any=None ):
							UpperCamelCase_:							str		=  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
											return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None ):
							UpperCamelCase_:							Tuple		=  [self.sep_token_id]
							UpperCamelCase_:							List[str]		=  [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






