






'''simple docstring'''

import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase=False   )->			List[Any]:
      try:
            import torch  # noqa: F401
      except ImportError:
            logger.error(
                """Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see"""
                """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
                """ instructions."""   )
            raise

      if not is_sharded:
            UpperCamelCase      = os.path.abspath(__UpperCamelCase   )
            logger.info(F"Loading PyTorch weights from {pt_path}"   )

            UpperCamelCase      = torch.load(__UpperCamelCase		,						map_location="""cpu"""   )
            logger.info(F"PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()   ):,} parameters."   )

            UpperCamelCase      = convert_pytorch_state_dict_to_flax(__UpperCamelCase		,						__UpperCamelCase   )
      else:
            # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
            UpperCamelCase      = convert_pytorch_sharded_state_dict_to_flax(__UpperCamelCase		,						__UpperCamelCase   )
      return flax_state_dict







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase		,						)->			(Tuple[str], np.ndarray):

      def is_key_or_prefix_key_in_dict(__UpperCamelCase   ) -> bool:
            return len(set(__UpperCamelCase   ) & {key, (model_prefix,) + key}   ) > 0

      # layer norm
      UpperCamelCase      = pt_tuple_key[:-1] + ("""scale""",)
      if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            return renamed_pt_tuple_key, pt_tensor

      # batch norm layer mean
      UpperCamelCase      = pt_tuple_key[:-1] + ("""mean""",)
      if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            return renamed_pt_tuple_key, pt_tensor

      # batch norm layer var
      UpperCamelCase      = pt_tuple_key[:-1] + ("""var""",)
      if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            return renamed_pt_tuple_key, pt_tensor

      # embedding
      UpperCamelCase      = pt_tuple_key[:-1] + ("""embedding""",)
      if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            return renamed_pt_tuple_key, pt_tensor

      # conv layer
      UpperCamelCase      = pt_tuple_key[:-1] + ("""kernel""",)
      if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            UpperCamelCase      = pt_tensor.transpose(2		,						3		,						1		,						0   )
            return renamed_pt_tuple_key, pt_tensor

      # linear layer
      UpperCamelCase      = pt_tuple_key[:-1] + ("""kernel""",)
      if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(__UpperCamelCase   ):
            UpperCamelCase      = pt_tensor.T
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm weight
      UpperCamelCase      = pt_tuple_key[:-1] + ("""weight""",)
      if pt_tuple_key[-1] == "gamma":
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm bias
      UpperCamelCase      = pt_tuple_key[:-1] + ("""bias""",)
      if pt_tuple_key[-1] == "beta":
            return renamed_pt_tuple_key, pt_tensor

      # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
      UpperCamelCase      = None
      if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
            UpperCamelCase      = pt_tuple_key[-2] + """_g"""
      elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
            UpperCamelCase      = pt_tuple_key[-2] + """_v"""
      if name is not None:
            UpperCamelCase      = pt_tuple_key[:-3] + (name,)
            return renamed_pt_tuple_key, pt_tensor

      return pt_tuple_key, pt_tensor







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase   )->			int:
      # convert pytorch tensor to numpy
      UpperCamelCase      = {k: v.numpy() for k, v in pt_state_dict.items()}

      UpperCamelCase      = flax_model.base_model_prefix

      # use params dict if the model contains batch norm layers
      if "params" in flax_model.params:
            UpperCamelCase      = flax_model.params["""params"""]
      else:
            UpperCamelCase      = flax_model.params
      UpperCamelCase      = flatten_dict(__UpperCamelCase   )

      # add batch_stats keys,values to dict
      if "batch_stats" in flax_model.params:
            UpperCamelCase      = flatten_dict(flax_model.params["""batch_stats"""]   )
            random_flax_state_dict.update(__UpperCamelCase   )

      UpperCamelCase      = {}

      UpperCamelCase      = (model_prefix not in flax_model_params) and (
          model_prefix in {k.split("""."""   )[0] for k in pt_state_dict.keys()}
      )
      UpperCamelCase      = (model_prefix in flax_model_params) and (
          model_prefix not in {k.split("""."""   )[0] for k in pt_state_dict.keys()}
      )

      # Need to change some parameters name to match Flax names
      for pt_key, pt_tensor in pt_state_dict.items():
            UpperCamelCase      = tuple(pt_key.split("""."""   )   )

            # remove base model prefix if necessary
            UpperCamelCase      = pt_tuple_key[0] == model_prefix
            if load_model_with_head_into_base_model and has_base_model_prefix:
                  UpperCamelCase      = pt_tuple_key[1:]

            # Correctly rename weight parameters
            UpperCamelCase  ,UpperCamelCase      = rename_key_and_reshape_tensor(
                __UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase   )

            # add model prefix if necessary
            UpperCamelCase      = (model_prefix,) + flax_key in random_flax_state_dict
            if load_base_model_into_model_with_head and require_base_model_prefix:
                  UpperCamelCase      = (model_prefix,) + flax_key

            if flax_key in random_flax_state_dict:
                  if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                        raise ValueError(
                            F"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
                            F"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."   )

        # add batch stats if the model contains batchnorm layers
            if "batch_stats" in flax_model.params:
                  if "mean" in flax_key[-1] or "var" in flax_key[-1]:
                        UpperCamelCase      = jnp.asarray(__UpperCamelCase   )
                        continue
                  # remove num_batches_tracked key
                  if "num_batches_tracked" in flax_key[-1]:
                        flax_state_dict.pop(__UpperCamelCase		,						__UpperCamelCase   )
                        continue

                  # also add unexpected weight so that warning is thrown
                  UpperCamelCase      = jnp.asarray(__UpperCamelCase   )

            else:
                  # also add unexpected weight so that warning is thrown
                  UpperCamelCase      = jnp.asarray(__UpperCamelCase   )

      return unflatten_dict(__UpperCamelCase   )







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase   )->			int:
      import torch

      # Load the index
      UpperCamelCase      = {}
      for shard_file in shard_filenames:
            # load using msgpack utils
            UpperCamelCase      = torch.load(__UpperCamelCase   )
            UpperCamelCase      = {k: v.numpy() for k, v in pt_state_dict.items()}

            UpperCamelCase      = flax_model.base_model_prefix

            # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
            if "batch_stats" in flax_model.params:
                  UpperCamelCase      = flax_model.params["""params"""]

                  UpperCamelCase      = flatten_dict(__UpperCamelCase   )
                  random_flax_state_dict.update(flatten_dict(flax_model.params["""batch_stats"""]   )   )
            else:
                  UpperCamelCase      = flax_model.params
                  UpperCamelCase      = flatten_dict(__UpperCamelCase   )

            UpperCamelCase      = (model_prefix not in flax_model_params) and (
                model_prefix in {k.split("""."""   )[0] for k in pt_state_dict.keys()}
            )
            UpperCamelCase      = (model_prefix in flax_model_params) and (
                model_prefix not in {k.split("""."""   )[0] for k in pt_state_dict.keys()}
            )
            # Need to change some parameters name to match Flax names
            for pt_key, pt_tensor in pt_state_dict.items():
                  UpperCamelCase      = tuple(pt_key.split("""."""   )   )

                  # remove base model prefix if necessary
                  UpperCamelCase      = pt_tuple_key[0] == model_prefix
                  if load_model_with_head_into_base_model and has_base_model_prefix:
                        UpperCamelCase      = pt_tuple_key[1:]

                  # Correctly rename weight parameters
                  UpperCamelCase  ,UpperCamelCase      = rename_key_and_reshape_tensor(
                      __UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase		,						__UpperCamelCase   )
                  # add model prefix if necessary
                  UpperCamelCase      = (model_prefix,) + flax_key in random_flax_state_dict
                  if load_base_model_into_model_with_head and require_base_model_prefix:
                        UpperCamelCase      = (model_prefix,) + flax_key

                  if flax_key in random_flax_state_dict:
                        if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                              raise ValueError(
                                  F"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
                                  F"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."   )

            # add batch stats if the model contains batchnorm layers
                  if "batch_stats" in flax_model.params:
                        if "mean" in flax_key[-1]:
                              UpperCamelCase      = jnp.asarray(__UpperCamelCase   )
                              continue
                        if "var" in flax_key[-1]:
                              UpperCamelCase      = jnp.asarray(__UpperCamelCase   )
                              continue
                        # remove num_batches_tracked key
                        if "num_batches_tracked" in flax_key[-1]:
                              flax_state_dict.pop(__UpperCamelCase		,						__UpperCamelCase   )
                              continue

                        # also add unexpected weight so that warning is thrown
                        UpperCamelCase      = jnp.asarray(__UpperCamelCase   )

                  else:
                        # also add unexpected weight so that warning is thrown
                        UpperCamelCase      = jnp.asarray(__UpperCamelCase   )
      return unflatten_dict(__UpperCamelCase   )







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase   )->			Tuple:
      UpperCamelCase      = os.path.abspath(__UpperCamelCase   )
      logger.info(F"Loading Flax weights from {flax_checkpoint_path}"   )

      # import correct flax class
      UpperCamelCase      = getattr(__UpperCamelCase		,						"""Flax""" + model.__class__.__name__   )

      # load flax weight dict
      with open(__UpperCamelCase		,						"""rb"""   ) as state_f:
            try:
                  UpperCamelCase      = from_bytes(__UpperCamelCase		,						state_f.read()   )
            except UnpicklingError:
                  raise EnvironmentError(F"Unable to convert {flax_checkpoint_path} to Flax deserializable object. "   )

      return load_flax_weights_in_pytorch_model(__UpperCamelCase		,						__UpperCamelCase   )







def    lowercase__			(				__UpperCamelCase		,						__UpperCamelCase   )->			Any:

      try:
            import torch  # noqa: F401
      except ImportError:
            logger.error(
                """Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see"""
                """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
                """ instructions."""   )
            raise

      # check if we have bf16 weights
      UpperCamelCase      = flatten_dict(jax.tree_util.tree_map(lambda __UpperCamelCase   : x.dtype == jnp.bfloataa		,						__UpperCamelCase   )   ).values()
      if any(__UpperCamelCase   ):
            # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
            # and bf16 is not fully supported in PT yet.
            logger.warning(
                """Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` """
                """before loading those in PyTorch model."""   )
            UpperCamelCase      = jax.tree_util.tree_map(
                lambda __UpperCamelCase   : params.astype(np.floataa   ) if params.dtype == jnp.bfloataa else params		,						__UpperCamelCase   )

      UpperCamelCase      = flatten_dict(__UpperCamelCase   )
      UpperCamelCase      = pt_model.state_dict()

      UpperCamelCase      = (pt_model.base_model_prefix in flax_state) and (
          pt_model.base_model_prefix not in {k.split("""."""   )[0] for k in pt_model_dict.keys()}
      )
      UpperCamelCase      = (pt_model.base_model_prefix not in flax_state) and (
          pt_model.base_model_prefix in {k.split("""."""   )[0] for k in pt_model_dict.keys()}
      )

      # keep track of unexpected & missing keys
      UpperCamelCase      = []
      UpperCamelCase      = set(pt_model_dict.keys()   )

      for flax_key_tuple, flax_tensor in flax_state_dict.items():
            UpperCamelCase      = flax_key_tuple[0] == pt_model.base_model_prefix
            UpperCamelCase      = """.""".join((pt_model.base_model_prefix,) + flax_key_tuple   ) in pt_model_dict

            # adapt flax_key to prepare for loading from/to base model only
            if load_model_with_head_into_base_model and has_base_model_prefix:
                  UpperCamelCase      = flax_key_tuple[1:]
            elif load_base_model_into_model_with_head and require_base_model_prefix:
                  UpperCamelCase      = (pt_model.base_model_prefix,) + flax_key_tuple

            # rename flax weights to PyTorch format
            if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(__UpperCamelCase   ) not in pt_model_dict:
                  # conv layer
                  UpperCamelCase      = flax_key_tuple[:-1] + ("""weight""",)
                  UpperCamelCase      = jnp.transpose(__UpperCamelCase		,						(3, 2, 0, 1)   )
            elif flax_key_tuple[-1] == "kernel" and ".".join(__UpperCamelCase   ) not in pt_model_dict:
                  # linear layer
                  UpperCamelCase      = flax_key_tuple[:-1] + ("""weight""",)
                  UpperCamelCase      = flax_tensor.T
            elif flax_key_tuple[-1] in ["scale", "embedding"]:
                  UpperCamelCase      = flax_key_tuple[:-1] + ("""weight""",)

            # adding batch stats from flax batch norm to pt
            elif "mean" in flax_key_tuple[-1]:
                  UpperCamelCase      = flax_key_tuple[:-1] + ("""running_mean""",)
            elif "var" in flax_key_tuple[-1]:
                  UpperCamelCase      = flax_key_tuple[:-1] + ("""running_var""",)

            if "batch_stats" in flax_state:
                  UpperCamelCase      = """.""".join(flax_key_tuple[1:]   )  # Remove the params/batch_stats header
            else:
                  UpperCamelCase      = """.""".join(__UpperCamelCase   )

            # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
            UpperCamelCase      = {}
            # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
            for key in pt_model_dict:
                  UpperCamelCase      = key.split("""."""   )
                  UpperCamelCase      = None
                  if key_components[-3::2] == ["parametrizations", "original0"]:
                        UpperCamelCase      = key_components[-2] + """_g"""
                  elif key_components[-3::2] == ["parametrizations", "original1"]:
                        UpperCamelCase      = key_components[-2] + """_v"""
                  if name is not None:
                        UpperCamelCase      = key_components[:-3] + [name]
                        UpperCamelCase      = """.""".join(__UpperCamelCase   )
                        UpperCamelCase      = key

            if flax_key in special_pt_names:
                  UpperCamelCase      = special_pt_names[flax_key]

            if flax_key in pt_model_dict:
                  if flax_tensor.shape != pt_model_dict[flax_key].shape:
                        raise ValueError(
                            F"Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected "
                            F"to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."   )
                  else:
                        # add weight to pytorch dict
                        UpperCamelCase      = np.asarray(__UpperCamelCase   ) if not isinstance(__UpperCamelCase		,						np.ndarray   ) else flax_tensor
                        UpperCamelCase      = torch.from_numpy(__UpperCamelCase   )
                        # remove from missing keys
                        missing_keys.remove(__UpperCamelCase   )
            else:
                  # weight is not expected by PyTorch model
                  unexpected_keys.append(__UpperCamelCase   )

      pt_model.load_state_dict(__UpperCamelCase   )

      # re-transform missing_keys to list
      UpperCamelCase      = list(__UpperCamelCase   )

      if len(__UpperCamelCase   ) > 0:
            logger.warning(
                """Some weights of the Flax model were not used when initializing the PyTorch model"""
                F" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"
                F" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"
                """ (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"""
                F" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"
                """ to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"""
                """ FlaxBertForSequenceClassification model)."""   )
      else:
            logger.warning(F"All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n"   )
      if len(__UpperCamelCase   ) > 0:
            logger.warning(
                F"Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"
                F" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"
                """ use it for predictions and inference."""   )
      else:
            logger.warning(
                F"All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"
                """If your task is similar to the task the model of the checkpoint was trained on, """
                F"you can already use {pt_model.__class__.__name__} for predictions without further training."   )

      return pt_model








'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'spm_char.model'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

SCREAMING_SNAKE_CASE__     =       {
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}






class     a_						(      lowerCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				None:



        """simple docstring"""






        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=_SCREAMING_SNAKE_CASE      ,		eos_token=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**_SCREAMING_SNAKE_CASE      ,		)

        UpperCamelCase      = vocab_file

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.get_piece_size()

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def __getstate__(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        return self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,		out_type=_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.piece_to_id(_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE				)
        return token

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				) + token
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE				)
        out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				)
        return out_string.strip()

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None				)  ->				List[int]:



        """simple docstring"""






        if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_a + token_ids_a + [self.eos_token_id]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_SCREAMING_SNAKE_CASE      ,		token_ids_a=_SCREAMING_SNAKE_CASE      ,		already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = [1]
        if token_ids_a is None:
              return ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones
        return ([0] * len(_SCREAMING_SNAKE_CASE				)) + ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            _SCREAMING_SNAKE_CASE      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(_SCREAMING_SNAKE_CASE				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		_SCREAMING_SNAKE_CASE				)
        elif not os.path.isfile(self.vocab_file				):
              with open(_SCREAMING_SNAKE_CASE      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE				)

        return (out_vocab_file,)
