


"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				"""▁"""

__lowercase    =				{"""vocab_file""": """sentencepiece.bpe.model"""}

__lowercase    =				{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

__lowercase    =				{
    """facebook/mbart-large-50-one-to-many-mmt""": 1024,
}

# fmt: off
__lowercase    =				["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							List[str]     =	VOCAB_FILES_NAMES
    UpperCAmelCase	:							Optional[Any]     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase	:							List[str]     =	PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase	:							Any     =	["""input_ids""", """attention_mask"""]

    UpperCAmelCase	:							List[Any]     =	[]
    UpperCAmelCase	:							Union[str, Any]     =	[]






    def __init__( self    :   int    ,			__UpperCAmelCase    :   Tuple    ,			__UpperCAmelCase    :   str=None    ,			__UpperCAmelCase    :   str=None    ,			__UpperCAmelCase    :   List[Any]="</s>"    ,			__UpperCAmelCase    :   List[str]="</s>"    ,			__UpperCAmelCase    :   Union[str, Any]="<s>"    ,			__UpperCAmelCase    :   Optional[Any]="<unk>"    ,			__UpperCAmelCase    :   Any="<pad>"    ,			__UpperCAmelCase    :   Any="<mask>"    ,			__UpperCAmelCase    :   List[Any] = None    ,			**__UpperCAmelCase    :   Any    ,			):
      # Mask token behave like a normal word, i.e. include the space before it
      a						:		Optional[int]											=      AddedToken(lowerCamelCase_    ,			lstrip=lowerCamelCase_    ,			rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_    ,			lowerCamelCase_) else mask_token

      a						:		Optional[int]											=      {} if sp_model_kwargs is None else sp_model_kwargs

      a						:		List[Any]											=      kwargs.get("additional_special_tokens"    ,			[])
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=lowerCamelCase_    ,			tgt_lang=lowerCamelCase_    ,			eos_token=lowerCamelCase_    ,			unk_token=lowerCamelCase_    ,			sep_token=lowerCamelCase_    ,			cls_token=lowerCamelCase_    ,			pad_token=lowerCamelCase_    ,			mask_token=lowerCamelCase_    ,			sp_model_kwargs=self.sp_model_kwargs    ,			**lowerCamelCase_    ,			)

      a						:		str											=      spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(str(lowerCamelCase_))
      a						:		str											=      vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      a						:		str											=      {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      a						:		Optional[Any]											=      1

      a						:		List[Any]											=      len(self.sp_model)
      a						:		Optional[int]											=      {
          code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(lowerCamelCase_)
      }
      a						:		Dict											=      {v: k for k, v in self.lang_code_to_id.items()}
      a						:		List[Any]											=      len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

      self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
      a						:		Tuple											=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}

      a						:		Optional[Any]											=      src_lang if src_lang is not None else "en_XX"
      a						:		str											=      self.lang_code_to_id[self._src_lang]
      a						:		List[Any]											=      tgt_lang
      self.set_src_lang_special_tokens(self._src_lang)






    @property
    def      __snake_case							( self    :   Any):
      return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token






    @property
    def      __snake_case							( self    :   Tuple):
      return self._src_lang






    @src_lang.setter
    def      __snake_case							( self    :   Tuple    ,			__UpperCAmelCase    :   Dict):
      a						:		Tuple											=      new_src_lang
      self.set_src_lang_special_tokens(self._src_lang)






    def __getstate__( self    :   Dict):
      a						:		Optional[int]											=      self.__dict__.copy()
      a						:		str											=      None
      return state






    def __setstate__( self    :   Optional[Any]    ,			__UpperCAmelCase    :   Optional[Any]):
      a						:		List[Any]											=      d

      # for backward compatibility
      if not hasattr(self    ,			"sp_model_kwargs"):
        a						:		Union[str, Any]											=      {}

      a						:		Optional[int]											=      spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.vocab_file)






    def      __snake_case							( self    :   Optional[int]):
      a						:		int											=      {self.convert_ids_to_tokens(lowerCamelCase_): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   Dict):
      return self.sp_model.encode(lowerCamelCase_    ,			out_type=lowerCamelCase_)






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   Tuple):
      if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
      a						:		List[str]											=      self.sp_model.PieceToId(lowerCamelCase_)

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   List[str]):
      if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset)






    def      __snake_case							( self    :   int    ,			__UpperCAmelCase    :   Optional[Any]):
      a						:		Tuple											=      []
      a						:		List[str]											=      ""
      a						:		List[Any]											=      False
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          if not prev_is_special:
            out_string += " "
          out_string += self.sp_model.decode(lowerCamelCase_) + token
          a						:		Dict											=      True
          a						:		str											=      []
        else:
          current_sub_tokens.append(lowerCamelCase_)
          a						:		int											=      False
      out_string += self.sp_model.decode(lowerCamelCase_)
      return out_string.strip()






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   Optional[Any]    ,			__UpperCAmelCase    :   Tuple = None):
      if not os.path.isdir(lowerCamelCase_):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
        return
      a						:		Union[str, Any]											=      os.path.join(
          lowerCamelCase_    ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

      if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCamelCase_) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file    ,			lowerCamelCase_)
      elif not os.path.isfile(self.vocab_file):
        with open(lowerCamelCase_    ,			"wb") as fi:
          a						:		Dict											=      self.sp_model.serialized_model_proto()
          fi.write(lowerCamelCase_)

      return (out_vocab_file,)






    def      __snake_case							( self    :   Optional[int]    ,			__UpperCAmelCase    :   str    ,			__UpperCAmelCase    :   int = None    ,			__UpperCAmelCase    :   Optional[Any] = False):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCamelCase_    ,			token_ids_a=lowerCamelCase_    ,			already_has_special_tokens=lowerCamelCase_)

      a						:		str											=      [1] * len(self.prefix_tokens)
      a						:		str											=      [1] * len(self.suffix_tokens)
      if token_ids_a is None:
        return prefix_ones + ([0] * len(lowerCamelCase_)) + suffix_ones
      return prefix_ones + ([0] * len(lowerCamelCase_)) + ([0] * len(lowerCamelCase_)) + suffix_ones






    def      __snake_case							( self    :   Any    ,			__UpperCAmelCase    :   List[str]    ,			__UpperCAmelCase    :   Optional[int] = None):
      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   Tuple    ,			__UpperCAmelCase    :   Dict    ,			__UpperCAmelCase    :   Any    ,			__UpperCAmelCase    :   List[str]    ,			**__UpperCAmelCase    :   Optional[Any]):
      if src_lang is None or tgt_lang is None:
        raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
      a						:		Any											=      src_lang
      a						:		Optional[int]											=      self(lowerCamelCase_    ,			add_special_tokens=lowerCamelCase_    ,			return_tensors=lowerCamelCase_    ,			**lowerCamelCase_)
      a						:		Dict											=      self.convert_tokens_to_ids(lowerCamelCase_)
      a						:		List[str]											=      tgt_lang_id
      return inputs






    def      __snake_case							( self    :   str    ,			__UpperCAmelCase    :   Optional[int]    ,			__UpperCAmelCase    :   Any = "en_XX"    ,			__UpperCAmelCase    :   Optional[Any] = None    ,			__UpperCAmelCase    :   int = "ro_RO"    ,			**__UpperCAmelCase    :   Union[str, Any]    ,			):
      a						:		Tuple											=      src_lang
      a						:		str											=      tgt_lang
      return super().prepare_seqaseq_batch(lowerCamelCase_    ,			lowerCamelCase_    ,			**lowerCamelCase_)






    def      __snake_case							( self    :   str):
      return self.set_src_lang_special_tokens(self.src_lang)






    def      __snake_case							( self    :   int):
      return self.set_tgt_lang_special_tokens(self.tgt_lang)






    def      __snake_case							( self    :   int    ,			__UpperCAmelCase    :   str):
      a						:		int											=      self.lang_code_to_id[src_lang]
      a						:		Optional[int]											=      [self.cur_lang_code_id]
      a						:		Dict											=      [self.eos_token_id]






    def      __snake_case							( self    :   Tuple    ,			__UpperCAmelCase    :   List[Any]):
      a						:		str											=      self.lang_code_to_id[tgt_lang]
      a						:		Dict											=      [self.cur_lang_code_id]
      a						:		Union[str, Any]											=      [self.eos_token_id]










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json""",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Union[str, Any]     =	"""gpt_neox"""






    def __init__( self    :   List[str]    ,			__UpperCAmelCase    :   Tuple=50432    ,			__UpperCAmelCase    :   str=6144    ,			__UpperCAmelCase    :   Any=44    ,			__UpperCAmelCase    :   Union[str, Any]=64    ,			__UpperCAmelCase    :   Dict=24576    ,			__UpperCAmelCase    :   Any="gelu"    ,			__UpperCAmelCase    :   List[Any]=0.25    ,			__UpperCAmelCase    :   Optional[Any]=10000    ,			__UpperCAmelCase    :   Tuple=0.0    ,			__UpperCAmelCase    :   Union[str, Any]=0.0    ,			__UpperCAmelCase    :   Optional[int]=0.1    ,			__UpperCAmelCase    :   List[Any]=2048    ,			__UpperCAmelCase    :   Optional[Any]=0.02    ,			__UpperCAmelCase    :   Optional[Any]=1e-5    ,			__UpperCAmelCase    :   Dict=True    ,			__UpperCAmelCase    :   Union[str, Any]=0    ,			__UpperCAmelCase    :   Union[str, Any]=2    ,			__UpperCAmelCase    :   Any=False    ,			__UpperCAmelCase    :   List[Any]=True    ,			__UpperCAmelCase    :   int=None    ,			**__UpperCAmelCase    :   Tuple    ,			):
      super().__init__(bos_token_id=__UpperCAmelCase    ,			eos_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase)
      a						:		List[Any]											=      vocab_size
      a						:		Optional[int]											=      max_position_embeddings
      a						:		List[Any]											=      hidden_size
      a						:		Union[str, Any]											=      num_hidden_layers
      a						:		int											=      num_attention_heads
      a						:		Union[str, Any]											=      intermediate_size
      a						:		Optional[Any]											=      hidden_act
      a						:		Dict											=      rotary_pct
      a						:		Any											=      rotary_emb_base
      a						:		Dict											=      attention_dropout
      a						:		List[str]											=      hidden_dropout
      a						:		List[str]											=      classifier_dropout
      a						:		Any											=      initializer_range
      a						:		Union[str, Any]											=      layer_norm_eps
      a						:		int											=      use_cache
      a						:		int											=      tie_word_embeddings
      a						:		str											=      use_parallel_residual
      a						:		Union[str, Any]											=      rope_scaling
      self._rope_scaling_validation()

      if self.hidden_size % self.num_attention_heads != 0:
        raise ValueError(
            "The hidden size is not divisble by the number of attention heads! Make sure to update them!")






    def      __snake_case							( self    :   Any):
      if self.rope_scaling is None:
        return

      if not isinstance(self.rope_scaling    ,			__UpperCAmelCase) or len(self.rope_scaling) != 2:
        raise ValueError(
            "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
            f'''got {self.rope_scaling}''')
      a						:		str											=      self.rope_scaling.get("type"    ,			__UpperCAmelCase)
      a						:		List[str]											=      self.rope_scaling.get("factor"    ,			__UpperCAmelCase)
      if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''')
      if rope_scaling_factor is None or not isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) or rope_scaling_factor <= 1.0:
        raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''')






