




"""simple docstring"""


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_A													=      logging.get_logger(__name__)

_A													=      """▁"""

_A													=      {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

_A													=      {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

_A													=      {
    """facebook/m2m100_418M""": 1_0_2_4,
}

# fmt: off
_A													=      {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}


class _lowerCamelCase    (					a_						):
   _lowerCamelCase    :List[Any]     				=     VOCAB_FILES_NAMES
   _lowerCamelCase    :Union[str, Any]     				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowerCamelCase    :List[Any]     				=     PRETRAINED_VOCAB_FILES_MAP
   _lowerCamelCase    :Optional[int]     				=     ["input_ids", "attention_mask"]

   _lowerCamelCase    :List[int]     				=     []
   _lowerCamelCase    :List[int]     				=     []






   def __init__(		self       :		Optional[int]   ,     UpperCamelCase       :		Optional[int]   ,     UpperCamelCase       :		str   ,     UpperCamelCase       :		Optional[Any]=None   ,     UpperCamelCase       :		List[str]=None   ,     UpperCamelCase       :		Union[str, Any]="<s>"   ,     UpperCamelCase       :		str="</s>"   ,     UpperCamelCase       :		Optional[Any]="</s>"   ,     UpperCamelCase       :		int="<pad>"   ,     UpperCamelCase       :		List[str]="<unk>"   ,     UpperCamelCase       :		Union[str, Any]="m2m100"   ,     UpperCamelCase       :		Optional[Dict[str, Any]] = None   ,     UpperCamelCase       :		List[Any]=8   ,     **UpperCamelCase       :		Dict   ,     )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	{} if sp_model_kwargs is None else sp_model_kwargs

         lowerCAmelCase__      :			Any   =	language_codes
         lowerCAmelCase__      :			List[Any]   =	FAIRSEQ_LANGUAGE_CODES[language_codes]
         lowerCAmelCase__      :			Tuple   =	{lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

         lowerCAmelCase__      :			int   =	kwargs.get("""additional_special_tokens"""   ,     []   )
         kwargs["additional_special_tokens"] += [
             self.get_lang_token(UpperCamelCase   )
             for lang_code in fairseq_language_code
             if self.get_lang_token(UpperCamelCase   ) not in kwargs["additional_special_tokens"]
         ]

         super().__init__(
             src_lang=UpperCamelCase   ,     tgt_lang=UpperCamelCase   ,     bos_token=UpperCamelCase   ,     eos_token=UpperCamelCase   ,     sep_token=UpperCamelCase   ,     unk_token=UpperCamelCase   ,     pad_token=UpperCamelCase   ,     language_codes=UpperCamelCase   ,     sp_model_kwargs=self.sp_model_kwargs   ,     num_madeup_words=UpperCamelCase   ,     **UpperCamelCase   ,     )

         lowerCAmelCase__      :			Dict   =	vocab_file
         lowerCAmelCase__      :			List[Any]   =	load_json(UpperCamelCase   )
         lowerCAmelCase__      :			Tuple   =	{v: k for k, v in self.encoder.items()}
         lowerCAmelCase__      :			Optional[Any]   =	spm_file
         lowerCAmelCase__      :			List[str]   =	load_spm(UpperCamelCase   ,     self.sp_model_kwargs   )

         lowerCAmelCase__      :			Any   =	len(self.encoder   )

         lowerCAmelCase__      :			Dict   =	{
             self.get_lang_token(UpperCamelCase   ): self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase   )
         }
         lowerCAmelCase__      :			List[str]   =	{lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase   )}
         lowerCAmelCase__      :			Optional[int]   =	{v: k for k, v in self.lang_token_to_id.items()}

         lowerCAmelCase__      :			Optional[Any]   =	src_lang if src_lang is not None else """en"""
         lowerCAmelCase__      :			Optional[int]   =	tgt_lang
         lowerCAmelCase__      :			Dict   =	self.get_lang_id(self._src_lang   )
         self.set_src_lang_special_tokens(self._src_lang   )

         lowerCAmelCase__      :			Any   =	num_madeup_words






   @property
   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						int:

         """simple docstring"""
         return len(self.encoder   ) + len(self.lang_token_to_id   )






   @property
   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						str:

         """simple docstring"""
         return self._src_lang






   @src_lang.setter
   def 	_lowerCAmelCase   (		self       :		Any   ,     UpperCamelCase       :		str   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	new_src_lang
         self.set_src_lang_special_tokens(self._src_lang   )






   def 	_lowerCAmelCase   (		self       :		int   ,     UpperCamelCase       :		str   )			->						List[str]:

         """simple docstring"""
         return self.sp_model.encode(UpperCamelCase   ,     out_type=UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		Union[str, Any]   )			->						str:

         """simple docstring"""
         if token in self.lang_token_to_id:
               return self.lang_token_to_id[token]
         return self.encoder.get(UpperCamelCase   ,     self.encoder[self.unk_token]   )






   def 	_lowerCAmelCase   (		self       :		Any   ,     UpperCamelCase       :		int   )			->						str:

         """simple docstring"""
         if index in self.id_to_lang_token:
               return self.id_to_lang_token[index]
         return self.decoder.get(UpperCamelCase   ,     self.unk_token   )






   def 	_lowerCAmelCase   (		self       :		Optional[int]   ,     UpperCamelCase       :		Tuple   )			->						Tuple:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	[]
         lowerCAmelCase__      :			Any   =	""""""
         for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                     out_string += self.sp_model.decode(UpperCamelCase   ) + token
                     lowerCAmelCase__      :			Optional[Any]   =	[]
               else:
                     current_sub_tokens.append(UpperCamelCase   )
         out_string += self.sp_model.decode(UpperCamelCase   )
         return out_string.strip()






   def 	_lowerCAmelCase   (		self       :		Optional[int]   ,     UpperCamelCase       :		List[int]   ,     UpperCamelCase       :		Optional[List[int]] = None   ,     UpperCamelCase       :		bool = False   )			->						List[int]:

         """simple docstring"""

         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=UpperCamelCase   ,     token_ids_a=UpperCamelCase   ,     already_has_special_tokens=UpperCamelCase   )

         lowerCAmelCase__      :			Dict   =	[1] * len(self.prefix_tokens   )
         lowerCAmelCase__      :			Any   =	[1] * len(self.suffix_tokens   )
         if token_ids_a is None:
               return prefix_ones + ([0] * len(UpperCamelCase   )) + suffix_ones
         return prefix_ones + ([0] * len(UpperCamelCase   )) + ([0] * len(UpperCamelCase   )) + suffix_ones






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		List[int]   ,     UpperCamelCase       :		Optional[List[int]] = None   )			->						List[int]:

         """simple docstring"""
         if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Dict:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[int]   =	{self.convert_ids_to_tokens(UpperCamelCase   ): i for i in range(self.vocab_size   )}
         vocab.update(self.added_tokens_encoder   )
         return vocab






   def __getstate__(		self       :		Optional[Any]   )			->						Dict:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	self.__dict__.copy()
         lowerCAmelCase__      :			Union[str, Any]   =	None
         return state






   def __setstate__(		self       :		Optional[int]   ,     UpperCamelCase       :		Dict   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	d

         # for backward compatibility
         if not hasattr(self   ,     """sp_model_kwargs"""   ):
               lowerCAmelCase__      :			Optional[int]   =	{}

         lowerCAmelCase__      :			List[str]   =	load_spm(self.spm_file   ,     self.sp_model_kwargs   )






   def 	_lowerCAmelCase   (		self       :		Dict   ,     UpperCamelCase       :		str   ,     UpperCamelCase       :		Optional[str] = None   )			->						Tuple[str]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	Path(UpperCamelCase   )
         if not save_dir.is_dir():
               raise OSError(f"""{save_directory} should be a directory"""   )
         lowerCAmelCase__      :			str   =	save_dir / (
             (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
         )
         lowerCAmelCase__      :			List[Any]   =	save_dir / (
             (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
         )

         save_json(self.encoder   ,     UpperCamelCase   )

         if os.path.abspath(self.spm_file   ) != os.path.abspath(UpperCamelCase   ) and os.path.isfile(self.spm_file   ):
               copyfile(self.spm_file   ,     UpperCamelCase   )
         elif not os.path.isfile(self.spm_file   ):
               with open(UpperCamelCase   ,     """wb"""   ) as fi:
                     lowerCAmelCase__      :			Union[str, Any]   =	self.sp_model.serialized_model_proto()
                     fi.write(UpperCamelCase   )

         return (str(UpperCamelCase   ), str(UpperCamelCase   ))






   def 	_lowerCAmelCase   (		self       :		int   ,     UpperCamelCase       :		List[str]   ,     UpperCamelCase       :		str = "en"   ,     UpperCamelCase       :		Optional[List[str]] = None   ,     UpperCamelCase       :		str = "ro"   ,     **UpperCamelCase       :		Any   ,     )			->						BatchEncoding:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	src_lang
         lowerCAmelCase__      :			Optional[Any]   =	tgt_lang
         self.set_src_lang_special_tokens(self.src_lang   )
         return super().prepare_seqaseq_batch(UpperCamelCase   ,     UpperCamelCase   ,     **UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		Optional[Any]   ,     UpperCamelCase       :		Optional[str]   ,     UpperCamelCase       :		Optional[str]   ,     **UpperCamelCase       :		Dict   )			->						str:

         """simple docstring"""
         if src_lang is None or tgt_lang is None:
               raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""   )
         lowerCAmelCase__      :			List[Any]   =	src_lang
         lowerCAmelCase__      :			Optional[int]   =	self(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   ,     **UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	self.get_lang_id(UpperCamelCase   )
         lowerCAmelCase__      :			List[str]   =	tgt_lang_id
         return inputs






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						List[Any]:

         """simple docstring"""
         self.set_src_lang_special_tokens(self.src_lang   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						List[Any]:

         """simple docstring"""
         self.set_tgt_lang_special_tokens(self.tgt_lang   )






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		str   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	self.get_lang_token(UpperCamelCase   )
         lowerCAmelCase__      :			int   =	self.lang_token_to_id[lang_token]
         lowerCAmelCase__      :			Union[str, Any]   =	[self.cur_lang_id]
         lowerCAmelCase__      :			Tuple   =	[self.eos_token_id]






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		str   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	self.get_lang_token(UpperCamelCase   )
         lowerCAmelCase__      :			Optional[int]   =	self.lang_token_to_id[lang_token]
         lowerCAmelCase__      :			Tuple   =	[self.cur_lang_id]
         lowerCAmelCase__      :			Any   =	[self.eos_token_id]






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		str   )			->						str:

         """simple docstring"""
         return self.lang_code_to_token[lang]






   def 	_lowerCAmelCase   (		self       :		Dict   ,     UpperCamelCase       :		str   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	self.get_lang_token(UpperCamelCase   )
         return self.lang_token_to_id[lang_token]






def 	lowercase_				(			__UpperCAmelCase    ,		__UpperCAmelCase				)						->    sentencepiece.SentencePieceProcessor:
      lowerCAmelCase__      :			Dict   =	sentencepiece.SentencePieceProcessor(**__UpperCAmelCase				)
      spm.Load(str(__UpperCAmelCase				)				)
      return spm



def 	lowercase_				(			__UpperCAmelCase				)						->    Union[Dict, List]:
      with open(__UpperCAmelCase    ,		"""r"""				) as f:
            return json.load(__UpperCAmelCase				)



def 	lowercase_				(			__UpperCAmelCase    ,		__UpperCAmelCase				)						->    None:
      with open(__UpperCAmelCase    ,		"""w"""				) as f:
            json.dump(__UpperCAmelCase    ,		__UpperCAmelCase    ,		indent=2				)











"""simple docstring"""


import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A													=      logging.get_logger(__name__)

_A													=      {"""vocab_file""": """vocab.json"""}

_A													=      {
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

_A													=      {"""mgp-str""": 2_7}


class _lowerCamelCase    (					a_						):
   _lowerCamelCase    :Dict     				=     VOCAB_FILES_NAMES
   _lowerCamelCase    :Union[str, Any]     				=     PRETRAINED_VOCAB_FILES_MAP
   _lowerCamelCase    :Optional[int]     				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






   def __init__(		self       :		Optional[int]   ,     UpperCamelCase       :		int   ,     UpperCamelCase       :		Union[str, Any]="[GO]"   ,     UpperCamelCase       :		Any="[GO]"   ,     UpperCamelCase       :		Tuple="[s]"   ,     UpperCamelCase       :		List[Any]="[GO]"   ,     **UpperCamelCase       :		Dict   )			->						Union[str, Any]:

         """simple docstring"""
         super().__init__(
             unk_token=UpperCamelCase   ,     bos_token=UpperCamelCase   ,     eos_token=UpperCamelCase   ,     pad_token=UpperCamelCase   ,     **UpperCamelCase   ,     )

         with open(UpperCamelCase   ,     encoding="""utf-8"""   ) as vocab_handle:
               lowerCAmelCase__      :			Optional[Any]   =	json.load(UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	{v: k for k, v in self.vocab.items()}






   @property
   def 	_lowerCAmelCase   (		self       :		int   )			->						Dict:

         """simple docstring"""
         return len(self.vocab   )






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Any:

         """simple docstring"""
         return dict(self.vocab   ,     **self.added_tokens_encoder   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		Tuple   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	[]
         for s in text:
               char_tokens.extend(UpperCamelCase   )
         return char_tokens






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		str   )			->						Tuple:

         """simple docstring"""
         return self.vocab.get(UpperCamelCase   ,     self.vocab.get(self.unk_token   )   )






   def 	_lowerCAmelCase   (		self       :		List[str]   ,     UpperCamelCase       :		int   )			->						Any:

         """simple docstring"""
         return self.decoder.get(UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Dict   ,     UpperCamelCase       :		str   ,     UpperCamelCase       :		Optional[str] = None   )			->						Tuple[str]:

         """simple docstring"""
         if not os.path.isdir(UpperCamelCase   ):
               logger.error("""Vocabulary path ({}) should be a directory""".format(UpperCamelCase   )   )
               return
         lowerCAmelCase__      :			Union[str, Any]   =	os.path.join(
             UpperCamelCase   ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

         with open(UpperCamelCase   ,     """w"""   ,     encoding="""utf-8"""   ) as f:
               f.write(json.dumps(self.vocab   ,     indent=2   ,     sort_keys=UpperCamelCase   ,     ensure_ascii=UpperCamelCase   ) + """\n"""   )

         return (vocab_file,)





