


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  :List[Any]             =							logging.get_logger(__name__)

lowerCAmelCase__  :Optional[int]             =							'▁'

lowerCAmelCase__  :List[Any]             =							{'vocab_file': 'sentencepiece.bpe.model'}

lowerCAmelCase__  :Any             =							{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

lowerCAmelCase__  :List[str]             =							{
    'facebook/mbart-large-en-ro': 1_0_2_4,
    'facebook/mbart-large-cc25': 1_0_2_4,
}

# fmt: off
lowerCAmelCase__  :Tuple             =							['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']

class 		__a				(    SCREAMING_SNAKE_CASE__      ):
		_a     :  Optional[Any]           =     VOCAB_FILES_NAMES
		_a     :  int           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     :  Any           =     PRETRAINED_VOCAB_FILES_MAP
		_a     :  Optional[int]           =     ['input_ids', 'attention_mask']

		_a     :  Union[str, Any]           =     []
		_a     :  Optional[Any]           =     []
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE="<s>"      ,   _SCREAMING_SNAKE_CASE="</s>"      ,   _SCREAMING_SNAKE_CASE="</s>"      ,   _SCREAMING_SNAKE_CASE="<s>"      ,   _SCREAMING_SNAKE_CASE="<unk>"      ,   _SCREAMING_SNAKE_CASE="<pad>"      ,   _SCREAMING_SNAKE_CASE="<mask>"      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE = None      ,   _SCREAMING_SNAKE_CASE=None      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      AddedToken(_SCREAMING_SNAKE_CASE      ,   lstrip=_SCREAMING_SNAKE_CASE      ,   rstrip=_SCREAMING_SNAKE_CASE						) if isinstance(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						) else mask_token

									_UpperCAmelCase					=      {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_SCREAMING_SNAKE_CASE      ,   eos_token=_SCREAMING_SNAKE_CASE      ,   unk_token=_SCREAMING_SNAKE_CASE      ,   sep_token=_SCREAMING_SNAKE_CASE      ,   cls_token=_SCREAMING_SNAKE_CASE      ,   pad_token=_SCREAMING_SNAKE_CASE      ,   mask_token=_SCREAMING_SNAKE_CASE      ,   tokenizer_file=_SCREAMING_SNAKE_CASE      ,   src_lang=_SCREAMING_SNAKE_CASE      ,   tgt_lang=_SCREAMING_SNAKE_CASE      ,   additional_special_tokens=_SCREAMING_SNAKE_CASE      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **_SCREAMING_SNAKE_CASE      ,   )

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(str(_SCREAMING_SNAKE_CASE						)						)
									_UpperCAmelCase					=      vocab_file

									# Original fairseq vocab and spm vocab must be "aligned":
									# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
									# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
									# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
									# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

									# Mimic fairseq token-to-id alignment for the first 4 token
									_UpperCAmelCase					=      {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

									# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
									_UpperCAmelCase					=      1

									_UpperCAmelCase					=      len(self.sp_model						)
									_UpperCAmelCase					=      {
									    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_SCREAMING_SNAKE_CASE						)
									}
									_UpperCAmelCase					=      {v: k for k, v in self.lang_code_to_id.items()}
									_UpperCAmelCase					=      len(self.sp_model						) + len(self.lang_code_to_id						) + self.fairseq_offset

									self.fairseq_tokens_to_ids.update(self.lang_code_to_id						)
									_UpperCAmelCase					=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}
									_UpperCAmelCase					=      list(self.lang_code_to_id.keys()						)

									if additional_special_tokens is not None:
																# Only add those special tokens if they are not already there.
																self._additional_special_tokens.extend(
																    [t for t in additional_special_tokens if t not in self._additional_special_tokens]						)

									_UpperCAmelCase					=      src_lang if src_lang is not None else 'en_XX'
									_UpperCAmelCase					=      self.lang_code_to_id[self._src_lang]
									_UpperCAmelCase					=      tgt_lang
									self.set_src_lang_special_tokens(self._src_lang						)
		def __getstate__(      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.__dict__.copy()
									_UpperCAmelCase					=      None
									_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
									return state
		def __setstate__(      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      d

									# for backward compatibility
									if not hasattr(self      ,   'sp_model_kwargs'						):
																_UpperCAmelCase					=      {}

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)
		@property
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									return len(self.sp_model						) + len(self.lang_code_to_id						) + self.fairseq_offset + 1  # Plus 1 for the mask token
		@property
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									return self._src_lang
		@src_lang.setter
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	None:




									"""simple docstring"""





									_UpperCAmelCase					=      new_src_lang
									self.set_src_lang_special_tokens(self._src_lang						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None      ,   _SCREAMING_SNAKE_CASE = False						)			->	List[int]:




									"""simple docstring"""





									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_SCREAMING_SNAKE_CASE      ,   token_ids_a=_SCREAMING_SNAKE_CASE      ,   already_has_special_tokens=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      [1] * len(self.prefix_tokens						)
									_UpperCAmelCase					=      [1] * len(self.suffix_tokens						)
									if token_ids_a is None:
																return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE						)) + suffix_ones
									return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE						)) + ([0] * len(_SCREAMING_SNAKE_CASE						)) + suffix_ones
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep						) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	List[Any]:




									"""simple docstring"""





									if src_lang is None or tgt_lang is None:
																raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'						)
									_UpperCAmelCase					=      src_lang
									_UpperCAmelCase					=      self(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE      ,   return_tensors=_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tgt_lang_id
									return inputs
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						): i for i in range(self.vocab_size						)}
									vocab.update(self.added_tokens_encoder						)
									return vocab
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									return self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,   out_type=_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Union[str, Any]:




									"""simple docstring"""





									if token in self.fairseq_tokens_to_ids:
																return self.fairseq_tokens_to_ids[token]
									_UpperCAmelCase					=      self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE						)

									# Need to return unknown token if the SP model returned 0
									return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									if index in self.fairseq_ids_to_tokens:
																return self.fairseq_ids_to_tokens[index]
									return self.sp_model.IdToPiece(index - self.fairseq_offset						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      ''.join(_SCREAMING_SNAKE_CASE						).replace(_SCREAMING_SNAKE_CASE      ,   ' '						).strip()
									return out_string
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	Tuple[str]:




									"""simple docstring"""





									if not os.path.isdir(_SCREAMING_SNAKE_CASE						):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																return
									_UpperCAmelCase					=      os.path.join(
									    _SCREAMING_SNAKE_CASE      ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(_SCREAMING_SNAKE_CASE						) and os.path.isfile(self.vocab_file						):
																copyfile(self.vocab_file      ,   _SCREAMING_SNAKE_CASE						)
									elif not os.path.isfile(self.vocab_file						):
																with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as fi:
																							_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
																							fi.write(_SCREAMING_SNAKE_CASE						)

									return (out_vocab_file,)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = "en_XX"      ,   _SCREAMING_SNAKE_CASE = None      ,   _SCREAMING_SNAKE_CASE = "ro_RO"      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	BatchEncoding:




									"""simple docstring"""





									_UpperCAmelCase					=      src_lang
									_UpperCAmelCase					=      tgt_lang
									return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									return self.set_src_lang_special_tokens(self.src_lang						)
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									return self.set_tgt_lang_special_tokens(self.tgt_lang						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	None:




									"""simple docstring"""





									_UpperCAmelCase					=      self.lang_code_to_id[src_lang]
									_UpperCAmelCase					=      []
									_UpperCAmelCase					=      [self.eos_token_id, self.cur_lang_code]






		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	None:




									"""simple docstring"""





									_UpperCAmelCase					=      self.lang_code_to_id[lang]
									_UpperCAmelCase					=      []
									_UpperCAmelCase					=      [self.eos_token_id, self.cur_lang_code]




from importlib import import_module

from .logging import get_logger


lowerCAmelCase__  :Optional[Any]             =							get_logger(__name__)

class 		__a				:
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      attrs or []
									if module is not None:
																for key in module.__dict__:
																							if key in attrs or not key.startswith('__'						):
																														setattr(self      ,   _SCREAMING_SNAKE_CASE      ,   getattr(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)						)
									_UpperCAmelCase					=      module._original_module if isinstance(_SCREAMING_SNAKE_CASE      ,   _PatchedModuleObj						) else module



class 		__a				:
		_a     :  Any           =     []
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      obj
									_UpperCAmelCase					=      target
									_UpperCAmelCase					=      new
									_UpperCAmelCase					=      target.split('.'						)[0]
									_UpperCAmelCase					=      {}
									_UpperCAmelCase					=      attrs or []
		def __enter__(      self						)			->	int:




									"""simple docstring"""





									*_UpperCAmelCase		,	_UpperCAmelCase					=      self.target.split('.'						)

									# Patch modules:
									# it's used to patch attributes of submodules like "os.path.join";
									# in this case we need to patch "os" and "os.path"

									for i in range(len(_SCREAMING_SNAKE_CASE						)						):
																try:
																							_UpperCAmelCase					=      import_module('.'.join(submodules[: i + 1]						)						)
																except ModuleNotFoundError:
																							continue
																# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																for attr in self.obj.__dir__():
																							_UpperCAmelCase					=      getattr(self.obj      ,   _SCREAMING_SNAKE_CASE						)
																							# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																							# This allows to patch renamed modules like "from os import path as ospath".
																							if obj_attr is submodule or (
																							    (isinstance(_SCREAMING_SNAKE_CASE      ,   _PatchedModuleObj						) and obj_attr._original_module is submodule)
																							):
																														_UpperCAmelCase					=      obj_attr
																														# patch at top level
																														setattr(self.obj      ,   _SCREAMING_SNAKE_CASE      ,   _PatchedModuleObj(_SCREAMING_SNAKE_CASE      ,   attrs=self.attrs						)						)
																														_UpperCAmelCase					=      getattr(self.obj      ,   _SCREAMING_SNAKE_CASE						)
																														# construct lower levels patches
																														for key in submodules[i + 1 :]:
																																					setattr(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _PatchedModuleObj(getattr(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)      ,   attrs=self.attrs						)						)
																																					_UpperCAmelCase					=      getattr(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
																														# finally set the target attribute
																														setattr(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   self.new						)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

									if submodules:  # if it's an attribute of a submodule like "os.path.join"
																try:
																							_UpperCAmelCase					=      getattr(import_module('.'.join(_SCREAMING_SNAKE_CASE						)						)      ,   _SCREAMING_SNAKE_CASE						)
																except (AttributeError, ModuleNotFoundError):
																							return
																# We iterate over all the globals in self.obj in case we find "os.path.join"
																for attr in self.obj.__dir__():
																							# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																							# This allows to patch renamed attributes like "from os.path import join as pjoin".
																							if getattr(self.obj      ,   _SCREAMING_SNAKE_CASE						) is attr_value:
																														_UpperCAmelCase					=      getattr(self.obj      ,   _SCREAMING_SNAKE_CASE						)
																														setattr(self.obj      ,   _SCREAMING_SNAKE_CASE      ,   self.new						)
									elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																_UpperCAmelCase					=      globals()['__builtins__'][target_attr]
																setattr(self.obj      ,   _SCREAMING_SNAKE_CASE      ,   self.new						)
									else:
																raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''						)
		def __exit__(      self      ,   *_SCREAMING_SNAKE_CASE						)			->	Union[str, Any]:




									"""simple docstring"""





									for attr in list(self.original						):
																setattr(self.obj      ,   _SCREAMING_SNAKE_CASE      ,   self.original.pop(_SCREAMING_SNAKE_CASE						)						)
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									self.__enter__()
									self._active_patches.append(self						)






		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									try:
																self._active_patches.remove(self						)
									except ValueError:
																# If the patch hasn't been started this will fail
																return None

									return self.__exit__()
