



'''simple docstring'''
import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
     from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
     from tensorflow.keras.optimizers import Adam






class 							lowercase__     (							tf.keras.optimizers.schedules.LearningRateSchedule       ):





 def __init__(			self :    List[Any]			,lowerCamelCase__ :    float			,lowerCamelCase__ :    Callable			,lowerCamelCase__ :    int			,lowerCamelCase__ :    float = 1.0			,lowerCamelCase__ :    str = None			,):







       '''simple docstring'''
       super().__init__()
       _UpperCamelCase :		Union[str, Any]			    =       initial_learning_rate
       _UpperCamelCase :		Optional[Any]			    =       warmup_steps
       _UpperCamelCase :		Tuple			    =       power
       _UpperCamelCase :		Union[str, Any]			    =       decay_schedule_fn
       _UpperCamelCase :		Any			    =       name





 def __call__(			self :    List[Any]			,lowerCamelCase__ :    Union[str, Any]			):







       '''simple docstring'''
       with tf.name_scope(self.name or 'WarmUp'			) as name:
             # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
             # learning rate will be `global_step/num_warmup_steps * init_lr`.
             _UpperCamelCase :		int			    =       tf.cast(lowerCamelCase__			,tf.floataa			)
             _UpperCamelCase :		Optional[int]			    =       tf.cast(self.warmup_steps			,tf.floataa			)
             _UpperCamelCase :		Tuple			    =       global_step_float / warmup_steps_float
             _UpperCamelCase :		Optional[Any]			    =       self.initial_learning_rate * tf.math.pow(lowerCamelCase__			,self.power			)
             return tf.cond(
                 global_step_float < warmup_steps_float			,lambda: warmup_learning_rate			,lambda: self.decay_schedule_fn(step - self.warmup_steps			)			,name=lowerCamelCase__			,)





 def      UpperCamelCase_  (			self :    Dict			):







       '''simple docstring'''
       return {
           "initial_learning_rate": self.initial_learning_rate,
           "decay_schedule_fn": self.decay_schedule_fn,
           "warmup_steps": self.warmup_steps,
           "power": self.power,
           "name": self.name,
       }






def A__   (					UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_ = 0.0  ,  UpperCAmelCase_ = 0.9  ,  UpperCAmelCase_ = 0.999  ,  UpperCAmelCase_ = 1E-8  ,  UpperCAmelCase_ = None  ,  UpperCAmelCase_ = None  ,  UpperCAmelCase_ = 0.0  ,  UpperCAmelCase_ = 1.0  ,  UpperCAmelCase_ = None  ,  ):
      _UpperCamelCase :		Optional[int]			    =       tf.keras.optimizers.schedules.PolynomialDecay(
          initial_learning_rate=UpperCAmelCase_  ,  decay_steps=num_train_steps - num_warmup_steps  ,  end_learning_rate=init_lr * min_lr_ratio  ,  power=UpperCAmelCase_  ,  )
      if num_warmup_steps:
            _UpperCamelCase :		Optional[int]			    =       WarmUp(
                initial_learning_rate=UpperCAmelCase_  ,  decay_schedule_fn=UpperCAmelCase_  ,  warmup_steps=UpperCAmelCase_  ,  )
      if weight_decay_rate > 0.0:
            _UpperCamelCase :		Dict			    =       AdamWeightDecay(
                learning_rate=UpperCAmelCase_  ,  weight_decay_rate=UpperCAmelCase_  ,  beta_a=UpperCAmelCase_  ,  beta_a=UpperCAmelCase_  ,  epsilon=UpperCAmelCase_  ,  clipnorm=UpperCAmelCase_  ,  global_clipnorm=UpperCAmelCase_  ,  exclude_from_weight_decay=['LayerNorm', 'layer_norm', 'bias']  ,  include_in_weight_decay=UpperCAmelCase_  ,  )
      else:
            _UpperCamelCase :		Union[str, Any]			    =       tf.keras.optimizers.Adam(
                learning_rate=UpperCAmelCase_  ,  beta_a=UpperCAmelCase_  ,  beta_a=UpperCAmelCase_  ,  epsilon=UpperCAmelCase_  ,  clipnorm=UpperCAmelCase_  ,  global_clipnorm=UpperCAmelCase_  ,  )
      # We return the optimizer and the LR scheduler in order to better track the
      # evolution of the LR independently of the optimizer.
      return optimizer, lr_schedule






class 							lowercase__     (							lowercase       ):





 def __init__(			self :    int			,lowerCamelCase__ :    Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.0_0_1			,lowerCamelCase__ :    float = 0.9			,lowerCamelCase__ :    float = 0.9_9_9			,lowerCamelCase__ :    float = 1E-7			,lowerCamelCase__ :    bool = False			,lowerCamelCase__ :    float = 0.0			,lowerCamelCase__ :    Optional[List[str]] = None			,lowerCamelCase__ :    Optional[List[str]] = None			,lowerCamelCase__ :    str = "AdamWeightDecay"			,**lowerCamelCase__ :    Optional[Any]			,):







       '''simple docstring'''
       super().__init__(lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,**lowerCamelCase__			)
       _UpperCamelCase :		Union[str, Any]			    =       weight_decay_rate
       _UpperCamelCase :		Tuple			    =       include_in_weight_decay
       _UpperCamelCase :		int			    =       exclude_from_weight_decay





 @classmethod
 def      UpperCamelCase_  (			cls :    int			,lowerCamelCase__ :    List[Any]			):







       '''simple docstring'''
       _UpperCamelCase :		Tuple			    =       {'WarmUp': WarmUp}
       return super(lowerCamelCase__			,cls			).from_config(lowerCamelCase__			,custom_objects=lowerCamelCase__			)





 def      UpperCamelCase_  (			self :    List[str]			,lowerCamelCase__ :    List[Any]			,lowerCamelCase__ :    List[str]			,lowerCamelCase__ :    Union[str, Any]			):







       '''simple docstring'''
       super(lowerCamelCase__			,self			)._prepare_local(lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			)
       _UpperCamelCase :		List[Any]			    =       tf.constant(
           self.weight_decay_rate			,name='adam_weight_decay_rate'			)





 def      UpperCamelCase_  (			self :    str			,lowerCamelCase__ :    Union[str, Any]			,lowerCamelCase__ :    Optional[Any]			,lowerCamelCase__ :    Union[str, Any]			):







       '''simple docstring'''
       _UpperCamelCase :		Any			    =       self._do_use_weight_decay(var.name			)
       if do_decay:
             return var.assign_sub(
                 learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]['weight_decay_rate']			,use_locking=self._use_locking			,)
       return tf.no_op()





 def      UpperCamelCase_  (			self :    Union[str, Any]			,lowerCamelCase__ :    Union[str, Any]			,lowerCamelCase__ :    Union[str, Any]=None			,**lowerCamelCase__ :    Dict			):







       '''simple docstring'''
       _UpperCamelCase :		Tuple			    =       list(zip(*lowerCamelCase__			)			)
       return super(lowerCamelCase__			,self			).apply_gradients(zip(lowerCamelCase__			,lowerCamelCase__			)			,name=lowerCamelCase__			,**lowerCamelCase__			)





 def      UpperCamelCase_  (			self :    Union[str, Any]			,lowerCamelCase__ :    List[str]			,lowerCamelCase__ :    Tuple			,lowerCamelCase__ :    Dict			):







       '''simple docstring'''
       if apply_state is None:
             return self._decayed_lr_t[var_dtype], {}

       _UpperCamelCase :		Tuple			    =       apply_state or {}
       _UpperCamelCase :		Any			    =       apply_state.get((var_device, var_dtype)			)
       if coefficients is None:
             _UpperCamelCase :		Dict			    =       self._fallback_apply_state(lowerCamelCase__			,lowerCamelCase__			)
             _UpperCamelCase :		int			    =       coefficients

       return coefficients["lr_t"], {"apply_state": apply_state}





 def      UpperCamelCase_  (			self :    int			,lowerCamelCase__ :    Optional[Any]			,lowerCamelCase__ :    Any			,lowerCamelCase__ :    Union[str, Any]=None			):







       '''simple docstring'''
       _UpperCamelCase :		str			    =       self._get_lr(var.device			,var.dtype.base_dtype			,lowerCamelCase__			)
       _UpperCamelCase :		Optional[int]			    =       self._decay_weights_op(lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			)
       with tf.control_dependencies([decay]			):
             return super(lowerCamelCase__			,self			)._resource_apply_dense(lowerCamelCase__			,lowerCamelCase__			,**lowerCamelCase__			)





 def      UpperCamelCase_  (			self :    Optional[int]			,lowerCamelCase__ :    Dict			,lowerCamelCase__ :    int			,lowerCamelCase__ :    Dict			,lowerCamelCase__ :    Tuple=None			):







       '''simple docstring'''
       _UpperCamelCase :		int			    =       self._get_lr(var.device			,var.dtype.base_dtype			,lowerCamelCase__			)
       _UpperCamelCase :		Any			    =       self._decay_weights_op(lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			)
       with tf.control_dependencies([decay]			):
             return super(lowerCamelCase__			,self			)._resource_apply_sparse(lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,**lowerCamelCase__			)





 def      UpperCamelCase_  (			self :    str			):







       '''simple docstring'''
       _UpperCamelCase :		Tuple			    =       super().get_config()
       config.update({'weight_decay_rate': self.weight_decay_rate}			)
       return config





 def      UpperCamelCase_  (			self :    Union[str, Any]			,lowerCamelCase__ :    str			):







       '''simple docstring'''
       if self.weight_decay_rate == 0:
             return False

       if self._include_in_weight_decay:
             for r in self._include_in_weight_decay:
                   if re.search(lowerCamelCase__			,lowerCamelCase__			) is not None:
                         return True

       if self._exclude_from_weight_decay:
             for r in self._exclude_from_weight_decay:
                   if re.search(lowerCamelCase__			,lowerCamelCase__			) is not None:
                         return False
       return True






class 							lowercase__     (							lowercase       ):





 def __init__(			self :    Optional[Any]			):







       '''simple docstring'''
       _UpperCamelCase :		Any			    =       []
       _UpperCamelCase :		Optional[Any]			    =       None





 @property
 def      UpperCamelCase_  (			self :    Optional[int]			):







       '''simple docstring'''
       if self._accum_steps is None:
             _UpperCamelCase :		Union[str, Any]			    =       tf.Variable(
                 tf.constant(0			,dtype=tf.intaa			)			,trainable=lowerCamelCase__			,synchronization=tf.VariableSynchronization.ON_READ			,aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA			,)

       return self._accum_steps.value()





 @property
 def      UpperCamelCase_  (			self :    Tuple			):







       '''simple docstring'''
       if not self._gradients:
             raise ValueError('The accumulator should be called first to initialize the gradients'			)
       return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]





 def __call__(			self :    Optional[int]			,lowerCamelCase__ :    Optional[Any]			):







       '''simple docstring'''
       if not self._gradients:
             _UpperCamelCase :		Any			    =       self.step  # Create the step variable.
             self._gradients.extend(
                 [
                     tf.Variable(
                         tf.zeros_like(lowerCamelCase__			)			,trainable=lowerCamelCase__			,synchronization=tf.VariableSynchronization.ON_READ			,aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA			,)
                     if gradient is not None
                     else gradient
                     for gradient in gradients
                 ]			)
       if len(lowerCamelCase__			) != len(self._gradients			):
             raise ValueError(F'Expected {len(self._gradients			)} gradients, but got {len(lowerCamelCase__			)}'			)

       for accum_gradient, gradient in zip(self._gradients			,lowerCamelCase__			):
             if accum_gradient is not None and gradient is not None:
                   accum_gradient.assign_add(lowerCamelCase__			)

       self._accum_steps.assign_add(1			)





 def      UpperCamelCase_  (			self :    Optional[int]			):







       '''simple docstring'''
       if not self._gradients:
             return
       self._accum_steps.assign(0			)
       for gradient in self._gradients:
             if gradient is not None:
                   gradient.assign(tf.zeros_like(lowerCamelCase__			)			)







'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_nllb import NllbTokenizer
else:
					snake_case_			:     List[Any]  	=  None


snake_case_			:     Dict  	=  logging.get_logger(__name__)


snake_case_			:     Dict  	=  {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_			:     List[str]  	=  {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

snake_case_			:     str  	=  {
    'facebook/nllb-large-en-ro': 1024,
    'facebook/nllb-200-distilled-600M': 1024,
}

# fmt: off
snake_case_			:     Optional[Any]  	=  ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']






class 							lowercase__     (							lowercase       ):
	lowercase__      =      VOCAB_FILES_NAMES
	lowercase__      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__      =      PRETRAINED_VOCAB_FILES_MAP
	lowercase__      =      ["""input_ids""", """attention_mask"""]
	lowercase__      =      NllbTokenizer

	lowercase__      =      []
	lowercase__      =      []





	def __init__(			self :    List[Any]			,lowerCamelCase__ :    Optional[Any]=None			,lowerCamelCase__ :    Dict=None			,lowerCamelCase__ :    List[Any]="<s>"			,lowerCamelCase__ :    Dict="</s>"			,lowerCamelCase__ :    List[Any]="</s>"			,lowerCamelCase__ :    Union[str, Any]="<s>"			,lowerCamelCase__ :    List[Any]="<unk>"			,lowerCamelCase__ :    Any="<pad>"			,lowerCamelCase__ :    Optional[Any]="<mask>"			,lowerCamelCase__ :    Optional[Any]=None			,lowerCamelCase__ :    str=None			,lowerCamelCase__ :    Tuple=None			,lowerCamelCase__ :    Union[str, Any]=False			,**lowerCamelCase__ :    Optional[Any]			,):







							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it
							_UpperCamelCase :		Optional[int]			    =       AddedToken(lowerCamelCase__			,lstrip=lowerCamelCase__			,rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__			,lowerCamelCase__			) else mask_token
							_UpperCamelCase :		Union[str, Any]			    =       legacy_behaviour
							super().__init__(
							    vocab_file=lowerCamelCase__			,tokenizer_file=lowerCamelCase__			,bos_token=lowerCamelCase__			,eos_token=lowerCamelCase__			,sep_token=lowerCamelCase__			,cls_token=lowerCamelCase__			,unk_token=lowerCamelCase__			,pad_token=lowerCamelCase__			,mask_token=lowerCamelCase__			,src_lang=lowerCamelCase__			,tgt_lang=lowerCamelCase__			,additional_special_tokens=lowerCamelCase__			,legacy_behaviour=lowerCamelCase__			,**lowerCamelCase__			,)

							_UpperCamelCase :		int			    =       vocab_file
							_UpperCamelCase :		int			    =       False if not self.vocab_file else True

							_UpperCamelCase :		Dict			    =       FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
													# Only add those special tokens if they are not already there.
													_additional_special_tokens.extend(
													    [t for t in additional_special_tokens if t not in _additional_special_tokens]			)

							self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}			)
							_UpperCamelCase :		List[str]			    =       {
							    lang_code: self.convert_tokens_to_ids(lowerCamelCase__			) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							_UpperCamelCase :		List[str]			    =       src_lang if src_lang is not None else 'eng_Latn'
							_UpperCamelCase :		int			    =       self.convert_tokens_to_ids(self._src_lang			)
							_UpperCamelCase :		Dict			    =       tgt_lang
							self.set_src_lang_special_tokens(self._src_lang			)





	@property
	def      UpperCamelCase_  (			self :    Tuple			):







							'''simple docstring'''
							return self._src_lang





	@src_lang.setter
	def      UpperCamelCase_  (			self :    Union[str, Any]			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							_UpperCamelCase :		str			    =       new_src_lang
							self.set_src_lang_special_tokens(self._src_lang			)





	def      UpperCamelCase_  (			self :    List[str]			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			):







							'''simple docstring'''
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def      UpperCamelCase_  (			self :    Dict			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       [self.sep_token_id]
							_UpperCamelCase :		List[str]			    =       [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]





	def      UpperCamelCase_  (			self :    Union[str, Any]			,lowerCamelCase__ :    str			,lowerCamelCase__ :    str			,lowerCamelCase__ :    Optional[str]			,lowerCamelCase__ :    Optional[str]			,**lowerCamelCase__ :    Dict			):







							'''simple docstring'''
							if src_lang is None or tgt_lang is None:
													raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'			)
							_UpperCamelCase :		Tuple			    =       src_lang
							_UpperCamelCase :		Optional[Any]			    =       self(lowerCamelCase__			,add_special_tokens=lowerCamelCase__			,return_tensors=lowerCamelCase__			,**lowerCamelCase__			)
							_UpperCamelCase :		Tuple			    =       self.convert_tokens_to_ids(lowerCamelCase__			)
							_UpperCamelCase :		str			    =       tgt_lang_id
							return inputs





	def      UpperCamelCase_  (			self :    int			,lowerCamelCase__ :    List[str]			,lowerCamelCase__ :    str = "eng_Latn"			,lowerCamelCase__ :    Optional[List[str]] = None			,lowerCamelCase__ :    str = "fra_Latn"			,**lowerCamelCase__ :    Union[str, Any]			,):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       src_lang
							_UpperCamelCase :		List[str]			    =       tgt_lang
							return super().prepare_seqaseq_batch(lowerCamelCase__			,lowerCamelCase__			,**lowerCamelCase__			)





	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							return self.set_src_lang_special_tokens(self.src_lang			)





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							return self.set_tgt_lang_special_tokens(self.tgt_lang			)





	def      UpperCamelCase_  (			self :    str			,lowerCamelCase__ :    List[Any]			):







							'''simple docstring'''
							_UpperCamelCase :		int			    =       self.convert_tokens_to_ids(lowerCamelCase__			)

							if self.legacy_behaviour:
													_UpperCamelCase :		Optional[Any]			    =       []
													_UpperCamelCase :		int			    =       [self.eos_token_id, self.cur_lang_code]
							else:
													_UpperCamelCase :		List[Any]			    =       [self.cur_lang_code]
													_UpperCamelCase :		List[Any]			    =       [self.eos_token_id]

							_UpperCamelCase :		Union[str, Any]			    =       self.convert_ids_to_tokens(self.prefix_tokens			)
							_UpperCamelCase :		int			    =       self.convert_ids_to_tokens(self.suffix_tokens			)

							_UpperCamelCase :		Any			    =       processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,self.prefix_tokens + self.suffix_tokens			)			)			,)





	def      UpperCamelCase_  (			self :    int			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							_UpperCamelCase :		Any			    =       self.convert_tokens_to_ids(lowerCamelCase__			)
							if self.legacy_behaviour:
													_UpperCamelCase :		Tuple			    =       []
													_UpperCamelCase :		str			    =       [self.eos_token_id, self.cur_lang_code]
							else:
													_UpperCamelCase :		Tuple			    =       [self.cur_lang_code]
													_UpperCamelCase :		Optional[Any]			    =       [self.eos_token_id]

							_UpperCamelCase :		int			    =       self.convert_ids_to_tokens(self.prefix_tokens			)
							_UpperCamelCase :		List[Any]			    =       self.convert_ids_to_tokens(self.suffix_tokens			)

							_UpperCamelCase :		Optional[int]			    =       processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str			,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str			,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str			,self.prefix_tokens + self.suffix_tokens			)			)			,)





	def      UpperCamelCase_  (			self :    Any			,lowerCamelCase__ :    str			,lowerCamelCase__ :    Optional[str] = None			):







							'''simple docstring'''
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
													    'tokenizer.'			)

							if not os.path.isdir(lowerCamelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory.'			)
													return
							_UpperCamelCase :		List[Any]			    =       os.path.join(
							    lowerCamelCase__			,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCamelCase__			):
													copyfile(self.vocab_file			,lowerCamelCase__			)

							return (out_vocab_file,)


