

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		lowerCamelCase						=		None


lowerCamelCase						=		logging.get_logger(__name__)


lowerCamelCase						=		{'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase						=		{
    '''vocab_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase						=		{
    '''facebook/nllb-large-en-ro''': 1024,
    '''facebook/nllb-200-distilled-600M''': 1024,
}

# fmt: off
lowerCamelCase						=		['''ace_Arab''', '''ace_Latn''', '''acm_Arab''', '''acq_Arab''', '''aeb_Arab''', '''afr_Latn''', '''ajp_Arab''', '''aka_Latn''', '''amh_Ethi''', '''apc_Arab''', '''arb_Arab''', '''ars_Arab''', '''ary_Arab''', '''arz_Arab''', '''asm_Beng''', '''ast_Latn''', '''awa_Deva''', '''ayr_Latn''', '''azb_Arab''', '''azj_Latn''', '''bak_Cyrl''', '''bam_Latn''', '''ban_Latn''', '''bel_Cyrl''', '''bem_Latn''', '''ben_Beng''', '''bho_Deva''', '''bjn_Arab''', '''bjn_Latn''', '''bod_Tibt''', '''bos_Latn''', '''bug_Latn''', '''bul_Cyrl''', '''cat_Latn''', '''ceb_Latn''', '''ces_Latn''', '''cjk_Latn''', '''ckb_Arab''', '''crh_Latn''', '''cym_Latn''', '''dan_Latn''', '''deu_Latn''', '''dik_Latn''', '''dyu_Latn''', '''dzo_Tibt''', '''ell_Grek''', '''eng_Latn''', '''epo_Latn''', '''est_Latn''', '''eus_Latn''', '''ewe_Latn''', '''fao_Latn''', '''pes_Arab''', '''fij_Latn''', '''fin_Latn''', '''fon_Latn''', '''fra_Latn''', '''fur_Latn''', '''fuv_Latn''', '''gla_Latn''', '''gle_Latn''', '''glg_Latn''', '''grn_Latn''', '''guj_Gujr''', '''hat_Latn''', '''hau_Latn''', '''heb_Hebr''', '''hin_Deva''', '''hne_Deva''', '''hrv_Latn''', '''hun_Latn''', '''hye_Armn''', '''ibo_Latn''', '''ilo_Latn''', '''ind_Latn''', '''isl_Latn''', '''ita_Latn''', '''jav_Latn''', '''jpn_Jpan''', '''kab_Latn''', '''kac_Latn''', '''kam_Latn''', '''kan_Knda''', '''kas_Arab''', '''kas_Deva''', '''kat_Geor''', '''knc_Arab''', '''knc_Latn''', '''kaz_Cyrl''', '''kbp_Latn''', '''kea_Latn''', '''khm_Khmr''', '''kik_Latn''', '''kin_Latn''', '''kir_Cyrl''', '''kmb_Latn''', '''kon_Latn''', '''kor_Hang''', '''kmr_Latn''', '''lao_Laoo''', '''lvs_Latn''', '''lij_Latn''', '''lim_Latn''', '''lin_Latn''', '''lit_Latn''', '''lmo_Latn''', '''ltg_Latn''', '''ltz_Latn''', '''lua_Latn''', '''lug_Latn''', '''luo_Latn''', '''lus_Latn''', '''mag_Deva''', '''mai_Deva''', '''mal_Mlym''', '''mar_Deva''', '''min_Latn''', '''mkd_Cyrl''', '''plt_Latn''', '''mlt_Latn''', '''mni_Beng''', '''khk_Cyrl''', '''mos_Latn''', '''mri_Latn''', '''zsm_Latn''', '''mya_Mymr''', '''nld_Latn''', '''nno_Latn''', '''nob_Latn''', '''npi_Deva''', '''nso_Latn''', '''nus_Latn''', '''nya_Latn''', '''oci_Latn''', '''gaz_Latn''', '''ory_Orya''', '''pag_Latn''', '''pan_Guru''', '''pap_Latn''', '''pol_Latn''', '''por_Latn''', '''prs_Arab''', '''pbt_Arab''', '''quy_Latn''', '''ron_Latn''', '''run_Latn''', '''rus_Cyrl''', '''sag_Latn''', '''san_Deva''', '''sat_Beng''', '''scn_Latn''', '''shn_Mymr''', '''sin_Sinh''', '''slk_Latn''', '''slv_Latn''', '''smo_Latn''', '''sna_Latn''', '''snd_Arab''', '''som_Latn''', '''sot_Latn''', '''spa_Latn''', '''als_Latn''', '''srd_Latn''', '''srp_Cyrl''', '''ssw_Latn''', '''sun_Latn''', '''swe_Latn''', '''swh_Latn''', '''szl_Latn''', '''tam_Taml''', '''tat_Cyrl''', '''tel_Telu''', '''tgk_Cyrl''', '''tgl_Latn''', '''tha_Thai''', '''tir_Ethi''', '''taq_Latn''', '''taq_Tfng''', '''tpi_Latn''', '''tsn_Latn''', '''tso_Latn''', '''tuk_Latn''', '''tum_Latn''', '''tur_Latn''', '''twi_Latn''', '''tzm_Tfng''', '''uig_Arab''', '''ukr_Cyrl''', '''umb_Latn''', '''urd_Arab''', '''uzn_Latn''', '''vec_Latn''', '''vie_Latn''', '''war_Latn''', '''wol_Latn''', '''xho_Latn''', '''ydd_Hebr''', '''yor_Latn''', '''yue_Hant''', '''zho_Hans''', '''zho_Hant''', '''zul_Latn''']





class  _a							(    __snake_case):
	_a  :  Optional[int]				   =							VOCAB_FILES_NAMES
	_a  :  Dict				   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a  :  Tuple				   =							PRETRAINED_VOCAB_FILES_MAP
	_a  :  Optional[int]				   =							["input_ids", "attention_mask"]
	_a  :  List[Any]				   =							NllbTokenizer

	_a  :  List[int]				   =							[]
	_a  :  List[int]				   =							[]




	def __init__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		List[Any]=None		, _SCREAMING_SNAKE_CASE						:		Optional[int]=None		, _SCREAMING_SNAKE_CASE						:		Tuple="<s>"		, _SCREAMING_SNAKE_CASE						:		Any="</s>"		, _SCREAMING_SNAKE_CASE						:		str="</s>"		, _SCREAMING_SNAKE_CASE						:		List[Any]="<s>"		, _SCREAMING_SNAKE_CASE						:		Optional[Any]="<unk>"		, _SCREAMING_SNAKE_CASE						:		List[str]="<pad>"		, _SCREAMING_SNAKE_CASE						:		Any="<mask>"		, _SCREAMING_SNAKE_CASE						:		List[str]=None		, _SCREAMING_SNAKE_CASE						:		List[Any]=None		, _SCREAMING_SNAKE_CASE						:		List[Any]=None		, _SCREAMING_SNAKE_CASE						:		List[Any]=False		, **_SCREAMING_SNAKE_CASE						:		Tuple		, )->	Tuple:
			# Mask token behave like a normal word, i.e. include the space before it
			lowerCAmelCase__       :      int   					= AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_		) if isinstance(lowerCamelCase_		, lowerCamelCase_		) else mask_token
			lowerCAmelCase__       :      str   					= legacy_behaviour
			super().__init__(
			    vocab_file=lowerCamelCase_		, tokenizer_file=lowerCamelCase_		, bos_token=lowerCamelCase_		, eos_token=lowerCamelCase_		, sep_token=lowerCamelCase_		, cls_token=lowerCamelCase_		, unk_token=lowerCamelCase_		, pad_token=lowerCamelCase_		, mask_token=lowerCamelCase_		, src_lang=lowerCamelCase_		, tgt_lang=lowerCamelCase_		, additional_special_tokens=lowerCamelCase_		, legacy_behaviour=lowerCamelCase_		, **lowerCamelCase_		, )

			lowerCAmelCase__       :      Optional[int]   					= vocab_file
			lowerCAmelCase__       :      Dict   					= False if not self.vocab_file else True

			lowerCAmelCase__       :      List[Any]   					= FAIRSEQ_LANGUAGE_CODES.copy()

			if additional_special_tokens is not None:
					# Only add those special tokens if they are not already there.
					_additional_special_tokens.extend(
					    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

			self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}		)
			lowerCAmelCase__       :      str   					= {
			    lang_code: self.convert_tokens_to_ids(lowerCamelCase_		) for lang_code in FAIRSEQ_LANGUAGE_CODES
			}

			lowerCAmelCase__       :      List[Any]   					= src_lang if src_lang is not None else """eng_Latn"""
			lowerCAmelCase__       :      Dict   					= self.convert_tokens_to_ids(self._src_lang		)
			lowerCAmelCase__       :      Dict   					= tgt_lang
			self.set_src_lang_special_tokens(self._src_lang		)




	@property
	def 							UpperCAmelCase__( self						:		Optional[Any]		)->	str:
			return self._src_lang




	@src_lang.setter
	def 							UpperCAmelCase__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		str		)->	None:
			lowerCAmelCase__       :      Optional[int]   					= new_src_lang
			self.set_src_lang_special_tokens(self._src_lang		)




	def 							UpperCAmelCase__( self						:		Optional[int]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
			# We don't expect to process pairs, but leave the pair logic for API consistency
			return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




	def 							UpperCAmelCase__( self						:		Tuple		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      Union[str, Any]   					= [self.sep_token_id]
			lowerCAmelCase__       :      Dict   					= [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep		) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]




	def 							UpperCAmelCase__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		List[str]		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str]		, _SCREAMING_SNAKE_CASE						:		Optional[str]		, **_SCREAMING_SNAKE_CASE						:		int		)->	Tuple:
			if src_lang is None or tgt_lang is None:
					raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
			lowerCAmelCase__       :      int   					= src_lang
			lowerCAmelCase__       :      List[str]   					= self(lowerCamelCase_		, add_special_tokens=lowerCamelCase_		, return_tensors=lowerCamelCase_		, **lowerCamelCase_		)
			lowerCAmelCase__       :      Tuple   					= self.convert_tokens_to_ids(lowerCamelCase_		)
			lowerCAmelCase__       :      int   					= tgt_lang_id
			return inputs




	def 							UpperCAmelCase__( self						:		int		, _SCREAMING_SNAKE_CASE						:		List[str]		, _SCREAMING_SNAKE_CASE						:		str = "eng_Latn"		, _SCREAMING_SNAKE_CASE						:		Optional[List[str]] = None		, _SCREAMING_SNAKE_CASE						:		str = "fra_Latn"		, **_SCREAMING_SNAKE_CASE						:		Optional[Any]		, )->	BatchEncoding:
			lowerCAmelCase__       :      Optional[Any]   					= src_lang
			lowerCAmelCase__       :      Optional[Any]   					= tgt_lang
			return super().prepare_seqaseq_batch(lowerCamelCase_		, lowerCamelCase_		, **lowerCamelCase_		)




	def 							UpperCAmelCase__( self						:		int		)->	Optional[int]:
			return self.set_src_lang_special_tokens(self.src_lang		)




	def 							UpperCAmelCase__( self						:		str		)->	str:
			return self.set_tgt_lang_special_tokens(self.tgt_lang		)




	def 							UpperCAmelCase__( self						:		Tuple		, _SCREAMING_SNAKE_CASE						:		int		)->	None:
			lowerCAmelCase__       :      Union[str, Any]   					= self.convert_tokens_to_ids(lowerCamelCase_		)

			if self.legacy_behaviour:
					lowerCAmelCase__       :      str   					= []
					lowerCAmelCase__       :      int   					= [self.eos_token_id, self.cur_lang_code]
			else:
					lowerCAmelCase__       :      Union[str, Any]   					= [self.cur_lang_code]
					lowerCAmelCase__       :      int   					= [self.eos_token_id]

			lowerCAmelCase__       :      Tuple   					= self.convert_ids_to_tokens(self.prefix_tokens		)
			lowerCAmelCase__       :      Union[str, Any]   					= self.convert_ids_to_tokens(self.suffix_tokens		)

			lowerCAmelCase__       :      Dict   					= processors.TemplateProcessing(
			    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str		, pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str		, special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		, self.prefix_tokens + self.suffix_tokens		)		)		, )




	def 							UpperCAmelCase__( self						:		Any		, _SCREAMING_SNAKE_CASE						:		str		)->	None:
			lowerCAmelCase__       :      Any   					= self.convert_tokens_to_ids(lowerCamelCase_		)
			if self.legacy_behaviour:
					lowerCAmelCase__       :      Any   					= []
					lowerCAmelCase__       :      Any   					= [self.eos_token_id, self.cur_lang_code]
			else:
					lowerCAmelCase__       :      Dict   					= [self.cur_lang_code]
					lowerCAmelCase__       :      List[str]   					= [self.eos_token_id]

			lowerCAmelCase__       :      str   					= self.convert_ids_to_tokens(self.prefix_tokens		)
			lowerCAmelCase__       :      Dict   					= self.convert_ids_to_tokens(self.suffix_tokens		)

			lowerCAmelCase__       :      List[str]   					= processors.TemplateProcessing(
			    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str		, pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str		, special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		, self.prefix_tokens + self.suffix_tokens		)		)		, )




	def 							UpperCAmelCase__( self						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str] = None		)->	Tuple[str]:
			if not self.can_save_slow_tokenizer:
					raise ValueError(
					    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
					    '''tokenizer.'''		)

			if not os.path.isdir(lowerCamelCase_		):
					logger.error(F'Vocabulary path ({save_directory}) should be a directory.'		)
					return
			lowerCAmelCase__       :      Optional[Any]   					= os.path.join(
			    lowerCamelCase_		, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

			if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase_		):
					copyfile(self.vocab_file		, lowerCamelCase_		)

			return (out_vocab_file,)








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase						=		[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase						=		{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}





class  _a							(    _lowercase):
	_a  :  Tuple				   =							VOCAB_FILES_NAMES
	_a  :  Dict				   =							PRETRAINED_VOCAB_FILES_MAP
	_a  :  Dict				   =							PRETRAINED_INIT_CONFIGURATION
	_a  :  Union[str, Any]				   =							FunnelTokenizer
	_a  :  int				   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a  :  int				   =							2




	def __init__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Any="<unk>"		, _SCREAMING_SNAKE_CASE						:		Dict="<sep>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"		, _SCREAMING_SNAKE_CASE						:		str="<cls>"		, _SCREAMING_SNAKE_CASE						:		List[str]="<mask>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<s>"		, _SCREAMING_SNAKE_CASE						:		Dict="</s>"		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Dict=True		, _SCREAMING_SNAKE_CASE						:		Tuple=None		, _SCREAMING_SNAKE_CASE						:		str="##"		, **_SCREAMING_SNAKE_CASE						:		List[str]		, )->	List[str]:
			super().__init__(
			    _SCREAMING_SNAKE_CASE		, tokenizer_file=_SCREAMING_SNAKE_CASE		, do_lower_case=_SCREAMING_SNAKE_CASE		, unk_token=_SCREAMING_SNAKE_CASE		, sep_token=_SCREAMING_SNAKE_CASE		, pad_token=_SCREAMING_SNAKE_CASE		, cls_token=_SCREAMING_SNAKE_CASE		, mask_token=_SCREAMING_SNAKE_CASE		, bos_token=_SCREAMING_SNAKE_CASE		, eos_token=_SCREAMING_SNAKE_CASE		, clean_text=_SCREAMING_SNAKE_CASE		, tokenize_chinese_chars=_SCREAMING_SNAKE_CASE		, strip_accents=_SCREAMING_SNAKE_CASE		, wordpieces_prefix=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		, )

			lowerCAmelCase__       :      Any   					= json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
			if (
			    normalizer_state.get('''lowercase'''		, _SCREAMING_SNAKE_CASE		) != do_lower_case
			    or normalizer_state.get('''strip_accents'''		, _SCREAMING_SNAKE_CASE		) != strip_accents
			    or normalizer_state.get('''handle_chinese_chars'''		, _SCREAMING_SNAKE_CASE		) != tokenize_chinese_chars
			):
					lowerCAmelCase__       :      int   					= getattr(_SCREAMING_SNAKE_CASE		, normalizer_state.pop('''type'''		)		)
					lowerCAmelCase__       :      Dict   					= do_lower_case
					lowerCAmelCase__       :      str   					= strip_accents
					lowerCAmelCase__       :      Dict   					= tokenize_chinese_chars
					lowerCAmelCase__       :      str   					= normalizer_class(**_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      List[Any]   					= do_lower_case




	def 							UpperCAmelCase__( self						:		Any		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=None		)->	Optional[int]:
			lowerCAmelCase__       :      Tuple   					= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output




	def 							UpperCAmelCase__( self						:		Optional[Any]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      str   					= [self.sep_token_id]
			lowerCAmelCase__       :      Optional[int]   					= [self.cls_token_id]
			if token_ids_a is None:
					return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
			return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




	def 							UpperCAmelCase__( self						:		str		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str] = None		)->	Tuple[str]:
			lowerCAmelCase__       :      Any   					= self._tokenizer.model.save(_SCREAMING_SNAKE_CASE		, name=_SCREAMING_SNAKE_CASE		)
			return tuple(_SCREAMING_SNAKE_CASE		)




