
"""simple docstring"""

import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


_A						=			'\n@inproceedings{xu-etal-2016-optimizing,\n    title = {Optimizing Statistical Machine Translation for Text Simplification},\n    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},\n    journal = {Transactions of the Association for Computational Linguistics},\n    volume = {4},\n    year={2016},\n    url = {https://www.aclweb.org/anthology/Q16-1029},\n    pages = {401--415\n},\n@inproceedings{post-2018-call,\n    title = "A Call for Clarity in Reporting {BLEU} Scores",\n    author = "Post, Matt",\n    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",\n    month = oct,\n    year = "2018",\n    address = "Belgium, Brussels",\n    publisher = "Association for Computational Linguistics",\n    url = "https://www.aclweb.org/anthology/W18-6319",\n    pages = "186--191",\n}\n'

_A						=			'\\nWIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU\nIt can be used to evaluate the quality of machine-generated texts.\n'


_A						=			'\nCalculates sari score (between 0 and 100) given a list of source and predicted\nsentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.\nArgs:\n    sources: list of source sentences where each sentence should be a string.\n    predictions: list of predicted sentences where each sentence should be a string.\n    references: list of lists of reference sentences where each sentence should be a string.\nReturns:\n    sari: sari score\n    sacrebleu: sacrebleu score\n    exact: exact score\n\nExamples:\n    >>> sources=["About 95 species are currently accepted ."]\n    >>> predictions=["About 95 you now get in ."]\n    >>> references=[["About 95 species are currently known ."]]\n    >>> wiki_split = datasets.load_metric("wiki_split")\n    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)\n    >>> print(results)\n    {\'sari\': 21.805555555555557, \'sacrebleu\': 14.535768424205482, \'exact\': 0.0}\n'
def 		UpperCAmelCase (		a_      ):





		'''simple docstring'''


		def remove_articles(a_      ):
				lowerCamelCase					:      Dict          =   re.compile(r'\b(a|an|the)\b',						re.UNICODE      )
				return re.sub(a_,						' ',						a_      )

		def white_space_fix(a_      ):
				return " ".join(text.split()      )

		def remove_punc(a_      ):
				lowerCamelCase					:      Any          =   set(string.punctuation      )
				return "".join(ch for ch in text if ch not in exclude      )

		def lower(a_      ):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(a_      )      )      )      )
def 		UpperCAmelCase (		a_,						a_      ):





		'''simple docstring'''


		return int(normalize_answer(a_      ) == normalize_answer(a_      )      )
def 		UpperCAmelCase (		a_,						a_      ):





		'''simple docstring'''


		lowerCamelCase					:      Dict          =   [any(compute_exact(a_,						a_      ) for ref in refs      ) for pred, refs in zip(a_,						a_      )]
		return (sum(a_      ) / len(a_      )) * 100
def 		UpperCAmelCase (		a_,						a_,						a_,						a_      ):





		'''simple docstring'''


		lowerCamelCase					:      Union[str, Any]          =   [rgram for rgrams in rgramslist for rgram in rgrams]
		lowerCamelCase					:      Union[str, Any]          =   Counter(a_      )

		lowerCamelCase					:      Optional[int]          =   Counter(a_      )
		lowerCamelCase					:      List[Any]          =   Counter()
		for sgram, scount in sgramcounter.items():
				lowerCamelCase					:      List[Any]          =   scount * numref

		lowerCamelCase					:      Optional[Any]          =   Counter(a_      )
		lowerCamelCase					:      int          =   Counter()
		for cgram, ccount in cgramcounter.items():
				lowerCamelCase					:      Optional[int]          =   ccount * numref

		# KEEP
		lowerCamelCase					:      Optional[Any]          =   sgramcounter_rep & cgramcounter_rep
		lowerCamelCase					:      Any          =   keepgramcounter_rep & rgramcounter
		lowerCamelCase					:      int          =   sgramcounter_rep & rgramcounter

		lowerCamelCase					:      Dict          =   0
		lowerCamelCase					:      str          =   0
		for keepgram in keepgramcountergood_rep:
				keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
				# Fix an alleged bug [2] in the keep score computation.
				# keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
				keeptmpscorea += keepgramcountergood_rep[keepgram]
		# Define 0/0=1 instead of 0 to give higher scores for predictions that match
		#      a target exactly.
		lowerCamelCase					:      Optional[Any]          =   1
		lowerCamelCase					:      List[Any]          =   1
		if len(a_      ) > 0:
				lowerCamelCase					:      Dict          =   keeptmpscorea / len(a_      )
		if len(a_      ) > 0:
				# Fix an alleged bug [2] in the keep score computation.
				# keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
				lowerCamelCase					:      Any          =   keeptmpscorea / sum(keepgramcounterall_rep.values()      )
		lowerCamelCase					:      List[Any]          =   0
		if keepscore_precision > 0 or keepscore_recall > 0:
				lowerCamelCase					:      Union[str, Any]          =   2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

		# DELETION
		lowerCamelCase					:      Optional[int]          =   sgramcounter_rep - cgramcounter_rep
		lowerCamelCase					:      Optional[Any]          =   delgramcounter_rep - rgramcounter
		lowerCamelCase					:      Tuple          =   sgramcounter_rep - rgramcounter
		lowerCamelCase					:      List[str]          =   0
		lowerCamelCase					:      Optional[Any]          =   0
		for delgram in delgramcountergood_rep:
				deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
				deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
		# Define 0/0=1 instead of 0 to give higher scores for predictions that match
		# a target exactly.
		lowerCamelCase					:      List[str]          =   1
		if len(a_      ) > 0:
				lowerCamelCase					:      str          =   deltmpscorea / len(a_      )

		# ADDITION
		lowerCamelCase					:      List[Any]          =   set(a_      ) - set(a_      )
		lowerCamelCase					:      List[str]          =   set(a_      ) & set(a_      )
		lowerCamelCase					:      Tuple          =   set(a_      ) - set(a_      )

		lowerCamelCase					:      List[str]          =   0
		for addgram in addgramcountergood:
				addtmpscore += 1

		# Define 0/0=1 instead of 0 to give higher scores for predictions that match
		# a target exactly.
		lowerCamelCase					:      Tuple          =   1
		lowerCamelCase					:      int          =   1
		if len(a_      ) > 0:
				lowerCamelCase					:      Tuple          =   addtmpscore / len(a_      )
		if len(a_      ) > 0:
				lowerCamelCase					:      int          =   addtmpscore / len(a_      )
		lowerCamelCase					:      Any          =   0
		if addscore_precision > 0 or addscore_recall > 0:
				lowerCamelCase					:      Tuple          =   2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

		return (keepscore, delscore_precision, addscore)
def 		UpperCAmelCase (		a_,						a_,						a_      ):





		'''simple docstring'''


		lowerCamelCase					:      List[Any]          =   len(a_      )

		lowerCamelCase					:      str          =   ssent.split(' '      )
		lowerCamelCase					:      Union[str, Any]          =   csent.split(' '      )
		lowerCamelCase					:      Any          =   []
		lowerCamelCase					:      List[Any]          =   []
		lowerCamelCase					:      Union[str, Any]          =   []
		lowerCamelCase					:      Any          =   []
		lowerCamelCase					:      Any          =   []
		lowerCamelCase					:      Tuple          =   []

		lowerCamelCase					:      Union[str, Any]          =   []
		lowerCamelCase					:      int          =   []
		lowerCamelCase					:      Tuple          =   []
		lowerCamelCase					:      Tuple          =   []
		for rsent in rsents:
				lowerCamelCase					:      Any          =   rsent.split(' '      )
				lowerCamelCase					:      Union[str, Any]          =   []
				lowerCamelCase					:      List[str]          =   []
				lowerCamelCase					:      Any          =   []
				ragramslist.append(a_      )
				for i in range(0,						len(a_      ) - 1      ):
						if i < len(a_      ) - 1:
								lowerCamelCase					:      Tuple          =   ragrams[i] + ' ' + ragrams[i + 1]
								ragrams.append(a_      )
						if i < len(a_      ) - 2:
								lowerCamelCase					:      str          =   ragrams[i] + ' ' + ragrams[i + 1] + ' ' + ragrams[i + 2]
								ragrams.append(a_      )
						if i < len(a_      ) - 3:
								lowerCamelCase					:      Any          =   ragrams[i] + ' ' + ragrams[i + 1] + ' ' + ragrams[i + 2] + ' ' + ragrams[i + 3]
								ragrams.append(a_      )
				ragramslist.append(a_      )
				ragramslist.append(a_      )
				ragramslist.append(a_      )

		for i in range(0,						len(a_      ) - 1      ):
				if i < len(a_      ) - 1:
						lowerCamelCase					:      Tuple          =   sagrams[i] + ' ' + sagrams[i + 1]
						sagrams.append(a_      )
				if i < len(a_      ) - 2:
						lowerCamelCase					:      int          =   sagrams[i] + ' ' + sagrams[i + 1] + ' ' + sagrams[i + 2]
						sagrams.append(a_      )
				if i < len(a_      ) - 3:
						lowerCamelCase					:      Optional[Any]          =   sagrams[i] + ' ' + sagrams[i + 1] + ' ' + sagrams[i + 2] + ' ' + sagrams[i + 3]
						sagrams.append(a_      )

		for i in range(0,						len(a_      ) - 1      ):
				if i < len(a_      ) - 1:
						lowerCamelCase					:      Tuple          =   cagrams[i] + ' ' + cagrams[i + 1]
						cagrams.append(a_      )
				if i < len(a_      ) - 2:
						lowerCamelCase					:      Optional[int]          =   cagrams[i] + ' ' + cagrams[i + 1] + ' ' + cagrams[i + 2]
						cagrams.append(a_      )
				if i < len(a_      ) - 3:
						lowerCamelCase					:      Union[str, Any]          =   cagrams[i] + ' ' + cagrams[i + 1] + ' ' + cagrams[i + 2] + ' ' + cagrams[i + 3]
						cagrams.append(a_      )

		((lowerCamelCase)     ,   (lowerCamelCase)     ,   (lowerCamelCase))					:      Optional[int]          =   SARIngram(a_,						a_,						a_,						a_      )
		((lowerCamelCase)     ,   (lowerCamelCase)     ,   (lowerCamelCase))					:      str          =   SARIngram(a_,						a_,						a_,						a_      )
		((lowerCamelCase)     ,   (lowerCamelCase)     ,   (lowerCamelCase))					:      str          =   SARIngram(a_,						a_,						a_,						a_      )
		((lowerCamelCase)     ,   (lowerCamelCase)     ,   (lowerCamelCase))					:      Dict          =   SARIngram(a_,						a_,						a_,						a_      )
		lowerCamelCase					:      Dict          =   sum([keepascore, keepascore, keepascore, keepascore]      ) / 4
		lowerCamelCase					:      List[str]          =   sum([delascore, delascore, delascore, delascore]      ) / 4
		lowerCamelCase					:      Optional[int]          =   sum([addascore, addascore, addascore, addascore]      ) / 4
		lowerCamelCase					:      int          =   (avgkeepscore + avgdelscore + avgaddscore) / 3
		return finalscore
def 		UpperCAmelCase (		a_,						a_ = True,						a_ = "13a",						a_ = True      ):





		'''simple docstring'''


		if lowercase:
				lowerCamelCase					:      Optional[int]          =   sentence.lower()

		if tokenizer in ["13a", "intl"]:
				if version.parse(sacrebleu.__version__      ).major >= 2:
						lowerCamelCase					:      Optional[Any]          =   sacrebleu.metrics.bleu._get_tokenizer(a_      )()(a_      )
				else:
						lowerCamelCase					:      Tuple          =   sacrebleu.TOKENIZERS[tokenizer]()(a_      )
		elif tokenizer == "moses":
				lowerCamelCase					:      Optional[Any]          =   sacremoses.MosesTokenizer().tokenize(a_,						return_str=a_,						escape=a_      )
		elif tokenizer == "penn":
				lowerCamelCase					:      List[str]          =   sacremoses.MosesTokenizer().penn_tokenize(a_,						return_str=a_      )
		else:
				lowerCamelCase					:      str          =   sentence

		if not return_str:
				lowerCamelCase					:      str          =   normalized_sent.split()

		return normalized_sent
def 		UpperCAmelCase (		a_,						a_,						a_      ):





		'''simple docstring'''


		if not (len(a_      ) == len(a_      ) == len(a_      )):
				raise ValueError('Sources length must match predictions and references lengths.'      )
		lowerCamelCase					:      List[str]          =   0
		for src, pred, refs in zip(a_,						a_,						a_      ):
				sari_score += SARIsent(normalize(a_      ),						normalize(a_      ),						[normalize(a_      ) for sent in refs]      )
		lowerCamelCase					:      Dict          =   sari_score / len(a_      )
		return 100 * sari_score
def 		UpperCAmelCase (		a_,						a_,						a_="exp",						a_=None,						a_=False,						a_=False,						a_=False,						):





		'''simple docstring'''


		lowerCamelCase					:      Union[str, Any]          =   len(references[0]      )
		if any(len(a_      ) != references_per_prediction for refs in references      ):
				raise ValueError('Sacrebleu requires the same number of references for each prediction'      )
		lowerCamelCase					:      Any          =   [[refs[i] for refs in references] for i in range(a_      )]
		lowerCamelCase					:      str          =   sacrebleu.corpus_bleu(
		    a_,						a_,						smooth_method=a_,						smooth_value=a_,						force=a_,						lowercase=a_,						use_effective_order=a_,						)
		return output.score






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,	_KWARGS_DESCRIPTION		)
class 	_lowercase       (    datasets.Metric		):




	def       _UpperCamelCase   (			self )		->							Any:
			return datasets.MetricInfo(
			    description=_DESCRIPTION		,			citation=_CITATION		,			inputs_description=_KWARGS_DESCRIPTION		,			features=datasets.Features(
			        {
			            'predictions': datasets.Value('string'		,			id='sequence' ),
			            'references': datasets.Sequence(datasets.Value('string'		,			id='sequence' )		,			id='references' ),
			        } )		,			codebase_urls=[
			        'https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py',
			        'https://github.com/cocoxu/simplification/blob/master/SARI.py',
			        'https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py',
			        'https://github.com/mjpost/sacreBLEU',
			    ]		,			reference_urls=[
			        'https://www.aclweb.org/anthology/Q16-1029.pdf',
			        'https://github.com/mjpost/sacreBLEU',
			        'https://en.wikipedia.org/wiki/BLEU',
			        'https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213',
			    ]		,			)




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							int:
			lowerCamelCase					:      int          =   {}
			result.update({'sari': compute_sari(sources=UpperCAmelCase_		,			predictions=UpperCAmelCase_		,			references=UpperCAmelCase_ )} )
			result.update({'sacrebleu': compute_sacrebleu(predictions=UpperCAmelCase_		,			references=UpperCAmelCase_ )} )
			result.update({'exact': compute_em(predictions=UpperCAmelCase_		,			references=UpperCAmelCase_ )} )
			return result








"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_A						=			logging.get_logger(__name__)


_A						=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
_A						=			{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

_A						=			{
    'facebook/bart-base': 1_0_2_4,
    'facebook/bart-large': 1_0_2_4,
    'facebook/bart-large-mnli': 1_0_2_4,
    'facebook/bart-large-cnn': 1_0_2_4,
    'facebook/bart-large-xsum': 1_0_2_4,
    'yjernite/bart_eli5': 1_0_2_4,
}






class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     VOCAB_FILES_NAMES
	lowercase_				=     PRETRAINED_VOCAB_FILES_MAP
	lowercase_				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_				=     ['input_ids', 'attention_mask']
	lowercase_				=     BartTokenizer




	def __init__(			self		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_="replace"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="<unk>"		,			UpperCAmelCase_="<pad>"		,			UpperCAmelCase_="<mask>"		,			UpperCAmelCase_=False		,			UpperCAmelCase_=True		,			**UpperCAmelCase_		,			)		->							Union[str, Any]:
			super().__init__(
			    UpperCAmelCase_		,			UpperCAmelCase_		,			tokenizer_file=UpperCAmelCase_		,			errors=UpperCAmelCase_		,			bos_token=UpperCAmelCase_		,			eos_token=UpperCAmelCase_		,			sep_token=UpperCAmelCase_		,			cls_token=UpperCAmelCase_		,			unk_token=UpperCAmelCase_		,			pad_token=UpperCAmelCase_		,			mask_token=UpperCAmelCase_		,			add_prefix_space=UpperCAmelCase_		,			trim_offsets=UpperCAmelCase_		,			**UpperCAmelCase_		,			)

			lowerCamelCase					:      List[Any]          =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
			if pre_tok_state.get('add_prefix_space'		,			UpperCAmelCase_ ) != add_prefix_space:
					lowerCamelCase					:      Tuple          =   getattr(UpperCAmelCase_		,			pre_tok_state.pop('type' ) )
					lowerCamelCase					:      Optional[Any]          =   add_prefix_space
					lowerCamelCase					:      str          =   pre_tok_class(**UpperCAmelCase_ )

			lowerCamelCase					:      Optional[Any]          =   add_prefix_space

			# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
			lowerCamelCase					:      Dict          =   'post_processor'
			lowerCamelCase					:      str          =   getattr(self.backend_tokenizer		,			UpperCAmelCase_		,			UpperCAmelCase_ )
			if tokenizer_component_instance:
					lowerCamelCase					:      Optional[Any]          =   json.loads(tokenizer_component_instance.__getstate__() )

					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
					if "sep" in state:
							lowerCamelCase					:      int          =   tuple(state['sep'] )
					if "cls" in state:
							lowerCamelCase					:      str          =   tuple(state['cls'] )

					lowerCamelCase					:      Optional[Any]          =   False

					if state.get('add_prefix_space'		,			UpperCAmelCase_ ) != add_prefix_space:
							lowerCamelCase					:      Dict          =   add_prefix_space
							lowerCamelCase					:      Tuple          =   True

					if state.get('trim_offsets'		,			UpperCAmelCase_ ) != trim_offsets:
							lowerCamelCase					:      Tuple          =   trim_offsets
							lowerCamelCase					:      Dict          =   True

					if changes_to_apply:
							lowerCamelCase					:      Optional[int]          =   getattr(UpperCAmelCase_		,			state.pop('type' ) )
							lowerCamelCase					:      Any          =   component_class(**UpperCAmelCase_ )
							setattr(self.backend_tokenizer		,			UpperCAmelCase_		,			UpperCAmelCase_ )




	@property
	def       _UpperCamelCase   (			self )		->							str:
			if self._mask_token is None:
					if self.verbose:
							logger.error('Using mask_token, but it is not set yet.' )
					return None
			return str(self._mask_token )




	@mask_token.setter
	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							List[Any]:
			lowerCamelCase					:      Optional[int]          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else value
			lowerCamelCase					:      int          =   value




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							BatchEncoding:
			lowerCamelCase					:      str          =   kwargs.get('is_split_into_words'		,			UpperCAmelCase_ )

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    'to use it with pretokenized inputs.' )

			return super()._batch_encode_plus(*UpperCAmelCase_		,			**UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							BatchEncoding:
			lowerCamelCase					:      Optional[Any]          =   kwargs.get('is_split_into_words'		,			UpperCAmelCase_ )

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    'to use it with pretokenized inputs.' )

			return super()._encode_plus(*UpperCAmelCase_		,			**UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							Tuple[str]:
			lowerCamelCase					:      Any          =   self._tokenizer.model.save(UpperCAmelCase_		,			name=UpperCAmelCase_ )
			return tuple(UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=None )		->							List[Any]:
			lowerCamelCase					:      Union[str, Any]          =   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
			if token_ids_a is None:
					return output

			return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							List[int]:
			lowerCamelCase					:      List[Any]          =   [self.sep_token_id]
			lowerCamelCase					:      str          =   [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep ) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






