





import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


_lowerCamelCase :    List[str]  							= '''
@inproceedings{xu-etal-2016-optimizing,
    title = {Optimizing Statistical Machine Translation for Text Simplification},
    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},
    journal = {Transactions of the Association for Computational Linguistics},
    volume = {4},
    year={2016},
    url = {https://www.aclweb.org/anthology/Q16-1029},
    pages = {401--415
},
@inproceedings{post-2018-call,
    title = "A Call for Clarity in Reporting {BLEU} Scores",
    author = "Post, Matt",
    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
    month = oct,
    year = "2018",
    address = "Belgium, Brussels",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/W18-6319",
    pages = "186--191",
}
'''

_lowerCamelCase :    Union[str, Any]  							= '''\
WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU
It can be used to evaluate the quality of machine-generated texts.
'''


_lowerCamelCase :    Optional[Any]  							= '''
Calculates sari score (between 0 and 100) given a list of source and predicted
sentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.
Args:
    sources: list of source sentences where each sentence should be a string.
    predictions: list of predicted sentences where each sentence should be a string.
    references: list of lists of reference sentences where each sentence should be a string.
Returns:
    sari: sari score
    sacrebleu: sacrebleu score
    exact: exact score

Examples:
    >>> sources=["About 95 species are currently accepted ."]
    >>> predictions=["About 95 you now get in ."]
    >>> references=[["About 95 species are currently known ."]]
    >>> wiki_split = datasets.load_metric("wiki_split")
    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)
    >>> print(results)
    {\'sari\': 21.805555555555557, \'sacrebleu\': 14.535768424205482, \'exact\': 0.0}
'''




def _a (  SCREAMING_SNAKE_CASE__      :  Any       )	->    Union[str, Any]:

       '''simple docstring'''






       def remove_articles(SCREAMING_SNAKE_CASE__      :  Dict       ):
              SCREAMING_SNAKE_CASE__       :      Any     =						re.compile(R"\b(a|an|the)\b"			,				re.UNICODE       )
              return re.sub(SCREAMING_SNAKE_CASE__			,				" "			,				SCREAMING_SNAKE_CASE__       )

       def white_space_fix(SCREAMING_SNAKE_CASE__      :  Optional[Any]       ):
              return " ".join(text.split()       )

       def remove_punc(SCREAMING_SNAKE_CASE__      :  Union[str, Any]       ):
              SCREAMING_SNAKE_CASE__       :      List[str]     =						set(string.punctuation       )
              return "".join(ch for ch in text if ch not in exclude       )

       def lower(SCREAMING_SNAKE_CASE__      :  int       ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE__       )       )       )       )




def _a (  SCREAMING_SNAKE_CASE__      :  Dict			,				SCREAMING_SNAKE_CASE__      :  Dict       )	->    int:

       '''simple docstring'''






       return int(normalize_answer(SCREAMING_SNAKE_CASE__       ) == normalize_answer(SCREAMING_SNAKE_CASE__       )       )




def _a (  SCREAMING_SNAKE_CASE__      :  Union[str, Any]			,				SCREAMING_SNAKE_CASE__      :  Dict       )	->    int:

       '''simple docstring'''






       SCREAMING_SNAKE_CASE__       :      List[str]     =						[any(compute_exact(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       ) for ref in refs       ) for pred, refs in zip(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       )]
       return (sum(SCREAMING_SNAKE_CASE__       ) / len(SCREAMING_SNAKE_CASE__       )) * 1_00




def _a (  SCREAMING_SNAKE_CASE__      :  Union[str, Any]			,				SCREAMING_SNAKE_CASE__      :  Optional[Any]			,				SCREAMING_SNAKE_CASE__      :  Optional[int]			,				SCREAMING_SNAKE_CASE__      :  int       )	->    int:

       '''simple docstring'''






       SCREAMING_SNAKE_CASE__       :      List[Any]     =						[rgram for rgrams in rgramslist for rgram in rgrams]
       SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						Counter(SCREAMING_SNAKE_CASE__       )

       SCREAMING_SNAKE_CASE__       :      Any     =						Counter(SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      int     =						Counter()
       for sgram, scount in sgramcounter.items():
              SCREAMING_SNAKE_CASE__       :      int     =						scount * numref

       SCREAMING_SNAKE_CASE__       :      int     =						Counter(SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      Any     =						Counter()
       for cgram, ccount in cgramcounter.items():
              SCREAMING_SNAKE_CASE__       :      Any     =						ccount * numref

       # KEEP
       SCREAMING_SNAKE_CASE__       :      Dict     =						sgramcounter_rep & cgramcounter_rep
       SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						keepgramcounter_rep & rgramcounter
       SCREAMING_SNAKE_CASE__       :      Optional[int]     =						sgramcounter_rep & rgramcounter

       SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						0
       SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						0
       for keepgram in keepgramcountergood_rep:
              keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
              # Fix an alleged bug [2] in the keep score computation.
              # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
              keeptmpscorea += keepgramcountergood_rep[keepgram]
       # Define 0/0=1 instead of 0 to give higher scores for predictions that match
       #      a target exactly.
       SCREAMING_SNAKE_CASE__       :      List[str]     =						1
       SCREAMING_SNAKE_CASE__       :      List[Any]     =						1
       if len(SCREAMING_SNAKE_CASE__       ) > 0:
              SCREAMING_SNAKE_CASE__       :      str     =						keeptmpscorea / len(SCREAMING_SNAKE_CASE__       )
       if len(SCREAMING_SNAKE_CASE__       ) > 0:
              # Fix an alleged bug [2] in the keep score computation.
              # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
              SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						keeptmpscorea / sum(keepgramcounterall_rep.values()       )
       SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						0
       if keepscore_precision > 0 or keepscore_recall > 0:
              SCREAMING_SNAKE_CASE__       :      str     =						2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

       # DELETION
       SCREAMING_SNAKE_CASE__       :      Dict     =						sgramcounter_rep - cgramcounter_rep
       SCREAMING_SNAKE_CASE__       :      int     =						delgramcounter_rep - rgramcounter
       SCREAMING_SNAKE_CASE__       :      Tuple     =						sgramcounter_rep - rgramcounter
       SCREAMING_SNAKE_CASE__       :      List[str]     =						0
       SCREAMING_SNAKE_CASE__       :      List[str]     =						0
       for delgram in delgramcountergood_rep:
              deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
              deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
       # Define 0/0=1 instead of 0 to give higher scores for predictions that match
       # a target exactly.
       SCREAMING_SNAKE_CASE__       :      Tuple     =						1
       if len(SCREAMING_SNAKE_CASE__       ) > 0:
              SCREAMING_SNAKE_CASE__       :      Tuple     =						deltmpscorea / len(SCREAMING_SNAKE_CASE__       )

       # ADDITION
       SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						set(SCREAMING_SNAKE_CASE__       ) - set(SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      Dict     =						set(SCREAMING_SNAKE_CASE__       ) & set(SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      Optional[int]     =						set(SCREAMING_SNAKE_CASE__       ) - set(SCREAMING_SNAKE_CASE__       )

       SCREAMING_SNAKE_CASE__       :      List[str]     =						0
       for addgram in addgramcountergood:
              addtmpscore += 1

       # Define 0/0=1 instead of 0 to give higher scores for predictions that match
       # a target exactly.
       SCREAMING_SNAKE_CASE__       :      Optional[int]     =						1
       SCREAMING_SNAKE_CASE__       :      Tuple     =						1
       if len(SCREAMING_SNAKE_CASE__       ) > 0:
              SCREAMING_SNAKE_CASE__       :      Dict     =						addtmpscore / len(SCREAMING_SNAKE_CASE__       )
       if len(SCREAMING_SNAKE_CASE__       ) > 0:
              SCREAMING_SNAKE_CASE__       :      Tuple     =						addtmpscore / len(SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      List[Any]     =						0
       if addscore_precision > 0 or addscore_recall > 0:
              SCREAMING_SNAKE_CASE__       :      List[Any]     =						2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

       return (keepscore, delscore_precision, addscore)




def _a (  SCREAMING_SNAKE_CASE__      :  List[str]			,				SCREAMING_SNAKE_CASE__      :  Dict			,				SCREAMING_SNAKE_CASE__      :  str       )	->    Any:

       '''simple docstring'''






       SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						len(SCREAMING_SNAKE_CASE__       )

       SCREAMING_SNAKE_CASE__       :      Dict     =						ssent.split(" "       )
       SCREAMING_SNAKE_CASE__       :      str     =						csent.split(" "       )
       SCREAMING_SNAKE_CASE__       :      str     =						[]
       SCREAMING_SNAKE_CASE__       :      Tuple     =						[]
       SCREAMING_SNAKE_CASE__       :      int     =						[]
       SCREAMING_SNAKE_CASE__       :      int     =						[]
       SCREAMING_SNAKE_CASE__       :      str     =						[]
       SCREAMING_SNAKE_CASE__       :      List[Any]     =						[]

       SCREAMING_SNAKE_CASE__       :      Any     =						[]
       SCREAMING_SNAKE_CASE__       :      List[str]     =						[]
       SCREAMING_SNAKE_CASE__       :      List[str]     =						[]
       SCREAMING_SNAKE_CASE__       :      List[Any]     =						[]
       for rsent in rsents:
              SCREAMING_SNAKE_CASE__       :      List[Any]     =						rsent.split(" "       )
              SCREAMING_SNAKE_CASE__       :      Tuple     =						[]
              SCREAMING_SNAKE_CASE__       :      Dict     =						[]
              SCREAMING_SNAKE_CASE__       :      int     =						[]
              ragramslist.append(SCREAMING_SNAKE_CASE__       )
              for i in range(0			,				len(SCREAMING_SNAKE_CASE__       ) - 1       ):
                     if i < len(SCREAMING_SNAKE_CASE__       ) - 1:
                            SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						ragrams[i] + " " + ragrams[i + 1]
                            ragrams.append(SCREAMING_SNAKE_CASE__       )
                     if i < len(SCREAMING_SNAKE_CASE__       ) - 2:
                            SCREAMING_SNAKE_CASE__       :      List[str]     =						ragrams[i] + " " + ragrams[i + 1] + " " + ragrams[i + 2]
                            ragrams.append(SCREAMING_SNAKE_CASE__       )
                     if i < len(SCREAMING_SNAKE_CASE__       ) - 3:
                            SCREAMING_SNAKE_CASE__       :      Dict     =						ragrams[i] + " " + ragrams[i + 1] + " " + ragrams[i + 2] + " " + ragrams[i + 3]
                            ragrams.append(SCREAMING_SNAKE_CASE__       )
              ragramslist.append(SCREAMING_SNAKE_CASE__       )
              ragramslist.append(SCREAMING_SNAKE_CASE__       )
              ragramslist.append(SCREAMING_SNAKE_CASE__       )

       for i in range(0			,				len(SCREAMING_SNAKE_CASE__       ) - 1       ):
              if i < len(SCREAMING_SNAKE_CASE__       ) - 1:
                     SCREAMING_SNAKE_CASE__       :      Dict     =						sagrams[i] + " " + sagrams[i + 1]
                     sagrams.append(SCREAMING_SNAKE_CASE__       )
              if i < len(SCREAMING_SNAKE_CASE__       ) - 2:
                     SCREAMING_SNAKE_CASE__       :      Tuple     =						sagrams[i] + " " + sagrams[i + 1] + " " + sagrams[i + 2]
                     sagrams.append(SCREAMING_SNAKE_CASE__       )
              if i < len(SCREAMING_SNAKE_CASE__       ) - 3:
                     SCREAMING_SNAKE_CASE__       :      List[Any]     =						sagrams[i] + " " + sagrams[i + 1] + " " + sagrams[i + 2] + " " + sagrams[i + 3]
                     sagrams.append(SCREAMING_SNAKE_CASE__       )

       for i in range(0			,				len(SCREAMING_SNAKE_CASE__       ) - 1       ):
              if i < len(SCREAMING_SNAKE_CASE__       ) - 1:
                     SCREAMING_SNAKE_CASE__       :      Any     =						cagrams[i] + " " + cagrams[i + 1]
                     cagrams.append(SCREAMING_SNAKE_CASE__       )
              if i < len(SCREAMING_SNAKE_CASE__       ) - 2:
                     SCREAMING_SNAKE_CASE__       :      Dict     =						cagrams[i] + " " + cagrams[i + 1] + " " + cagrams[i + 2]
                     cagrams.append(SCREAMING_SNAKE_CASE__       )
              if i < len(SCREAMING_SNAKE_CASE__       ) - 3:
                     SCREAMING_SNAKE_CASE__       :      List[Any]     =						cagrams[i] + " " + cagrams[i + 1] + " " + cagrams[i + 2] + " " + cagrams[i + 3]
                     cagrams.append(SCREAMING_SNAKE_CASE__       )

       (SCREAMING_SNAKE_CASE__)       :      Tuple     =						SARIngram(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       )
       (SCREAMING_SNAKE_CASE__)       :      Union[str, Any]     =						SARIngram(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       )
       (SCREAMING_SNAKE_CASE__)       :      List[Any]     =						SARIngram(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       )
       (SCREAMING_SNAKE_CASE__)       :      List[str]     =						SARIngram(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       )
       SCREAMING_SNAKE_CASE__       :      str     =						sum([keepascore, keepascore, keepascore, keepascore]       ) / 4
       SCREAMING_SNAKE_CASE__       :      Any     =						sum([delascore, delascore, delascore, delascore]       ) / 4
       SCREAMING_SNAKE_CASE__       :      Optional[int]     =						sum([addascore, addascore, addascore, addascore]       ) / 4
       SCREAMING_SNAKE_CASE__       :      Optional[int]     =						(avgkeepscore + avgdelscore + avgaddscore) / 3
       return finalscore




def _a (  SCREAMING_SNAKE_CASE__      :  str			,				SCREAMING_SNAKE_CASE__      :  bool = True			,				SCREAMING_SNAKE_CASE__      :  str = "13a"			,				SCREAMING_SNAKE_CASE__      :  bool = True       )	->    List[str]:

       '''simple docstring'''






       if lowercase:
              SCREAMING_SNAKE_CASE__       :      Dict     =						sentence.lower()

       if tokenizer in ["13a", "intl"]:
              if version.parse(sacrebleu.__version__       ).major >= 2:
                     SCREAMING_SNAKE_CASE__       :      Tuple     =						sacrebleu.metrics.bleu._get_tokenizer(SCREAMING_SNAKE_CASE__       )()(SCREAMING_SNAKE_CASE__       )
              else:
                     SCREAMING_SNAKE_CASE__       :      List[str]     =						sacrebleu.TOKENIZERS[tokenizer]()(SCREAMING_SNAKE_CASE__       )
       elif tokenizer == "moses":
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						sacremoses.MosesTokenizer().tokenize(SCREAMING_SNAKE_CASE__			,				return_str=SCREAMING_SNAKE_CASE__			,				escape=SCREAMING_SNAKE_CASE__       )
       elif tokenizer == "penn":
              SCREAMING_SNAKE_CASE__       :      Dict     =						sacremoses.MosesTokenizer().penn_tokenize(SCREAMING_SNAKE_CASE__			,				return_str=SCREAMING_SNAKE_CASE__       )
       else:
              SCREAMING_SNAKE_CASE__       :      Dict     =						sentence

       if not return_str:
              SCREAMING_SNAKE_CASE__       :      str     =						normalized_sent.split()

       return normalized_sent




def _a (  SCREAMING_SNAKE_CASE__      :  Union[str, Any]			,				SCREAMING_SNAKE_CASE__      :  Any			,				SCREAMING_SNAKE_CASE__      :  int       )	->    Dict:

       '''simple docstring'''






       if not (len(SCREAMING_SNAKE_CASE__       ) == len(SCREAMING_SNAKE_CASE__       ) == len(SCREAMING_SNAKE_CASE__       )):
              raise ValueError("Sources length must match predictions and references lengths."       )
       SCREAMING_SNAKE_CASE__       :      List[str]     =						0
       for src, pred, refs in zip(SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__       ):
              sari_score += SARIsent(normalize(SCREAMING_SNAKE_CASE__       )			,				normalize(SCREAMING_SNAKE_CASE__       )			,				[normalize(SCREAMING_SNAKE_CASE__       ) for sent in refs]       )
       SCREAMING_SNAKE_CASE__       :      int     =						sari_score / len(SCREAMING_SNAKE_CASE__       )
       return 1_00 * sari_score




def _a (  SCREAMING_SNAKE_CASE__      :  int			,				SCREAMING_SNAKE_CASE__      :  Optional[int]			,				SCREAMING_SNAKE_CASE__      :  Tuple="exp"			,				SCREAMING_SNAKE_CASE__      :  Any=None			,				SCREAMING_SNAKE_CASE__      :  Optional[Any]=False			,				SCREAMING_SNAKE_CASE__      :  Any=False			,				SCREAMING_SNAKE_CASE__      :  Optional[Any]=False			,				)	->    Optional[int]:

       '''simple docstring'''






       SCREAMING_SNAKE_CASE__       :      Dict     =						len(references[0]       )
       if any(len(SCREAMING_SNAKE_CASE__       ) != references_per_prediction for refs in references       ):
              raise ValueError("Sacrebleu requires the same number of references for each prediction"       )
       SCREAMING_SNAKE_CASE__       :      List[str]     =						[[refs[i] for refs in references] for i in range(SCREAMING_SNAKE_CASE__       )]
       SCREAMING_SNAKE_CASE__       :      str     =						sacrebleu.corpus_bleu(
           SCREAMING_SNAKE_CASE__			,				SCREAMING_SNAKE_CASE__			,				smooth_method=SCREAMING_SNAKE_CASE__			,				smooth_value=SCREAMING_SNAKE_CASE__			,				force=SCREAMING_SNAKE_CASE__			,				lowercase=SCREAMING_SNAKE_CASE__			,				use_effective_order=SCREAMING_SNAKE_CASE__			,				)
       return output.score







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,     _KWARGS_DESCRIPTION      )
class 						lowerCamelCase       (datasets.Metric      ):







       """simple docstring"""




       def  A_     ( self     :					Dict						)				->			int:





              """simple docstring"""
              return datasets.MetricInfo(
                  description=_DESCRIPTION,			citation=_CITATION,			inputs_description=_KWARGS_DESCRIPTION,			features=datasets.Features(
                      {
                          "predictions": datasets.Value("string",			id="sequence"						),
                          "references": datasets.Sequence(datasets.Value("string",			id="sequence"						),			id="references"						),
                      }						),			codebase_urls=[
                      "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py",
                      "https://github.com/cocoxu/simplification/blob/master/SARI.py",
                      "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py",
                      "https://github.com/mjpost/sacreBLEU",
                  ],			reference_urls=[
                      "https://www.aclweb.org/anthology/Q16-1029.pdf",
                      "https://github.com/mjpost/sacreBLEU",
                      "https://en.wikipedia.org/wiki/BLEU",
                      "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213",
                  ],			)






       def  A_     ( self     :					Any,			_UpperCAmelCase     :					str,			_UpperCAmelCase     :					Optional[Any],			_UpperCAmelCase     :					int						)				->			str:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      Optional[int]     =						{}
              result.update({"sari": compute_sari(sources=_UpperCAmelCase,			predictions=_UpperCAmelCase,			references=_UpperCAmelCase						)}						)
              result.update({"sacrebleu": compute_sacrebleu(predictions=_UpperCAmelCase,			references=_UpperCAmelCase						)}						)
              result.update({"exact": compute_em(predictions=_UpperCAmelCase,			references=_UpperCAmelCase						)}						)
              return result













import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase :    int  							= logging.get_logger(__name__)

_lowerCamelCase :    Optional[Any]  							= '''▁'''

_lowerCamelCase :    Dict  							= {'''vocab_file''': '''sentencepiece.bpe.model'''}

_lowerCamelCase :    int  							= {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_lowerCamelCase :    Optional[Any]  							= {
    '''xlm-roberta-base''': 5_1_2,
    '''xlm-roberta-large''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_1_2,
}


class 						lowerCamelCase       (__lowerCamelCase      ):







							"""simple docstring"""




							UpperCAmelCase_      						=   VOCAB_FILES_NAMES
							UpperCAmelCase_      						=   PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_      						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_      						=   ["input_ids", "attention_mask"]






							def __init__( self     :					Dict,			_UpperCAmelCase     :					str,			_UpperCAmelCase     :					Optional[int]="<s>",			_UpperCAmelCase     :					Optional[int]="</s>",			_UpperCAmelCase     :					Dict="</s>",			_UpperCAmelCase     :					List[Any]="<s>",			_UpperCAmelCase     :					Union[str, Any]="<unk>",			_UpperCAmelCase     :					List[Any]="<pad>",			_UpperCAmelCase     :					str="<mask>",			_UpperCAmelCase     :					Optional[Dict[str, Any]] = None,			**_UpperCAmelCase     :					List[Any],			)				->			None:





														"""simple docstring"""
														# Mask token behave like a normal word, i.e. include the space before it
														SCREAMING_SNAKE_CASE__       :      int     =						AddedToken(_UpperCAmelCase,			lstrip=_UpperCAmelCase,			rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase,			_UpperCAmelCase						) else mask_token

														SCREAMING_SNAKE_CASE__       :      List[str]     =						{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=_UpperCAmelCase,			eos_token=_UpperCAmelCase,			unk_token=_UpperCAmelCase,			sep_token=_UpperCAmelCase,			cls_token=_UpperCAmelCase,			pad_token=_UpperCAmelCase,			mask_token=_UpperCAmelCase,			sp_model_kwargs=self.sp_model_kwargs,			**_UpperCAmelCase,			)

														SCREAMING_SNAKE_CASE__       :      str     =						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
														self.sp_model.Load(str(_UpperCAmelCase						)						)
														SCREAMING_SNAKE_CASE__       :      Tuple     =						vocab_file

														# Original fairseq vocab and spm vocab must be "aligned":
														# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
														# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
														# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
														# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

														# Mimic fairseq token-to-id alignment for the first 4 token
														SCREAMING_SNAKE_CASE__       :      List[str]     =						{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

														# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
														SCREAMING_SNAKE_CASE__       :      Dict     =						1

														SCREAMING_SNAKE_CASE__       :      int     =						len(self.sp_model						) + self.fairseq_offset
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}






							def __getstate__( self     :					str						)				->			Optional[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						self.__dict__.copy()
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						None
														SCREAMING_SNAKE_CASE__       :      Dict     =						self.sp_model.serialized_model_proto()
														return state






							def __setstate__( self     :					int,			_UpperCAmelCase     :					List[Any]						)				->			Any:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Tuple     =						d

														# for backward compatibility
														if not hasattr(self,			"sp_model_kwargs"						):
																					SCREAMING_SNAKE_CASE__       :      Dict     =						{}

														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
														self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)






							def  A_     ( self     :					Any,			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None						)				->			List[int]:





														"""simple docstring"""

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														SCREAMING_SNAKE_CASE__       :      List[str]     =						[self.cls_token_id]
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep






							def  A_     ( self     :					List[Any],			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None,			_UpperCAmelCase     :					bool = False						)				->			List[int]:





														"""simple docstring"""

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=_UpperCAmelCase,			token_ids_a=_UpperCAmelCase,			already_has_special_tokens=_UpperCAmelCase						)

														if token_ids_a is None:
																					return [1] + ([0] * len(_UpperCAmelCase						)) + [1]
														return [1] + ([0] * len(_UpperCAmelCase						)) + [1, 1] + ([0] * len(_UpperCAmelCase						)) + [1]






							def  A_     ( self     :					Union[str, Any],			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None						)				->			List[int]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						[self.sep_token_id]
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






							@property
							def  A_     ( self     :					List[str]						)				->			List[str]:





														"""simple docstring"""
														return len(self.sp_model						) + self.fairseq_offset + 1  # Add the <mask> token






							def  A_     ( self     :					List[Any]						)				->			Optional[int]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						{self.convert_ids_to_tokens(_UpperCAmelCase						): i for i in range(self.vocab_size						)}
														vocab.update(self.added_tokens_encoder						)
														return vocab






							def  A_     ( self     :					List[str],			_UpperCAmelCase     :					str						)				->			List[str]:





														"""simple docstring"""
														return self.sp_model.encode(_UpperCAmelCase,			out_type=_UpperCAmelCase						)






							def  A_     ( self     :					Optional[Any],			_UpperCAmelCase     :					Union[str, Any]						)				->			Union[str, Any]:





														"""simple docstring"""
														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						self.sp_model.PieceToId(_UpperCAmelCase						)

														# Need to return unknown token if the SP model returned 0
														return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






							def  A_     ( self     :					Tuple,			_UpperCAmelCase     :					List[str]						)				->			List[str]:





														"""simple docstring"""
														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset						)






							def  A_     ( self     :					Any,			_UpperCAmelCase     :					int						)				->			List[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      int     =						"".join(_UpperCAmelCase						).replace(_UpperCAmelCase,			" "						).strip()
														return out_string






							def  A_     ( self     :					Union[str, Any],			_UpperCAmelCase     :					str,			_UpperCAmelCase     :					Optional[str] = None						)				->			Tuple[str]:





														"""simple docstring"""
														if not os.path.isdir(_UpperCAmelCase						):
																					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
																					return
														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						os.path.join(
														    _UpperCAmelCase,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

														if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						) and os.path.isfile(self.vocab_file						):
																					copyfile(self.vocab_file,			_UpperCAmelCase						)
														elif not os.path.isfile(self.vocab_file						):
																					with open(_UpperCAmelCase,			"wb"						) as fi:
																												SCREAMING_SNAKE_CASE__       :      Any     =						self.sp_model.serialized_model_proto()
																												fi.write(_UpperCAmelCase						)

														return (out_vocab_file,)






