






"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase    :   Optional[Any]   			=						logging.get_logger(__name__)


_UpperCamelCase    :   Optional[Any]   			=						'▁'

_UpperCamelCase    :   Tuple   			=						{'vocab_file': 'spiece.model'}

_UpperCamelCase    :   List[Any]   			=						{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    }
}

_UpperCamelCase    :   Optional[int]   			=						{
    'google/reformer-crime-and-punishment': 52_4288,
}






class 						snake_case  (  _lowercase	):
				__magic_name__   					=					VOCAB_FILES_NAMES
				__magic_name__   					=					PRETRAINED_VOCAB_FILES_MAP
				__magic_name__   					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__   					=					['''input_ids''', '''attention_mask''']





				def __init__(							self     : Union[str, Any]    ,		A     : Union[str, Any]    ,		A     : Dict="</s>"    ,		A     : Any="<unk>"    ,		A     : Any=[]    ,		A     : Optional[Dict[str, Any]] = None    ,		**A     : Optional[int]    ,		):


						'''simple docstring'''




						a					:      Union[str, Any]    =       {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=__UpperCamelCase    ,		unk_token=__UpperCamelCase    ,		additional_special_tokens=__UpperCamelCase    ,		sp_model_kwargs=self.sp_model_kwargs    ,		**__UpperCamelCase    ,		)

						a					:      Tuple    =       vocab_file
						a					:      Optional[Any]    =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
						self.sp_model.Load(__UpperCamelCase      )





				@property
				def     lowerCamelCase__				(							self     : Union[str, Any]      ):


						'''simple docstring'''




						return self.sp_model.get_piece_size()





				def     lowerCamelCase__				(							self     : Union[str, Any]      ):


						'''simple docstring'''




						a					:      Dict    =       {self.convert_ids_to_tokens(__UpperCamelCase      ): i for i in range(self.vocab_size      )}
						vocab.update(self.added_tokens_encoder      )
						return vocab





				def __getstate__(							self     : Optional[Any]      ):


						'''simple docstring'''




						a					:      str    =       self.__dict__.copy()
						a					:      str    =       None
						return state





				def __setstate__(							self     : Optional[Any]    ,		A     : Union[str, Any]      ):


						'''simple docstring'''




						a					:      str    =       d

						# for backward compatibility
						if not hasattr(self    ,		'sp_model_kwargs'      ):
								a					:      List[Any]    =       {}

						a					:      Union[str, Any]    =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
						self.sp_model.Load(self.vocab_file      )





				def     lowerCamelCase__				(							self     : List[str]    ,		A     : str      ):


						'''simple docstring'''




						return self.sp_model.encode(__UpperCamelCase    ,		out_type=__UpperCamelCase      )





				def     lowerCamelCase__				(							self     : List[str]    ,		A     : Any      ):


						'''simple docstring'''




						return self.sp_model.piece_to_id(__UpperCamelCase      )





				def     lowerCamelCase__				(							self     : Union[str, Any]    ,		A     : Optional[Any]      ):


						'''simple docstring'''




						if index < self.sp_model.get_piece_size():
								a					:      Optional[Any]    =       self.sp_model.IdToPiece(__UpperCamelCase      )
						return token





				def     lowerCamelCase__				(							self     : int    ,		A     : Union[str, Any]      ):


						'''simple docstring'''




						a					:      Any    =       []
						a					:      List[Any]    =       ''
						for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
										out_string += self.sp_model.decode(__UpperCamelCase      ) + token
										a					:      str    =       []
								else:
										current_sub_tokens.append(__UpperCamelCase      )
						out_string += self.sp_model.decode(__UpperCamelCase      )
						return out_string.strip()





				def     lowerCamelCase__				(							self     : List[Any]    ,		A     : str    ,		A     : Optional[str] = None      ):


						'''simple docstring'''




						if not os.path.isdir(__UpperCamelCase      ):
								logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
								return
						a					:      Optional[int]    =       os.path.join(
						    __UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

						if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCamelCase      ) and os.path.isfile(self.vocab_file      ):
								copyfile(self.vocab_file    ,		__UpperCamelCase      )
						elif not os.path.isfile(self.vocab_file      ):
								with open(__UpperCamelCase    ,		'wb'      ) as fi:
										a					:      Union[str, Any]    =       self.sp_model.serialized_model_proto()
										fi.write(__UpperCamelCase      )

						return (out_vocab_file,)












"""simple docstring"""


from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels
from .squad import SquadExample, SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features
from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor
from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels




