


"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							:      Any						=      '▁'

__SCREAMING_SNAKE_CASE							:      str						=      {'vocab_file': 'spiece.model'}

__SCREAMING_SNAKE_CASE							:      Any						=      {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'}
}

__SCREAMING_SNAKE_CASE							:      int						=      {
    'google/pegasus-xsum': 512,
}


__SCREAMING_SNAKE_CASE							:      int						=      logging.get_logger(__name__)





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						Union[str, Any]							 =      VOCAB_FILES_NAMES

							__lowercase:						Union[str, Any]							 =      VOCAB_FILES_NAMES
							__lowercase:						str							 =      PRETRAINED_VOCAB_FILES_MAP
							__lowercase:						Dict							 =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase:						Tuple							 =      ["""input_ids""", """attention_mask"""]



							def __init__(			self							: List[Any]						,    UpperCAmelCase_							: Union[str, Any]						,    UpperCAmelCase_							: Dict="<pad>"						,    UpperCAmelCase_							: Optional[Any]="</s>"						,    UpperCAmelCase_							: List[str]="<unk>"						,    UpperCAmelCase_							: List[str]="<mask_2>"						,    UpperCAmelCase_							: Optional[Any]="<mask_1>"						,    UpperCAmelCase_							: Dict=None						,    UpperCAmelCase_							: List[Any]=103						,    UpperCAmelCase_							: Optional[Dict[str, Any]] = None						,    **UpperCAmelCase_							: Dict						,    )   ->None:



														"""simple docstring"""



														snake_case_          =						offset
														if additional_special_tokens is not None:
																					if not isinstance(UpperCAmelCase_						,    UpperCAmelCase_    ):
																												raise TypeError(
																												    F"""additional_special_tokens should be of type {type(UpperCAmelCase_    )}, but is"""
																												    F""" {type(UpperCAmelCase_    )}"""    )

																					snake_case_          =						(
																					    ([mask_token_sent] + additional_special_tokens)
																					    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																					    else additional_special_tokens
																					)
																					# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																					additional_special_tokens_extended += [
																					    F"""<unk_{i}>""" for i in range(len(UpperCAmelCase_    )						,    self.offset - 1    )
																					]

																					if len(set(UpperCAmelCase_    )    ) != len(UpperCAmelCase_    ):
																												raise ValueError(
																												    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
																												    F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""    )
																					snake_case_          =						additional_special_tokens_extended
														else:
																					snake_case_          =						[mask_token_sent] if mask_token_sent is not None else []
																					additional_special_tokens += [F"""<unk_{i}>""" for i in range(2						,    self.offset    )]

														snake_case_          =						{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    eos_token=UpperCAmelCase_						,    unk_token=UpperCAmelCase_						,    mask_token=UpperCAmelCase_						,    pad_token=UpperCAmelCase_						,    mask_token_sent=UpperCAmelCase_						,    offset=UpperCAmelCase_						,    additional_special_tokens=UpperCAmelCase_						,    sp_model_kwargs=self.sp_model_kwargs						,    **UpperCAmelCase_						,    )
														snake_case_          =						mask_token_sent
														snake_case_          =						vocab_file
														snake_case_          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(UpperCAmelCase_    )

														# add special tokens to encoder dict
														snake_case_          =						{
														    0: self.pad_token,
														    1: self.eos_token,
														}

														if self.mask_token_sent is not None:
																					self.encoder.update(
																					    {
																					        2: self.mask_token_sent,
																					        3: self.mask_token,
																					    }    )

														if self.offset > 0:
																					# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
																					# mask_token_sent is already added to list -> so start at 1
																					self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1						,    self.offset - 1    )}    )

														snake_case_          =						{v: k for k, v in self.encoder.items()}



							@property
							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->int:



														"""simple docstring"""



														return len(self.sp_model    ) + self.offset



							def 						lowerCAmelCase    (			self							: List[Any]    )   ->Dict[str, int]:



														"""simple docstring"""



														snake_case_          =						{self.convert_ids_to_tokens(UpperCAmelCase_    ): i for i in range(self.vocab_size    )}
														vocab.update(self.added_tokens_encoder    )
														return vocab



							def __getstate__(			self							: int    )   ->str:



														"""simple docstring"""



														snake_case_          =						self.__dict__.copy()
														snake_case_          =						None
														return state



							def __setstate__(			self							: Tuple						,    UpperCAmelCase_							: int    )   ->Any:



														"""simple docstring"""



														snake_case_          =						d

														# for backward compatibility
														if not hasattr(self						,    """sp_model_kwargs"""    ):
																					snake_case_          =						{}

														snake_case_          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(self.vocab_file    )



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: str    )   ->List[str]:



														"""simple docstring"""



														return self.sp_model.encode(UpperCAmelCase_						,    out_type=UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]						,    UpperCAmelCase_							: str    )   ->int:



														"""simple docstring"""



														if token in self.decoder:
																					return self.decoder[token]
														elif token in self.added_tokens_decoder:
																					return self.added_tokens_decoder[token]
														snake_case_          =						self.sp_model.piece_to_id(UpperCAmelCase_    )
														return sp_id + self.offset



							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: int    )   ->str:



														"""simple docstring"""



														if index in self.encoder:
																					return self.encoder[index]
														elif index in self.added_tokens_encoder:
																					return self.added_tokens_encoder[index]
														else:
																					snake_case_          =						self.sp_model.IdToPiece(index - self.offset    )
														return token



							def 						lowerCAmelCase    (			self							: Optional[Any]						,    UpperCAmelCase_							: Optional[Any]    )   ->Any:



														"""simple docstring"""



														snake_case_          =						[]
														snake_case_          =						""""""
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												out_string += self.sp_model.decode(UpperCAmelCase_    ) + token
																												snake_case_          =						[]
																					else:
																												current_sub_tokens.append(UpperCAmelCase_    )
														out_string += self.sp_model.decode(UpperCAmelCase_    )
														return out_string.strip()



							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: List[Any]=False    )   ->List[str]:



														"""simple docstring"""



														return 1



							def 						lowerCAmelCase    (			self							: Union[str, Any]						,    UpperCAmelCase_							: List[Any]    )   ->Any:



														"""simple docstring"""



														snake_case_          =						set(self.all_special_ids    )  # call it once instead of inside list comp
														all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

														return [1 if x in all_special_ids else 0 for x in seq]



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: List						,    UpperCAmelCase_							: Optional[List] = None						,    UpperCAmelCase_							: bool = False    )   ->List[int]:



														"""simple docstring"""



														if already_has_special_tokens:
																					return self._special_token_mask(UpperCAmelCase_    )
														elif token_ids_a is None:
																					return self._special_token_mask(UpperCAmelCase_    ) + [1]
														else:
																					return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]



							def 						lowerCAmelCase    (			self							: List[Any]						,    UpperCAmelCase_							: Tuple						,    UpperCAmelCase_							: str=None    )   ->List[int]:



														"""simple docstring"""



														if token_ids_a is None:
																					return token_ids_a + [self.eos_token_id]
														# We don't expect to process pairs, but leave the pair logic for API consistency
														return token_ids_a + token_ids_a + [self.eos_token_id]



							def 						lowerCAmelCase    (			self							: List[Any]						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[str] = None    )   ->Tuple[str]:



														"""simple docstring"""



														if not os.path.isdir(UpperCAmelCase_    ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
																					return
														snake_case_          =						os.path.join(
														    UpperCAmelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

														if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCAmelCase_    ) and os.path.isfile(self.vocab_file    ):
																					copyfile(self.vocab_file						,    UpperCAmelCase_    )
														elif not os.path.isfile(self.vocab_file    ):
																					with open(UpperCAmelCase_						,    """wb"""    ) as fi:
																												snake_case_          =						self.sp_model.serialized_model_proto()
																												fi.write(UpperCAmelCase_    )

														return (out_vocab_file,)







"""simple docstring"""
def   _a			( _SCREAMING_SNAKE_CASE			)      ->			int:
							assert column_title.isupper()
							snake_case_          =						0
							snake_case_          =						len(_SCREAMING_SNAKE_CASE			) - 1
							snake_case_          =						0

							while index >= 0:
														snake_case_          =						(ord(column_title[index]			) - 64) * pow(26			,	_SCREAMING_SNAKE_CASE			)
														answer += value
														power += 1
														index -= 1

							return answer


if __name__ == "__main__":
				from doctest import testmod

				testmod()



