






"""simple docstring"""



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__	:    Dict			 = logging.get_logger(__name__)
A__	:    List[Any]			 = {'vocab_file': 'spiece.model'}

A__	:    List[Any]			 = {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A__	:    Optional[Any]			 = {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

A__	:    int			 = '▁'







class        lowercase__      (							snake_case_       ):
			_UpperCAmelCase		:Union[str, Any]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:Optional[Any]             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:Optional[Any]             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


			def __init__(				self					:  int       ,	snake_case__					:  Any       ,	snake_case__					:  Union[str, Any]=True       ,	snake_case__					:  int=True       ,	snake_case__					:  Optional[int]=False       ,	snake_case__					:  int="[CLS]"       ,	snake_case__					:  List[str]="[SEP]"       ,	snake_case__					:  Optional[int]="<unk>"       ,	snake_case__					:  str="[SEP]"       ,	snake_case__					:  List[str]="<pad>"       ,	snake_case__					:  Dict="[CLS]"       ,	snake_case__					:  Any="[MASK]"       ,	snake_case__					:  Optional[Dict[str, Any]] = None       ,	**snake_case__					:  Optional[Any]       ,	):
										lowerCamelCase_	:							Optional[int]									=(
										    AddedToken(snake_case__       ,	lstrip=snake_case__       ,	rstrip=snake_case__       ,	normalized=snake_case__ )
										    if isinstance(snake_case__       ,	snake_case__ )
										    else mask_token
										)

										lowerCamelCase_	:							Optional[Any]									={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=snake_case__       ,	remove_space=snake_case__       ,	keep_accents=snake_case__       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	unk_token=snake_case__       ,	sep_token=snake_case__       ,	pad_token=snake_case__       ,	cls_token=snake_case__       ,	mask_token=snake_case__       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**snake_case__       ,	)

										lowerCamelCase_	:							Optional[int]									=do_lower_case
										lowerCamelCase_	:							Union[str, Any]									=remove_space
										lowerCamelCase_	:							Any									=keep_accents
										lowerCamelCase_	:							Dict									=vocab_file

										lowerCamelCase_	:							Any									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(snake_case__ )


			@property
			def      UpperCAmelCase__						(				self					:  Tuple ):
										return len(self.sp_model )


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										lowerCamelCase_	:							Dict									={self.convert_ids_to_tokens(snake_case__ ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab


			def __getstate__(				self					:  Tuple ):
										lowerCamelCase_	:							Optional[int]									=self.__dict__.copy()
										lowerCamelCase_	:							List[str]									=None
										return state


			def __setstate__(				self					:  Tuple       ,	snake_case__					:  Optional[int] ):
										lowerCamelCase_	:							Any									=d

										# for backward compatibility
										if not hasattr(self       ,	"sp_model_kwargs" ):
																	lowerCamelCase_	:							List[Any]									={}

										lowerCamelCase_	:							List[str]									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(self.vocab_file )


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  Dict ):





										if self.remove_space:
																	lowerCamelCase_	:							Tuple									=' '.join(inputs.strip().split() )
										else:
																	lowerCamelCase_	:							Union[str, Any]									=inputs
										lowerCamelCase_	:							Tuple									=outputs.replace("``"       ,	"\"" ).replace("\'\'"       ,	"\"" )

										if not self.keep_accents:
																	lowerCamelCase_	:							Union[str, Any]									=unicodedata.normalize("NFKD"       ,	snake_case__ )
																	lowerCamelCase_	:							Union[str, Any]									=''.join([c for c in outputs if not unicodedata.combining(snake_case__ )] )
										if self.do_lower_case:
																	lowerCamelCase_	:							int									=outputs.lower()

										return outputs


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  str ):
										lowerCamelCase_	:							List[str]									=self.preprocess_text(snake_case__ )
										lowerCamelCase_	:							List[Any]									=self.sp_model.encode(snake_case__       ,	out_type=snake_case__ )
										lowerCamelCase_	:							Optional[Any]									=[]
										for piece in pieces:
																	if len(snake_case__ ) > 1 and piece[-1] == str("," ) and piece[-2].isdigit():
																								lowerCamelCase_	:							str									=self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case__       ,	"" ) )
																								if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																															if len(cur_pieces[0] ) == 1:
																																						lowerCamelCase_	:							Optional[int]									=cur_pieces[1:]
																															else:
																																						lowerCamelCase_	:							Union[str, Any]									=cur_pieces[0][1:]
																								cur_pieces.append(piece[-1] )
																								new_pieces.extend(snake_case__ )
																	else:
																								new_pieces.append(snake_case__ )

										return new_pieces


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  List[Any] ):
										return self.sp_model.PieceToId(snake_case__ )


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  Union[str, Any] ):
										return self.sp_model.IdToPiece(snake_case__ )


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							Any									=[]
										lowerCamelCase_	:							Dict									=''
										lowerCamelCase_	:							List[str]									=False
										for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																								if not prev_is_special:
																															out_string += " "
																								out_string += self.sp_model.decode(snake_case__ ) + token
																								lowerCamelCase_	:							Dict									=True
																								lowerCamelCase_	:							Any									=[]
																	else:
																								current_sub_tokens.append(snake_case__ )
																								lowerCamelCase_	:							Dict									=False
										out_string += self.sp_model.decode(snake_case__ )
										return out_string.strip()


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None ):
										lowerCamelCase_	:							Optional[int]									=[self.sep_token_id]
										lowerCamelCase_	:							Tuple									=[self.cls_token_id]
										if token_ids_a is None:
																	return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None       ,	snake_case__					:  bool = False ):





										if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case__       ,	token_ids_a=snake_case__       ,	already_has_special_tokens=snake_case__ )

										if token_ids_a is not None:
																	return [1] + ([0] * len(snake_case__ )) + [1] + ([0] * len(snake_case__ )) + [1]
										return [1] + ([0] * len(snake_case__ )) + [1]


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None ):
										lowerCamelCase_	:							int									=[self.sep_token_id]
										lowerCamelCase_	:							List[str]									=[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]


			def      UpperCAmelCase__						(				self					:  Any       ,	snake_case__					:  str       ,	snake_case__					:  Optional[str] = None ):





										if not os.path.isdir(snake_case__ ):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
																	return
										lowerCamelCase_	:							Optional[int]									=os.path.join(
										    snake_case__       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case__ ) and os.path.isfile(self.vocab_file ):
																	copyfile(self.vocab_file       ,	snake_case__ )
										elif not os.path.isfile(self.vocab_file ):
																	with open(snake_case__       ,	"wb" ) as fi:
																								lowerCamelCase_	:							Union[str, Any]									=self.sp_model.serialized_model_proto()
																								fi.write(snake_case__ )

										return (out_vocab_file,)











"""simple docstring"""



def 			_snake_case    (					lowerCamelCase__     :   int  )    ->     int:

							if not isinstance(lowerCamelCase__    ,						lowerCamelCase__  ):
														raise TypeError("only integers accepted as input"  )
							else:
														lowerCamelCase_	:							str									=str(abs(lowerCamelCase__  )  )
														lowerCamelCase_	:							Tuple									=[list(lowerCamelCase__  ) for char in range(len(lowerCamelCase__  )  )]
														for index in range(len(lowerCamelCase__  )  ):
																					num_transpositions[index].pop(lowerCamelCase__  )
														return max(
														    int("".join(list(lowerCamelCase__  )  )  ) for transposition in num_transpositions  )


if __name__ == "__main__":
						__import__('doctest').testmod()



