


"""simple docstring"""







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__    :							Dict            =					logging.get_logger(__name__)

a__    :							Dict            =					'''▁'''

a__    :							str            =					{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

a__    :							List[Any]            =					{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

a__    :							str            =					{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

a__    :							List[Any]            =					{
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

a__    :							int            =					{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}




class 			UpperCamelCase_     (			UpperCamelCase):

	"""simple docstring"""





	snake_case__    :  List[str]    =      ["input_ids"]

	snake_case__    :  Any    =      VOCAB_FILES_NAMES
	snake_case__    :  Union[str, Any]    =      PRETRAINED_INIT_CONFIGURATION
	snake_case__    :  List[Any]    =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__    :  List[Any]    =      PRETRAINED_VOCAB_FILES_MAP
	snake_case__    :  Dict    =      RESOURCE_FILES_NAMES




	def __init__(			self      :    Optional[int]      ,      UpperCAmelCase__      :    str      ,      UpperCAmelCase__      :    str=None      ,      UpperCAmelCase__      :    List[Any]=False      ,      UpperCAmelCase__      :    Dict="utf8"      ,      UpperCAmelCase__      :    List[str]="[UNK]"      ,      UpperCAmelCase__      :    List[str]="[SEP]"      ,      UpperCAmelCase__      :    Any="[PAD]"      ,      UpperCAmelCase__      :    Optional[int]="[CLS]"      ,      UpperCAmelCase__      :    Union[str, Any]="[MASK]"      ,      UpperCAmelCase__      :    Optional[Dict[str, Any]] = None      ,      **UpperCAmelCase__      :    Tuple      ,      )	->     None:
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.

						__SCREAMING_SNAKE_CASE						       =    {} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=UpperCAmelCase__      ,      unk_token=UpperCAmelCase__      ,      sep_token=UpperCAmelCase__      ,      pad_token=UpperCAmelCase__      ,      cls_token=UpperCAmelCase__      ,      mask_token=UpperCAmelCase__      ,      vocab_file=UpperCAmelCase__      ,      encoding=UpperCAmelCase__      ,      sp_model_kwargs=self.sp_model_kwargs      ,      **UpperCAmelCase__      ,      )
						__SCREAMING_SNAKE_CASE						       =    do_lower_case
						__SCREAMING_SNAKE_CASE						       =    sentencepiece_model_ckpt
						__SCREAMING_SNAKE_CASE						       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(UpperCAmelCase__       )

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
											__SCREAMING_SNAKE_CASE						       =    self.load_vocab(filepath=UpperCAmelCase__       )
						else:
											__SCREAMING_SNAKE_CASE						       =    {self.sp_model.id_to_piece(UpperCAmelCase__       ): id for id in range(self.sp_model.get_piece_size()       )}
						__SCREAMING_SNAKE_CASE						       =    {v: k for k, v in self.vocab.items()}




	def   UpperCAmelCase_				(			self      :    str      ,      UpperCAmelCase__      :    List[Any]       )	->     Union[str, Any]:
						if text is None:
											return None

						__SCREAMING_SNAKE_CASE						       =    self.tokenize(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    "", []

						for i, ch in enumerate(UpperCAmelCase__       ):
											if ch in self.SP_CHAR_MAPPING:
																__SCREAMING_SNAKE_CASE						       =    self.SP_CHAR_MAPPING.get(UpperCAmelCase__       )
											else:
																__SCREAMING_SNAKE_CASE						       =    unicodedata.normalize("NFKC"      ,      UpperCAmelCase__       )
											if self.is_whitespace(UpperCAmelCase__       ):
																continue
											normalized_text += ch
											char_mapping.extend([i] * len(UpperCAmelCase__       )       )

						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    normalized_text, [], 0

						if self.do_lower_case:
											__SCREAMING_SNAKE_CASE						       =    text.lower()

						for token in split_tokens:
											if token[:1] == "▁":
																__SCREAMING_SNAKE_CASE						       =    token[1:]
											__SCREAMING_SNAKE_CASE						       =    text[offset:].index(UpperCAmelCase__       ) + offset
											__SCREAMING_SNAKE_CASE						       =    start + len(UpperCAmelCase__       )

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
											__SCREAMING_SNAKE_CASE						       =    end
						return token_mapping




	@property
	def   UpperCAmelCase_				(			self      :    Optional[int]       )	->     str:
						return len(self.vocab       )




	def   UpperCAmelCase_				(			self      :    Optional[int]       )	->     str:
						return dict(self.vocab      ,      **self.added_tokens_encoder       )




	def __getstate__(			self      :    Dict       )	->     List[Any]:
						__SCREAMING_SNAKE_CASE						       =    self.__dict__.copy()
						__SCREAMING_SNAKE_CASE						       =    None
						return state




	def __setstate__(			self      :    int      ,      UpperCAmelCase__      :    str       )	->     Any:
						__SCREAMING_SNAKE_CASE						       =    d

						# for backward compatibility
						if not hasattr(self      ,      "sp_model_kwargs"       ):
											__SCREAMING_SNAKE_CASE						       =    {}

						__SCREAMING_SNAKE_CASE						       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(self.sentencepiece_model_ckpt       )




	def   UpperCAmelCase_				(			self      :    Optional[Any]      ,      UpperCAmelCase__      :    int       )	->     str:
						return "".join((self.SP_CHAR_MAPPING.get(UpperCAmelCase__      ,      UpperCAmelCase__       ) for c in text)       )




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    Dict      ,      UpperCAmelCase__      :    List[Any]=False      ,      UpperCAmelCase__      :    Optional[int]=6_4      ,      UpperCAmelCase__      :    Dict=0.1       )	->     List[Any]:

						if self.sp_model_kwargs.get("enable_sampling"       ) is True:
											__SCREAMING_SNAKE_CASE						       =    True
						if self.sp_model_kwargs.get("alpha"       ) is not None:
											__SCREAMING_SNAKE_CASE						       =    self.sp_model_kwargs.get("alpha"       )
						if self.sp_model_kwargs.get("nbest_size"       ) is not None:
											__SCREAMING_SNAKE_CASE						       =    self.sp_model_kwargs.get("nbest_size"       )

						if not enable_sampling:
											__SCREAMING_SNAKE_CASE						       =    self.sp_model.EncodeAsPieces(UpperCAmelCase__       )
						else:
											__SCREAMING_SNAKE_CASE						       =    self.sp_model.SampleEncodeAsPieces(UpperCAmelCase__      ,      UpperCAmelCase__      ,      UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    []
						for pi, piece in enumerate(UpperCAmelCase__       ):
											if piece == SPIECE_UNDERLINE:
																if not pieces[pi + 1].startswith(UpperCAmelCase__       ) and pi != 0:
																					new_pieces.append(UpperCAmelCase__       )
																					continue
																else:
																					continue
											__SCREAMING_SNAKE_CASE						       =    0
											for i, chunk in enumerate(UpperCAmelCase__       ):
																if chunk == SPIECE_UNDERLINE:
																					continue
																if self.is_ch_char(UpperCAmelCase__       ) or self.is_punct(UpperCAmelCase__       ):
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]       )
																					new_pieces.append(UpperCAmelCase__       )
																					__SCREAMING_SNAKE_CASE						       =    i + 1
																elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]       )
																					__SCREAMING_SNAKE_CASE						       =    i
																elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]       )
																					__SCREAMING_SNAKE_CASE						       =    i
											if len(UpperCAmelCase__       ) > lst_i:
																new_pieces.append(piece[lst_i:]       )
						return new_pieces




	def   UpperCAmelCase_				(			self      :    Union[str, Any]      ,      UpperCAmelCase__      :    Optional[int]       )	->     List[Any]:
						__SCREAMING_SNAKE_CASE						       =    "".join(UpperCAmelCase__       ).replace(UpperCAmelCase__      ,      " "       ).strip()
						return out_string




	def   UpperCAmelCase_				(			self      :    Any      ,      UpperCAmelCase__      :    Tuple       )	->     int:
						__SCREAMING_SNAKE_CASE						       =    self.convert_ids_to_tokens(UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    "".join(UpperCAmelCase__       ).replace(UpperCAmelCase__      ,      " "       ).strip()
						return out_string




	def   UpperCAmelCase_				(			self      :    str      ,      UpperCAmelCase__      :    Dict       )	->     str:
						return self.vocab.get(UpperCAmelCase__      ,      self.vocab.get(self.unk_token       )       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]      ,      UpperCAmelCase__      :    Union[str, Any]       )	->     Optional[Any]:
						return self.reverse_vocab.get(UpperCAmelCase__      ,      self.unk_token       )




	def   UpperCAmelCase_				(			self      :    Optional[Any]      ,      UpperCAmelCase__      :    Any      ,      UpperCAmelCase__      :    Optional[int]=None       )	->     str:
						if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__SCREAMING_SNAKE_CASE						       =    [self.cls_token_id]
						__SCREAMING_SNAKE_CASE						       =    [self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    Optional[Any]      ,      UpperCAmelCase__      :    List[str]=None       )	->     List[Any]:
						if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




	def   UpperCAmelCase_				(			self      :    int      ,      UpperCAmelCase__      :    Dict      ,      UpperCAmelCase__      :    str=None      ,      UpperCAmelCase__      :    Any=False       )	->     Optional[int]:

						if already_has_special_tokens:
											if token_ids_a is not None:
																raise ValueError(
																    "You should not supply a second sequence if the provided sequence of "
																    "ids is already formatted with special tokens for the model."       )
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
											return [1] + ([0] * len(UpperCAmelCase__       )) + [1, 1] + ([0] * len(UpperCAmelCase__       )) + [1]
						return [1] + ([0] * len(UpperCAmelCase__       )) + [1]




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    List[int]      ,      UpperCAmelCase__      :    Optional[List[int]] = None       )	->     List[int]:
						# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
						if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(UpperCAmelCase__       ) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(UpperCAmelCase__       ) + 1) + [1] * (len(UpperCAmelCase__       ) + 3)




	def   UpperCAmelCase_				(			self      :    int      ,      UpperCAmelCase__      :    str       )	->     Union[str, Any]:
						if "\u4e00" <= char <= "\u9fff":
											return True
						return False




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    List[str]       )	->     Tuple:
						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
						return False




	def   UpperCAmelCase_				(			self      :    List[str]      ,      UpperCAmelCase__      :    Dict       )	->     Union[str, Any]:
						if char in ",;:.?!~，；：。？！《》【】":
											return True
						return False




	def   UpperCAmelCase_				(			self      :    Any      ,      UpperCAmelCase__      :    Any       )	->     Optional[Any]:
						if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
						if len(UpperCAmelCase__       ) == 1:
											__SCREAMING_SNAKE_CASE						       =    unicodedata.category(UpperCAmelCase__       )
											if cat == "Zs":
																return True
						return False




	def   UpperCAmelCase_				(			self      :    Dict      ,      UpperCAmelCase__      :    List[Any]       )	->     Optional[int]:
						__SCREAMING_SNAKE_CASE						       =    {}
						with io.open(UpperCAmelCase__      ,      "r"      ,      encoding="utf-8"       ) as f:
											for index, line in enumerate(UpperCAmelCase__       ):
																__SCREAMING_SNAKE_CASE						       =    line.rstrip("\n"       )
																__SCREAMING_SNAKE_CASE						       =    int(UpperCAmelCase__       )

						return token_to_idx




	def   UpperCAmelCase_				(			self      :    Union[str, Any]      ,      UpperCAmelCase__      :    str      ,      UpperCAmelCase__      :    Optional[str] = None       )	->     Tuple[str]:
						__SCREAMING_SNAKE_CASE						       =    0
						if os.path.isdir(UpperCAmelCase__       ):
											__SCREAMING_SNAKE_CASE						       =    os.path.join(
											    UpperCAmelCase__      ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
						else:
											__SCREAMING_SNAKE_CASE						       =    (filename_prefix + "-" if filename_prefix else "") + save_directory
						with open(UpperCAmelCase__      ,      "w"      ,      encoding="utf-8"       ) as writer:
											for token, token_index in sorted(self.vocab.items()      ,      key=lambda UpperCAmelCase__       : kv[1]       ):
																if index != token_index:
																					logger.warning(
																					    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																					    " Please check that the vocabulary is not corrupted!"       )
																					__SCREAMING_SNAKE_CASE						       =    token_index
																writer.write(token + "\n"       )
																index += 1

						__SCREAMING_SNAKE_CASE						       =    os.path.join(UpperCAmelCase__      ,      "sentencepiece.bpe.model"       )
						with open(UpperCAmelCase__      ,      "wb"       ) as fi:
											__SCREAMING_SNAKE_CASE						       =    self.sp_model.serialized_model_proto()
											fi.write(UpperCAmelCase__       )

						return (vocab_file,)




"""simple docstring"""







from __future__ import annotations




def        UpperCAmelCase__	(lowerCAmelCase_       ):




					'''simple docstring'''


					if not nums:
										return 0
					__SCREAMING_SNAKE_CASE						       =    nums[0]
					__SCREAMING_SNAKE_CASE						       =    0
					for num in nums[1:]:
										__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    (
										    max_excluding + num,
										    max(lowerCAmelCase_  ,						lowerCAmelCase_       ),
										)
					return max(lowerCAmelCase_  ,						lowerCAmelCase_       )


if __name__ == "__main__":
		import doctest

		doctest.testmod()
