






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  :Optional[Any]             =							logging.get_logger(__name__)

lowerCAmelCase__  :Dict             =							'''▁'''

lowerCAmelCase__  :Dict             =							{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

lowerCAmelCase__  :Dict             =							{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

lowerCAmelCase__  :int             =							{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

lowerCAmelCase__  :Optional[int]             =							{
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

lowerCAmelCase__  :Any             =							{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}

class 		__a				(    UpperCAmelCase      ):
		_a     :  List[str]           =     ["input_ids"]

		_a     :  List[str]           =     VOCAB_FILES_NAMES
		_a     :  str           =     PRETRAINED_INIT_CONFIGURATION
		_a     :  Tuple           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     :  Any           =     PRETRAINED_VOCAB_FILES_MAP
		_a     :  Tuple           =     RESOURCE_FILES_NAMES
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=False      ,   _SCREAMING_SNAKE_CASE="utf8"      ,   _SCREAMING_SNAKE_CASE="[UNK]"      ,   _SCREAMING_SNAKE_CASE="[SEP]"      ,   _SCREAMING_SNAKE_CASE="[PAD]"      ,   _SCREAMING_SNAKE_CASE="[CLS]"      ,   _SCREAMING_SNAKE_CASE="[MASK]"      ,   _SCREAMING_SNAKE_CASE = None      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	None:




									"""simple docstring"""





									_UpperCAmelCase					=      {} if sp_model_kwargs is None else sp_model_kwargs
									super().__init__(
									    do_lower_case=_SCREAMING_SNAKE_CASE      ,   unk_token=_SCREAMING_SNAKE_CASE      ,   sep_token=_SCREAMING_SNAKE_CASE      ,   pad_token=_SCREAMING_SNAKE_CASE      ,   cls_token=_SCREAMING_SNAKE_CASE      ,   mask_token=_SCREAMING_SNAKE_CASE      ,   vocab_file=_SCREAMING_SNAKE_CASE      ,   encoding=_SCREAMING_SNAKE_CASE      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **_SCREAMING_SNAKE_CASE      ,   )
									_UpperCAmelCase					=      do_lower_case
									_UpperCAmelCase					=      sentencepiece_model_ckpt
									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(_SCREAMING_SNAKE_CASE						)

									# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
									if vocab_file is not None:
																_UpperCAmelCase					=      self.load_vocab(filepath=_SCREAMING_SNAKE_CASE						)
									else:
																_UpperCAmelCase					=      {self.sp_model.id_to_piece(_SCREAMING_SNAKE_CASE						): id for id in range(self.sp_model.get_piece_size()						)}
									_UpperCAmelCase					=      {v: k for k, v in self.vocab.items()}
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Dict:




									"""simple docstring"""





									if text is None:
																return None

									_UpperCAmelCase					=      self.tokenize(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase		,	_UpperCAmelCase					=      '', []

									for i, ch in enumerate(_SCREAMING_SNAKE_CASE						):
																if ch in self.SP_CHAR_MAPPING:
																							_UpperCAmelCase					=      self.SP_CHAR_MAPPING.get(_SCREAMING_SNAKE_CASE						)
																else:
																							_UpperCAmelCase					=      unicodedata.normalize('NFKC'      ,   _SCREAMING_SNAKE_CASE						)
																if self.is_whitespace(_SCREAMING_SNAKE_CASE						):
																							continue
																normalized_text += ch
																char_mapping.extend([i] * len(_SCREAMING_SNAKE_CASE						)						)

									_UpperCAmelCase		,	_UpperCAmelCase		,	_UpperCAmelCase					=      normalized_text, [], 0

									if self.do_lower_case:
																_UpperCAmelCase					=      text.lower()

									for token in split_tokens:
																if token[:1] == "▁":
																							_UpperCAmelCase					=      token[1:]
																_UpperCAmelCase					=      text[offset:].index(_SCREAMING_SNAKE_CASE						) + offset
																_UpperCAmelCase					=      start + len(_SCREAMING_SNAKE_CASE						)

																token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
																_UpperCAmelCase					=      end
									return token_mapping
		@property
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									return len(self.vocab						)
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									return dict(self.vocab      ,   **self.added_tokens_encoder						)
		def __getstate__(      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.__dict__.copy()
									_UpperCAmelCase					=      None
									return state
		def __setstate__(      self      ,   _SCREAMING_SNAKE_CASE						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      d

									# for backward compatibility
									if not hasattr(self      ,   'sp_model_kwargs'						):
																_UpperCAmelCase					=      {}

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(self.sentencepiece_model_ckpt						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Union[str, Any]:




									"""simple docstring"""





									return "".join((self.SP_CHAR_MAPPING.get(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						) for c in text)						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=False      ,   _SCREAMING_SNAKE_CASE=64      ,   _SCREAMING_SNAKE_CASE=0.1						)			->	str:




									"""simple docstring"""





									if self.sp_model_kwargs.get('enable_sampling'						) is True:
																_UpperCAmelCase					=      True
									if self.sp_model_kwargs.get('alpha'						) is not None:
																_UpperCAmelCase					=      self.sp_model_kwargs.get('alpha'						)
									if self.sp_model_kwargs.get('nbest_size'						) is not None:
																_UpperCAmelCase					=      self.sp_model_kwargs.get('nbest_size'						)

									if not enable_sampling:
																_UpperCAmelCase					=      self.sp_model.EncodeAsPieces(_SCREAMING_SNAKE_CASE						)
									else:
																_UpperCAmelCase					=      self.sp_model.SampleEncodeAsPieces(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      []
									for pi, piece in enumerate(_SCREAMING_SNAKE_CASE						):
																if piece == SPIECE_UNDERLINE:
																							if not pieces[pi + 1].startswith(_SCREAMING_SNAKE_CASE						) and pi != 0:
																														new_pieces.append(_SCREAMING_SNAKE_CASE						)
																														continue
																							else:
																														continue
																_UpperCAmelCase					=      0
																for i, chunk in enumerate(_SCREAMING_SNAKE_CASE						):
																							if chunk == SPIECE_UNDERLINE:
																														continue
																							if self.is_ch_char(_SCREAMING_SNAKE_CASE						) or self.is_punct(_SCREAMING_SNAKE_CASE						):
																														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]						)
																														new_pieces.append(_SCREAMING_SNAKE_CASE						)
																														_UpperCAmelCase					=      i + 1
																							elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]						)
																														_UpperCAmelCase					=      i
																							elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																														if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]						)
																														_UpperCAmelCase					=      i
																if len(_SCREAMING_SNAKE_CASE						) > lst_i:
																							new_pieces.append(piece[lst_i:]						)
									return new_pieces
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      ''.join(_SCREAMING_SNAKE_CASE						).replace(_SCREAMING_SNAKE_CASE      ,   ' '						).strip()
									return out_string
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      ''.join(_SCREAMING_SNAKE_CASE						).replace(_SCREAMING_SNAKE_CASE      ,   ' '						).strip()
									return out_string
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									return self.vocab.get(_SCREAMING_SNAKE_CASE      ,   self.vocab.get(self.unk_token						)						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Dict:




									"""simple docstring"""





									return self.reverse_vocab.get(_SCREAMING_SNAKE_CASE      ,   self.unk_token						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None						)			->	List[Any]:




									"""simple docstring"""





									if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]
									_UpperCAmelCase					=      [self.sep_token_id]
									return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None						)			->	int:




									"""simple docstring"""





									if offset_mapping_a is None:
																return [(0, 0)] + offset_mapping_a + [(0, 0)]

									return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=False						)			->	Any:




									"""simple docstring"""





									if already_has_special_tokens:
																if token_ids_a is not None:
																							raise ValueError(
																							    'You should not supply a second sequence if the provided sequence of '
																							    'ids is already formatted with special tokens for the model.'						)
																return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

									if token_ids_a is not None:
																return [1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1, 1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1]
									return [1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1]
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									if token_ids_a is None:
																# [CLS] X [SEP]
																return (len(_SCREAMING_SNAKE_CASE						) + 2) * [0]

									# [CLS] A [SEP] [SEP] B [SEP]
									return [0] * (len(_SCREAMING_SNAKE_CASE						) + 1) + [1] * (len(_SCREAMING_SNAKE_CASE						) + 3)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Any:




									"""simple docstring"""





									if "\u4e00" <= char <= "\u9fff":
																return True
									return False
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																return True
									return False
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[int]:




									"""simple docstring"""





									if char in ",;:.?!~，；：。？！《》【】":
																return True
									return False
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[Any]:




									"""simple docstring"""





									if char == " " or char == "\t" or char == "\n" or char == "\r":
																return True
									if len(_SCREAMING_SNAKE_CASE						) == 1:
																_UpperCAmelCase					=      unicodedata.category(_SCREAMING_SNAKE_CASE						)
																if cat == "Zs":
																							return True
									return False
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      {}
									with io.open(_SCREAMING_SNAKE_CASE      ,   'r'      ,   encoding='utf-8'						) as f:
																for index, line in enumerate(_SCREAMING_SNAKE_CASE						):
																							_UpperCAmelCase					=      line.rstrip('\n'						)
																							_UpperCAmelCase					=      int(_SCREAMING_SNAKE_CASE						)

									return token_to_idx






		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	Tuple[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      0
									if os.path.isdir(_SCREAMING_SNAKE_CASE						):
																_UpperCAmelCase					=      os.path.join(
																    _SCREAMING_SNAKE_CASE      ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
									else:
																_UpperCAmelCase					=      (filename_prefix + '-' if filename_prefix else '') + save_directory
									with open(_SCREAMING_SNAKE_CASE      ,   'w'      ,   encoding='utf-8'						) as writer:
																for token, token_index in sorted(self.vocab.items()      ,   key=lambda _SCREAMING_SNAKE_CASE						: kv[1]						):
																							if index != token_index:
																														logger.warning(
																														    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																														    ' Please check that the vocabulary is not corrupted!'						)
																														_UpperCAmelCase					=      token_index
																							writer.write(token + '\n'						)
																							index += 1

									_UpperCAmelCase					=      os.path.join(_SCREAMING_SNAKE_CASE      ,   'sentencepiece.bpe.model'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as fi:
																_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
																fi.write(_SCREAMING_SNAKE_CASE						)

									return (vocab_file,)




from __future__ import annotations

import time
from collections.abc import Sequence
from random import randint

from matplotlib import pyplot as plt

def 	lowerCAmelCase__			(      a__:		Sequence[float]      ,  a__:		int      ,  a__:		int	)  ->		tuple[int | None, int | None, float]:


							'''simple docstring'''





							if not arr:
														return None, None, 0
							if low == high:
														return low, high, arr[low]

							_UpperCAmelCase					=      (low + high) // 2
							_UpperCAmelCase		,	_UpperCAmelCase		,	_UpperCAmelCase					=      max_subarray(a__      ,  a__      ,  a__	)
							_UpperCAmelCase		,	_UpperCAmelCase		,	_UpperCAmelCase					=      max_subarray(a__      ,  mid + 1      ,  a__	)
							_UpperCAmelCase		,	_UpperCAmelCase		,	_UpperCAmelCase					=      max_cross_sum(a__      ,  a__      ,  a__      ,  a__	)
							if left_sum >= right_sum and left_sum >= cross_sum:
														return left_low, left_high, left_sum
							elif right_sum >= left_sum and right_sum >= cross_sum:
														return right_low, right_high, right_sum
							return cross_left, cross_right, cross_sum

def 	lowerCAmelCase__			(      a__:		Sequence[float]      ,  a__:		int      ,  a__:		int      ,  a__:		int	)  ->		tuple[int, int, float]:


							'''simple docstring'''





							_UpperCAmelCase		,	_UpperCAmelCase					=      float('-inf'	), -1
							_UpperCAmelCase		,	_UpperCAmelCase					=      float('-inf'	), -1

							_UpperCAmelCase					=      0
							for i in range(a__      ,  low - 1      ,  -1	):
														summ += arr[i]
														if summ > left_sum:
																					_UpperCAmelCase					=      summ
																					_UpperCAmelCase					=      i

							_UpperCAmelCase					=      0
							for i in range(mid + 1      ,  high + 1	):
														summ += arr[i]
														if summ > right_sum:
																					_UpperCAmelCase					=      summ
																					_UpperCAmelCase					=      i

							return max_left, max_right, (left_sum + right_sum)

def 	lowerCAmelCase__			(      a__:		int	)  ->		float:


							'''simple docstring'''





							_UpperCAmelCase					=      [randint(1      ,  a__	) for _ in range(a__	)]
							_UpperCAmelCase					=      time.time()
							max_subarray(a__      ,  0      ,  input_size - 1	)
							_UpperCAmelCase					=      time.time()
							return end - start

def 	lowerCAmelCase__			(      )  ->		None:


							'''simple docstring'''





							_UpperCAmelCase					=      [1_0, 1_0_0, 1_0_0_0, 1_0_0_0_0, 5_0_0_0_0, 1_0_0_0_0_0, 2_0_0_0_0_0, 3_0_0_0_0_0, 4_0_0_0_0_0, 5_0_0_0_0_0]
							_UpperCAmelCase					=      [time_max_subarray(a__	) for input_size in input_sizes]
							print('No of Inputs\t\tTime Taken'	)
							for input_size, runtime in zip(a__      ,  a__	):
														print(a__      ,  '\t\t'      ,  a__	)
							plt.plot(a__      ,  a__	)
							plt.xlabel('Number of Inputs'	)
							plt.ylabel('Time taken in seconds'	)
							plt.show()


if __name__ == "__main__":
		from doctest import testmod

		testmod()
