


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_	   =     logging.get_logger(__name__)

lowercase_	   =     '''▁'''

lowercase_	   =     {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

lowercase_	   =     {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

lowercase_	   =     {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

lowercase_	   =     {
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

lowercase_	   =     {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}


class 	A_ (	a_   ):




      '''simple docstring'''


      __snake_case 	=       ["""input_ids"""]

      __snake_case 	=       VOCAB_FILES_NAMES
      __snake_case 	=       PRETRAINED_INIT_CONFIGURATION
      __snake_case 	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case 	=       PRETRAINED_VOCAB_FILES_MAP
      __snake_case 	=       RESOURCE_FILES_NAMES




      def __init__(    self:	Optional[Any]						,				a:	Any						,				a:	Union[str, Any]=None						,				a:	Optional[Any]=False						,				a:	str="utf8"						,				a:	Tuple="[UNK]"						,				a:	Tuple="[SEP]"						,				a:	Optional[Any]="[PAD]"						,				a:	List[Any]="[CLS]"						,				a:	Optional[int]="[MASK]"						,				a:	Optional[Dict[str, Any]] = None						,				**a:	Optional[Any]						,				):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.

       __lowerCamelCase		:							Optional[Any]		 =						{} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=lowercase_						,				unk_token=lowercase_						,				sep_token=lowercase_						,				pad_token=lowercase_						,				cls_token=lowercase_						,				mask_token=lowercase_						,				vocab_file=lowercase_						,				encoding=lowercase_						,				sp_model_kwargs=self.sp_model_kwargs						,				**lowercase_						,				)
       __lowerCamelCase		:							List[str]		 =						do_lower_case
       __lowerCamelCase		:							Tuple		 =						sentencepiece_model_ckpt
       __lowerCamelCase		:							str		 =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(lowercase_    )

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
        __lowerCamelCase		:							int		 =						self.load_vocab(filepath=lowercase_    )
       else:
        __lowerCamelCase		:							int		 =						{self.sp_model.id_to_piece(lowercase_    ): id for id in range(self.sp_model.get_piece_size()    )}
       __lowerCamelCase		:							Any		 =						{v: k for k, v in self.vocab.items()}




      def 					_snake_case     (    self:	str						,				a:	Dict    ):
       if text is None:
        return None

       __lowerCamelCase		:							Optional[int]		 =						self.tokenize(lowercase_    )
       __lowerCamelCase		:							List[Any]		 =						'''''', []

       for i, ch in enumerate(lowercase_    ):
        if ch in self.SP_CHAR_MAPPING:
         __lowerCamelCase		:							Dict		 =						self.SP_CHAR_MAPPING.get(lowercase_    )
        else:
         __lowerCamelCase		:							Tuple		 =						unicodedata.normalize('NFKC'						,				lowercase_    )
        if self.is_whitespace(lowercase_    ):
         continue
        normalized_text += ch
        char_mapping.extend([i] * len(lowercase_    )    )

       __lowerCamelCase		:							int		 =						normalized_text, [], 0

       if self.do_lower_case:
        __lowerCamelCase		:							Union[str, Any]		 =						text.lower()

       for token in split_tokens:
        if token[:1] == "▁":
         __lowerCamelCase		:							Optional[Any]		 =						token[1:]
        __lowerCamelCase		:							int		 =						text[offset:].index(lowercase_    ) + offset
        __lowerCamelCase		:							Union[str, Any]		 =						start + len(lowercase_    )

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
        __lowerCamelCase		:							Dict		 =						end
       return token_mapping




      @property
      def 					_snake_case     (    self:	Tuple    ):
       return len(self.vocab    )




      def 					_snake_case     (    self:	List[Any]    ):
       return dict(self.vocab						,				**self.added_tokens_encoder    )




      def __getstate__(    self:	str    ):
       __lowerCamelCase		:							Dict		 =						self.__dict__.copy()
       __lowerCamelCase		:							Dict		 =						None
       return state




      def __setstate__(    self:	Any						,				a:	str    ):
       __lowerCamelCase		:							Any		 =						d

       # for backward compatibility
       if not hasattr(self						,				'sp_model_kwargs'    ):
        __lowerCamelCase		:							Tuple		 =						{}

       __lowerCamelCase		:							Optional[int]		 =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(self.sentencepiece_model_ckpt    )




      def 					_snake_case     (    self:	Union[str, Any]						,				a:	List[Any]    ):
       return "".join((self.SP_CHAR_MAPPING.get(lowercase_						,				lowercase_    ) for c in text)    )




      def 					_snake_case     (    self:	Tuple						,				a:	Tuple						,				a:	Dict=False						,				a:	Optional[int]=64						,				a:	Optional[int]=0.1    ):

       if self.sp_model_kwargs.get('enable_sampling'    ) is True:
        __lowerCamelCase		:							str		 =						True
       if self.sp_model_kwargs.get('alpha'    ) is not None:
        __lowerCamelCase		:							Optional[Any]		 =						self.sp_model_kwargs.get('alpha'    )
       if self.sp_model_kwargs.get('nbest_size'    ) is not None:
        __lowerCamelCase		:							Optional[Any]		 =						self.sp_model_kwargs.get('nbest_size'    )

       if not enable_sampling:
        __lowerCamelCase		:							Union[str, Any]		 =						self.sp_model.EncodeAsPieces(lowercase_    )
       else:
        __lowerCamelCase		:							Optional[Any]		 =						self.sp_model.SampleEncodeAsPieces(lowercase_						,				lowercase_						,				lowercase_    )
       __lowerCamelCase		:							Optional[int]		 =						[]
       for pi, piece in enumerate(lowercase_    ):
        if piece == SPIECE_UNDERLINE:
         if not pieces[pi + 1].startswith(lowercase_    ) and pi != 0:
          new_pieces.append(lowercase_    )
          continue
         else:
          continue
        __lowerCamelCase		:							str		 =						0
        for i, chunk in enumerate(lowercase_    ):
         if chunk == SPIECE_UNDERLINE:
          continue
         if self.is_ch_char(lowercase_    ) or self.is_punct(lowercase_    ):
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]    )
          new_pieces.append(lowercase_    )
          __lowerCamelCase		:							Dict		 =						i + 1
         elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]    )
          __lowerCamelCase		:							Any		 =						i
         elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]    )
          __lowerCamelCase		:							Optional[int]		 =						i
        if len(lowercase_    ) > lst_i:
         new_pieces.append(piece[lst_i:]    )
       return new_pieces




      def 					_snake_case     (    self:	List[str]						,				a:	List[str]    ):
       __lowerCamelCase		:							int		 =						''''''.join(lowercase_    ).replace(lowercase_						,				' '    ).strip()
       return out_string




      def 					_snake_case     (    self:	List[Any]						,				a:	Union[str, Any]    ):
       __lowerCamelCase		:							Tuple		 =						self.convert_ids_to_tokens(lowercase_    )
       __lowerCamelCase		:							Tuple		 =						''''''.join(lowercase_    ).replace(lowercase_						,				' '    ).strip()
       return out_string




      def 					_snake_case     (    self:	List[Any]						,				a:	int    ):
       return self.vocab.get(lowercase_						,				self.vocab.get(self.unk_token    )    )




      def 					_snake_case     (    self:	List[str]						,				a:	List[str]    ):
       return self.reverse_vocab.get(lowercase_						,				self.unk_token    )




      def 					_snake_case     (    self:	Union[str, Any]						,				a:	Optional[Any]						,				a:	Any=None    ):
       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       __lowerCamelCase		:							Optional[int]		 =						[self.cls_token_id]
       __lowerCamelCase		:							Optional[int]		 =						[self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




      def 					_snake_case     (    self:	int						,				a:	int						,				a:	str=None    ):
       if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




      def 					_snake_case     (    self:	Any						,				a:	List[str]						,				a:	int=None						,				a:	Dict=False    ):

       if already_has_special_tokens:
        if token_ids_a is not None:
         raise ValueError(
             'You should not supply a second sequence if the provided sequence of '
             'ids is already formatted with special tokens for the model.'    )
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
        return [1] + ([0] * len(lowercase_    )) + [1, 1] + ([0] * len(lowercase_    )) + [1]
       return [1] + ([0] * len(lowercase_    )) + [1]




      def 					_snake_case     (    self:	Tuple						,				a:	List[int]						,				a:	Optional[List[int]] = None    ):
       # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
       if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(lowercase_    ) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(lowercase_    ) + 1) + [1] * (len(lowercase_    ) + 3)




      def 					_snake_case     (    self:	Optional[int]						,				a:	Tuple    ):
       if "\u4e00" <= char <= "\u9fff":
        return True
       return False




      def 					_snake_case     (    self:	List[str]						,				a:	List[Any]    ):
       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
       return False




      def 					_snake_case     (    self:	List[str]						,				a:	List[Any]    ):
       if char in ",;:.?!~，；：。？！《》【】":
        return True
       return False




      def 					_snake_case     (    self:	int						,				a:	Optional[int]    ):
       if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
       if len(lowercase_    ) == 1:
        __lowerCamelCase		:							Tuple		 =						unicodedata.category(lowercase_    )
        if cat == "Zs":
         return True
       return False




      def 					_snake_case     (    self:	Tuple						,				a:	Union[str, Any]    ):
       __lowerCamelCase		:							Tuple		 =						{}
       with io.open(lowercase_						,				'r'						,				encoding='utf-8'    ) as f:
        for index, line in enumerate(lowercase_    ):
         __lowerCamelCase		:							Dict		 =						line.rstrip('\n'    )
         __lowerCamelCase		:							List[Any]		 =						int(lowercase_    )

       return token_to_idx




      def 					_snake_case     (    self:	str						,				a:	str						,				a:	Optional[str] = None    ):
       __lowerCamelCase		:							str		 =						0
       if os.path.isdir(lowercase_    ):
        __lowerCamelCase		:							Optional[Any]		 =						os.path.join(
            lowercase_						,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
       else:
        __lowerCamelCase		:							List[str]		 =						(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
       with open(lowercase_						,				'w'						,				encoding='utf-8'    ) as writer:
        for token, token_index in sorted(self.vocab.items()						,				key=lambda a    : kv[1]    ):
         if index != token_index:
          logger.warning(
              F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
              ' Please check that the vocabulary is not corrupted!'    )
          __lowerCamelCase		:							str		 =						token_index
         writer.write(token + '\n'    )
         index += 1

       __lowerCamelCase		:							int		 =						os.path.join(lowercase_						,				'sentencepiece.bpe.model'    )
       with open(lowercase_						,				'wb'    ) as fi:
        __lowerCamelCase		:							Any		 =						self.sp_model.serialized_model_proto()
        fi.write(lowercase_    )

       return (vocab_file,)





from __future__ import annotations
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Optional[Any]		 =						0.00
 __lowerCamelCase		:							Tuple		 =						0
 for resistor in resistors:
  if resistor <= 0:
   __lowerCamelCase		:							Union[str, Any]		 =						f'Resistor at index {index} has a negative or zero value!'
   raise ValueError(SCREAMING_SNAKE_CASE__						)
  first_sum += 1 / float(SCREAMING_SNAKE_CASE__						)
  index += 1
 return 1 / first_sum
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Union[str, Any]		 =						0.00
 __lowerCamelCase		:							str		 =						0
 for resistor in resistors:
  sum_r += resistor
  if resistor < 0:
   __lowerCamelCase		:							Any		 =						f'Resistor at index {index} has a negative value!'
   raise ValueError(SCREAMING_SNAKE_CASE__						)
  index += 1
 return sum_r


if __name__ == "__main__":
       import doctest

       doctest.testmod()

