





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case  :   int						   =  logging.get_logger(__name__)

_snake_case  :   str						   =  '▁'

_snake_case  :   Any						   =  {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

_snake_case  :   Dict						   =  {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

_snake_case  :   Dict						   =  {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

_snake_case  :   Any						   =  {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

_snake_case  :   str						   =  {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}



class     _UpperCAmelCase   (   lowerCamelCase__   ):

  """simple docstring"""


  a_      =			["input_ids"]

  a_      =			VOCAB_FILES_NAMES
  a_      =			PRETRAINED_INIT_CONFIGURATION
  a_      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_      =			PRETRAINED_VOCAB_FILES_MAP
  a_      =			RESOURCE_FILES_NAMES






  def __init__(    self			:		Dict ,  lowerCAmelCase_			:		int ,  lowerCAmelCase_			:		Any=None ,  lowerCAmelCase_			:		Optional[Any]=False ,  lowerCAmelCase_			:		Tuple="utf8" ,  lowerCAmelCase_			:		Dict="[UNK]" ,  lowerCAmelCase_			:		Optional[Any]="[SEP]" ,  lowerCAmelCase_			:		int="[PAD]" ,  lowerCAmelCase_			:		List[Any]="[CLS]" ,  lowerCAmelCase_			:		Optional[int]="[MASK]" ,  lowerCAmelCase_			:		Optional[Dict[str, Any]] = None ,  **lowerCAmelCase_			:		Optional[Any] ,  )							->						Optional[Any]:
     # Mask token behave like a normal word, i.e. include the space before it and
     # is included in the raw text, there should be a match in a non-normalized sentence.

     __lowerCAmelCase							     =     {} if sp_model_kwargs is None else sp_model_kwargs
     super().__init__(
         do_lower_case=lowercase__ ,  unk_token=lowercase__ ,  sep_token=lowercase__ ,  pad_token=lowercase__ ,  cls_token=lowercase__ ,  mask_token=lowercase__ ,  vocab_file=lowercase__ ,  encoding=lowercase__ ,  sp_model_kwargs=self.sp_model_kwargs ,  **lowercase__ ,  )
     __lowerCAmelCase							     =     do_lower_case
     __lowerCAmelCase							     =     sentencepiece_model_ckpt
     __lowerCAmelCase							     =     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
     self.sp_model.Load(lowercase__					)

     # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
     if vocab_file is not None:
        __lowerCAmelCase							     =     self.load_vocab(filepath=lowercase__					)
     else:
        __lowerCAmelCase							     =     {self.sp_model.id_to_piece(lowercase__					): id for id in range(self.sp_model.get_piece_size()					)}
     __lowerCAmelCase							     =     {v: k for k, v in self.vocab.items()}






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		Optional[int]					)							->						Any:
     if text is None:
        return None

     __lowerCAmelCase							     =     self.tokenize(lowercase__					)
     __lowerCAmelCase     ,     __lowerCAmelCase							     =     '', []

     for i, ch in enumerate(lowercase__					):
        if ch in self.SP_CHAR_MAPPING:
           __lowerCAmelCase							     =     self.SP_CHAR_MAPPING.get(lowercase__					)
        else:
           __lowerCAmelCase							     =     unicodedata.normalize('NFKC' ,  lowercase__					)
        if self.is_whitespace(lowercase__					):
           continue
        normalized_text += ch
        char_mapping.extend([i] * len(lowercase__					)					)

     __lowerCAmelCase     ,     __lowerCAmelCase     ,     __lowerCAmelCase							     =     normalized_text, [], 0

     if self.do_lower_case:
        __lowerCAmelCase							     =     text.lower()

     for token in split_tokens:
        if token[:1] == "▁":
           __lowerCAmelCase							     =     token[1:]
        __lowerCAmelCase							     =     text[offset:].index(lowercase__					) + offset
        __lowerCAmelCase							     =     start + len(lowercase__					)

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
        __lowerCAmelCase							     =     end
     return token_mapping






  @property
  def     lowercase		(    self			:		Optional[int]					)							->						List[Any]:
     return len(self.vocab					)






  def     lowercase		(    self			:		int					)							->						Union[str, Any]:
     return dict(self.vocab ,  **self.added_tokens_encoder					)






  def __getstate__(    self			:		str					)							->						Dict:
     __lowerCAmelCase							     =     self.__dict__.copy()
     __lowerCAmelCase							     =     None
     return state






  def __setstate__(    self			:		str ,  lowerCAmelCase_			:		Any					)							->						List[str]:
     __lowerCAmelCase							     =     d

     # for backward compatibility
     if not hasattr(self ,  'sp_model_kwargs'					):
        __lowerCAmelCase							     =     {}

     __lowerCAmelCase							     =     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
     self.sp_model.Load(self.sentencepiece_model_ckpt					)






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		List[str]					)							->						Tuple:
     return "".join((self.SP_CHAR_MAPPING.get(lowercase__ ,  lowercase__					) for c in text)					)






  def     lowercase		(    self			:		Tuple ,  lowerCAmelCase_			:		Union[str, Any] ,  lowerCAmelCase_			:		Union[str, Any]=False ,  lowerCAmelCase_			:		Any=6_4 ,  lowerCAmelCase_			:		Any=0.1					)							->						Tuple:

     if self.sp_model_kwargs.get('enable_sampling'					) is True:
        __lowerCAmelCase							     =     True
     if self.sp_model_kwargs.get('alpha'					) is not None:
        __lowerCAmelCase							     =     self.sp_model_kwargs.get('alpha'					)
     if self.sp_model_kwargs.get('nbest_size'					) is not None:
        __lowerCAmelCase							     =     self.sp_model_kwargs.get('nbest_size'					)

     if not enable_sampling:
        __lowerCAmelCase							     =     self.sp_model.EncodeAsPieces(lowercase__					)
     else:
        __lowerCAmelCase							     =     self.sp_model.SampleEncodeAsPieces(lowercase__ ,  lowercase__ ,  lowercase__					)
     __lowerCAmelCase							     =     []
     for pi, piece in enumerate(lowercase__					):
        if piece == SPIECE_UNDERLINE:
           if not pieces[pi + 1].startswith(lowercase__					) and pi != 0:
              new_pieces.append(lowercase__					)
              continue
           else:
              continue
        __lowerCAmelCase							     =     0
        for i, chunk in enumerate(lowercase__					):
           if chunk == SPIECE_UNDERLINE:
              continue
           if self.is_ch_char(lowercase__					) or self.is_punct(lowercase__					):
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                 new_pieces.append(piece[lst_i:i]					)
              new_pieces.append(lowercase__					)
              __lowerCAmelCase							     =     i + 1
           elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                 new_pieces.append(piece[lst_i:i]					)
              __lowerCAmelCase							     =     i
           elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                 new_pieces.append(piece[lst_i:i]					)
              __lowerCAmelCase							     =     i
        if len(lowercase__					) > lst_i:
           new_pieces.append(piece[lst_i:]					)
     return new_pieces






  def     lowercase		(    self			:		int ,  lowerCAmelCase_			:		int					)							->						Optional[Any]:
     __lowerCAmelCase							     =     ''.join(lowercase__					).replace(lowercase__ ,  ' '					).strip()
     return out_string






  def     lowercase		(    self			:		List[str] ,  lowerCAmelCase_			:		Union[str, Any]					)							->						Union[str, Any]:
     __lowerCAmelCase							     =     self.convert_ids_to_tokens(lowercase__					)
     __lowerCAmelCase							     =     ''.join(lowercase__					).replace(lowercase__ ,  ' '					).strip()
     return out_string






  def     lowercase		(    self			:		Tuple ,  lowerCAmelCase_			:		List[str]					)							->						int:
     return self.vocab.get(lowercase__ ,  self.vocab.get(self.unk_token					)					)






  def     lowercase		(    self			:		int ,  lowerCAmelCase_			:		Optional[int]					)							->						Tuple:
     return self.reverse_vocab.get(lowercase__ ,  self.unk_token					)






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		List[Any] ,  lowerCAmelCase_			:		Dict=None					)							->						Optional[int]:
     if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     __lowerCAmelCase							     =     [self.cls_token_id]
     __lowerCAmelCase							     =     [self.sep_token_id]
     return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






  def     lowercase		(    self			:		Optional[int] ,  lowerCAmelCase_			:		Tuple ,  lowerCAmelCase_			:		Optional[Any]=None					)							->						Tuple:
     if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

     return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






  def     lowercase		(    self			:		Union[str, Any] ,  lowerCAmelCase_			:		Union[str, Any] ,  lowerCAmelCase_			:		Optional[Any]=None ,  lowerCAmelCase_			:		Optional[int]=False					)							->						Union[str, Any]:

     if already_has_special_tokens:
        if token_ids_a is not None:
           raise ValueError(
               'You should not supply a second sequence if the provided sequence of '
               'ids is already formatted with special tokens for the model.'					)
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

     if token_ids_a is not None:
        return [1] + ([0] * len(lowercase__					)) + [1, 1] + ([0] * len(lowercase__					)) + [1]
     return [1] + ([0] * len(lowercase__					)) + [1]






  def     lowercase		(    self			:		Tuple ,  lowerCAmelCase_			:		List[int] ,  lowerCAmelCase_			:		Optional[List[int]] = None					)							->						List[Any]:
     # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
     if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(lowercase__					) + 2) * [0]

     # [CLS] A [SEP] [SEP] B [SEP]
     return [0] * (len(lowercase__					) + 1) + [1] * (len(lowercase__					) + 3)






  def     lowercase		(    self			:		Any ,  lowerCAmelCase_			:		int					)							->						Any:
     if "\u4e00" <= char <= "\u9fff":
        return True
     return False






  def     lowercase		(    self			:		List[str] ,  lowerCAmelCase_			:		str					)							->						List[str]:
     if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
     return False






  def     lowercase		(    self			:		Union[str, Any] ,  lowerCAmelCase_			:		Any					)							->						Union[str, Any]:
     if char in ",;:.?!~，；：。？！《》【】":
        return True
     return False






  def     lowercase		(    self			:		Dict ,  lowerCAmelCase_			:		Tuple					)							->						Any:
     if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
     if len(lowercase__					) == 1:
        __lowerCAmelCase							     =     unicodedata.category(lowercase__					)
        if cat == "Zs":
           return True
     return False






  def     lowercase		(    self			:		Optional[int] ,  lowerCAmelCase_			:		List[Any]					)							->						int:
     __lowerCAmelCase							     =     {}
     with io.open(lowercase__ ,  'r' ,  encoding='utf-8'					) as f:
        for index, line in enumerate(lowercase__					):
           __lowerCAmelCase							     =     line.rstrip('\n'					)
           __lowerCAmelCase							     =     int(lowercase__					)

     return token_to_idx






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		str ,  lowerCAmelCase_			:		Optional[str] = None					)							->						List[str]:
     __lowerCAmelCase							     =     0
     if os.path.isdir(lowercase__					):
        __lowerCAmelCase							     =     os.path.join(
            lowercase__ ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
     else:
        __lowerCAmelCase							     =     (filename_prefix + '-' if filename_prefix else '') + save_directory
     with open(lowercase__ ,  'w' ,  encoding='utf-8'					) as writer:
        for token, token_index in sorted(self.vocab.items() ,  key=lambda lowerCAmelCase_					: kv[1]					):
           if index != token_index:
              logger.warning(
                  f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                  ' Please check that the vocabulary is not corrupted!'					)
              __lowerCAmelCase							     =     token_index
           writer.write(token + '\n'					)
           index += 1

     __lowerCAmelCase							     =     os.path.join(lowercase__ ,  'sentencepiece.bpe.model'					)
     with open(lowercase__ ,  'wb'					) as fi:
        __lowerCAmelCase							     =     self.sp_model.serialized_model_proto()
        fi.write(lowercase__					)

     return (vocab_file,)








class     _UpperCAmelCase   :

  """simple docstring"""


  def __init__(    self			:		Union[str, Any] ,  lowerCAmelCase_			:		List[str] ,  lowerCAmelCase_			:		int ,  lowerCAmelCase_			:		Dict					)							->						List[str]:
     __lowerCAmelCase							     =     name
     __lowerCAmelCase							     =     value
     __lowerCAmelCase							     =     weight






  def __repr__(    self			:		Union[str, Any]					)							->						List[str]:
     return f"""{self.__class__.__name__}({self.name}, {self.value}, {self.weight})"""






  def     lowercase		(    self			:		int					)							->						Optional[int]:
     return self.value






  def     lowercase		(    self			:		Optional[Any]					)							->						Union[str, Any]:
     return self.name






  def     lowercase		(    self			:		List[Any]					)							->						Tuple:
     return self.weight






  def     lowercase		(    self			:		int					)							->						Dict:
     return self.value / self.weight





def   a_   (			lowerCAmelCase_				:      Dict,     lowerCAmelCase_				:      int,     lowerCAmelCase_				:      Optional[int]     ):
   __lowerCAmelCase							     =     []
   for i in range(len(lowerCAmelCase_     )     ):
      menu.append(Things(name[i],     value[i],     weight[i]     )     )
   return menu

def   a_   (			lowerCAmelCase_				:      Optional[int],     lowerCAmelCase_				:      List[Any],     lowerCAmelCase_				:      Optional[Any]     ):
   __lowerCAmelCase							     =     sorted(lowerCAmelCase_,     key=lowerCAmelCase_,     reverse=lowerCAmelCase_     )
   __lowerCAmelCase							     =     []
   __lowerCAmelCase     ,     __lowerCAmelCase							     =     0.0, 0.0
   for i in range(len(lowerCAmelCase_     )     ):
      if (total_cost + items_copy[i].get_weight()) <= max_cost:
         result.append(items_copy[i]     )
         total_cost += items_copy[i].get_weight()
         total_value += items_copy[i].get_value()
   return (result, total_value)

def   a_   (			):
   pass


if __name__ == "__main__":
     import doctest

     doctest.testmod()

