






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase     :							List[Any]    =logging.get_logger(__name__)

lowerCamelCase     :							str    ={'''vocab_file''': '''spiece.model'''}

lowerCamelCase     :							Any    ={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

lowerCamelCase     :							int    ={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
lowerCamelCase     :							List[Any]    =0
lowerCamelCase     :							Union[str, Any]    =1
lowerCamelCase     :							str    =2
lowerCamelCase     :							Any    =3
lowerCamelCase     :							List[Any]    =4





class       __a  (   A__	):
 _lowerCAmelCase	:	Optional[int]				=			VOCAB_FILES_NAMES
 _lowerCAmelCase	:	List[Any]				=			PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase	:	List[str]				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase	:	int				=			'''left'''







 def __init__(		self :       Tuple							,       SCREAMING_SNAKE_CASE :       Optional[int]							,       SCREAMING_SNAKE_CASE :       List[Any]=False							,       SCREAMING_SNAKE_CASE :       Optional[int]=True							,       SCREAMING_SNAKE_CASE :       Any=False							,       SCREAMING_SNAKE_CASE :       List[Any]="<s>"							,       SCREAMING_SNAKE_CASE :       Tuple="</s>"							,       SCREAMING_SNAKE_CASE :       Dict="<unk>"							,       SCREAMING_SNAKE_CASE :       int="<sep>"							,       SCREAMING_SNAKE_CASE :       List[str]="<pad>"							,       SCREAMING_SNAKE_CASE :       Optional[int]="<cls>"							,       SCREAMING_SNAKE_CASE :       Union[str, Any]="<mask>"							,       SCREAMING_SNAKE_CASE :       Dict=["<eop>", "<eod>"]							,       SCREAMING_SNAKE_CASE :       Optional[Dict[str, Any]] = None							,       **SCREAMING_SNAKE_CASE :       Optional[int]							,       ):


      '''simple docstring'''






      UpperCamelCase__      :       Union[str, Any]	    =  AddedToken(SCREAMING_SNAKE_CASE							,       lstrip=SCREAMING_SNAKE_CASE							,       rstrip=SCREAMING_SNAKE_CASE						) if isinstance(SCREAMING_SNAKE_CASE							,       SCREAMING_SNAKE_CASE						) else mask_token

      UpperCamelCase__      :       Optional[int]	    =  {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=SCREAMING_SNAKE_CASE							,       remove_space=SCREAMING_SNAKE_CASE							,       keep_accents=SCREAMING_SNAKE_CASE							,       bos_token=SCREAMING_SNAKE_CASE							,       eos_token=SCREAMING_SNAKE_CASE							,       unk_token=SCREAMING_SNAKE_CASE							,       sep_token=SCREAMING_SNAKE_CASE							,       pad_token=SCREAMING_SNAKE_CASE							,       cls_token=SCREAMING_SNAKE_CASE							,       mask_token=SCREAMING_SNAKE_CASE							,       additional_special_tokens=SCREAMING_SNAKE_CASE							,       sp_model_kwargs=self.sp_model_kwargs							,       **SCREAMING_SNAKE_CASE							,       )

      UpperCamelCase__      :       List[Any]	    =  3

      UpperCamelCase__      :       Optional[int]	    =  do_lower_case
      UpperCamelCase__      :       Optional[Any]	    =  remove_space
      UpperCamelCase__      :       Any	    =  keep_accents
      UpperCamelCase__      :       Dict	    =  vocab_file

      UpperCamelCase__      :       List[Any]	    =  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(SCREAMING_SNAKE_CASE						)







 @property
 def  __lowercase (		self :       Optional[Any]						):


      '''simple docstring'''






      return len(self.sp_model						)







 def  __lowercase (		self :       Dict						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE						): i for i in range(self.vocab_size						)}
      vocab.update(self.added_tokens_encoder						)
      return vocab







 def __getstate__(		self :       int						):


      '''simple docstring'''






      UpperCamelCase__      :       int	    =  self.__dict__.copy()
      UpperCamelCase__      :       Tuple	    =  None
      return state







 def __setstate__(		self :       Dict							,       SCREAMING_SNAKE_CASE :       Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__      :       List[str]	    =  d

      # for backward compatibility
      if not hasattr(self							,       "sp_model_kwargs"						):
           UpperCamelCase__      :       List[str]	    =  {}

      UpperCamelCase__      :       Dict	    =  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(self.vocab_file						)







 def  __lowercase (		self :       List[str]							,       SCREAMING_SNAKE_CASE :       Any						):


      '''simple docstring'''






      if self.remove_space:
           UpperCamelCase__      :       Optional[int]	    =  " ".join(inputs.strip().split()						)
      else:
           UpperCamelCase__      :       str	    =  inputs
      UpperCamelCase__      :       int	    =  outputs.replace("``"							,       "\""						).replace("''"							,       "\""						)

      if not self.keep_accents:
           UpperCamelCase__      :       str	    =  unicodedata.normalize("NFKD"							,       SCREAMING_SNAKE_CASE						)
           UpperCamelCase__      :       Dict	    =  "".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE						)]						)
      if self.do_lower_case:
           UpperCamelCase__      :       Optional[Any]	    =  outputs.lower()

      return outputs







 def  __lowercase (		self :       List[str]							,       SCREAMING_SNAKE_CASE :       str						):


      '''simple docstring'''






      UpperCamelCase__      :       str	    =  self.preprocess_text(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       Dict	    =  self.sp_model.encode(SCREAMING_SNAKE_CASE							,       out_type=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       str	    =  []
      for piece in pieces:
           if len(SCREAMING_SNAKE_CASE						) > 1 and piece[-1] == str(","						) and piece[-2].isdigit():
                UpperCamelCase__      :       Optional[int]	    =  self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE							,       ""						)						)
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                     if len(cur_pieces[0]						) == 1:
                          UpperCamelCase__      :       str	    =  cur_pieces[1:]
                     else:
                          UpperCamelCase__      :       str	    =  cur_pieces[0][1:]
                cur_pieces.append(piece[-1]						)
                new_pieces.extend(SCREAMING_SNAKE_CASE						)
           else:
                new_pieces.append(SCREAMING_SNAKE_CASE						)

      return new_pieces







 def  __lowercase (		self :       Union[str, Any]							,       SCREAMING_SNAKE_CASE :       int						):


      '''simple docstring'''






      return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       Optional[int]							,       SCREAMING_SNAKE_CASE :       Dict						):


      '''simple docstring'''






      return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       str							,       SCREAMING_SNAKE_CASE :       List[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  "".join(SCREAMING_SNAKE_CASE						).replace(SCREAMING_SNAKE_CASE							,       " "						).strip()
      return out_string







 def  __lowercase (		self :       Tuple							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       bool = False							,       SCREAMING_SNAKE_CASE :       bool = None							,       SCREAMING_SNAKE_CASE :       bool = True							,       **SCREAMING_SNAKE_CASE :       List[Any]							,       ):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  kwargs.pop("use_source_tokenizer"							,       SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       int	    =  self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE							,       skip_special_tokens=SCREAMING_SNAKE_CASE						)

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      UpperCamelCase__      :       str	    =  []
      UpperCamelCase__      :       Any	    =  []
      for token in filtered_tokens:
           if skip_special_tokens and token in self.all_special_ids:
                continue
           if token in self.added_tokens_encoder:
                if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE						)						)
                     UpperCamelCase__      :       Any	    =  []
                sub_texts.append(SCREAMING_SNAKE_CASE						)
           else:
                current_sub_text.append(SCREAMING_SNAKE_CASE						)
      if current_sub_text:
           sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE						)						)

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      UpperCamelCase__      :       Any	    =  "".join(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Any	    =  (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
           UpperCamelCase__      :       List[str]	    =  self.clean_up_tokenization(SCREAMING_SNAKE_CASE						)
           return clean_text
      else:
           return text







 def  __lowercase (		self :       Tuple							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       Optional[List[int]] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       int	    =  [self.sep_token_id]
      UpperCamelCase__      :       Tuple	    =  [self.cls_token_id]
      if token_ids_a is None:
           return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls







 def  __lowercase (		self :       Union[str, Any]							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       Optional[List[int]] = None							,       SCREAMING_SNAKE_CASE :       bool = False						):


      '''simple docstring'''






      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=SCREAMING_SNAKE_CASE							,       token_ids_a=SCREAMING_SNAKE_CASE							,       already_has_special_tokens=SCREAMING_SNAKE_CASE						)

      if token_ids_a is not None:
           return ([0] * len(SCREAMING_SNAKE_CASE						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE						)) + [1, 1]
      return ([0] * len(SCREAMING_SNAKE_CASE						)) + [1, 1]







 def  __lowercase (		self :       Dict							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       Optional[List[int]] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  [self.sep_token_id]
      UpperCamelCase__      :       Union[str, Any]	    =  [2]

      if token_ids_a is None:
           return len(token_ids_a + sep						) * [0] + cls_segment_id
      return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







 def  __lowercase (		self :       Optional[Any]							,       SCREAMING_SNAKE_CASE :       str							,       SCREAMING_SNAKE_CASE :       Optional[str] = None						):


      '''simple docstring'''






      if not os.path.isdir(SCREAMING_SNAKE_CASE						):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
           return
      UpperCamelCase__      :       Any	    =  os.path.join(
          SCREAMING_SNAKE_CASE							,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

      if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE						) and os.path.isfile(self.vocab_file						):
           copyfile(self.vocab_file							,       SCREAMING_SNAKE_CASE						)
      elif not os.path.isfile(self.vocab_file						):
           with open(SCREAMING_SNAKE_CASE							,       "wb"						) as fi:
                UpperCamelCase__      :       Optional[Any]	    =  self.sp_model.serialized_model_proto()
                fi.write(SCREAMING_SNAKE_CASE						)

      return (out_vocab_file,)







from collections.abc import Generator
from math import sin







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     if len(__lowerCAmelCase      ) != 32:
          raise ValueError("Input must be of length 32"      )

     UpperCamelCase__      :       Optional[Any]	    =  B""
     for i in [3, 2, 1, 0]:
          little_endian += string_aa[8 * i : 8 * i + 8]
     return little_endian







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     if i < 0:
          raise ValueError("Input must be non-negative"      )

     UpperCamelCase__      :       int	    =  format(__lowerCAmelCase ,   "08x"      )[-8:]
     UpperCamelCase__      :       Union[str, Any]	    =  B""
     for i in [3, 2, 1, 0]:
          little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode("utf-8"      )
     return little_endian_hex







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     UpperCamelCase__      :       Optional[Any]	    =  B""
     for char in message:
          bit_string += format(__lowerCAmelCase ,   "08b"      ).encode("utf-8"      )
     UpperCamelCase__      :       Any	    =  format(len(__lowerCAmelCase      ) ,   "064b"      ).encode("utf-8"      )

     # Pad bit_string to a multiple of 512 chars
     bit_string += b"1"
     while len(__lowerCAmelCase      ) % 512 != 448:
          bit_string += b"0"
     bit_string += to_little_endian(start_len[32:]      ) + to_little_endian(start_len[:32]      )

     return bit_string







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     Generator[list[int], None, None]:
     if len(__lowerCAmelCase      ) % 512 != 0:
          raise ValueError("Input must have length that's a multiple of 512"      )

     for pos in range(0 ,   len(__lowerCAmelCase      ) ,   512      ):
          UpperCamelCase__      :       Tuple	    =  bit_string[pos : pos + 512]
          UpperCamelCase__      :       Tuple	    =  []
          for i in range(0 ,   512 ,   32      ):
               block_words.append(int(to_little_endian(block[i : i + 32]      ) ,   2      )      )
          yield block_words







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     int:
     if i < 0:
          raise ValueError("Input must be non-negative"      )

     UpperCamelCase__      :       str	    =  format(__lowerCAmelCase ,   "032b"      )
     UpperCamelCase__      :       Optional[int]	    =  ""
     for c in i_str:
          new_str += "1" if c == "0" else "0"
     return int(__lowerCAmelCase ,   2      )







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase ,   __lowerCAmelCase      )			->     int:
     return (a + b) % 2**32







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase ,   __lowerCAmelCase      )			->     int:
     if i < 0:
          raise ValueError("Input must be non-negative"      )
     if shift < 0:
          raise ValueError("Shift must be non-negative"      )
     return ((i << shift) ^ (i >> (32 - shift))) % 2**32







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     UpperCamelCase__      :       int	    =  preprocess(__lowerCAmelCase      )

     UpperCamelCase__      :       Optional[Any]	    =  [int(2**32 * abs(sin(i + 1      )      )      ) for i in range(64      )]

     # Starting states
     UpperCamelCase__      :       Any	    =  0x6745_2301
     UpperCamelCase__      :       Optional[int]	    =  0xEFCD_AB89
     UpperCamelCase__      :       Union[str, Any]	    =  0x98BA_DCFE
     UpperCamelCase__      :       Any	    =  0x1032_5476

     UpperCamelCase__      :       List[Any]	    =  [
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
     ]

     # Process bit string in chunks, each with 16 32-char words
     for block_words in get_block_words(__lowerCAmelCase      ):
          UpperCamelCase__      :       Union[str, Any]	    =  aa
          UpperCamelCase__      :       int	    =  ba
          UpperCamelCase__      :       Union[str, Any]	    =  ca
          UpperCamelCase__      :       str	    =  da

          # Hash current chunk
          for i in range(64      ):
               if i <= 15:
                    # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                    UpperCamelCase__      :       Any	    =  d ^ (b & (c ^ d))
                    UpperCamelCase__      :       Optional[int]	    =  i
               elif i <= 31:
                    # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                    UpperCamelCase__      :       Any	    =  c ^ (d & (b ^ c))
                    UpperCamelCase__      :       str	    =  (5 * i + 1) % 16
               elif i <= 47:
                    UpperCamelCase__      :       Dict	    =  b ^ c ^ d
                    UpperCamelCase__      :       Union[str, Any]	    =  (3 * i + 5) % 16
               else:
                    UpperCamelCase__      :       Optional[int]	    =  c ^ (b | not_aa(__lowerCAmelCase      ))
                    UpperCamelCase__      :       Optional[int]	    =  (7 * i) % 16
               UpperCamelCase__      :       List[str]	    =  (f + a + added_consts[i] + block_words[g]) % 2**32
               UpperCamelCase__      :       Optional[int]	    =  d
               UpperCamelCase__      :       Dict	    =  c
               UpperCamelCase__      :       List[Any]	    =  b
               UpperCamelCase__      :       List[Any]	    =  sum_aa(__lowerCAmelCase ,   left_rotate_aa(__lowerCAmelCase ,   shift_amounts[i]      )      )

          # Add hashed chunk to running total
          UpperCamelCase__      :       Optional[int]	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       Any	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       Dict	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       List[str]	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )

     UpperCamelCase__      :       List[str]	    =  reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      )
     return digest


if __name__ == "__main__":
 import doctest

 doctest.testmod()