






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase     :							Any    =logging.get_logger(__name__)

lowerCamelCase     :							Tuple    ={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase     :							Dict    ={
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase     :							List[Any]    ={
    '''bert-base-uncased''': 512,
    '''bert-large-uncased''': 512,
    '''bert-base-cased''': 512,
    '''bert-large-cased''': 512,
    '''bert-base-multilingual-uncased''': 512,
    '''bert-base-multilingual-cased''': 512,
    '''bert-base-chinese''': 512,
    '''bert-base-german-cased''': 512,
    '''bert-large-uncased-whole-word-masking''': 512,
    '''bert-large-cased-whole-word-masking''': 512,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 512,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 512,
    '''bert-base-cased-finetuned-mrpc''': 512,
    '''bert-base-german-dbmdz-cased''': 512,
    '''bert-base-german-dbmdz-uncased''': 512,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 512,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 512,
    '''wietsedv/bert-base-dutch-cased''': 512,
}

lowerCamelCase     :							Optional[Any]    ={
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}





class       __a  (   A__	):
 _lowerCAmelCase	:	str				=			VOCAB_FILES_NAMES
 _lowerCAmelCase	:	Optional[Any]				=			PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase	:	List[str]				=			PRETRAINED_INIT_CONFIGURATION
 _lowerCAmelCase	:	Optional[int]				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase	:	Optional[int]				=			BertTokenizer







 def __init__(		self :       Optional[int]							,       SCREAMING_SNAKE_CASE :       Dict=None							,       SCREAMING_SNAKE_CASE :       Optional[Any]=None							,       SCREAMING_SNAKE_CASE :       List[Any]=True							,       SCREAMING_SNAKE_CASE :       Tuple="[UNK]"							,       SCREAMING_SNAKE_CASE :       int="[SEP]"							,       SCREAMING_SNAKE_CASE :       int="[PAD]"							,       SCREAMING_SNAKE_CASE :       Tuple="[CLS]"							,       SCREAMING_SNAKE_CASE :       List[str]="[MASK]"							,       SCREAMING_SNAKE_CASE :       Union[str, Any]=True							,       SCREAMING_SNAKE_CASE :       Any=None							,       **SCREAMING_SNAKE_CASE :       str							,       ):


      '''simple docstring'''






      super().__init__(
          SCREAMING_SNAKE_CASE							,       tokenizer_file=SCREAMING_SNAKE_CASE							,       do_lower_case=SCREAMING_SNAKE_CASE							,       unk_token=SCREAMING_SNAKE_CASE							,       sep_token=SCREAMING_SNAKE_CASE							,       pad_token=SCREAMING_SNAKE_CASE							,       cls_token=SCREAMING_SNAKE_CASE							,       mask_token=SCREAMING_SNAKE_CASE							,       tokenize_chinese_chars=SCREAMING_SNAKE_CASE							,       strip_accents=SCREAMING_SNAKE_CASE							,       **SCREAMING_SNAKE_CASE							,       )

      UpperCamelCase__      :       Optional[Any]	    =  json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
      if (
          normalizer_state.get("lowercase"							,       SCREAMING_SNAKE_CASE						) != do_lower_case
          or normalizer_state.get("strip_accents"							,       SCREAMING_SNAKE_CASE						) != strip_accents
          or normalizer_state.get("handle_chinese_chars"							,       SCREAMING_SNAKE_CASE						) != tokenize_chinese_chars
      ):
           UpperCamelCase__      :       Optional[int]	    =  getattr(SCREAMING_SNAKE_CASE							,       normalizer_state.pop("type"						)						)
           UpperCamelCase__      :       Tuple	    =  do_lower_case
           UpperCamelCase__      :       Any	    =  strip_accents
           UpperCamelCase__      :       Optional[int]	    =  tokenize_chinese_chars
           UpperCamelCase__      :       Any	    =  normalizer_class(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       List[str]	    =  do_lower_case







 def  __lowercase (		self :       Union[str, Any]							,       SCREAMING_SNAKE_CASE :       Optional[int]							,       SCREAMING_SNAKE_CASE :       Any=None						):


      '''simple docstring'''






      UpperCamelCase__      :       List[Any]	    =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output







 def  __lowercase (		self :       str							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       Optional[List[int]] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  [self.sep_token_id]
      UpperCamelCase__      :       Tuple	    =  [self.cls_token_id]
      if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
      return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







 def  __lowercase (		self :       str							,       SCREAMING_SNAKE_CASE :       str							,       SCREAMING_SNAKE_CASE :       Optional[str] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       Union[str, Any]	    =  self._tokenizer.model.save(SCREAMING_SNAKE_CASE							,       name=SCREAMING_SNAKE_CASE						)
      return tuple(SCREAMING_SNAKE_CASE						)







from collections.abc import Generator
from math import sin







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     if len(__lowerCAmelCase      ) != 32:
          raise ValueError("Input must be of length 32"      )

     UpperCamelCase__      :       Optional[Any]	    =  B""
     for i in [3, 2, 1, 0]:
          little_endian += string_aa[8 * i : 8 * i + 8]
     return little_endian







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     if i < 0:
          raise ValueError("Input must be non-negative"      )

     UpperCamelCase__      :       int	    =  format(__lowerCAmelCase ,   "08x"      )[-8:]
     UpperCamelCase__      :       Union[str, Any]	    =  B""
     for i in [3, 2, 1, 0]:
          little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode("utf-8"      )
     return little_endian_hex







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     UpperCamelCase__      :       Optional[Any]	    =  B""
     for char in message:
          bit_string += format(__lowerCAmelCase ,   "08b"      ).encode("utf-8"      )
     UpperCamelCase__      :       Any	    =  format(len(__lowerCAmelCase      ) ,   "064b"      ).encode("utf-8"      )

     # Pad bit_string to a multiple of 512 chars
     bit_string += b"1"
     while len(__lowerCAmelCase      ) % 512 != 448:
          bit_string += b"0"
     bit_string += to_little_endian(start_len[32:]      ) + to_little_endian(start_len[:32]      )

     return bit_string







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     Generator[list[int], None, None]:
     if len(__lowerCAmelCase      ) % 512 != 0:
          raise ValueError("Input must have length that's a multiple of 512"      )

     for pos in range(0 ,   len(__lowerCAmelCase      ) ,   512      ):
          UpperCamelCase__      :       Tuple	    =  bit_string[pos : pos + 512]
          UpperCamelCase__      :       Tuple	    =  []
          for i in range(0 ,   512 ,   32      ):
               block_words.append(int(to_little_endian(block[i : i + 32]      ) ,   2      )      )
          yield block_words







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     int:
     if i < 0:
          raise ValueError("Input must be non-negative"      )

     UpperCamelCase__      :       str	    =  format(__lowerCAmelCase ,   "032b"      )
     UpperCamelCase__      :       Optional[int]	    =  ""
     for c in i_str:
          new_str += "1" if c == "0" else "0"
     return int(__lowerCAmelCase ,   2      )







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase ,   __lowerCAmelCase      )			->     int:
     return (a + b) % 2**32







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase ,   __lowerCAmelCase      )			->     int:
     if i < 0:
          raise ValueError("Input must be non-negative"      )
     if shift < 0:
          raise ValueError("Shift must be non-negative"      )
     return ((i << shift) ^ (i >> (32 - shift))) % 2**32







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     bytes:
     UpperCamelCase__      :       int	    =  preprocess(__lowerCAmelCase      )

     UpperCamelCase__      :       Optional[Any]	    =  [int(2**32 * abs(sin(i + 1      )      )      ) for i in range(64      )]

     # Starting states
     UpperCamelCase__      :       Any	    =  0x6745_2301
     UpperCamelCase__      :       Optional[int]	    =  0xEFCD_AB89
     UpperCamelCase__      :       Union[str, Any]	    =  0x98BA_DCFE
     UpperCamelCase__      :       Any	    =  0x1032_5476

     UpperCamelCase__      :       List[Any]	    =  [
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
     ]

     # Process bit string in chunks, each with 16 32-char words
     for block_words in get_block_words(__lowerCAmelCase      ):
          UpperCamelCase__      :       Union[str, Any]	    =  aa
          UpperCamelCase__      :       int	    =  ba
          UpperCamelCase__      :       Union[str, Any]	    =  ca
          UpperCamelCase__      :       str	    =  da

          # Hash current chunk
          for i in range(64      ):
               if i <= 15:
                    # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                    UpperCamelCase__      :       Any	    =  d ^ (b & (c ^ d))
                    UpperCamelCase__      :       Optional[int]	    =  i
               elif i <= 31:
                    # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                    UpperCamelCase__      :       Any	    =  c ^ (d & (b ^ c))
                    UpperCamelCase__      :       str	    =  (5 * i + 1) % 16
               elif i <= 47:
                    UpperCamelCase__      :       Dict	    =  b ^ c ^ d
                    UpperCamelCase__      :       Union[str, Any]	    =  (3 * i + 5) % 16
               else:
                    UpperCamelCase__      :       Optional[int]	    =  c ^ (b | not_aa(__lowerCAmelCase      ))
                    UpperCamelCase__      :       Optional[int]	    =  (7 * i) % 16
               UpperCamelCase__      :       List[str]	    =  (f + a + added_consts[i] + block_words[g]) % 2**32
               UpperCamelCase__      :       Optional[int]	    =  d
               UpperCamelCase__      :       Dict	    =  c
               UpperCamelCase__      :       List[Any]	    =  b
               UpperCamelCase__      :       List[Any]	    =  sum_aa(__lowerCAmelCase ,   left_rotate_aa(__lowerCAmelCase ,   shift_amounts[i]      )      )

          # Add hashed chunk to running total
          UpperCamelCase__      :       Optional[int]	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       Any	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       Dict	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )
          UpperCamelCase__      :       List[str]	    =  sum_aa(__lowerCAmelCase ,   __lowerCAmelCase      )

     UpperCamelCase__      :       List[str]	    =  reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      ) + reformat_hex(__lowerCAmelCase      )
     return digest


if __name__ == "__main__":
 import doctest

 doctest.testmod()