





'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__   :				List[str]      =   logging.get_logger(__name__)

lowercase__   :				Dict      =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase__   :				str      =   {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase__   :				Optional[Any]      =   {
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

lowercase__   :				str      =   {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}


class 			SCREAMING_SNAKE_CASE   (a__							):
     lowerCAmelCase		      =	VOCAB_FILES_NAMES
     lowerCAmelCase		      =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase		      =	PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase		      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase		      =	BertTokenizer


     def __init__(						self						,     _UpperCAmelCase=None						,     _UpperCAmelCase=None						,     _UpperCAmelCase=True						,     _UpperCAmelCase="[UNK]"						,     _UpperCAmelCase="[SEP]"						,     _UpperCAmelCase="[PAD]"						,     _UpperCAmelCase="[CLS]"						,     _UpperCAmelCase="[MASK]"						,     _UpperCAmelCase=True						,     _UpperCAmelCase=None						,     **_UpperCAmelCase						,     ):

          '''simple docstring'''






          super().__init__(
              _UpperCAmelCase						,     tokenizer_file=_UpperCAmelCase						,     do_lower_case=_UpperCAmelCase						,     unk_token=_UpperCAmelCase						,     sep_token=_UpperCAmelCase						,     pad_token=_UpperCAmelCase						,     cls_token=_UpperCAmelCase						,     mask_token=_UpperCAmelCase						,     tokenize_chinese_chars=_UpperCAmelCase						,     strip_accents=_UpperCAmelCase						,     **_UpperCAmelCase						,     )

          __A						:			Dict								=	json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              normalizer_state.get('lowercase'						,     _UpperCAmelCase) != do_lower_case
              or normalizer_state.get('strip_accents'						,     _UpperCAmelCase) != strip_accents
              or normalizer_state.get('handle_chinese_chars'						,     _UpperCAmelCase) != tokenize_chinese_chars
          ):
               __A						:			int								=	getattr(_UpperCAmelCase						,     normalizer_state.pop('type'))
               __A						:			Tuple								=	do_lower_case
               __A						:			Optional[int]								=	strip_accents
               __A						:			Any								=	tokenize_chinese_chars
               __A						:			Optional[Any]								=	normalizer_class(**_UpperCAmelCase)

          __A						:			List[str]								=	do_lower_case


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase=None):

          '''simple docstring'''






          __A						:			int								=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			str								=	[self.sep_token_id]
          __A						:			Tuple								=	[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			List[str]								=	self._tokenizer.model.save(_UpperCAmelCase						,     name=_UpperCAmelCase)
          return tuple(_UpperCAmelCase)






'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowercase__   :				Optional[Any]      =   logging.get_logger(__name__)

lowercase__   :				str      =   {
    '''andreasmadsen/efficient_mlm_m0.40''': (
        '''https://huggingface.co/andreasmadsen/efficient_mlm_m0.40/resolve/main/config.json'''
    ),
}


class 			SCREAMING_SNAKE_CASE   (a__							):
     lowerCAmelCase		      =	'''roberta-prelayernorm'''


     def __init__(						self						,     _UpperCAmelCase=5_0265						,     _UpperCAmelCase=768						,     _UpperCAmelCase=12						,     _UpperCAmelCase=12						,     _UpperCAmelCase=3072						,     _UpperCAmelCase="gelu"						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=512						,     _UpperCAmelCase=2						,     _UpperCAmelCase=0.02						,     _UpperCAmelCase=1e-1_2						,     _UpperCAmelCase=1						,     _UpperCAmelCase=0						,     _UpperCAmelCase=2						,     _UpperCAmelCase="absolute"						,     _UpperCAmelCase=True						,     _UpperCAmelCase=None						,     **_UpperCAmelCase						,     ):

          '''simple docstring'''






          super().__init__(pad_token_id=_UpperCAmelCase						,     bos_token_id=_UpperCAmelCase						,     eos_token_id=_UpperCAmelCase						,     **_UpperCAmelCase)

          __A						:			Union[str, Any]								=	vocab_size
          __A						:			List[str]								=	hidden_size
          __A						:			Tuple								=	num_hidden_layers
          __A						:			List[str]								=	num_attention_heads
          __A						:			Tuple								=	hidden_act
          __A						:			Optional[Any]								=	intermediate_size
          __A						:			List[Any]								=	hidden_dropout_prob
          __A						:			Union[str, Any]								=	attention_probs_dropout_prob
          __A						:			Dict								=	max_position_embeddings
          __A						:			Any								=	type_vocab_size
          __A						:			Optional[int]								=	initializer_range
          __A						:			List[str]								=	layer_norm_eps
          __A						:			Optional[int]								=	position_embedding_type
          __A						:			Dict								=	use_cache
          __A						:			List[str]								=	classifier_dropout




class 			SCREAMING_SNAKE_CASE   (a__							):


     @property
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.task == "multiple-choice":
               __A						:			List[str]								=	{0: 'batch', 1: 'choice', 2: 'sequence'}
          else:
               __A						:			Tuple								=	{0: 'batch', 1: 'sequence'}
          return OrderedDict(
              [
                  ('input_ids', dynamic_axis),
                  ('attention_mask', dynamic_axis),
              ])