




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCamelCase      			=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase      			=						{
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

_lowerCamelCase      			=						{
    'google/electra-small-generator': 5_12,
    'google/electra-base-generator': 5_12,
    'google/electra-large-generator': 5_12,
    'google/electra-small-discriminator': 5_12,
    'google/electra-base-discriminator': 5_12,
    'google/electra-large-discriminator': 5_12,
}

_lowerCamelCase      			=						{
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}



class       a					(       _A      ):

     '''simple docstring'''





     lowerCAmelCase    :	str       		=  VOCAB_FILES_NAMES
     lowerCAmelCase    :	Optional[int]       		=  PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase    :	Dict       		=  PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase    :	Any       		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase    :	List[Any]       		=  ElectraTokenizer
     def __init__(     self       :      List[Any] ,   __snake_case       :      Union[str, Any]=None ,   __snake_case       :      str=None ,   __snake_case       :      List[Any]=True ,   __snake_case       :      int="[UNK]" ,   __snake_case       :      Tuple="[SEP]" ,   __snake_case       :      Optional[int]="[PAD]" ,   __snake_case       :      int="[CLS]" ,   __snake_case       :      int="[MASK]" ,   __snake_case       :      Optional[Any]=True ,   __snake_case       :      str=None ,   **__snake_case       :      Dict ,   ):
         super().__init__(
             __snake_case ,   tokenizer_file=__snake_case ,   do_lower_case=__snake_case ,   unk_token=__snake_case ,   sep_token=__snake_case ,   pad_token=__snake_case ,   cls_token=__snake_case ,   mask_token=__snake_case ,   tokenize_chinese_chars=__snake_case ,   strip_accents=__snake_case ,   **__snake_case ,   )

         UpperCAmelCase_										=					json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
         if (
             normalizer_state.get('''lowercase''' ,   __snake_case    ) != do_lower_case
             or normalizer_state.get('''strip_accents''' ,   __snake_case    ) != strip_accents
             or normalizer_state.get('''handle_chinese_chars''' ,   __snake_case    ) != tokenize_chinese_chars
         ):
             UpperCAmelCase_										=					getattr(__snake_case ,   normalizer_state.pop('''type'''    )    )
             UpperCAmelCase_										=					do_lower_case
             UpperCAmelCase_										=					strip_accents
             UpperCAmelCase_										=					tokenize_chinese_chars
             UpperCAmelCase_										=					normalizer_class(**__snake_case    )

         UpperCAmelCase_										=					do_lower_case
     def   lowerCamelCase_       (     self       :      Any ,   __snake_case       :      Optional[int] ,   __snake_case       :      List[Any]=None    ):
         UpperCAmelCase_										=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output
     def   lowerCamelCase_       (     self       :      List[Any] ,   __snake_case       :      List[int] ,   __snake_case       :      Optional[List[int]] = None    ):
         UpperCAmelCase_										=					[self.sep_token_id]
         UpperCAmelCase_										=					[self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep    ) * [0]
         return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




     def   lowerCamelCase_       (     self       :      List[Any] ,   __snake_case       :      str ,   __snake_case       :      Optional[str] = None    ):
         UpperCAmelCase_										=					self._tokenizer.model.save(__snake_case ,   name=__snake_case    )
         return tuple(__snake_case    )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCamelCase      			=						{
    'configuration_roberta': ['ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RobertaConfig', 'RobertaOnnxConfig'],
    'tokenization_roberta': ['RobertaTokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCamelCase      			=						['RobertaTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCamelCase      			=						[
     'ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'RobertaForCausalLM',
     'RobertaForMaskedLM',
     'RobertaForMultipleChoice',
     'RobertaForQuestionAnswering',
     'RobertaForSequenceClassification',
     'RobertaForTokenClassification',
     'RobertaModel',
     'RobertaPreTrainedModel',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCamelCase      			=						[
     'TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFRobertaForCausalLM',
     'TFRobertaForMaskedLM',
     'TFRobertaForMultipleChoice',
     'TFRobertaForQuestionAnswering',
     'TFRobertaForSequenceClassification',
     'TFRobertaForTokenClassification',
     'TFRobertaMainLayer',
     'TFRobertaModel',
     'TFRobertaPreTrainedModel',
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCamelCase      			=						[
     'FlaxRobertaForCausalLM',
     'FlaxRobertaForMaskedLM',
     'FlaxRobertaForMultipleChoice',
     'FlaxRobertaForQuestionAnswering',
     'FlaxRobertaForSequenceClassification',
     'FlaxRobertaForTokenClassification',
     'FlaxRobertaModel',
     'FlaxRobertaPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, RobertaOnnxConfig
 from .tokenization_roberta import RobertaTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_roberta_fast import RobertaTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_roberta import (
      ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      RobertaForCausalLM,
      RobertaForMaskedLM,
      RobertaForMultipleChoice,
      RobertaForQuestionAnswering,
      RobertaForSequenceClassification,
      RobertaForTokenClassification,
      RobertaModel,
      RobertaPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_roberta import (
      TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFRobertaForCausalLM,
      TFRobertaForMaskedLM,
      TFRobertaForMultipleChoice,
      TFRobertaForQuestionAnswering,
      TFRobertaForSequenceClassification,
      TFRobertaForTokenClassification,
      TFRobertaMainLayer,
      TFRobertaModel,
      TFRobertaPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_roberta import (
      FlaxRobertaForCausalLM,
      FlaxRobertaForMaskedLM,
      FlaxRobertaForMultipleChoice,
      FlaxRobertaForQuestionAnswering,
      FlaxRobertaForSequenceClassification,
      FlaxRobertaForTokenClassification,
      FlaxRobertaModel,
      FlaxRobertaPreTrainedModel,
  )

else:
 import sys

 _lowerCamelCase      			=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





