



"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase    : str							=							{
    "configuration_xlm_roberta": [
        "XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "XLMRobertaConfig",
        "XLMRobertaOnnxConfig",
    ],
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase    : int							=							["XLMRobertaTokenizer"]

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase    : Any							=							["XLMRobertaTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase    : Any							=							[
     "XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
     "XLMRobertaForCausalLM",
     "XLMRobertaForMaskedLM",
     "XLMRobertaForMultipleChoice",
     "XLMRobertaForQuestionAnswering",
     "XLMRobertaForSequenceClassification",
     "XLMRobertaForTokenClassification",
     "XLMRobertaModel",
     "XLMRobertaPreTrainedModel",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase    : Union[str, Any]							=							[
     "TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFXLMRobertaForCausalLM",
     "TFXLMRobertaForMaskedLM",
     "TFXLMRobertaForMultipleChoice",
     "TFXLMRobertaForQuestionAnswering",
     "TFXLMRobertaForSequenceClassification",
     "TFXLMRobertaForTokenClassification",
     "TFXLMRobertaModel",
     "TFXLMRobertaPreTrainedModel",
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase    : List[Any]							=							[
     "FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
     "FlaxXLMRobertaForMaskedLM",
     "FlaxXLMRobertaForCausalLM",
     "FlaxXLMRobertaForMultipleChoice",
     "FlaxXLMRobertaForQuestionAnswering",
     "FlaxXLMRobertaForSequenceClassification",
     "FlaxXLMRobertaForTokenClassification",
     "FlaxXLMRobertaModel",
     "FlaxXLMRobertaPreTrainedModel",
 ]

if TYPE_CHECKING:
 from .configuration_xlm_roberta import (
     XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
     XLMRobertaConfig,
     XLMRobertaOnnxConfig,
 )

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_xlm_roberta import XLMRobertaTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_xlm_roberta_fast import XLMRobertaTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_xlm_roberta import (
      XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      XLMRobertaForCausalLM,
      XLMRobertaForMaskedLM,
      XLMRobertaForMultipleChoice,
      XLMRobertaForQuestionAnswering,
      XLMRobertaForSequenceClassification,
      XLMRobertaForTokenClassification,
      XLMRobertaModel,
      XLMRobertaPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_xlm_roberta import (
      TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMRobertaForCausalLM,
      TFXLMRobertaForMaskedLM,
      TFXLMRobertaForMultipleChoice,
      TFXLMRobertaForQuestionAnswering,
      TFXLMRobertaForSequenceClassification,
      TFXLMRobertaForTokenClassification,
      TFXLMRobertaModel,
      TFXLMRobertaPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_xlm_roberta import (
      FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      FlaxXLMRobertaForCausalLM,
      FlaxXLMRobertaForMaskedLM,
      FlaxXLMRobertaForMultipleChoice,
      FlaxXLMRobertaForQuestionAnswering,
      FlaxXLMRobertaForSequenceClassification,
      FlaxXLMRobertaForTokenClassification,
      FlaxXLMRobertaModel,
      FlaxXLMRobertaPreTrainedModel,
  )

else:
 import sys

 _lowerCAmelCase    : Optional[Any]							=							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCAmelCase    : List[str]							=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase    : int							=							{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCAmelCase    : List[Any]							=							{
    "google/electra-small-generator": 5_12,
    "google/electra-base-generator": 5_12,
    "google/electra-large-generator": 5_12,
    "google/electra-small-discriminator": 5_12,
    "google/electra-base-discriminator": 5_12,
    "google/electra-large-discriminator": 5_12,
}

_lowerCAmelCase    : Optional[Any]							=							{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}






class      UpperCAmelCase_  (    _UpperCamelCase						):
      __SCREAMING_SNAKE_CASE						:       Optional[Any]								=			VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE						:       Any								=			PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_INIT_CONFIGURATION
      __SCREAMING_SNAKE_CASE						:       List[str]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE						:       Tuple								=			ElectraTokenizer



      def __init__(					self					:     Dict					,     A					:     Dict=None					,     A					:     Optional[int]=None					,     A					:     Dict=True					,     A					:     Optional[Any]="[UNK]"					,     A					:     Any="[SEP]"					,     A					:     str="[PAD]"					,     A					:     Tuple="[CLS]"					,     A					:     Optional[Any]="[MASK]"					,     A					:     Any=True					,     A					:     Tuple=None					,     **A					:     Any					,     ):
           super().__init__(
               A					,     tokenizer_file=A					,     do_lower_case=A					,     unk_token=A					,     sep_token=A					,     pad_token=A					,     cls_token=A					,     mask_token=A					,     tokenize_chinese_chars=A					,     strip_accents=A					,     **A					,     )

           _UpperCAmelCase      :     List[str]        =   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
           if (
               normalizer_state.get("lowercase"					,     A						) != do_lower_case
               or normalizer_state.get("strip_accents"					,     A						) != strip_accents
               or normalizer_state.get("handle_chinese_chars"					,     A						) != tokenize_chinese_chars
           ):
                _UpperCAmelCase      :     Union[str, Any]        =   getattr(A					,     normalizer_state.pop("type"						)						)
                _UpperCAmelCase      :     Dict        =   do_lower_case
                _UpperCAmelCase      :     Optional[int]        =   strip_accents
                _UpperCAmelCase      :     Any        =   tokenize_chinese_chars
                _UpperCAmelCase      :     Optional[Any]        =   normalizer_class(**A						)

           _UpperCAmelCase      :     int        =   do_lower_case



      def 						snake_case_ (					self					:     Tuple					,     A					:     str					,     A					:     int=None						):
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

           return output



      def 						snake_case_ (					self					:     Any					,     A					:     List[int]					,     A					:     Optional[List[int]] = None						):
           _UpperCAmelCase      :     Any        =   [self.sep_token_id]
           _UpperCAmelCase      :     Dict        =   [self.cls_token_id]
           if token_ids_a is None:
                return len(cls + token_ids_a + sep						) * [0]
           return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



      def 						snake_case_ (					self					:     Any					,     A					:     str					,     A					:     Optional[str] = None						):
           _UpperCAmelCase      :     List[Any]        =   self._tokenizer.model.save(A					,     name=A						)
           return tuple(A						)
