






'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__     =       {
    'configuration_distilbert': [
        'DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'DistilBertConfig',
        'DistilBertOnnxConfig',
    ],
    'tokenization_distilbert': ['DistilBertTokenizer'],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       ['DistilBertTokenizerFast']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       [
         'DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
         'DistilBertForMaskedLM',
         'DistilBertForMultipleChoice',
         'DistilBertForQuestionAnswering',
         'DistilBertForSequenceClassification',
         'DistilBertForTokenClassification',
         'DistilBertModel',
         'DistilBertPreTrainedModel',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       [
         'TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
         'TFDistilBertForMaskedLM',
         'TFDistilBertForMultipleChoice',
         'TFDistilBertForQuestionAnswering',
         'TFDistilBertForSequenceClassification',
         'TFDistilBertForTokenClassification',
         'TFDistilBertMainLayer',
         'TFDistilBertModel',
         'TFDistilBertPreTrainedModel',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       [
         'FlaxDistilBertForMaskedLM',
         'FlaxDistilBertForMultipleChoice',
         'FlaxDistilBertForQuestionAnswering',
         'FlaxDistilBertForSequenceClassification',
         'FlaxDistilBertForTokenClassification',
         'FlaxDistilBertModel',
         'FlaxDistilBertPreTrainedModel',
     ]


if TYPE_CHECKING:
     from .configuration_distilbert import (
         DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DistilBertConfig,
         DistilBertOnnxConfig,
     )
     from .tokenization_distilbert import DistilBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_distilbert_fast import DistilBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_distilbert import (
              DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
              DistilBertModel,
              DistilBertPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_distilbert import (
              TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFDistilBertForMaskedLM,
              TFDistilBertForMultipleChoice,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertMainLayer,
              TFDistilBertModel,
              TFDistilBertPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_distilbert import (
              FlaxDistilBertForMaskedLM,
              FlaxDistilBertForMultipleChoice,
              FlaxDistilBertForQuestionAnswering,
              FlaxDistilBertForSequenceClassification,
              FlaxDistilBertForTokenClassification,
              FlaxDistilBertModel,
              FlaxDistilBertPreTrainedModel,
          )

else:
     import sys

     SCREAMING_SNAKE_CASE__     =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)








'''simple docstring'''

import os
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'vocab.txt'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'facebook/esm2_t6_8M_UR50D': 'https://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txt',
        'facebook/esm2_t12_35M_UR50D': 'https://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt',
    },
}

SCREAMING_SNAKE_CASE__     =       {
    'facebook/esm2_t6_8M_UR50D': 1_0_2_4,
    'facebook/esm2_t12_35M_UR50D': 1_0_2_4,
}







def    lowercase__			(				__UpperCamelCase   )->			Any:
      with open(__UpperCamelCase		,						"""r"""   ) as f:
            UpperCamelCase      = f.read().splitlines()
            return [l.strip() for l in lines]






class     a_						(      lowerCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<cls>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE="<mask>"      ,		_SCREAMING_SNAKE_CASE="<eos>"      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				Any:



        """simple docstring"""






        super().__init__(**_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = load_vocab_file(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = dict(enumerate(self.all_tokens				)				)
        UpperCamelCase      = {tok: ind for ind, tok in enumerate(self.all_tokens				)}
        UpperCamelCase      = unk_token
        UpperCamelCase      = cls_token
        UpperCamelCase      = pad_token
        UpperCamelCase      = mask_token
        UpperCamelCase      = eos_token
        UpperCamelCase      = self.all_tokens
        self._create_trie(self.unique_no_split_tokens				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        return self._id_to_token.get(_SCREAMING_SNAKE_CASE      ,		self.unk_token				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        return self._token_to_id.get(_SCREAMING_SNAKE_CASE      ,		self._token_to_id.get(self.unk_token				)				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        return text.split()

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE=False				)  ->				Dict:



        """simple docstring"""






        return len(self._id_to_token				)

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return {token: i for i, token in enumerate(self.all_tokens				)}

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        return self._token_to_id.get(_SCREAMING_SNAKE_CASE      ,		self._token_to_id.get(self.unk_token				)				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        return self._id_to_token.get(_SCREAMING_SNAKE_CASE      ,		self.unk_token				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[int]:



        """simple docstring"""






        UpperCamelCase      = [self.cls_token_id]
        UpperCamelCase      = [self.eos_token_id]  # No sep token in ESM vocabulary
        if token_ids_a is None:
              if self.eos_token_id is None:
                    return cls + token_ids_a
              else:
                    return cls + token_ids_a + sep
        elif self.eos_token_id is None:
              raise ValueError("""Cannot tokenize multiple sequences when EOS token is not set!"""				)
        return cls + token_ids_a + sep + token_ids_a + sep  # Multiple inputs always have an EOS token

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



        """simple docstring"""






        if already_has_special_tokens:
              if token_ids_a is not None:
                    raise ValueError(
                        """You should not supply a second sequence if the provided sequence of """
                        """ids is already formatted with special tokens for the model."""				)

              return [1 if token in self.all_special_ids else 0 for token in token_ids_a]
        UpperCamelCase      = [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1]
        if token_ids_a is not None:
              mask += [0] * len(_SCREAMING_SNAKE_CASE				) + [1]
        return mask

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = os.path.join(_SCREAMING_SNAKE_CASE      ,		(filename_prefix + """-""" if filename_prefix else """""") + """vocab.txt"""				)
        with open(_SCREAMING_SNAKE_CASE      ,		"""w"""				) as f:
              f.write("""\n""".join(self.all_tokens				)				)
        return (vocab_file,)

  @property
  def   A__						(				self				)  ->				int:



        """simple docstring"""






        return self.get_vocab_size(with_added_tokens=_SCREAMING_SNAKE_CASE				)


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = False				)  ->				int:



        """simple docstring"""






        return super()._add_tokens(_SCREAMING_SNAKE_CASE      ,		special_tokens=_SCREAMING_SNAKE_CASE				)
