






'''simple docstring'''

import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     =       '▁'

SCREAMING_SNAKE_CASE__     =       get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class     a_						(      lowerCamelCase     ,	unittest.TestCase    ):
  lowercase							=				BigBirdTokenizer
  lowercase							=				BigBirdTokenizerFast
  lowercase							=				True
  lowercase							=				True

  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        super().setUp()

        UpperCamelCase      = self.tokenizer_class(_SCREAMING_SNAKE_CASE      ,		keep_accents=_SCREAMING_SNAKE_CASE				)
        tokenizer.save_pretrained(self.tmpdirname				)

  def   A__						(				self				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = """<s>"""
        UpperCamelCase      = 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE				)      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE				)      ,		_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = list(self.get_tokenizer().get_vocab().keys()				)

        self.assertEqual(vocab_keys[0]      ,		"""<unk>"""				)
        self.assertEqual(vocab_keys[1]      ,		"""<s>"""				)
        self.assertEqual(vocab_keys[-1]      ,		"""[MASK]"""				)
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		1004				)

  def   A__						(				self				)  ->				Union[str, Any]:



        """simple docstring"""






        self.assertEqual(self.get_tokenizer().vocab_size      ,		1000				)

  def   A__						(				self				)  ->				str:



        """simple docstring"""






        if not self.test_rust_tokenizer:
              return

        UpperCamelCase      = self.get_tokenizer()
        UpperCamelCase      = self.get_rust_tokenizer()

        UpperCamelCase      = """I was born in 92000, and this is falsé."""

        UpperCamelCase      = tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = tokenizer.encode(_SCREAMING_SNAKE_CASE      ,		add_special_tokens=_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = rust_tokenizer.encode(_SCREAMING_SNAKE_CASE      ,		add_special_tokens=_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = self.get_rust_tokenizer()
        UpperCamelCase      = tokenizer.encode(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        UpperCamelCase      = BigBirdTokenizer(_SCREAMING_SNAKE_CASE      ,		keep_accents=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = tokenizer.tokenize("""This is a test"""				)
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)      ,		[285, 46, 10, 170, 382]      ,		)

        UpperCamelCase      = tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]      ,		)
        UpperCamelCase      = tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,		)

        UpperCamelCase      = tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]      ,		)

  @cached_property
  def   A__						(				self				)  ->				Any:



        """simple docstring"""






        return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""				)

  @slow
  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase      = """Hello World!"""
        UpperCamelCase      = [65, 18536, 2260, 101, 66]

        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE				)				)

  @slow
  def   A__						(				self				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = (
            """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
            """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
        )
        # fmt: off
        UpperCamelCase      = [65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
        # fmt: on
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE				)				)

  @require_torch
  @slow
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        import torch

        from transformers import BigBirdConfig, BigBirdModel

        # Build sequence
        UpperCamelCase      = list(self.big_tokenizer.get_vocab().keys()				)[:10]
        UpperCamelCase      = """ """.join(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = self.big_tokenizer.encode_plus(_SCREAMING_SNAKE_CASE      ,		return_tensors="""pt"""      ,		return_token_type_ids=_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = self.big_tokenizer.batch_encode_plus(
            [sequence + """ """ + sequence]      ,		return_tensors="""pt"""      ,		return_token_type_ids=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = BigBirdConfig(attention_type="""original_full"""				)
        UpperCamelCase      = BigBirdModel(_SCREAMING_SNAKE_CASE				)

        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
              model(**_SCREAMING_SNAKE_CASE				)
              model(**_SCREAMING_SNAKE_CASE				)

  @slow
  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase      = BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""				)
        UpperCamelCase      = tokenizer.decode(tokenizer("""Paris is the [MASK]."""				).input_ids				)

        self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""				)


  @slow
  def   A__						(				self				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = {"""input_ids""": [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=_SCREAMING_SNAKE_CASE      ,		model_name="""google/bigbird-roberta-base"""      ,		revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""      ,		)








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__     =       {
    'configuration_mobilebert': [
        'MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MobileBertConfig',
        'MobileBertOnnxConfig',
    ],
    'tokenization_mobilebert': ['MobileBertTokenizer'],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       ['MobileBertTokenizerFast']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       [
         'MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
         'MobileBertForMaskedLM',
         'MobileBertForMultipleChoice',
         'MobileBertForNextSentencePrediction',
         'MobileBertForPreTraining',
         'MobileBertForQuestionAnswering',
         'MobileBertForSequenceClassification',
         'MobileBertForTokenClassification',
         'MobileBertLayer',
         'MobileBertModel',
         'MobileBertPreTrainedModel',
         'load_tf_weights_in_mobilebert',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE__     =       [
         'TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
         'TFMobileBertForMaskedLM',
         'TFMobileBertForMultipleChoice',
         'TFMobileBertForNextSentencePrediction',
         'TFMobileBertForPreTraining',
         'TFMobileBertForQuestionAnswering',
         'TFMobileBertForSequenceClassification',
         'TFMobileBertForTokenClassification',
         'TFMobileBertMainLayer',
         'TFMobileBertModel',
         'TFMobileBertPreTrainedModel',
     ]


if TYPE_CHECKING:
     from .configuration_mobilebert import (
         MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         MobileBertConfig,
         MobileBertOnnxConfig,
     )
     from .tokenization_mobilebert import MobileBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_mobilebert_fast import MobileBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mobilebert import (
              MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              MobileBertForMaskedLM,
              MobileBertForMultipleChoice,
              MobileBertForNextSentencePrediction,
              MobileBertForPreTraining,
              MobileBertForQuestionAnswering,
              MobileBertForSequenceClassification,
              MobileBertForTokenClassification,
              MobileBertLayer,
              MobileBertModel,
              MobileBertPreTrainedModel,
              load_tf_weights_in_mobilebert,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_mobilebert import (
              TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFMobileBertForMaskedLM,
              TFMobileBertForMultipleChoice,
              TFMobileBertForNextSentencePrediction,
              TFMobileBertForPreTraining,
              TFMobileBertForQuestionAnswering,
              TFMobileBertForSequenceClassification,
              TFMobileBertForTokenClassification,
              TFMobileBertMainLayer,
              TFMobileBertModel,
              TFMobileBertPreTrainedModel,
          )

else:
     import sys

     SCREAMING_SNAKE_CASE__     =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
