


'''simple docstring'''







import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			__lowerCamelCase			(	a_    , unittest.TestCase	):
    """simple docstring"""
    a													=   MobileBertTokenizer
    a													=   MobileBertTokenizerFast
    a													=   True
    a													=   True
    a													=   filter_non_english
    a													=   "google/mobilebert-uncased"


    def        A   (  self      :							Union[str, Any]):
           super().setUp()

           _A					:     Optional[int]     = [
               '[UNK]',
               '[CLS]',
               '[SEP]',
               '[PAD]',
               '[MASK]',
               'want',
               '##want',
               '##ed',
               'wa',
               'un',
               'runn',
               '##ing',
               ',',
               'low',
               'lowest',
           ]
           _A					:     Dict     = os.path.join(self.tmpdirname				,							VOCAB_FILES_NAMES['vocab_file'])
           with open(self.vocab_file				,							'w'				,							encoding='utf-8') as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

           _A					:     Optional[Any]     = [
               (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
               for tokenizer_def in self.tokenizers_list
           ]


    def        A   (  self      :							Tuple				,							SCREAMING_SNAKE_CASE      :							Tuple):
           _A					:     Any     = 'UNwant\u00E9d,running'
           _A					:     Optional[Any]     = 'unwanted, running'
           return input_text, output_text


    def        A   (  self      :							List[str]):
           _A					:     Dict     = self.tokenizer_class(self.vocab_file)

           _A					:     Optional[int]     = tokenizer.tokenize('UNwant\u00E9d,running')
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							['un', '##want', '##ed', ',', 'runn', '##ing'])
           self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE)				,							[9, 6, 7, 12, 10, 11])


    def        A   (  self      :							Tuple):
           if not self.test_rust_tokenizer:
                  return

           _A					:     str     = self.get_tokenizer()
           _A					:     List[str]     = self.get_rust_tokenizer()

           _A					:     Optional[int]     = 'UNwant\u00E9d,running'

           _A					:     List[Any]     = tokenizer.tokenize(SCREAMING_SNAKE_CASE)
           _A					:     Optional[Any]     = rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           _A					:     int     = tokenizer.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
           _A					:     Dict     = rust_tokenizer.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           _A					:     str     = self.get_rust_tokenizer()
           _A					:     Optional[int]     = tokenizer.encode(SCREAMING_SNAKE_CASE)
           _A					:     Union[str, Any]     = rust_tokenizer.encode(SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           # With lower casing
           _A					:     List[str]     = self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE)
           _A					:     Tuple     = self.get_rust_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE)

           _A					:     List[str]     = 'UNwant\u00E9d,running'

           _A					:     Any     = tokenizer.tokenize(SCREAMING_SNAKE_CASE)
           _A					:     str     = rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           _A					:     int     = tokenizer.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
           _A					:     Tuple     = rust_tokenizer.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           _A					:     Tuple     = self.get_rust_tokenizer()
           _A					:     Any     = tokenizer.encode(SCREAMING_SNAKE_CASE)
           _A					:     str     = rust_tokenizer.encode(SCREAMING_SNAKE_CASE)
           self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)


    def        A   (  self      :							Any):
           _A					:     int     = BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')				,							['ah', '\u535A', '\u63A8', 'zz'])


    def        A   (  self      :							List[Any]):
           _A					:     Tuple     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')				,							['hello', '!', 'how', 'are', 'you', '?'])
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo')				,							['hello'])


    def        A   (  self      :							int):
           _A					:     Any     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE				,							strip_accents=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')				,							['hällo', '!', 'how', 'are', 'you', '?'])
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo')				,							['h\u00E9llo'])


    def        A   (  self      :							Optional[int]):
           _A					:     Optional[Any]     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE				,							strip_accents=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')				,							['hallo', '!', 'how', 'are', 'you', '?'])
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo')				,							['hello'])


    def        A   (  self      :							Optional[int]):
           _A					:     Dict     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')				,							['hallo', '!', 'how', 'are', 'you', '?'])
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo')				,							['hello'])


    def        A   (  self      :							Optional[int]):
           _A					:     str     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')				,							['HeLLo', '!', 'how', 'Are', 'yoU', '?'])


    def        A   (  self      :							Optional[Any]):
           _A					:     Tuple     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE				,							strip_accents=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')				,							['HäLLo', '!', 'how', 'Are', 'yoU', '?'])


    def        A   (  self      :							Union[str, Any]):
           _A					:     List[Any]     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE				,							strip_accents=SCREAMING_SNAKE_CASE)

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')				,							['HaLLo', '!', 'how', 'Are', 'yoU', '?'])


    def        A   (  self      :							Tuple):
           _A					:     List[str]     = BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE				,							never_split=['[UNK]'])

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')				,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])


    def        A   (  self      :							Optional[int]):
           _A					:     Tuple     = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

           _A					:     int     = {}
           for i, token in enumerate(SCREAMING_SNAKE_CASE):
                  _A					:     Any     = i
           _A					:     List[str]     = WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE				,							unk_token='[UNK]')

           self.assertListEqual(tokenizer.tokenize('')				,							[])

           self.assertListEqual(tokenizer.tokenize('unwanted running')				,							['un', '##want', '##ed', 'runn', '##ing'])

           self.assertListEqual(tokenizer.tokenize('unwantedX running')				,							['[UNK]', 'runn', '##ing'])


    def        A   (  self      :							Any):
           self.assertTrue(_is_whitespace(' '))
           self.assertTrue(_is_whitespace('\t'))
           self.assertTrue(_is_whitespace('\r'))
           self.assertTrue(_is_whitespace('\n'))
           self.assertTrue(_is_whitespace('\u00A0'))

           self.assertFalse(_is_whitespace('A'))
           self.assertFalse(_is_whitespace('-'))


    def        A   (  self      :							Optional[int]):
           self.assertTrue(_is_control('\u0005'))

           self.assertFalse(_is_control('A'))
           self.assertFalse(_is_control(' '))
           self.assertFalse(_is_control('\t'))
           self.assertFalse(_is_control('\r'))


    def        A   (  self      :							Optional[int]):
           self.assertTrue(_is_punctuation('-'))
           self.assertTrue(_is_punctuation('$'))
           self.assertTrue(_is_punctuation('`'))
           self.assertTrue(_is_punctuation('.'))

           self.assertFalse(_is_punctuation('A'))
           self.assertFalse(_is_punctuation(' '))


    def        A   (  self      :							int):
           _A					:     str     = self.get_tokenizer()
           _A					:     Optional[Any]     = self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE) for t in ['Test', '\xad', 'test']]				,							[['[UNK]'], [], ['[UNK]']])

           self.assertListEqual(
               [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE) for t in ['Test', '\xad', 'test']]				,							[['[UNK]'], [], ['[UNK]']])


    @slow
    def        A   (  self      :							int):
           _A					:     Optional[int]     = self.tokenizer_class.from_pretrained('google/mobilebert-uncased')

           _A					:     Tuple     = tokenizer.encode('sequence builders'				,							add_special_tokens=SCREAMING_SNAKE_CASE)
           _A					:     Dict     = tokenizer.encode('multi-sequence build'				,							add_special_tokens=SCREAMING_SNAKE_CASE)

           _A					:     Optional[int]     = tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE)
           _A					:     Tuple     = tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]


    def        A   (  self      :							Optional[int]):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                         _A					:     Optional[int]     = self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)

                         _A					:     int     = F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                         _A					:     Optional[int]     = tokenizer_r.encode_plus(
                             SCREAMING_SNAKE_CASE				,							return_attention_mask=SCREAMING_SNAKE_CASE				,							return_token_type_ids=SCREAMING_SNAKE_CASE				,							return_offsets_mapping=SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE				,							)

                         _A					:     int     = tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE				,							'do_lower_case') else False
                         _A					:     List[str]     = (
                             [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), 'A'),
                                 ((1, 2), ','),
                                 ((3, 5), 'na'),
                                 ((5, 6), '##ï'),
                                 ((6, 8), '##ve'),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), 'Allen'),
                                 ((21, 23), '##NL'),
                                 ((23, 24), '##P'),
                                 ((25, 33), 'sentence'),
                                 ((33, 34), '.'),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                             if not do_lower_case
                             else [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), 'a'),
                                 ((1, 2), ','),
                                 ((3, 8), 'naive'),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), 'allen'),
                                 ((21, 23), '##nl'),
                                 ((23, 24), '##p'),
                                 ((25, 33), 'sentence'),
                                 ((33, 34), '.'),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                         )

                         self.assertEqual(
                             [e[1] for e in expected_results]				,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
                         self.assertEqual([e[0] for e in expected_results]				,							tokens['offset_mapping'])


    def        A   (  self      :							str):
           _A					:     Dict     = ['的', '人', '有']
           _A					:     Union[str, Any]     = ''.join(SCREAMING_SNAKE_CASE)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                         _A					:     Optional[int]     = True
                         _A					:     Optional[Any]     = self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)
                         _A					:     List[Any]     = self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)

                         _A					:     str     = tokenizer_p.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
                         _A					:     List[Any]     = tokenizer_r.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)

                         _A					:     Dict     = tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE)
                         _A					:     List[Any]     = tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE)

                         # it is expected that each Chinese character is not preceded by "##"
                         self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)
                         self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)

                         _A					:     List[str]     = False
                         _A					:     int     = self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)
                         _A					:     int     = self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)

                         _A					:     List[Any]     = tokenizer_r.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)
                         _A					:     Dict     = tokenizer_p.encode(SCREAMING_SNAKE_CASE				,							add_special_tokens=SCREAMING_SNAKE_CASE)

                         _A					:     Union[str, Any]     = tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE)
                         _A					:     int     = tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE)

                         # it is expected that only the first Chinese character is not preceded by "##".
                         _A					:     Optional[Any]     = [
                             F'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE)
                         ]
                         self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)
                         self.assertListEqual(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A				:					List[str]    						=			{
    '''configuration_electra''': ['''ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ElectraConfig''', '''ElectraOnnxConfig'''],
    '''tokenization_electra''': ['''ElectraTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					int    						=			['''ElectraTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					List[str]    						=			[
       '''ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ElectraForCausalLM''',
       '''ElectraForMaskedLM''',
       '''ElectraForMultipleChoice''',
       '''ElectraForPreTraining''',
       '''ElectraForQuestionAnswering''',
       '''ElectraForSequenceClassification''',
       '''ElectraForTokenClassification''',
       '''ElectraModel''',
       '''ElectraPreTrainedModel''',
       '''load_tf_weights_in_electra''',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					Dict    						=			[
       '''TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFElectraForMaskedLM''',
       '''TFElectraForMultipleChoice''',
       '''TFElectraForPreTraining''',
       '''TFElectraForQuestionAnswering''',
       '''TFElectraForSequenceClassification''',
       '''TFElectraForTokenClassification''',
       '''TFElectraModel''',
       '''TFElectraPreTrainedModel''',
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					List[Any]    						=			[
       '''FlaxElectraForCausalLM''',
       '''FlaxElectraForMaskedLM''',
       '''FlaxElectraForMultipleChoice''',
       '''FlaxElectraForPreTraining''',
       '''FlaxElectraForQuestionAnswering''',
       '''FlaxElectraForSequenceClassification''',
       '''FlaxElectraForTokenClassification''',
       '''FlaxElectraModel''',
       '''FlaxElectraPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
   from .tokenization_electra import ElectraTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_electra_fast import ElectraTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_electra import (
          ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
          ElectraForCausalLM,
          ElectraForMaskedLM,
          ElectraForMultipleChoice,
          ElectraForPreTraining,
          ElectraForQuestionAnswering,
          ElectraForSequenceClassification,
          ElectraForTokenClassification,
          ElectraModel,
          ElectraPreTrainedModel,
          load_tf_weights_in_electra,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_electra import (
          TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFElectraForMaskedLM,
          TFElectraForMultipleChoice,
          TFElectraForPreTraining,
          TFElectraForQuestionAnswering,
          TFElectraForSequenceClassification,
          TFElectraForTokenClassification,
          TFElectraModel,
          TFElectraPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
          FlaxElectraPreTrainedModel,
      )

else:
   import sys

   A				:					Tuple    						=			_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






