"""simple docstring"""
import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     FlaubertForMultipleChoice,
     FlaubertForQuestionAnswering,
     FlaubertForQuestionAnsweringSimple,
     FlaubertForSequenceClassification,
     FlaubertForTokenClassification,
     FlaubertModel,
     FlaubertWithLMHeadModel,
 )
 from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class        __snake_case	(					__lowerCAmelCase    ):



      def __init__(						self   ,      lowercase   ,      lowercase=13   ,      lowercase=7   ,      lowercase=True   ,      lowercase=True   ,      lowercase=True   ,      lowercase=True   ,      lowercase=True   ,      lowercase=False   ,      lowercase=False   ,      lowercase=False   ,      lowercase=2   ,      lowercase=99   ,      lowercase=0   ,      lowercase=32   ,      lowercase=5   ,      lowercase=4   ,      lowercase=0.1   ,      lowercase=0.1   ,      lowercase=5_12   ,      lowercase=12   ,      lowercase=2   ,      lowercase=0.02   ,      lowercase=3   ,      lowercase=4   ,      lowercase="last"   ,      lowercase=None   ,      lowercase=None   ,      )     ->      Optional[int]:





        '''simple docstring'''





        a__:					List[str]      							=     parent
        a__:					Tuple      							=     batch_size
        a__:					Dict      							=     seq_length
        a__:					Union[str, Any]      							=     is_training
        a__:					Optional[Any]      							=     use_input_lengths
        a__:					Optional[int]      							=     use_token_type_ids
        a__:					List[str]      							=     use_labels
        a__:					Dict      							=     gelu_activation
        a__:					Optional[Any]      							=     sinusoidal_embeddings
        a__:					Optional[int]      							=     causal
        a__:					Tuple      							=     asm
        a__:					Optional[int]      							=     n_langs
        a__:					List[Any]      							=     vocab_size
        a__:					Dict      							=     n_special
        a__:					Dict      							=     hidden_size
        a__:					Optional[int]      							=     num_hidden_layers
        a__:					List[Any]      							=     num_attention_heads
        a__:					str      							=     hidden_dropout_prob
        a__:					Union[str, Any]      							=     attention_probs_dropout_prob
        a__:					int      							=     max_position_embeddings
        a__:					Optional[int]      							=     type_vocab_size
        a__:					Optional[int]      							=     type_sequence_label_size
        a__:					str      							=     initializer_range
        a__:					int      							=     num_labels
        a__:					Optional[int]      							=     num_choices
        a__:					Any      							=     summary_type
        a__:					Tuple      							=     use_proj
        a__:					Tuple      							=     scope



      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        a__:					str      							=     ids_tensor([self.batch_size, self.seq_length]   ,      self.vocab_size)
        a__:					str      							=     random_attention_mask([self.batch_size, self.seq_length])

        a__:					int      							=     None
        if self.use_input_lengths:
          a__:					str      							=     (
              ids_tensor([self.batch_size]   ,      vocab_size=2) + self.seq_length - 2
          )  # small variation of seq_length

        a__:					Tuple      							=     None
        if self.use_token_type_ids:
          a__:					Tuple      							=     ids_tensor([self.batch_size, self.seq_length]   ,      self.n_langs)

        a__:					Optional[Any]      							=     None
        a__:					Any      							=     None
        a__:					Tuple      							=     None
        if self.use_labels:
          a__:					int      							=     ids_tensor([self.batch_size]   ,      self.type_sequence_label_size)
          a__:					Dict      							=     ids_tensor([self.batch_size, self.seq_length]   ,      self.num_labels)
          a__:					Optional[Any]      							=     ids_tensor([self.batch_size]   ,      2).float()
          a__:					str      							=     ids_tensor([self.batch_size]   ,      self.num_choices)

        a__:					str      							=     self.get_config()

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )



      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        return FlaubertConfig(
            vocab_size=self.vocab_size   ,      n_special=self.n_special   ,      emb_dim=self.hidden_size   ,      n_layers=self.num_hidden_layers   ,      n_heads=self.num_attention_heads   ,      dropout=self.hidden_dropout_prob   ,      attention_dropout=self.attention_probs_dropout_prob   ,      gelu_activation=self.gelu_activation   ,      sinusoidal_embeddings=self.sinusoidal_embeddings   ,      asm=self.asm   ,      causal=self.causal   ,      n_langs=self.n_langs   ,      max_position_embeddings=self.max_position_embeddings   ,      initializer_range=self.initializer_range   ,      summary_type=self.summary_type   ,      use_proj=self.use_proj   ,      )



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      Optional[Any]:





        '''simple docstring'''





        a__:					Any      							=     FlaubertModel(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					Dict      							=     model(lowercase   ,      lengths=lowercase   ,      langs=lowercase)
        a__:					Dict      							=     model(lowercase   ,      langs=lowercase)
        a__:					Optional[Any]      							=     model(lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape   ,      (self.batch_size, self.seq_length, self.hidden_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      str:





        '''simple docstring'''





        a__:					Union[str, Any]      							=     FlaubertWithLMHeadModel(lowercase)
        model.to(lowercase)
        model.eval()

        a__:					Tuple      							=     model(lowercase   ,      token_type_ids=lowercase   ,      labels=lowercase)
        self.parent.assertEqual(result.loss.shape   ,      ())
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.seq_length, self.vocab_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      Dict:





        '''simple docstring'''





        a__:					Optional[Any]      							=     FlaubertForQuestionAnsweringSimple(lowercase)
        model.to(lowercase)
        model.eval()

        a__:					Any      							=     model(lowercase)

        a__:					List[str]      							=     model(lowercase   ,      start_positions=lowercase   ,      end_positions=lowercase)
        self.parent.assertEqual(result.start_logits.shape   ,      (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape   ,      (self.batch_size, self.seq_length))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      Tuple:





        '''simple docstring'''





        a__:					Optional[int]      							=     FlaubertForQuestionAnswering(lowercase)
        model.to(lowercase)
        model.eval()

        a__:					Optional[int]      							=     model(lowercase)

        a__:					Any      							=     model(
            lowercase   ,      start_positions=lowercase   ,      end_positions=lowercase   ,      cls_index=lowercase   ,      is_impossible=lowercase   ,      p_mask=lowercase   ,      )

        a__:					Optional[int]      							=     model(
            lowercase   ,      start_positions=lowercase   ,      end_positions=lowercase   ,      cls_index=lowercase   ,      is_impossible=lowercase   ,      )

        ((a__)							,   ):					Optional[int]      							=     result_with_labels.to_tuple()

        a__:					Union[str, Any]      							=     model(lowercase   ,      start_positions=lowercase   ,      end_positions=lowercase)

        ((a__)							,   ):					Optional[Any]      							=     result_with_labels.to_tuple()

        self.parent.assertEqual(result_with_labels.loss.shape   ,      ())
        self.parent.assertEqual(result.start_top_log_probs.shape   ,      (self.batch_size, model.config.start_n_top))
        self.parent.assertEqual(result.start_top_index.shape   ,      (self.batch_size, model.config.start_n_top))
        self.parent.assertEqual(
            result.end_top_log_probs.shape   ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top))
        self.parent.assertEqual(
            result.end_top_index.shape   ,      (self.batch_size, model.config.start_n_top * model.config.end_n_top))
        self.parent.assertEqual(result.cls_logits.shape   ,      (self.batch_size,))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      Optional[Any]:





        '''simple docstring'''





        a__:					Dict      							=     FlaubertForSequenceClassification(lowercase)
        model.to(lowercase)
        model.eval()

        a__:					Union[str, Any]      							=     model(lowercase)
        a__:					Any      							=     model(lowercase   ,      labels=lowercase)

        self.parent.assertEqual(result.loss.shape   ,      ())
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.type_sequence_label_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      List[str]:





        '''simple docstring'''





        a__:					str      							=     self.num_labels
        a__:					List[str]      							=     FlaubertForTokenClassification(lowercase)
        model.to(lowercase)
        model.eval()

        a__:					List[Any]      							=     model(lowercase   ,      attention_mask=lowercase   ,      labels=lowercase)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.seq_length, self.num_labels))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      )     ->      Tuple:





        '''simple docstring'''





        a__:					Tuple      							=     self.num_choices
        a__:					Tuple      							=     FlaubertForMultipleChoice(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					Tuple      							=     input_ids.unsqueeze(1).expand(-1   ,      self.num_choices   ,      -1).contiguous()
        a__:					Any      							=     token_type_ids.unsqueeze(1).expand(-1   ,      self.num_choices   ,      -1).contiguous()
        a__:					Union[str, Any]      							=     input_mask.unsqueeze(1).expand(-1   ,      self.num_choices   ,      -1).contiguous()
        a__:					str      							=     model(
            lowercase   ,      attention_mask=lowercase   ,      token_type_ids=lowercase   ,      labels=lowercase   ,      )
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_choices))



      def   lowerCamelCase_       (						self)     ->      List[Any]:





        '''simple docstring'''





        a__:					int      							=     self.prepare_config_and_inputs()
        (
            (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   (
            a__
        )							,   
        ):					Dict      							=     config_and_inputs
        a__:					Optional[Any]      							=     {
            'input_ids': input_ids,
            'token_type_ids': token_type_ids,
            'lengths': input_lengths,
            'attention_mask': input_mask,
        }
        return config, inputs_dict







@require_torch
class        __snake_case	(					__lowerCAmelCase	,				__lowerCAmelCase	,				unittest.TestCase    ):
      a__				=		(
          (
              FlaubertModel,
              FlaubertWithLMHeadModel,
              FlaubertForQuestionAnswering,
              FlaubertForQuestionAnsweringSimple,
              FlaubertForSequenceClassification,
              FlaubertForTokenClassification,
              FlaubertForMultipleChoice,
          )
          if is_torch_available()
          else ()
      )
      a__				=		(
          {
              """feature-extraction""": FlaubertModel,
              """fill-mask""": FlaubertWithLMHeadModel,
              """question-answering""": FlaubertForQuestionAnsweringSimple,
              """text-classification""": FlaubertForSequenceClassification,
              """token-classification""": FlaubertForTokenClassification,
              """zero-shot""": FlaubertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase)     ->      List[str]:





        '''simple docstring'''





        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith('Fast')
        ):
          # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
          # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
          # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
          return True

        return False



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase=False)     ->      Any:





        '''simple docstring'''





        a__:					Union[str, Any]      							=     super()._prepare_for_class(lowercase   ,      lowercase   ,      return_labels=lowercase)

        if return_labels:
          if model_class.__name__ == "FlaubertForQuestionAnswering":
            a__:					Any      							=     torch.zeros(
                self.model_tester.batch_size   ,      dtype=torch.long   ,      device=lowercase)
            a__:					Union[str, Any]      							=     torch.zeros(
                self.model_tester.batch_size   ,      dtype=torch.long   ,      device=lowercase)

        return inputs_dict



      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        a__:					Any      							=     FlaubertModelTester(self)
        a__:					Optional[Any]      							=     ConfigTester(self   ,      config_class=lowercase   ,      emb_dim=37)



      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        self.config_tester.run_common_tests()



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__:					str      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_model(*lowercase)



      def   lowerCamelCase_       (						self)     ->      int:





        '''simple docstring'''





        a__:					int      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_lm_head(*lowercase)



      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        a__:					Any      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_simple_qa(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        a__:					Tuple      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_qa(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__:					Dict      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_sequence_classif(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        a__:					Any      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_token_classif(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					Tuple      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_flaubert_multiple_choice(*lowercase)



      @slow
      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          a__:					int      							=     FlaubertModel.from_pretrained(lowercase)
          self.assertIsNotNone(lowercase)



      @slow
      @require_torch_gpu
      def   lowerCamelCase_       (						self)     ->      Optional[Any]:





        '''simple docstring'''





        a__							,   a__:					int      							=     self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes:
          # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
          if model_class == FlaubertForMultipleChoice:
            return

          a__:					List[str]      							=     True
          a__:					Any      							=     model_class(config=lowercase)

          a__:					Optional[Any]      							=     self._prepare_for_class(lowercase   ,      lowercase)
          a__:					List[Any]      							=     torch.jit.trace(
              lowercase   ,      (inputs_dict['input_ids'].to('cpu'), inputs_dict['attention_mask'].to('cpu')))

          with tempfile.TemporaryDirectory() as tmp:
            torch.jit.save(lowercase   ,      os.path.join(lowercase   ,      'traced_model.pt'))
            a__:					Any      							=     torch.jit.load(os.path.join(lowercase   ,      'traced_model.pt')   ,      map_location=lowercase)
            loaded(inputs_dict['input_ids'].to(lowercase)   ,      inputs_dict['attention_mask'].to(lowercase))







@require_torch
class        __snake_case	(					unittest.TestCase    ):



      @slow
      def   lowerCamelCase_       (						self)     ->      Tuple:





        '''simple docstring'''





        a__:					Optional[int]      							=     FlaubertModel.from_pretrained('flaubert/flaubert_base_cased')
        a__:					Any      							=     torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]])
        with torch.no_grad():
          a__:					Union[str, Any]      							=     model(lowercase)[0]
        a__:					Tuple      							=     torch.Size((1, 11, 7_68))
        self.assertEqual(output.shape   ,      lowercase)
        a__:					List[str]      							=     torch.tensor(
            [[[-2.6251, -1.4298, -0.0227], [-2.8510, -1.6387, 0.2258], [-2.8114, -1.1832, -0.3066]]])

        self.assertTrue(torch.allclose(output[:, :3, :3]   ,      lowercase   ,      atol=1e-4))



"""simple docstring"""
import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin




class        __snake_case	(					__lowerCAmelCase	,				unittest.TestCase    ):
      a__				=		DebertaTokenizer
      a__				=		True
      a__				=		DebertaTokenizerFast



      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        a__:					List[str]      							=     [
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            '\u0120',
            '\u0120l',
            '\u0120n',
            '\u0120lo',
            '\u0120low',
            'er',
            '\u0120lowest',
            '\u0120newer',
            '\u0120wider',
            '[UNK]',
        ]
        a__:					List[str]      							=     dict(zip(lowercase   ,      range(len(lowercase))))
        a__:					Any      							=     ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
        a__:					Optional[Any]      							=     {'unk_token': '[UNK]'}

        a__:					Any      							=     os.path.join(self.tmpdirname   ,      VOCAB_FILES_NAMES['vocab_file'])
        a__:					Dict      							=     os.path.join(self.tmpdirname   ,      VOCAB_FILES_NAMES['merges_file'])
        with open(self.vocab_file   ,      'w'   ,      encoding='utf-8') as fp:
          fp.write(json.dumps(lowercase) + '\n')
        with open(self.merges_file   ,      'w'   ,      encoding='utf-8') as fp:
          fp.write('\n'.join(lowercase))



      def   lowerCamelCase_       (						self   ,      **lowercase)     ->      int:





        '''simple docstring'''





        kwargs.update(self.special_tokens_map)
        return self.tokenizer_class.from_pretrained(self.tmpdirname   ,      **lowercase)



      def   lowerCamelCase_       (						self   ,      lowercase)     ->      Dict:





        '''simple docstring'''





        a__:					Tuple      							=     'lower newer'
        a__:					int      							=     'lower newer'
        return input_text, output_text



      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        a__:					Optional[int]      							=     self.get_tokenizer()
        a__:					List[Any]      							=     'lower newer'
        a__:					Optional[int]      							=     ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
        a__:					Optional[Any]      							=     tokenizer.tokenize(lowercase)
        self.assertListEqual(lowercase   ,      lowercase)

        a__:					int      							=     tokens + [tokenizer.unk_token]
        a__:					Optional[int]      							=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase)   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        a__:					int      							=     self.get_tokenizer()
        a__:					Any      							=     tokenizer('Hello'   ,      'World')
        a__:					Optional[Any]      							=     [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
        self.assertListEqual(tokd['token_type_ids']   ,      lowercase)



      @slow
      def   lowerCamelCase_       (						self)     ->      int:





        '''simple docstring'''





        a__:					Dict      							=     self.tokenizer_class.from_pretrained('microsoft/deberta-base')

        a__:					Union[str, Any]      							=     tokenizer.encode('sequence builders'   ,      add_special_tokens=lowercase)
        a__:					Optional[Any]      							=     tokenizer.encode('multi-sequence build'   ,      add_special_tokens=lowercase)

        a__:					List[str]      							=     tokenizer.encode(
            'sequence builders'   ,      add_special_tokens=lowercase   ,      add_prefix_space=lowercase)
        a__:					Any      							=     tokenizer.encode(
            'sequence builders'   ,      'multi-sequence build'   ,      add_special_tokens=lowercase   ,      add_prefix_space=lowercase)

        a__:					Optional[int]      							=     tokenizer.build_inputs_with_special_tokens(lowercase)
        a__:					Union[str, Any]      							=     tokenizer.build_inputs_with_special_tokens(lowercase   ,      lowercase)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode



      @slow
      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__:					List[Any]      							=     [self.tokenizer_class]
        if self.test_rust_tokenizer:
          tokenizer_classes.append(self.rust_tokenizer_class)

        for tokenizer_class in tokenizer_classes:
          a__:					int      							=     tokenizer_class.from_pretrained('microsoft/deberta-base')

          a__:					List[str]      							=     [
              'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
              'ALBERT incorporates two parameter reduction techniques',
              'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
              ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
              ' vocabulary embedding.',
          ]

          a__:					Dict      							=     tokenizer(lowercase   ,      padding=lowercase)
          a__:					Union[str, Any]      							=     [tokenizer.decode(lowercase   ,      skip_special_tokens=lowercase) for seq in encoding['input_ids']]

          # fmt: off
          a__:					Any      							=     {
              'input_ids': [
                  [1, 21_18, 1_11_26, 5_65, 35, 83, 2_51_91, 1_63, 1_88_54, 13, 1_21_56, 12, 1_61_01, 2_53_76, 1_38_07, 9, 2_22_05, 2_78_93, 16_35, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [1, 21_18, 1_11_26, 5_65, 2_45_36, 80, 4_37_97, 48_78, 73_73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [1, 1_33, 78, 65, 16, 10, 37_24, 15_38, 3_31_83, 1_13_03, 4_37_97, 19_38, 4, 8_70, 2_41_65, 2_91_05, 5, 7_39, 3_26_44, 3_31_83, 1_13_03, 3_61_73, 88, 80, 6_50, 78_21, 4_59_40, 6, 52, 25_59, 5, 18_36, 9, 5, 73_97, 1_31_71, 31, 5, 18_36, 9, 3_26_44, 3_31_83, 1_13_03, 4, 2]
              ],
              'token_type_ids': [
                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
              ],
              'attention_mask': [
                  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
              ]
          }
          # fmt: on

          a__:					str      							=     [
              'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
              'ALBERT incorporates two parameter reduction techniques',
              'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
              ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
              ' vocabulary embedding.',
          ]

          self.assertDictEqual(encoding.data   ,      lowercase)

          for expected, decoded in zip(lowercase   ,      lowercase):
            self.assertEqual(lowercase   ,      lowercase)


