


'''simple docstring'''




import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      AlbertForMaskedLM,
      AlbertForMultipleChoice,
      AlbertForPreTraining,
      AlbertForQuestionAnswering,
      AlbertForSequenceClassification,
      AlbertForTokenClassification,
      AlbertModel,
  )
  from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class    a__				:






     def __init__(			self		:		Optional[int]  ,    a		:		Union[str, Any]  ,    a		:		Optional[Any]=13  ,    a		:		Optional[Any]=7  ,    a		:		Optional[int]=True  ,    a		:		Tuple=True  ,    a		:		Optional[int]=True  ,    a		:		Union[str, Any]=True  ,    a		:		Any=99  ,    a		:		Union[str, Any]=16  ,    a		:		Optional[Any]=36  ,    a		:		Dict=6  ,    a		:		Dict=6  ,    a		:		Tuple=6  ,    a		:		Optional[Any]=37  ,    a		:		Optional[int]="gelu"  ,    a		:		int=0.1  ,    a		:		Optional[int]=0.1  ,    a		:		Dict=5_12  ,    a		:		Tuple=16  ,    a		:		Any=2  ,    a		:		Dict=0.02  ,    a		:		Optional[int]=3  ,    a		:		Union[str, Any]=4  ,    a		:		str=None  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  parent
         __lowerCamelCase          =  batch_size
         __lowerCamelCase          =  seq_length
         __lowerCamelCase          =  is_training
         __lowerCamelCase          =  use_input_mask
         __lowerCamelCase          =  use_token_type_ids
         __lowerCamelCase          =  use_labels
         __lowerCamelCase          =  vocab_size
         __lowerCamelCase          =  embedding_size
         __lowerCamelCase          =  hidden_size
         __lowerCamelCase          =  num_hidden_layers
         __lowerCamelCase          =  num_hidden_groups
         __lowerCamelCase          =  num_attention_heads
         __lowerCamelCase          =  intermediate_size
         __lowerCamelCase          =  hidden_act
         __lowerCamelCase          =  hidden_dropout_prob
         __lowerCamelCase          =  attention_probs_dropout_prob
         __lowerCamelCase          =  max_position_embeddings
         __lowerCamelCase          =  type_vocab_size
         __lowerCamelCase          =  type_sequence_label_size
         __lowerCamelCase          =  initializer_range
         __lowerCamelCase          =  num_labels
         __lowerCamelCase          =  num_choices
         __lowerCamelCase          =  scope






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size   )

         __lowerCamelCase          =  None
         if self.use_input_mask:
             __lowerCamelCase          =  random_attention_mask([self.batch_size, self.seq_length]   )

         __lowerCamelCase          =  None
         if self.use_token_type_ids:
             __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.type_vocab_size   )

         __lowerCamelCase          =  None
         __lowerCamelCase          =  None
         __lowerCamelCase          =  None
         if self.use_labels:
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    self.type_sequence_label_size   )
             __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels   )
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    self.num_choices   )

         __lowerCamelCase          =  self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         return AlbertConfig(
             vocab_size=self.vocab_size  ,    hidden_size=self.hidden_size  ,    num_hidden_layers=self.num_hidden_layers  ,    num_attention_heads=self.num_attention_heads  ,    intermediate_size=self.intermediate_size  ,    hidden_act=self.hidden_act  ,    hidden_dropout_prob=self.hidden_dropout_prob  ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,    max_position_embeddings=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    initializer_range=self.initializer_range  ,    num_hidden_groups=self.num_hidden_groups  ,    )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]  ,    a		:		List[str]  ,    a		:		List[Any]  ,    a		:		Tuple  ,    a		:		Tuple  ,    a		:		List[Any]  ,    a		:		Any  ,    a		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertModel(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase   )
         __lowerCamelCase          =  model(_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase   )
         __lowerCamelCase          =  model(_lowerCAmelCase   )
         self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.hidden_size)   )
         self.parent.assertEqual(result.pooler_output.shape  ,    (self.batch_size, self.hidden_size)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict  ,    a		:		Any  ,    a		:		int  ,    a		:		List[Any]  ,    a		:		str  ,    a		:		Optional[int]  ,    a		:		Union[str, Any]  ,    a		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertForPreTraining(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(
             _lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    labels=_lowerCAmelCase  ,    sentence_order_label=_lowerCAmelCase  ,    )
         self.parent.assertEqual(result.prediction_logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)   )
         self.parent.assertEqual(result.sop_logits.shape  ,    (self.batch_size, config.num_labels)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]  ,    a		:		int  ,    a		:		List[str]  ,    a		:		Union[str, Any]  ,    a		:		Optional[int]  ,    a		:		Any  ,    a		:		int  ,    a		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertForMaskedLM(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    labels=_lowerCAmelCase   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		List[str]  ,    a		:		int  ,    a		:		str  ,    a		:		List[str]  ,    a		:		List[str]  ,    a		:		Optional[int]  ,    a		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertForQuestionAnswering(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(
             _lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    start_positions=_lowerCAmelCase  ,    end_positions=_lowerCAmelCase  ,    )
         self.parent.assertEqual(result.start_logits.shape  ,    (self.batch_size, self.seq_length)   )
         self.parent.assertEqual(result.end_logits.shape  ,    (self.batch_size, self.seq_length)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		List[str]  ,    a		:		int  ,    a		:		str  ,    a		:		str  ,    a		:		Dict  ,    a		:		int  ,    a		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_labels
         __lowerCamelCase          =  AlbertForSequenceClassification(_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    labels=_lowerCAmelCase   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_labels)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		List[str]  ,    a		:		Union[str, Any]  ,    a		:		Any  ,    a		:		Any  ,    a		:		int  ,    a		:		str  ,    a		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_labels
         __lowerCamelCase          =  AlbertForTokenClassification(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    labels=_lowerCAmelCase   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.num_labels)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		str  ,    a		:		Any  ,    a		:		Union[str, Any]  ,    a		:		Any  ,    a		:		int  ,    a		:		Optional[Any]  ,    a		:		Tuple   ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_choices
         __lowerCamelCase          =  AlbertForMultipleChoice(config=_lowerCAmelCase   )
         model.to(_lowerCAmelCase   )
         model.eval()
         __lowerCamelCase          =  input_ids.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  token_type_ids.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  input_mask.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  model(
             _lowerCAmelCase  ,    attention_mask=_lowerCAmelCase  ,    token_type_ids=_lowerCAmelCase  ,    labels=_lowerCAmelCase  ,    )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_choices)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.prepare_config_and_inputs()
         (
             __lowerCamelCase				
         )          =  config_and_inputs
         __lowerCamelCase          =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
         return config, inputs_dict




@require_torch
class    a__				(       __UpperCamelCase							,       __UpperCamelCase							,       unittest.TestCase			):
     lowerCamelCase  :					List[str]							=(
         (
             AlbertModel,
             AlbertForPreTraining,
             AlbertForMaskedLM,
             AlbertForMultipleChoice,
             AlbertForSequenceClassification,
             AlbertForTokenClassification,
             AlbertForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     lowerCamelCase  :					Any							=(
         {
             "feature-extraction": AlbertModel,
             "fill-mask": AlbertForMaskedLM,
             "question-answering": AlbertForQuestionAnswering,
             "text-classification": AlbertForSequenceClassification,
             "token-classification": AlbertForTokenClassification,
             "zero-shot": AlbertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     lowerCamelCase  :					Tuple							=True






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]  ,    a		:		Optional[Any]  ,    a		:		Any  ,    a		:		Optional[int]=False   ):





         """simple docstring"""



         __lowerCamelCase          =  super()._prepare_for_class(_lowerCAmelCase  ,    _lowerCAmelCase  ,    return_labels=_lowerCAmelCase   )

         if return_labels:
             if model_class in get_values(_lowerCAmelCase   ):
                 __lowerCamelCase          =  torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.seq_length)  ,    dtype=torch.long  ,    device=_lowerCAmelCase   )
                 __lowerCamelCase          =  torch.zeros(
                     self.model_tester.batch_size  ,    dtype=torch.long  ,    device=_lowerCAmelCase   )
         return inputs_dict






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertModelTester(self   )
         __lowerCamelCase          =  ConfigTester(self  ,    config_class=_lowerCAmelCase  ,    hidden_size=37   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         self.config_tester.run_common_tests()






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
             __lowerCamelCase          =  type
             self.model_tester.create_and_check_model(*_lowerCAmelCase   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple   ):





         """simple docstring"""



         for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __lowerCamelCase          =  AlbertModel.from_pretrained(_lowerCAmelCase   )
             self.assertIsNotNone(_lowerCAmelCase   )




@require_torch
class    a__				(       unittest.TestCase			):






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  AlbertModel.from_pretrained('''albert-base-v2'''   )
         __lowerCamelCase          =  torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]   )
         __lowerCamelCase          =  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
         with torch.no_grad():
             __lowerCamelCase          =  model(_lowerCAmelCase  ,    attention_mask=_lowerCAmelCase   )[0]
         __lowerCamelCase          =  torch.Size((1, 11, 7_68)   )
         self.assertEqual(output.shape  ,    _lowerCAmelCase   )
         __lowerCamelCase          =  torch.tensor(
             [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]   )

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  ,    _lowerCAmelCase  ,    atol=1e-4   )   )

'''simple docstring'''




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase												=get_tests_dir("fixtures/spiece.model")




@require_sentencepiece
@require_tokenizers
class    a__				(       UpperCAmelCase__							,       unittest.TestCase			):
     lowerCamelCase  :					Optional[Any]							=DebertaVaTokenizer
     lowerCamelCase  :					Optional[int]							=DebertaVaTokenizerFast
     lowerCamelCase  :					Optional[Any]							=True
     lowerCamelCase  :					Tuple							=True






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    unk_token='''<unk>'''   )
         tokenizer.save_pretrained(self.tmpdirname   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  '''this is a test'''
         __lowerCamelCase          =  '''this is a test'''
         return input_text, output_text






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  '''<pad>'''
         __lowerCamelCase          =  0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(a   )  ,    a   )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(a   )  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  list(self.get_tokenizer().get_vocab().keys()   )
         self.assertEqual(vocab_keys[0]  ,    '''<pad>'''   )
         self.assertEqual(vocab_keys[1]  ,    '''<unk>'''   )
         self.assertEqual(vocab_keys[-1]  ,    '''[PAD]'''   )
         self.assertEqual(len(a   )  ,    3_00_01   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size  ,    3_00_00   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  ''' \tHeLLo!how  \n Are yoU?  '''
         __lowerCamelCase          =  ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    do_lower_case=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    do_lower_case=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )






     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''   )
     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         pass






     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''   )
     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         pass






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase          =  ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    split_by_punct=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    split_by_punct=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase          =  ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )
         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase          =  ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase          =  ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  ''' \tHeLLo!how  \n Are yoU?  '''
         __lowerCamelCase          =  ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
         # fmt: on

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    do_lower_case=a  ,    split_by_punct=a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )

         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.get_tokenizer()
         __lowerCamelCase          =  self.get_rust_tokenizer()

         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''

         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(tokenizer.encode(a  ,    add_special_tokens=a   )   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a  ,    add_special_tokens=a   )   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  tokenizer.encode(a  ,    add_special_tokens=a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  self.get_rust_tokenizer()
         __lowerCamelCase          =  tokenizer.encode(a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a   )
         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  '''This is a test'''
         __lowerCamelCase          =  [13, 1, 43_98, 25, 21, 12_89]
         __lowerCamelCase          =  ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
         __lowerCamelCase          =  ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

         __lowerCamelCase          =  DebertaVaTokenizer(a  ,    keep_accents=a   )
         __lowerCamelCase          =  DebertaVaTokenizerFast(a  ,    keep_accents=a   )

         __lowerCamelCase          =  tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  rust_tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  rust_tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(a   )
         self.assertListEqual(a  ,    a   )

         # fmt: off
         __lowerCamelCase          =  '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase          =  [13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
         __lowerCamelCase          =  ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
         __lowerCamelCase          =  ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
         # fmt: on

         __lowerCamelCase          =  tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  tokenizer.convert_ids_to_tokens(a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  rust_tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  rust_tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )
         __lowerCamelCase          =  rust_tokenizer.convert_ids_to_tokens(a   )
         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  DebertaVaTokenizer(a   )

         __lowerCamelCase          =  tokenizer.encode('''sequence builders'''   )
         __lowerCamelCase          =  tokenizer.encode('''multi-sequence build'''   )

         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a   )
         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a  ,    a   )

         self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]  ,    a   )
         self.assertEqual(
             [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]  ,    a  ,    )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  {'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=a  ,    model_name='''microsoft/deberta-v2-xlarge'''  ,    revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''  ,    )
