



import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    MobileBertForMaskedLM,
							    MobileBertForMultipleChoice,
							    MobileBertForNextSentencePrediction,
							    MobileBertForPreTraining,
							    MobileBertForQuestionAnswering,
							    MobileBertForSequenceClassification,
							    MobileBertForTokenClassification,
							    MobileBertModel,
							)
class 			_snake_case     :




				def __init__(  self   :  Union[str, Any],   __lowercase   :  Optional[Any],   __lowercase   :  str=13,   __lowercase   :  Union[str, Any]=7,   __lowercase   :  int=True,   __lowercase   :  Optional[int]=True,   __lowercase   :  Union[str, Any]=True,   __lowercase   :  Union[str, Any]=True,   __lowercase   :  Optional[int]=99,   __lowercase   :  Optional[int]=64,   __lowercase   :  Optional[int]=32,   __lowercase   :  Dict=5,   __lowercase   :  int=4,   __lowercase   :  List[str]=37,   __lowercase   :  Tuple="gelu",   __lowercase   :  int=0.1,   __lowercase   :  Tuple=0.1,   __lowercase   :  str=512,   __lowercase   :  Tuple=16,   __lowercase   :  Dict=2,   __lowercase   :  Dict=0.02,   __lowercase   :  Any=3,   __lowercase   :  Dict=4,   __lowercase   :  List[str]=None,   ):
							lowercase__   					=			parent
							lowercase__   					=			batch_size
							lowercase__   					=			seq_length
							lowercase__   					=			is_training
							lowercase__   					=			use_input_mask
							lowercase__   					=			use_token_type_ids
							lowercase__   					=			use_labels
							lowercase__   					=			vocab_size
							lowercase__   					=			hidden_size
							lowercase__   					=			embedding_size
							lowercase__   					=			num_hidden_layers
							lowercase__   					=			num_attention_heads
							lowercase__   					=			intermediate_size
							lowercase__   					=			hidden_act
							lowercase__   					=			hidden_dropout_prob
							lowercase__   					=			attention_probs_dropout_prob
							lowercase__   					=			max_position_embeddings
							lowercase__   					=			type_vocab_size
							lowercase__   					=			type_sequence_label_size
							lowercase__   					=			initializer_range
							lowercase__   					=			num_labels
							lowercase__   					=			num_choices
							lowercase__   					=			scope




				def 							A__ (  self   :  str  ):
							lowercase__   					=			ids_tensor([self.batch_size, self.seq_length],   self.vocab_size  )

							lowercase__   					=			None
							if self.use_input_mask:
										lowercase__   					=			random_attention_mask([self.batch_size, self.seq_length]  )

							lowercase__   					=			None
							if self.use_token_type_ids:
										lowercase__   					=			ids_tensor([self.batch_size, self.seq_length],   self.type_vocab_size  )

							lowercase__   					=			None
							lowercase__   					=			None
							lowercase__   					=			None
							if self.use_labels:
										lowercase__   					=			ids_tensor([self.batch_size],   self.type_sequence_label_size  )
										lowercase__   					=			ids_tensor([self.batch_size, self.seq_length],   self.num_labels  )
										lowercase__   					=			ids_tensor([self.batch_size],   self.num_choices  )

							lowercase__   					=			self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 							A__ (  self   :  str  ):
							return MobileBertConfig(
							    vocab_size=self.vocab_size,   hidden_size=self.hidden_size,   num_hidden_layers=self.num_hidden_layers,   num_attention_heads=self.num_attention_heads,   intermediate_size=self.intermediate_size,   embedding_size=self.embedding_size,   hidden_act=self.hidden_act,   hidden_dropout_prob=self.hidden_dropout_prob,   attention_probs_dropout_prob=self.attention_probs_dropout_prob,   max_position_embeddings=self.max_position_embeddings,   type_vocab_size=self.type_vocab_size,   is_decoder=__lowercase,   initializer_range=self.initializer_range,   )




				def 							A__ (  self   :  Any,   __lowercase   :  Optional[int],   __lowercase   :  Any,   __lowercase   :  List[Any],   __lowercase   :  List[str],   __lowercase   :  Dict,   __lowercase   :  Union[str, Any],   __lowercase   :  Dict  ):
							lowercase__   					=			MobileBertModel(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase  )
							lowercase__   					=			model(__lowercase,   token_type_ids=__lowercase  )
							lowercase__   					=			model(__lowercase  )

							self.parent.assertEqual(result.last_hidden_state.shape,   (self.batch_size, self.seq_length, self.hidden_size)  )
							self.parent.assertEqual(result.pooler_output.shape,   (self.batch_size, self.hidden_size)  )




				def 							A__ (  self   :  List[str],   __lowercase   :  int,   __lowercase   :  Tuple,   __lowercase   :  Dict,   __lowercase   :  str,   __lowercase   :  Dict,   __lowercase   :  Union[str, Any],   __lowercase   :  str  ):
							lowercase__   					=			MobileBertForMaskedLM(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.vocab_size)  )




				def 							A__ (  self   :  Dict,   __lowercase   :  Union[str, Any],   __lowercase   :  Union[str, Any],   __lowercase   :  Union[str, Any],   __lowercase   :  str,   __lowercase   :  Union[str, Any],   __lowercase   :  Optional[Any],   __lowercase   :  Optional[Any]  ):
							lowercase__   					=			MobileBertForNextSentencePrediction(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase,   )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, 2)  )




				def 							A__ (  self   :  Dict,   __lowercase   :  int,   __lowercase   :  Union[str, Any],   __lowercase   :  Optional[int],   __lowercase   :  Optional[int],   __lowercase   :  Tuple,   __lowercase   :  List[Any],   __lowercase   :  Tuple  ):
							lowercase__   					=			MobileBertForPreTraining(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase,   next_sentence_label=__lowercase,   )
							self.parent.assertEqual(result.prediction_logits.shape,   (self.batch_size, self.seq_length, self.vocab_size)  )
							self.parent.assertEqual(result.seq_relationship_logits.shape,   (self.batch_size, 2)  )




				def 							A__ (  self   :  str,   __lowercase   :  List[Any],   __lowercase   :  Any,   __lowercase   :  Dict,   __lowercase   :  Dict,   __lowercase   :  int,   __lowercase   :  Any,   __lowercase   :  Optional[Any]  ):
							lowercase__   					=			MobileBertForQuestionAnswering(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   start_positions=__lowercase,   end_positions=__lowercase,   )
							self.parent.assertEqual(result.start_logits.shape,   (self.batch_size, self.seq_length)  )
							self.parent.assertEqual(result.end_logits.shape,   (self.batch_size, self.seq_length)  )




				def 							A__ (  self   :  List[Any],   __lowercase   :  Dict,   __lowercase   :  Any,   __lowercase   :  List[Any],   __lowercase   :  int,   __lowercase   :  Union[str, Any],   __lowercase   :  Any,   __lowercase   :  Optional[int]  ):
							lowercase__   					=			self.num_labels
							lowercase__   					=			MobileBertForSequenceClassification(__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.num_labels)  )




				def 							A__ (  self   :  Union[str, Any],   __lowercase   :  Tuple,   __lowercase   :  Tuple,   __lowercase   :  Union[str, Any],   __lowercase   :  Optional[int],   __lowercase   :  Union[str, Any],   __lowercase   :  List[str],   __lowercase   :  Any  ):
							lowercase__   					=			self.num_labels
							lowercase__   					=			MobileBertForTokenClassification(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.num_labels)  )




				def 							A__ (  self   :  int,   __lowercase   :  str,   __lowercase   :  Optional[Any],   __lowercase   :  int,   __lowercase   :  Tuple,   __lowercase   :  List[str],   __lowercase   :  str,   __lowercase   :  Optional[Any]  ):
							lowercase__   					=			self.num_choices
							lowercase__   					=			MobileBertForMultipleChoice(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			input_ids.unsqueeze(1  ).expand(-1,   self.num_choices,   -1  ).contiguous()
							lowercase__   					=			token_type_ids.unsqueeze(1  ).expand(-1,   self.num_choices,   -1  ).contiguous()
							lowercase__   					=			input_mask.unsqueeze(1  ).expand(-1,   self.num_choices,   -1  ).contiguous()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   token_type_ids=__lowercase,   labels=__lowercase,   )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.num_choices)  )




				def 							A__ (  self   :  Any  ):
							lowercase__   					=			self.prepare_config_and_inputs()
							(
							    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    (
							    lowercase__
							)					,    
							)   					=			config_and_inputs
							lowercase__   					=			{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict
@require_torch
class 			_snake_case     (   lowercase__      ,		lowercase__      ,		unittest.TestCase):
				UpperCamelCase__			:   Any       						=(
				    (
				        MobileBertModel,
				        MobileBertForMaskedLM,
				        MobileBertForMultipleChoice,
				        MobileBertForNextSentencePrediction,
				        MobileBertForPreTraining,
				        MobileBertForQuestionAnswering,
				        MobileBertForSequenceClassification,
				        MobileBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				UpperCamelCase__			:   Any       						=(
				    {
				        """feature-extraction""": MobileBertModel,
				        """fill-mask""": MobileBertForMaskedLM,
				        """question-answering""": MobileBertForQuestionAnswering,
				        """text-classification""": MobileBertForSequenceClassification,
				        """token-classification""": MobileBertForTokenClassification,
				        """zero-shot""": MobileBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				UpperCamelCase__			:   Tuple       						=True




				def 							A__ (  self   :  List[Any],   __lowercase   :  Any,   __lowercase   :  Tuple,   __lowercase   :  Optional[Any]=False  ):
							lowercase__   					=			super()._prepare_for_class(__lowercase,   __lowercase,   return_labels=__lowercase  )

							if return_labels:
										if model_class in get_values(__lowercase  ):
													lowercase__   					=			torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length),   dtype=torch.long,   device=__lowercase  )
													lowercase__   					=			torch.zeros(
													    self.model_tester.batch_size,   dtype=torch.long,   device=__lowercase  )
							return inputs_dict




				def 							A__ (  self   :  str  ):
							lowercase__   					=			MobileBertModelTester(self  )
							lowercase__   					=			ConfigTester(self,   config_class=__lowercase,   hidden_size=37  )




				def 							A__ (  self   :  Dict  ):
							self.config_tester.run_common_tests()




				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_model(*__lowercase  )




				def 							A__ (  self   :  Union[str, Any]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_masked_lm(*__lowercase  )




				def 							A__ (  self   :  Any  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_multiple_choice(*__lowercase  )




				def 							A__ (  self   :  Tuple  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*__lowercase  )




				def 							A__ (  self   :  int  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_pretraining(*__lowercase  )




				def 							A__ (  self   :  List[str]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_question_answering(*__lowercase  )




				def 							A__ (  self   :  Dict  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_sequence_classification(*__lowercase  )




				def 							A__ (  self   :  Dict  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_mobilebert_for_token_classification(*__lowercase  )



def __lowerCAmelCase	(       SCREAMING_SNAKE_CASE_					):
			return torch.tensor(
			    SCREAMING_SNAKE_CASE_  ,					dtype=torch.long  ,					device=SCREAMING_SNAKE_CASE_  ,					)


lowercase_						=       1e-3




@require_torch
@require_sentencepiece
@require_tokenizers
class 			_snake_case     (   unittest.TestCase):




				@slow
				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			MobileBertModel.from_pretrained("google/mobilebert-uncased"  ).to(__lowercase  )
							lowercase__   					=			_long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]]  )
							with torch.no_grad():
										lowercase__   					=			model(__lowercase  )[0]
							lowercase__   					=			torch.Size((1, 9, 512)  )
							self.assertEqual(output.shape,   __lowercase  )
							lowercase__   					=			torch.tensor(
							    [
							        [
							            [-2.4_7_3_6_5_2_6e0_7, 8.2_6_9_1_6_5_6e0_4, 1.6_5_2_1_8_3_8e0_5],
							            [-5.7_5_4_1_7_0_4e-0_1, 3.9_0_5_6_0_2_2e0_0, 4.4_0_1_1_5_0_7e0_0],
							            [2.6_0_4_7_3_5_9e0_0, 1.5_6_7_7_6_5_2e0_0, -1.7_3_2_4_1_8_8e-0_1],
							        ]
							    ],   device=__lowercase,   )

							# MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
							# ~1 difference, it's therefore not a good idea to measure using addition.
							# Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
							# result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
							lowercase__   					=			torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE  )
							lowercase__   					=			torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE  )

							self.assertTrue(lower_bound and upper_bound  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_						=       logging.get_logger(__name__)

lowercase_						=       {
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}
class 			_snake_case     (   lowercase__):
				UpperCamelCase__			:   Optional[Any]       						="""transfo-xl"""
				UpperCamelCase__			:   Dict       						=["""mems"""]
				UpperCamelCase__			:   Optional[int]       						={
				    """n_token""": """vocab_size""",
				    """hidden_size""": """d_model""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}




				def __init__(  self   :  Optional[Any],   __lowercase   :  Optional[Any]=26_7735,   __lowercase   :  int=[2_0000, 4_0000, 20_0000],   __lowercase   :  Union[str, Any]=1024,   __lowercase   :  Tuple=1024,   __lowercase   :  Tuple=16,   __lowercase   :  Optional[Any]=64,   __lowercase   :  str=4096,   __lowercase   :  Optional[int]=4,   __lowercase   :  Union[str, Any]=False,   __lowercase   :  Union[str, Any]=18,   __lowercase   :  List[str]=1600,   __lowercase   :  List[Any]=1000,   __lowercase   :  Union[str, Any]=True,   __lowercase   :  Tuple=True,   __lowercase   :  Optional[Any]=0,   __lowercase   :  List[str]=-1,   __lowercase   :  int=True,   __lowercase   :  Dict=0.1,   __lowercase   :  Union[str, Any]=0.0,   __lowercase   :  str=True,   __lowercase   :  Optional[Any]="normal",   __lowercase   :  str=0.01,   __lowercase   :  Tuple=0.01,   __lowercase   :  List[Any]=0.02,   __lowercase   :  Any=1e-5,   __lowercase   :  Union[str, Any]=0,   **__lowercase   :  Union[str, Any],   ):
							lowercase__   					=			vocab_size
							lowercase__   					=			[]
							self.cutoffs.extend(__lowercase  )
							if proj_share_all_but_first:
										lowercase__   					=			[False] + [True] * len(self.cutoffs  )
							else:
										lowercase__   					=			[False] + [False] * len(self.cutoffs  )
							lowercase__   					=			d_model
							lowercase__   					=			d_embed
							lowercase__   					=			d_head
							lowercase__   					=			d_inner
							lowercase__   					=			div_val
							lowercase__   					=			pre_lnorm
							lowercase__   					=			n_layer
							lowercase__   					=			n_head
							lowercase__   					=			mem_len
							lowercase__   					=			same_length
							lowercase__   					=			attn_type
							lowercase__   					=			clamp_len
							lowercase__   					=			sample_softmax
							lowercase__   					=			adaptive
							lowercase__   					=			dropout
							lowercase__   					=			dropatt
							lowercase__   					=			untie_r
							lowercase__   					=			init
							lowercase__   					=			init_range
							lowercase__   					=			proj_init_std
							lowercase__   					=			init_std
							lowercase__   					=			layer_norm_epsilon
							super().__init__(eos_token_id=__lowercase,   **__lowercase  )




				@property
				def 							A__ (  self   :  Optional[Any]  ):
							# Message copied from Transformer-XL documentation
							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )
							return -1




				@max_position_embeddings.setter
				def 							A__ (  self   :  List[str],   __lowercase   :  Union[str, Any]  ):
							# Message copied from Transformer-XL documentation
							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )




