



import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
							    DistilBertForMaskedLM,
							    DistilBertForMultipleChoice,
							    DistilBertForQuestionAnswering,
							    DistilBertForSequenceClassification,
							    DistilBertForTokenClassification,
							    DistilBertModel,
							)
class 			_snake_case     (   lowercase__):




				def __init__(  self   :  List[Any],   __lowercase   :  Tuple,   __lowercase   :  List[Any]=13,   __lowercase   :  Dict=7,   __lowercase   :  int=True,   __lowercase   :  List[str]=True,   __lowercase   :  str=False,   __lowercase   :  str=True,   __lowercase   :  Dict=99,   __lowercase   :  Any=32,   __lowercase   :  Optional[Any]=5,   __lowercase   :  Union[str, Any]=4,   __lowercase   :  int=37,   __lowercase   :  int="gelu",   __lowercase   :  Optional[int]=0.1,   __lowercase   :  Any=0.1,   __lowercase   :  List[Any]=512,   __lowercase   :  Optional[Any]=16,   __lowercase   :  Tuple=2,   __lowercase   :  Dict=0.02,   __lowercase   :  Tuple=3,   __lowercase   :  Union[str, Any]=4,   __lowercase   :  List[str]=None,   ):
							lowercase__   					=			parent
							lowercase__   					=			batch_size
							lowercase__   					=			seq_length
							lowercase__   					=			is_training
							lowercase__   					=			use_input_mask
							lowercase__   					=			use_token_type_ids
							lowercase__   					=			use_labels
							lowercase__   					=			vocab_size
							lowercase__   					=			hidden_size
							lowercase__   					=			num_hidden_layers
							lowercase__   					=			num_attention_heads
							lowercase__   					=			intermediate_size
							lowercase__   					=			hidden_act
							lowercase__   					=			hidden_dropout_prob
							lowercase__   					=			attention_probs_dropout_prob
							lowercase__   					=			max_position_embeddings
							lowercase__   					=			type_vocab_size
							lowercase__   					=			type_sequence_label_size
							lowercase__   					=			initializer_range
							lowercase__   					=			num_labels
							lowercase__   					=			num_choices
							lowercase__   					=			scope




				def 							A__ (  self   :  List[str]  ):
							lowercase__   					=			ids_tensor([self.batch_size, self.seq_length],   self.vocab_size  )

							lowercase__   					=			None
							if self.use_input_mask:
										lowercase__   					=			random_attention_mask([self.batch_size, self.seq_length]  )

							lowercase__   					=			None
							lowercase__   					=			None
							lowercase__   					=			None
							if self.use_labels:
										lowercase__   					=			ids_tensor([self.batch_size],   self.type_sequence_label_size  )
										lowercase__   					=			ids_tensor([self.batch_size, self.seq_length],   self.num_labels  )
										lowercase__   					=			ids_tensor([self.batch_size],   self.num_choices  )

							lowercase__   					=			self.get_config()

							return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 							A__ (  self   :  int  ):
							return DistilBertConfig(
							    vocab_size=self.vocab_size,   dim=self.hidden_size,   n_layers=self.num_hidden_layers,   n_heads=self.num_attention_heads,   hidden_dim=self.intermediate_size,   hidden_act=self.hidden_act,   dropout=self.hidden_dropout_prob,   attention_dropout=self.attention_probs_dropout_prob,   max_position_embeddings=self.max_position_embeddings,   initializer_range=self.initializer_range,   )




				def 							A__ (  self   :  Union[str, Any],   __lowercase   :  Optional[Any],   __lowercase   :  Dict,   __lowercase   :  Union[str, Any],   __lowercase   :  Optional[Any],   __lowercase   :  Tuple,   __lowercase   :  List[Any]  ):
							lowercase__   					=			DistilBertModel(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   __lowercase  )
							lowercase__   					=			model(__lowercase  )
							self.parent.assertEqual(result.last_hidden_state.shape,   (self.batch_size, self.seq_length, self.hidden_size)  )




				def 							A__ (  self   :  Tuple,   __lowercase   :  List[Any],   __lowercase   :  Dict,   __lowercase   :  Any,   __lowercase   :  Union[str, Any],   __lowercase   :  List[Any],   __lowercase   :  Tuple  ):
							lowercase__   					=			DistilBertForMaskedLM(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.vocab_size)  )




				def 							A__ (  self   :  Optional[int],   __lowercase   :  int,   __lowercase   :  Optional[Any],   __lowercase   :  Optional[int],   __lowercase   :  List[str],   __lowercase   :  int,   __lowercase   :  Any  ):
							lowercase__   					=			DistilBertForQuestionAnswering(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   start_positions=__lowercase,   end_positions=__lowercase  )
							self.parent.assertEqual(result.start_logits.shape,   (self.batch_size, self.seq_length)  )
							self.parent.assertEqual(result.end_logits.shape,   (self.batch_size, self.seq_length)  )




				def 							A__ (  self   :  Any,   __lowercase   :  int,   __lowercase   :  str,   __lowercase   :  Optional[Any],   __lowercase   :  List[Any],   __lowercase   :  str,   __lowercase   :  Optional[Any]  ):
							lowercase__   					=			self.num_labels
							lowercase__   					=			DistilBertForSequenceClassification(__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.num_labels)  )




				def 							A__ (  self   :  Union[str, Any],   __lowercase   :  Union[str, Any],   __lowercase   :  Tuple,   __lowercase   :  List[str],   __lowercase   :  str,   __lowercase   :  Optional[Any],   __lowercase   :  str  ):
							lowercase__   					=			self.num_labels
							lowercase__   					=			DistilBertForTokenClassification(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()

							lowercase__   					=			model(__lowercase,   attention_mask=__lowercase,   labels=__lowercase  )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.num_labels)  )




				def 							A__ (  self   :  Optional[int],   __lowercase   :  List[Any],   __lowercase   :  List[str],   __lowercase   :  List[Any],   __lowercase   :  Optional[Any],   __lowercase   :  Optional[int],   __lowercase   :  Tuple  ):
							lowercase__   					=			self.num_choices
							lowercase__   					=			DistilBertForMultipleChoice(config=__lowercase  )
							model.to(__lowercase  )
							model.eval()
							lowercase__   					=			input_ids.unsqueeze(1  ).expand(-1,   self.num_choices,   -1  ).contiguous()
							lowercase__   					=			input_mask.unsqueeze(1  ).expand(-1,   self.num_choices,   -1  ).contiguous()
							lowercase__   					=			model(
							    __lowercase,   attention_mask=__lowercase,   labels=__lowercase,   )
							self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.num_choices)  )




				def 							A__ (  self   :  Any  ):
							lowercase__   					=			self.prepare_config_and_inputs()
							((lowercase__)					,    (lowercase__)					,    (lowercase__)					,    (lowercase__)					,    (lowercase__)					,    (lowercase__))   					=			config_and_inputs
							lowercase__   					=			{"input_ids": input_ids, "attention_mask": input_mask}
							return config, inputs_dict
@require_torch
class 			_snake_case     (   lowercase__      ,		lowercase__      ,		unittest.TestCase):
				UpperCamelCase__			:   Dict       						=(
				    (
				        DistilBertModel,
				        DistilBertForMaskedLM,
				        DistilBertForMultipleChoice,
				        DistilBertForQuestionAnswering,
				        DistilBertForSequenceClassification,
				        DistilBertForTokenClassification,
				    )
				    if is_torch_available()
				    else None
				)
				UpperCamelCase__			:   Any       						=(
				    {
				        """feature-extraction""": DistilBertModel,
				        """fill-mask""": DistilBertForMaskedLM,
				        """question-answering""": DistilBertForQuestionAnswering,
				        """text-classification""": DistilBertForSequenceClassification,
				        """token-classification""": DistilBertForTokenClassification,
				        """zero-shot""": DistilBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				UpperCamelCase__			:   Optional[Any]       						=True
				UpperCamelCase__			:   str       						=True
				UpperCamelCase__			:   Any       						=True
				UpperCamelCase__			:   List[str]       						=True




				def 							A__ (  self   :  Dict  ):
							lowercase__   					=			DistilBertModelTester(self  )
							lowercase__   					=			ConfigTester(self,   config_class=__lowercase,   dim=37  )




				def 							A__ (  self   :  Union[str, Any]  ):
							self.config_tester.run_common_tests()




				def 							A__ (  self   :  str  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_model(*__lowercase  )




				def 							A__ (  self   :  Optional[Any]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_masked_lm(*__lowercase  )




				def 							A__ (  self   :  List[str]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_question_answering(*__lowercase  )




				def 							A__ (  self   :  Union[str, Any]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_sequence_classification(*__lowercase  )




				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_token_classification(*__lowercase  )




				def 							A__ (  self   :  List[str]  ):
							lowercase__   					=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_multiple_choice(*__lowercase  )




				@slow
				def 							A__ (  self   :  Dict  ):
							for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowercase__   					=			DistilBertModel.from_pretrained(__lowercase  )
										self.assertIsNotNone(__lowercase  )




				@slow
				@require_torch_gpu
				def 							A__ (  self   :  List[str]  ):
							lowercase__					,    lowercase__   					=			self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
										# BertForMultipleChoice behaves incorrectly in JIT environments.
										if model_class == DistilBertForMultipleChoice:
													return

										lowercase__   					=			True
										lowercase__   					=			model_class(config=__lowercase  )

										lowercase__   					=			self._prepare_for_class(__lowercase,   __lowercase  )
										lowercase__   					=			torch.jit.trace(
										    __lowercase,   (inputs_dict["input_ids"].to("cpu"  ), inputs_dict["attention_mask"].to("cpu"  ))  )

										with tempfile.TemporaryDirectory() as tmp:
													torch.jit.save(__lowercase,   os.path.join(__lowercase,   "traced_model.pt"  )  )
													lowercase__   					=			torch.jit.load(os.path.join(__lowercase,   "traced_model.pt"  ),   map_location=__lowercase  )
													loaded(inputs_dict["input_ids"].to(__lowercase  ),   inputs_dict["attention_mask"].to(__lowercase  )  )
@require_torch
class 			_snake_case     (   unittest.TestCase):




				@slow
				def 							A__ (  self   :  str  ):
							lowercase__   					=			DistilBertModel.from_pretrained("distilbert-base-uncased"  )
							lowercase__   					=			torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]  )
							lowercase__   					=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]  )
							with torch.no_grad():
										lowercase__   					=			model(__lowercase,   attention_mask=__lowercase  )[0]
							lowercase__   					=			torch.Size((1, 11, 768)  )
							self.assertEqual(output.shape,   __lowercase  )
							lowercase__   					=			torch.tensor(
							    [[[-0.1639, 0.3299, 0.1648], [-0.1746, 0.3289, 0.1710], [-0.1884, 0.3357, 0.1810]]]  )

							self.assertTrue(torch.allclose(output[:, 1:4, 1:4],   __lowercase,   atol=1e-4  )  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_						=       logging.get_logger(__name__)

lowercase_						=       {
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}
class 			_snake_case     (   lowercase__):
				UpperCamelCase__			:   Optional[Any]       						="""transfo-xl"""
				UpperCamelCase__			:   Dict       						=["""mems"""]
				UpperCamelCase__			:   Optional[int]       						={
				    """n_token""": """vocab_size""",
				    """hidden_size""": """d_model""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}




				def __init__(  self   :  Optional[Any],   __lowercase   :  Optional[Any]=26_7735,   __lowercase   :  int=[2_0000, 4_0000, 20_0000],   __lowercase   :  Union[str, Any]=1024,   __lowercase   :  Tuple=1024,   __lowercase   :  Tuple=16,   __lowercase   :  Optional[Any]=64,   __lowercase   :  str=4096,   __lowercase   :  Optional[int]=4,   __lowercase   :  Union[str, Any]=False,   __lowercase   :  Union[str, Any]=18,   __lowercase   :  List[str]=1600,   __lowercase   :  List[Any]=1000,   __lowercase   :  Union[str, Any]=True,   __lowercase   :  Tuple=True,   __lowercase   :  Optional[Any]=0,   __lowercase   :  List[str]=-1,   __lowercase   :  int=True,   __lowercase   :  Dict=0.1,   __lowercase   :  Union[str, Any]=0.0,   __lowercase   :  str=True,   __lowercase   :  Optional[Any]="normal",   __lowercase   :  str=0.01,   __lowercase   :  Tuple=0.01,   __lowercase   :  List[Any]=0.02,   __lowercase   :  Any=1e-5,   __lowercase   :  Union[str, Any]=0,   **__lowercase   :  Union[str, Any],   ):
							lowercase__   					=			vocab_size
							lowercase__   					=			[]
							self.cutoffs.extend(__lowercase  )
							if proj_share_all_but_first:
										lowercase__   					=			[False] + [True] * len(self.cutoffs  )
							else:
										lowercase__   					=			[False] + [False] * len(self.cutoffs  )
							lowercase__   					=			d_model
							lowercase__   					=			d_embed
							lowercase__   					=			d_head
							lowercase__   					=			d_inner
							lowercase__   					=			div_val
							lowercase__   					=			pre_lnorm
							lowercase__   					=			n_layer
							lowercase__   					=			n_head
							lowercase__   					=			mem_len
							lowercase__   					=			same_length
							lowercase__   					=			attn_type
							lowercase__   					=			clamp_len
							lowercase__   					=			sample_softmax
							lowercase__   					=			adaptive
							lowercase__   					=			dropout
							lowercase__   					=			dropatt
							lowercase__   					=			untie_r
							lowercase__   					=			init
							lowercase__   					=			init_range
							lowercase__   					=			proj_init_std
							lowercase__   					=			init_std
							lowercase__   					=			layer_norm_epsilon
							super().__init__(eos_token_id=__lowercase,   **__lowercase  )




				@property
				def 							A__ (  self   :  Optional[Any]  ):
							# Message copied from Transformer-XL documentation
							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )
							return -1




				@max_position_embeddings.setter
				def 							A__ (  self   :  List[str],   __lowercase   :  Union[str, Any]  ):
							# Message copied from Transformer-XL documentation
							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''  )




