


"""simple docstring"""







import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch
		from torch import nn

		from transformers import (
		    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
		    VideoMAEForPreTraining,
		    VideoMAEForVideoClassification,
		    VideoMAEModel,
		)
		from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from transformers import VideoMAEImageProcessor




class 			UpperCamelCase_     :

	"""simple docstring"""





	def __init__(			self      :    Any      ,      UpperCAmelCase__      :    List[str]      ,      UpperCAmelCase__      :    Optional[Any]=1_3      ,      UpperCAmelCase__      :    str=1_0      ,      UpperCAmelCase__      :    Dict=3      ,      UpperCAmelCase__      :    Any=2      ,      UpperCAmelCase__      :    str=2      ,      UpperCAmelCase__      :    Any=2      ,      UpperCAmelCase__      :    Union[str, Any]=True      ,      UpperCAmelCase__      :    Any=True      ,      UpperCAmelCase__      :    Dict=3_2      ,      UpperCAmelCase__      :    Optional[int]=5      ,      UpperCAmelCase__      :    Tuple=4      ,      UpperCAmelCase__      :    Optional[int]=3_7      ,      UpperCAmelCase__      :    Dict="gelu"      ,      UpperCAmelCase__      :    Optional[int]=0.1      ,      UpperCAmelCase__      :    Dict=0.1      ,      UpperCAmelCase__      :    Union[str, Any]=1_0      ,      UpperCAmelCase__      :    List[Any]=0.02      ,      UpperCAmelCase__      :    str=0.9      ,      UpperCAmelCase__      :    Any=None      ,      )	->     Tuple:
						__SCREAMING_SNAKE_CASE						       =    parent
						__SCREAMING_SNAKE_CASE						       =    batch_size
						__SCREAMING_SNAKE_CASE						       =    image_size
						__SCREAMING_SNAKE_CASE						       =    num_channels
						__SCREAMING_SNAKE_CASE						       =    patch_size
						__SCREAMING_SNAKE_CASE						       =    tubelet_size
						__SCREAMING_SNAKE_CASE						       =    num_frames
						__SCREAMING_SNAKE_CASE						       =    is_training
						__SCREAMING_SNAKE_CASE						       =    use_labels
						__SCREAMING_SNAKE_CASE						       =    hidden_size
						__SCREAMING_SNAKE_CASE						       =    num_hidden_layers
						__SCREAMING_SNAKE_CASE						       =    num_attention_heads
						__SCREAMING_SNAKE_CASE						       =    intermediate_size
						__SCREAMING_SNAKE_CASE						       =    hidden_act
						__SCREAMING_SNAKE_CASE						       =    hidden_dropout_prob
						__SCREAMING_SNAKE_CASE						       =    attention_probs_dropout_prob
						__SCREAMING_SNAKE_CASE						       =    type_sequence_label_size
						__SCREAMING_SNAKE_CASE						       =    initializer_range
						__SCREAMING_SNAKE_CASE						       =    mask_ratio
						__SCREAMING_SNAKE_CASE						       =    scope

						# in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
						__SCREAMING_SNAKE_CASE						       =    (image_size // patch_size) ** 2
						__SCREAMING_SNAKE_CASE						       =    (num_frames // tubelet_size) * self.num_patches_per_frame

						# use this variable to define bool_masked_pos
						__SCREAMING_SNAKE_CASE						       =    int(mask_ratio * self.seq_length       )




	def   UpperCAmelCase_				(			self      :    Optional[Any]       )	->     Union[str, Any]:
						__SCREAMING_SNAKE_CASE						       =    floats_tensor(
						    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]       )

						__SCREAMING_SNAKE_CASE						       =    None
						if self.use_labels:
											__SCREAMING_SNAKE_CASE						       =    ids_tensor([self.batch_size]      ,      self.type_sequence_label_size       )

						__SCREAMING_SNAKE_CASE						       =    self.get_config()

						return config, pixel_values, labels




	def   UpperCAmelCase_				(			self      :    Optional[int]       )	->     Optional[int]:
						return VideoMAEConfig(
						    image_size=self.image_size      ,      patch_size=self.patch_size      ,      num_channels=self.num_channels      ,      num_frames=self.num_frames      ,      tubelet_size=self.tubelet_size      ,      hidden_size=self.hidden_size      ,      num_hidden_layers=self.num_hidden_layers      ,      num_attention_heads=self.num_attention_heads      ,      intermediate_size=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      is_decoder=UpperCAmelCase__      ,      initializer_range=self.initializer_range      ,      )




	def   UpperCAmelCase_				(			self      :    Any      ,      UpperCAmelCase__      :    Dict      ,      UpperCAmelCase__      :    Tuple      ,      UpperCAmelCase__      :    Tuple       )	->     List[Any]:
						__SCREAMING_SNAKE_CASE						       =    VideoMAEModel(config=UpperCAmelCase__       )
						model.to(UpperCAmelCase__       )
						model.eval()
						__SCREAMING_SNAKE_CASE						       =    model(UpperCAmelCase__       )
						self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)       )




	def   UpperCAmelCase_				(			self      :    List[str]      ,      UpperCAmelCase__      :    Optional[Any]      ,      UpperCAmelCase__      :    Union[str, Any]      ,      UpperCAmelCase__      :    List[Any]       )	->     List[str]:
						__SCREAMING_SNAKE_CASE						       =    VideoMAEForPreTraining(UpperCAmelCase__       )
						model.to(UpperCAmelCase__       )
						model.eval()
						# important: each video needs to have the same number of masked patches
						# hence we define a single mask, which we then repeat for each example in the batch
						__SCREAMING_SNAKE_CASE						       =    torch.ones((self.num_masks,)       )
						__SCREAMING_SNAKE_CASE						       =    torch.cat([mask, torch.zeros(self.seq_length - mask.size(0       )       )]       )
						__SCREAMING_SNAKE_CASE						       =    mask.expand(self.batch_size      ,      -1       ).bool()

						__SCREAMING_SNAKE_CASE						       =    model(UpperCAmelCase__      ,      UpperCAmelCase__       )
						# model only returns predictions for masked patches
						__SCREAMING_SNAKE_CASE						       =    mask.sum().item()
						__SCREAMING_SNAKE_CASE						       =    3 * self.tubelet_size * self.patch_size**2
						self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, num_masked_patches, decoder_num_labels)       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     List[str]:
						__SCREAMING_SNAKE_CASE						       =    self.prepare_config_and_inputs()
						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    config_and_inputs
						__SCREAMING_SNAKE_CASE						       =    {"pixel_values": pixel_values}
						return config, inputs_dict




@require_torch
class 			UpperCamelCase_     (			_lowerCamelCase      ,  _lowerCamelCase      ,  unittest.TestCase):

	"""simple docstring"""





	snake_case__    :  Union[str, Any]    =      (
	    (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
	)
	snake_case__    :  Optional[int]    =      (
	    {"feature-extraction": VideoMAEModel, "video-classification": VideoMAEForVideoClassification}
	    if is_torch_available()
	    else {}
	)

	snake_case__    :  Optional[int]    =      False
	snake_case__    :  Dict    =      False
	snake_case__    :  str    =      False
	snake_case__    :  Dict    =      False




	def   UpperCAmelCase_				(			self      :    List[str]       )	->     Dict:
						__SCREAMING_SNAKE_CASE						       =    VideoMAEModelTester(self       )
						__SCREAMING_SNAKE_CASE						       =    ConfigTester(self      ,      config_class=UpperCAmelCase__      ,      has_text_modality=UpperCAmelCase__      ,      hidden_size=3_7       )




	def   UpperCAmelCase_				(			self      :    str      ,      UpperCAmelCase__      :    Optional[int]      ,      UpperCAmelCase__      :    Dict      ,      UpperCAmelCase__      :    Union[str, Any]=False       )	->     Optional[int]:
						__SCREAMING_SNAKE_CASE						       =    copy.deepcopy(UpperCAmelCase__       )

						if model_class == VideoMAEForPreTraining:
											# important: each video needs to have the same number of masked patches
											# hence we define a single mask, which we then repeat for each example in the batch
											__SCREAMING_SNAKE_CASE						       =    torch.ones((self.model_tester.num_masks,)       )
											__SCREAMING_SNAKE_CASE						       =    torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0       )       )]       )
											__SCREAMING_SNAKE_CASE						       =    mask.expand(self.model_tester.batch_size      ,      -1       ).bool()
											__SCREAMING_SNAKE_CASE						       =    bool_masked_pos.to(UpperCAmelCase__       )

						if return_labels:
											if model_class in [
											    *get_values(UpperCAmelCase__       ),
											]:
																__SCREAMING_SNAKE_CASE						       =    torch.zeros(
																    self.model_tester.batch_size      ,      dtype=torch.long      ,      device=UpperCAmelCase__       )

						return inputs_dict




	def   UpperCAmelCase_				(			self      :    List[str]       )	->     Tuple:
						self.config_tester.run_common_tests()




	@unittest.skip(reason="VideoMAE does not use inputs_embeds"       )
	def   UpperCAmelCase_				(			self      :    Dict       )	->     int:
						pass




	def   UpperCAmelCase_				(			self      :    Tuple       )	->     Any:
						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
											self.assertIsInstance(model.get_input_embeddings()      ,      (nn.Module)       )
											__SCREAMING_SNAKE_CASE						       =    model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(UpperCAmelCase__      ,      nn.Linear       )       )




	def   UpperCAmelCase_				(			self      :    List[Any]       )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
											__SCREAMING_SNAKE_CASE						       =    inspect.signature(model.forward       )
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											__SCREAMING_SNAKE_CASE						       =    [*signature.parameters.keys()]

											__SCREAMING_SNAKE_CASE						       =    ["pixel_values"]
											self.assertListEqual(arg_names[:1]      ,      UpperCAmelCase__       )




	def   UpperCAmelCase_				(			self      :    List[str]       )	->     Union[str, Any]:
						__SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCAmelCase__       )




	def   UpperCAmelCase_				(			self      :    Union[str, Any]       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase__       )




	@slow
	def   UpperCAmelCase_				(			self      :    Tuple       )	->     Dict:


						for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__SCREAMING_SNAKE_CASE						       =    VideoMAEModel.from_pretrained(UpperCAmelCase__       )
											self.assertIsNotNone(UpperCAmelCase__       )




	def   UpperCAmelCase_				(			self      :    Tuple       )	->     List[Any]:


						if not self.has_attentions:
											pass

						else:
											__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs_for_common()
											__SCREAMING_SNAKE_CASE						       =    True

											for model_class in self.all_model_classes:
																__SCREAMING_SNAKE_CASE						       =    self.model_tester.seq_length - self.model_tester.num_masks
																__SCREAMING_SNAKE_CASE						       =    (
																    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
																)

																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    False
																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
																model.to(UpperCAmelCase__       )
																model.eval()
																with torch.no_grad():
																					__SCREAMING_SNAKE_CASE						       =    model(**self._prepare_for_class(UpperCAmelCase__      ,      UpperCAmelCase__       )       )
																__SCREAMING_SNAKE_CASE						       =    outputs.attentions
																self.assertEqual(len(UpperCAmelCase__       )      ,      self.model_tester.num_hidden_layers       )

																# check that output_attentions also work using config
																del inputs_dict["output_attentions"]
																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
																model.to(UpperCAmelCase__       )
																model.eval()
																with torch.no_grad():
																					__SCREAMING_SNAKE_CASE						       =    model(**self._prepare_for_class(UpperCAmelCase__      ,      UpperCAmelCase__       )       )
																__SCREAMING_SNAKE_CASE						       =    outputs.attentions
																self.assertEqual(len(UpperCAmelCase__       )      ,      self.model_tester.num_hidden_layers       )

																self.assertListEqual(
																    list(attentions[0].shape[-3:]       )      ,      [self.model_tester.num_attention_heads, seq_len, seq_len]      ,      )
																__SCREAMING_SNAKE_CASE						       =    len(UpperCAmelCase__       )

																# Check attention is always last and order is fine
																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    True
																__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
																model.to(UpperCAmelCase__       )
																model.eval()
																with torch.no_grad():
																					__SCREAMING_SNAKE_CASE						       =    model(**self._prepare_for_class(UpperCAmelCase__      ,      UpperCAmelCase__       )       )

																self.assertEqual(out_len + 1      ,      len(UpperCAmelCase__       )       )

																__SCREAMING_SNAKE_CASE						       =    outputs.attentions

																self.assertEqual(len(UpperCAmelCase__       )      ,      self.model_tester.num_hidden_layers       )
																self.assertListEqual(
																    list(self_attentions[0].shape[-3:]       )      ,      [self.model_tester.num_attention_heads, seq_len, seq_len]      ,      )




	def   UpperCAmelCase_				(			self      :    Optional[int]       )	->     int:


						def check_hidden_states_output(UpperCAmelCase__      :    Tuple      ,      UpperCAmelCase__      :    Optional[Any]      ,      UpperCAmelCase__      :    List[str]       ):
											__SCREAMING_SNAKE_CASE						       =    model_class(UpperCAmelCase__       )
											model.to(UpperCAmelCase__       )
											model.eval()

											with torch.no_grad():
																__SCREAMING_SNAKE_CASE						       =    model(**self._prepare_for_class(UpperCAmelCase__      ,      UpperCAmelCase__       )       )

											__SCREAMING_SNAKE_CASE						       =    outputs.hidden_states
											__SCREAMING_SNAKE_CASE						       =    self.model_tester.num_hidden_layers + 1
											self.assertEqual(len(UpperCAmelCase__       )      ,      UpperCAmelCase__       )

											__SCREAMING_SNAKE_CASE						       =    self.model_tester.seq_length - self.model_tester.num_masks
											__SCREAMING_SNAKE_CASE						       =    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

											self.assertListEqual(
											    list(hidden_states[0].shape[-2:]       )      ,      [seq_length, self.model_tester.hidden_size]      ,      )

						__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__SCREAMING_SNAKE_CASE						       =    True
											check_hidden_states_output(UpperCAmelCase__      ,      UpperCAmelCase__      ,      UpperCAmelCase__       )

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											__SCREAMING_SNAKE_CASE						       =    True

											check_hidden_states_output(UpperCAmelCase__      ,      UpperCAmelCase__      ,      UpperCAmelCase__       )




	@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."       )
	def   UpperCAmelCase_				(			self      :    Optional[Any]       )	->     Tuple:
						pass




def        UpperCAmelCase__	():




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						       =    hf_hub_download(
					    repo_id="hf-internal-testing/spaghetti-video"  ,						filename="eating_spaghetti.npy"  ,						repo_type="dataset"       )
					__SCREAMING_SNAKE_CASE						       =    np.load(UpperCAmelCase_       )
					return list(UpperCAmelCase_       )




@require_torch
@require_vision
class 			UpperCamelCase_     (			unittest.TestCase):

	"""simple docstring"""





	@cached_property
	def   UpperCAmelCase_				(			self      :    Tuple       )	->     List[str]:
						return (
						    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]      ,      image_std=[0.5, 0.5, 0.5]       )
						    if is_vision_available()
						    else None
						)




	@slow
	def   UpperCAmelCase_				(			self      :    Dict       )	->     Union[str, Any]:
						__SCREAMING_SNAKE_CASE						       =    VideoMAEForVideoClassification.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics"       ).to(
						    UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    self.default_image_processor
						__SCREAMING_SNAKE_CASE						       =    prepare_video()
						__SCREAMING_SNAKE_CASE						       =    image_processor(UpperCAmelCase__      ,      return_tensors="pt"       ).to(UpperCAmelCase__       )

						# forward pass
						with torch.no_grad():
											__SCREAMING_SNAKE_CASE						       =    model(**UpperCAmelCase__       )

						# verify the logits
						__SCREAMING_SNAKE_CASE						       =    torch.Size((1, 4_0_0)       )
						self.assertEqual(outputs.logits.shape      ,      UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    torch.tensor([0.3_669, -0.0_688, -0.2_421]       ).to(UpperCAmelCase__       )

						self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,      UpperCAmelCase__      ,      atol=1E-4       )       )




	@slow
	def   UpperCAmelCase_				(			self      :    Optional[int]       )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE						       =    VideoMAEForPreTraining.from_pretrained("MCG-NJU/videomae-base-short"       ).to(UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    self.default_image_processor
						__SCREAMING_SNAKE_CASE						       =    prepare_video()
						__SCREAMING_SNAKE_CASE						       =    image_processor(UpperCAmelCase__      ,      return_tensors="pt"       ).to(UpperCAmelCase__       )

						# add boolean mask, indicating which patches to mask
						__SCREAMING_SNAKE_CASE						       =    hf_hub_download(repo_id="hf-internal-testing/bool-masked-pos"      ,      filename="bool_masked_pos.pt"       )
						__SCREAMING_SNAKE_CASE						       =    torch.load(UpperCAmelCase__       )

						# forward pass
						with torch.no_grad():
											__SCREAMING_SNAKE_CASE						       =    model(**UpperCAmelCase__       )

						# verify the logits
						__SCREAMING_SNAKE_CASE						       =    torch.Size([1, 1_4_0_8, 1_5_3_6]       )
						__SCREAMING_SNAKE_CASE						       =    torch.tensor(
						    [[0.7_994, 0.9_612, 0.8_508], [0.7_401, 0.8_958, 0.8_302], [0.5_862, 0.7_468, 0.7_325]]      ,      device=UpperCAmelCase__       )
						self.assertEqual(outputs.logits.shape      ,      UpperCAmelCase__       )
						self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]      ,      UpperCAmelCase__      ,      atol=1E-4       )       )

						# verify the loss (`config.norm_pix_loss` = `True`)
						__SCREAMING_SNAKE_CASE						       =    torch.tensor([0.5_142]      ,      device=UpperCAmelCase__       )
						self.assertTrue(torch.allclose(outputs.loss      ,      UpperCAmelCase__      ,      atol=1E-4       )       )

						# verify the loss (`config.norm_pix_loss` = `False`)
						__SCREAMING_SNAKE_CASE						       =    VideoMAEForPreTraining.from_pretrained("MCG-NJU/videomae-base-short"      ,      norm_pix_loss=UpperCAmelCase__       ).to(
						    UpperCAmelCase__       )

						with torch.no_grad():
											__SCREAMING_SNAKE_CASE						       =    model(**UpperCAmelCase__       )

						__SCREAMING_SNAKE_CASE						       =    torch.tensor(torch.tensor([0.6_469]       )      ,      device=UpperCAmelCase__       )
						self.assertTrue(torch.allclose(outputs.loss      ,      UpperCAmelCase__      ,      atol=1E-4       )       )




"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__    :							int            =					logging.get_logger(__name__)

a__    :							Tuple            =					{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}




class 			UpperCamelCase_     (			UpperCamelCase):

	"""simple docstring"""





	snake_case__    :  Dict    =      "ctrl"
	snake_case__    :  int    =      ["past_key_values"]
	snake_case__    :  Tuple    =      {
	    "max_position_embeddings": "n_positions",
	    "hidden_size": "n_embd",
	    "num_attention_heads": "n_head",
	    "num_hidden_layers": "n_layer",
	}




	def __init__(			self      :    List[Any]      ,      UpperCAmelCase__      :    Any=2_4_6_5_3_4      ,      UpperCAmelCase__      :    List[Any]=2_5_6      ,      UpperCAmelCase__      :    Optional[int]=1_2_8_0      ,      UpperCAmelCase__      :    Optional[Any]=8_1_9_2      ,      UpperCAmelCase__      :    int=4_8      ,      UpperCAmelCase__      :    Optional[int]=1_6      ,      UpperCAmelCase__      :    Tuple=0.1      ,      UpperCAmelCase__      :    Optional[Any]=0.1      ,      UpperCAmelCase__      :    Any=1E-6      ,      UpperCAmelCase__      :    List[str]=0.02      ,      UpperCAmelCase__      :    Union[str, Any]=True      ,      **UpperCAmelCase__      :    Tuple      ,      )	->     str:
						__SCREAMING_SNAKE_CASE						       =    vocab_size
						__SCREAMING_SNAKE_CASE						       =    n_positions
						__SCREAMING_SNAKE_CASE						       =    n_embd
						__SCREAMING_SNAKE_CASE						       =    n_layer
						__SCREAMING_SNAKE_CASE						       =    n_head
						__SCREAMING_SNAKE_CASE						       =    dff
						__SCREAMING_SNAKE_CASE						       =    resid_pdrop
						__SCREAMING_SNAKE_CASE						       =    embd_pdrop
						__SCREAMING_SNAKE_CASE						       =    layer_norm_epsilon
						__SCREAMING_SNAKE_CASE						       =    initializer_range

						__SCREAMING_SNAKE_CASE						       =    use_cache

						super().__init__(**UpperCAmelCase__       )
