
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel

class     _UpperCamelCase			:






			'''simple docstring'''

			def __init__(    self	:    Dict  ,		snake_case_	:    str  ,		snake_case_	:    Tuple=13  ,		snake_case_	:    Tuple=7  ,		snake_case_	:    Optional[Any]=True  ,		snake_case_	:    int=True  ,		snake_case_	:    Dict=False  ,		snake_case_	:    Optional[Any]=True  ,		snake_case_	:    List[str]=99  ,		snake_case_	:    Dict=32  ,		snake_case_	:    Optional[Any]=5  ,		snake_case_	:    List[Any]=4  ,		snake_case_	:    Optional[Any]=37  ,		snake_case_	:    Tuple="gelu"  ,		snake_case_	:    Any=0.1  ,		snake_case_	:    Any=0.1  ,		snake_case_	:    Tuple=512  ,		snake_case_	:    List[Any]=16  ,		snake_case_	:    Optional[int]=2  ,		snake_case_	:    str=0.02  ,		snake_case_	:    List[str]=3  ,		snake_case_	:    Union[str, Any]=4  ,		snake_case_	:    Dict=None  ,		):
							UpperCamelCase_:							Optional[Any]		=  parent
							UpperCamelCase_:							Any		=  batch_size
							UpperCamelCase_:							Tuple		=  seq_length
							UpperCamelCase_:							Dict		=  is_training
							UpperCamelCase_:							List[Any]		=  use_input_mask
							UpperCamelCase_:							int		=  use_token_type_ids
							UpperCamelCase_:							Union[str, Any]		=  use_labels
							UpperCamelCase_:							Any		=  vocab_size
							UpperCamelCase_:							Optional[int]		=  hidden_size
							UpperCamelCase_:							Tuple		=  num_hidden_layers
							UpperCamelCase_:							Dict		=  num_attention_heads
							UpperCamelCase_:							Tuple		=  intermediate_size
							UpperCamelCase_:							Any		=  hidden_act
							UpperCamelCase_:							Any		=  hidden_dropout_prob
							UpperCamelCase_:							Optional[Any]		=  attention_probs_dropout_prob
							UpperCamelCase_:							str		=  max_position_embeddings
							UpperCamelCase_:							int		=  type_vocab_size
							UpperCamelCase_:							Tuple		=  type_sequence_label_size
							UpperCamelCase_:							Optional[int]		=  initializer_range
							UpperCamelCase_:							Dict		=  num_labels
							UpperCamelCase_:							Any		=  num_choices
							UpperCamelCase_:							Optional[Any]		=  scope

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Optional[int]		=  ids_tensor([self.batch_size, self.seq_length]  ,		self.vocab_size )

							UpperCamelCase_:							List[Any]		=  None
							if self.use_input_mask:
											UpperCamelCase_:							Tuple		=  random_attention_mask([self.batch_size, self.seq_length] )

							UpperCamelCase_:							List[str]		=  None
							if self.use_token_type_ids:
											UpperCamelCase_:							str		=  ids_tensor([self.batch_size, self.seq_length]  ,		self.type_vocab_size )

							UpperCamelCase_:							Dict		=  None
							UpperCamelCase_:							Any		=  None
							UpperCamelCase_:							List[Any]		=  None
							if self.use_labels:
											UpperCamelCase_:							Tuple		=  ids_tensor([self.batch_size]  ,		self.type_sequence_label_size )
											UpperCamelCase_:							Union[str, Any]		=  ids_tensor([self.batch_size, self.seq_length]  ,		self.num_labels )
											UpperCamelCase_:							Optional[Any]		=  ids_tensor([self.batch_size]  ,		self.num_choices )

							UpperCamelCase_:							int		=  self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							return OpenLlamaConfig(
							    vocab_size=self.vocab_size  ,		hidden_size=self.hidden_size  ,		num_hidden_layers=self.num_hidden_layers  ,		num_attention_heads=self.num_attention_heads  ,		intermediate_size=self.intermediate_size  ,		hidden_act=self.hidden_act  ,		hidden_dropout_prob=self.hidden_dropout_prob  ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,		max_position_embeddings=self.max_position_embeddings  ,		type_vocab_size=self.type_vocab_size  ,		is_decoder=__SCREAMING_SNAKE_CASE  ,		initializer_range=self.initializer_range  ,		use_stable_embedding=__SCREAMING_SNAKE_CASE  ,		)

			def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    str  ,		snake_case_	:    Any  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Tuple  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    int ):
							UpperCamelCase_:							Union[str, Any]		=  OpenLlamaModel(config=__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							Union[str, Any]		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							Any		=  model(__SCREAMING_SNAKE_CASE )
							self.parent.assertEqual(result.last_hidden_state.shape  ,		(self.batch_size, self.seq_length, self.hidden_size) )

			def  lowerCAmelCase__		(    self	:    int  ,		snake_case_	:    Optional[int]  ,		snake_case_	:    str  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Dict  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    Tuple  ,		):
							UpperCamelCase_:							Dict		=  True
							UpperCamelCase_:							int		=  OpenLlamaModel(__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							List[Any]		=  model(
							    __SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		encoder_hidden_states=__SCREAMING_SNAKE_CASE  ,		encoder_attention_mask=__SCREAMING_SNAKE_CASE  ,		)
							UpperCamelCase_:							Optional[Any]		=  model(
							    __SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		encoder_hidden_states=__SCREAMING_SNAKE_CASE  ,		)
							UpperCamelCase_:							str		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE )
							self.parent.assertEqual(result.last_hidden_state.shape  ,		(self.batch_size, self.seq_length, self.hidden_size) )

			def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    int  ,		snake_case_	:    Tuple  ,		snake_case_	:    int  ,		snake_case_	:    str  ,		snake_case_	:    str  ,		snake_case_	:    List[str]  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    List[Any]  ,		):
							UpperCamelCase_:							List[Any]		=  OpenLlamaForCausalLM(config=__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							Any		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		labels=__SCREAMING_SNAKE_CASE )
							self.parent.assertEqual(result.logits.shape  ,		(self.batch_size, self.seq_length, self.vocab_size) )

			def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Any  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    Dict  ,		snake_case_	:    List[str]  ,		snake_case_	:    int  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    List[str]  ,		):
							UpperCamelCase_:							Union[str, Any]		=  True
							UpperCamelCase_:							Tuple		=  True
							UpperCamelCase_:							int		=  OpenLlamaForCausalLM(config=__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()

							# first forward pass
							UpperCamelCase_:							Tuple		=  model(
							    __SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		encoder_hidden_states=__SCREAMING_SNAKE_CASE  ,		encoder_attention_mask=__SCREAMING_SNAKE_CASE  ,		use_cache=__SCREAMING_SNAKE_CASE  ,		)
							UpperCamelCase_:							Tuple		=  outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							UpperCamelCase_:							Optional[Any]		=  ids_tensor((self.batch_size, 3)  ,		config.vocab_size )
							UpperCamelCase_:							Union[str, Any]		=  ids_tensor((self.batch_size, 3)  ,		vocab_size=2 )

							# append to next input_ids and
							UpperCamelCase_:							Optional[Any]		=  torch.cat([input_ids, next_tokens]  ,		dim=-1 )
							UpperCamelCase_:							Any		=  torch.cat([input_mask, next_mask]  ,		dim=-1 )

							UpperCamelCase_:							int		=  model(
							    __SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		encoder_hidden_states=__SCREAMING_SNAKE_CASE  ,		encoder_attention_mask=__SCREAMING_SNAKE_CASE  ,		output_hidden_states=__SCREAMING_SNAKE_CASE  ,		)["""hidden_states"""][0]
							UpperCamelCase_:							Optional[int]		=  model(
							    __SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		encoder_hidden_states=__SCREAMING_SNAKE_CASE  ,		encoder_attention_mask=__SCREAMING_SNAKE_CASE  ,		past_key_values=__SCREAMING_SNAKE_CASE  ,		output_hidden_states=__SCREAMING_SNAKE_CASE  ,		)["""hidden_states"""][0]

							# select random slice
							UpperCamelCase_:							Union[str, Any]		=  ids_tensor((1,)  ,		output_from_past.shape[-1] ).item()
							UpperCamelCase_:							Any		=  output_from_no_past[:, -3:, random_slice_idx].detach()
							UpperCamelCase_:							int		=  output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE  ,		atol=1e-3 ) )




			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Optional[int]		=  self.prepare_config_and_inputs()
							(
							    (
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),				(
							    UpperCamelCase_	
							),					
							):							Dict		=  config_and_inputs
							UpperCamelCase_:							Any		=  {"""input_ids""": input_ids, """attention_mask""": input_mask}
							return config, inputs_dict




@require_torch
class     _UpperCamelCase			(   lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    unittest.TestCase					):






			'''simple docstring'''

			__UpperCamelCase :  Union[str, Any]    =	(
			    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
			)
			__UpperCamelCase :  Optional[Any]    =	(OpenLlamaForCausalLM,) if is_torch_available() else ()
			__UpperCamelCase :  Dict    =	(
			    {
			        """feature-extraction""": OpenLlamaModel,
			        """text-classification""": OpenLlamaForSequenceClassification,
			        """text-generation""": OpenLlamaForCausalLM,
			        """zero-shot""": OpenLlamaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__UpperCamelCase :  Dict    =	False
			__UpperCamelCase :  List[Any]    =	False

			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							UpperCamelCase_:							List[Any]		=  OpenLlamaModelTester(self )
							UpperCamelCase_:							Tuple		=  ConfigTester(self  ,		config_class=__SCREAMING_SNAKE_CASE  ,		hidden_size=37 )

			def  lowerCAmelCase__		(    self	:    List[str] ):
							self.config_tester.run_common_tests()

			def  lowerCAmelCase__		(    self	:    List[str] ):
							UpperCamelCase_:							Optional[Any]		=  self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE )

			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							UpperCamelCase_:							Union[str, Any]		=  self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
											UpperCamelCase_:							List[str]		=  type
											self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE )

			def  lowerCAmelCase__		(    self	:    str ):
							UpperCamelCase_,				UpperCamelCase_:							Optional[int]		=  self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase_:							List[str]		=  3
							UpperCamelCase_:							Union[str, Any]		=  input_dict["""input_ids"""]
							UpperCamelCase_:							List[Any]		=  input_ids.ne(1 ).to(__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							str		=  ids_tensor([self.model_tester.batch_size]  ,		self.model_tester.type_sequence_label_size )
							UpperCamelCase_:							Union[str, Any]		=  OpenLlamaForSequenceClassification(__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							Optional[Any]		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		labels=__SCREAMING_SNAKE_CASE )
							self.assertEqual(result.logits.shape  ,		(self.model_tester.batch_size, self.model_tester.num_labels) )

			def  lowerCAmelCase__		(    self	:    Any ):
							UpperCamelCase_,				UpperCamelCase_:							Dict		=  self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase_:							Dict		=  3
							UpperCamelCase_:							Optional[int]		=  """single_label_classification"""
							UpperCamelCase_:							int		=  input_dict["""input_ids"""]
							UpperCamelCase_:							str		=  input_ids.ne(1 ).to(__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							Dict		=  ids_tensor([self.model_tester.batch_size]  ,		self.model_tester.type_sequence_label_size )
							UpperCamelCase_:							int		=  OpenLlamaForSequenceClassification(__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							Optional[int]		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		labels=__SCREAMING_SNAKE_CASE )
							self.assertEqual(result.logits.shape  ,		(self.model_tester.batch_size, self.model_tester.num_labels) )

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_,				UpperCamelCase_:							Optional[Any]		=  self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase_:							Union[str, Any]		=  3
							UpperCamelCase_:							List[str]		=  """multi_label_classification"""
							UpperCamelCase_:							List[Any]		=  input_dict["""input_ids"""]
							UpperCamelCase_:							Any		=  input_ids.ne(1 ).to(__SCREAMING_SNAKE_CASE )
							UpperCamelCase_:							str		=  ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]  ,		self.model_tester.type_sequence_label_size ).to(torch.float )
							UpperCamelCase_:							Union[str, Any]		=  OpenLlamaForSequenceClassification(__SCREAMING_SNAKE_CASE )
							model.to(__SCREAMING_SNAKE_CASE )
							model.eval()
							UpperCamelCase_:							Dict		=  model(__SCREAMING_SNAKE_CASE  ,		attention_mask=__SCREAMING_SNAKE_CASE  ,		labels=__SCREAMING_SNAKE_CASE )
							self.assertEqual(result.logits.shape  ,		(self.model_tester.batch_size, self.model_tester.num_labels) )

			@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test""" )
			def  lowerCAmelCase__		(    self	:    List[Any] ):
							pass




			@parameterized.expand([("""linear""",), ("""dynamic""",)] )
			def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    str ):
							UpperCamelCase_,				UpperCamelCase_:							Optional[int]		=  self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase_:							Any		=  ids_tensor([1, 10]  ,		config.vocab_size )
							UpperCamelCase_:							Optional[int]		=  ids_tensor([1, int(config.max_position_embeddings * 1.5 )]  ,		config.vocab_size )

							set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase_:							int		=  OpenLlamaModel(__SCREAMING_SNAKE_CASE )
							original_model.to(__SCREAMING_SNAKE_CASE )
							original_model.eval()
							UpperCamelCase_:							str		=  original_model(__SCREAMING_SNAKE_CASE ).last_hidden_state
							UpperCamelCase_:							Dict		=  original_model(__SCREAMING_SNAKE_CASE ).last_hidden_state

							set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase_:							Dict		=  {"""type""": scaling_type, """factor""": 10.0}
							UpperCamelCase_:							Union[str, Any]		=  OpenLlamaModel(__SCREAMING_SNAKE_CASE )
							scaled_model.to(__SCREAMING_SNAKE_CASE )
							scaled_model.eval()
							UpperCamelCase_:							Tuple		=  scaled_model(__SCREAMING_SNAKE_CASE ).last_hidden_state
							UpperCamelCase_:							List[Any]		=  scaled_model(__SCREAMING_SNAKE_CASE ).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
											self.assertTrue(torch.allclose(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE  ,		atol=1e-5 ) )
							else:
											self.assertFalse(torch.allclose(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE  ,		atol=1e-5 ) )

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(__SCREAMING_SNAKE_CASE  ,		__SCREAMING_SNAKE_CASE  ,		atol=1e-5 ) )








from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_  :	Tuple 				=					logging.get_logger(__name__)

lowerCamelCase_  :	str 				=					{
    """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json""",
}

class     _UpperCamelCase			(   _A					):






			'''simple docstring'''

			__UpperCamelCase :  Any    =	"""lxmert"""
			__UpperCamelCase :  Optional[Any]    =	{}

			def __init__(    self	:    Dict  ,		snake_case_	:    List[str]=3_0522  ,		snake_case_	:    int=768  ,		snake_case_	:    str=12  ,		snake_case_	:    Tuple=9500  ,		snake_case_	:    Optional[Any]=1600  ,		snake_case_	:    str=400  ,		snake_case_	:    List[str]=3072  ,		snake_case_	:    Dict="gelu"  ,		snake_case_	:    Optional[Any]=0.1  ,		snake_case_	:    int=0.1  ,		snake_case_	:    Optional[int]=512  ,		snake_case_	:    Tuple=2  ,		snake_case_	:    Optional[int]=0.02  ,		snake_case_	:    Union[str, Any]=1e-12  ,		snake_case_	:    Tuple=9  ,		snake_case_	:    Tuple=5  ,		snake_case_	:    Union[str, Any]=5  ,		snake_case_	:    List[str]=2048  ,		snake_case_	:    Any=4  ,		snake_case_	:    int=6.67  ,		snake_case_	:    Dict=True  ,		snake_case_	:    Tuple=True  ,		snake_case_	:    int=True  ,		snake_case_	:    str=True  ,		snake_case_	:    List[Any]=True  ,		snake_case_	:    List[Any]=True  ,		snake_case_	:    List[str]=True  ,		**snake_case_	:    Optional[int]  ,		):
							UpperCamelCase_:							int		=  vocab_size
							UpperCamelCase_:							Tuple		=  hidden_size
							UpperCamelCase_:							Optional[Any]		=  num_attention_heads
							UpperCamelCase_:							List[str]		=  hidden_act
							UpperCamelCase_:							int		=  intermediate_size
							UpperCamelCase_:							Any		=  hidden_dropout_prob
							UpperCamelCase_:							List[Any]		=  attention_probs_dropout_prob
							UpperCamelCase_:							List[str]		=  max_position_embeddings
							UpperCamelCase_:							Optional[Any]		=  type_vocab_size
							UpperCamelCase_:							Optional[Any]		=  initializer_range
							UpperCamelCase_:							str		=  layer_norm_eps
							UpperCamelCase_:							List[str]		=  num_qa_labels
							UpperCamelCase_:							List[str]		=  num_object_labels
							UpperCamelCase_:							Tuple		=  num_attr_labels
							UpperCamelCase_:							List[Any]		=  l_layers
							UpperCamelCase_:							List[Any]		=  x_layers
							UpperCamelCase_:							Any		=  r_layers
							UpperCamelCase_:							str		=  visual_feat_dim
							UpperCamelCase_:							Union[str, Any]		=  visual_pos_dim
							UpperCamelCase_:							str		=  visual_loss_normalizer
							UpperCamelCase_:							Dict		=  task_matched
							UpperCamelCase_:							Tuple		=  task_mask_lm
							UpperCamelCase_:							List[str]		=  task_obj_predict
							UpperCamelCase_:							Any		=  task_qa
							UpperCamelCase_:							Dict		=  visual_obj_loss
							UpperCamelCase_:							Optional[Any]		=  visual_attr_loss
							UpperCamelCase_:							List[Any]		=  visual_feat_loss
							UpperCamelCase_:							Tuple		=  {"""vision""": r_layers, """cross_encoder""": x_layers, """language""": l_layers}
							super().__init__(**snake_case_ )






