





"""simple docstring"""






import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class 			__lowercase     :







					'''simple docstring'''




					def __init__(  self		,     _UpperCAmelCase		,     _UpperCAmelCase=13		,     _UpperCAmelCase=7		,     _UpperCAmelCase=True		,     _UpperCAmelCase=True		,     _UpperCAmelCase=False		,     _UpperCAmelCase=True		,     _UpperCAmelCase=99		,     _UpperCAmelCase=32		,     _UpperCAmelCase=5		,     _UpperCAmelCase=4		,     _UpperCAmelCase=37		,     _UpperCAmelCase="gelu"		,     _UpperCAmelCase=0.1		,     _UpperCAmelCase=0.1		,     _UpperCAmelCase=512		,     _UpperCAmelCase=16		,     _UpperCAmelCase=2		,     _UpperCAmelCase=0.0_2		,     _UpperCAmelCase=3		,     _UpperCAmelCase=4		,     _UpperCAmelCase=None		,     ):
									__a	:  Any         =					parent
									__a	:  str         =					batch_size
									__a	:  Union[str, Any]         =					seq_length
									__a	:  List[str]         =					is_training
									__a	:  int         =					use_input_mask
									__a	:  Union[str, Any]         =					use_token_type_ids
									__a	:  Any         =					use_labels
									__a	:  List[str]         =					vocab_size
									__a	:  Dict         =					hidden_size
									__a	:  Optional[int]         =					num_hidden_layers
									__a	:  Optional[Any]         =					num_attention_heads
									__a	:  List[str]         =					intermediate_size
									__a	:  Optional[int]         =					hidden_act
									__a	:  Optional[Any]         =					hidden_dropout_prob
									__a	:  Optional[Any]         =					attention_probs_dropout_prob
									__a	:  Optional[Any]         =					max_position_embeddings
									__a	:  str         =					type_vocab_size
									__a	:  Optional[Any]         =					type_sequence_label_size
									__a	:  Union[str, Any]         =					initializer_range
									__a	:  str         =					num_labels
									__a	:  str         =					num_choices
									__a	:  Dict         =					scope







					def 	_lowerCamelCase						(  self       ):
									__a	:  Union[str, Any]         =					ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size       )

									__a	:  Tuple         =					None
									if self.use_input_mask:
													__a	:  List[str]         =					random_attention_mask([self.batch_size, self.seq_length]       )

									__a	:  List[Any]         =					None
									if self.use_token_type_ids:
													__a	:  Optional[int]         =					ids_tensor([self.batch_size, self.seq_length]		,     self.type_vocab_size       )

									__a	:  Dict         =					None
									__a	:  Tuple         =					None
									__a	:  List[str]         =					None
									if self.use_labels:
													__a	:  List[Any]         =					ids_tensor([self.batch_size]		,     self.type_sequence_label_size       )
													__a	:  Optional[int]         =					ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels       )
													__a	:  Tuple         =					ids_tensor([self.batch_size]		,     self.num_choices       )

									__a	:  Union[str, Any]         =					self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







					def 	_lowerCamelCase						(  self       ):
									return OpenLlamaConfig(
									    vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     hidden_act=self.hidden_act		,     hidden_dropout_prob=self.hidden_dropout_prob		,     attention_probs_dropout_prob=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     type_vocab_size=self.type_vocab_size		,     is_decoder=_UpperCAmelCase		,     initializer_range=self.initializer_range		,     use_stable_embedding=_UpperCAmelCase		,     )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  int         =					OpenLlamaModel(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Dict         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase       )
									__a	:  List[Any]         =					model(_UpperCAmelCase       )
									self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     ):
									__a	:  Tuple         =					True
									__a	:  List[Any]         =					OpenLlamaModel(_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Optional[Any]         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     encoder_hidden_states=_UpperCAmelCase		,     encoder_attention_mask=_UpperCAmelCase		,     )
									__a	:  Any         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     encoder_hidden_states=_UpperCAmelCase		,     )
									__a	:  List[Any]         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase       )
									self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     ):
									__a	:  Optional[int]         =					OpenLlamaForCausalLM(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Optional[int]         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     ):
									__a	:  Optional[int]         =					True
									__a	:  Optional[int]         =					True
									__a	:  Optional[Any]         =					OpenLlamaForCausalLM(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()

									# first forward pass
									__a	:  Optional[int]         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     encoder_hidden_states=_UpperCAmelCase		,     encoder_attention_mask=_UpperCAmelCase		,     use_cache=_UpperCAmelCase		,     )
									__a	:  Optional[int]         =					outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									__a	:  Tuple         =					ids_tensor((self.batch_size, 3)		,     config.vocab_size       )
									__a	:  Dict         =					ids_tensor((self.batch_size, 3)		,     vocab_size=2       )

									# append to next input_ids and
									__a	:  Optional[Any]         =					torch.cat([input_ids, next_tokens]		,     dim=-1       )
									__a	:  Any         =					torch.cat([input_mask, next_mask]		,     dim=-1       )

									__a	:  List[Any]         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     encoder_hidden_states=_UpperCAmelCase		,     encoder_attention_mask=_UpperCAmelCase		,     output_hidden_states=_UpperCAmelCase		,     )['''hidden_states'''][0]
									__a	:  Dict         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     encoder_hidden_states=_UpperCAmelCase		,     encoder_attention_mask=_UpperCAmelCase		,     past_key_values=_UpperCAmelCase		,     output_hidden_states=_UpperCAmelCase		,     )['''hidden_states'''][0]

									# select random slice
									__a	:  List[str]         =					ids_tensor((1,)		,     output_from_past.shape[-1]       ).item()
									__a	:  List[Any]         =					output_from_no_past[:, -3:, random_slice_idx].detach()
									__a	:  Optional[int]         =					output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(_UpperCAmelCase		,     _UpperCAmelCase		,     atol=1e-3       )       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Any         =					self.prepare_config_and_inputs()
									(
									    (
									    __a
									)    ,  (
									    __a
									)    ,  (
									    __a
									)    ,  (
									    __a
									)    ,  (
									    __a
									)    ,  (
									    __a
									)    ,  (
									    __a
									)    ,  
									)	:  Dict         =					config_and_inputs
									__a	:  str         =					{'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict






@require_torch
class 			__lowercase     (     _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase					=					(
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					__lowerCAmelCase					=					(OpenLlamaForCausalLM,) if is_torch_available() else ()
					__lowerCAmelCase					=					(
					    {
					        '''feature-extraction''': OpenLlamaModel,
					        '''text-classification''': OpenLlamaForSequenceClassification,
					        '''text-generation''': OpenLlamaForCausalLM,
					        '''zero-shot''': OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					__lowerCAmelCase					=					False
					__lowerCAmelCase					=					False







					def 	_lowerCamelCase						(  self       ):
									__a	:  Union[str, Any]         =					OpenLlamaModelTester(self       )
									__a	:  str         =					ConfigTester(self		,     config_class=_UpperCAmelCase		,     hidden_size=37       )







					def 	_lowerCamelCase						(  self       ):
									self.config_tester.run_common_tests()







					def 	_lowerCamelCase						(  self       ):
									__a	:  str         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  str         =					self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
													__a	:  Optional[int]         =					type
													self.model_tester.create_and_check_model(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a    ,  __a	:  List[Any]         =					self.model_tester.prepare_config_and_inputs_for_common()
									__a	:  List[str]         =					3
									__a	:  Union[str, Any]         =					input_dict['''input_ids''']
									__a	:  Dict         =					input_ids.ne(1       ).to(_UpperCAmelCase       )
									__a	:  Optional[Any]         =					ids_tensor([self.model_tester.batch_size]		,     self.model_tester.type_sequence_label_size       )
									__a	:  Tuple         =					OpenLlamaForSequenceClassification(_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Union[str, Any]         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.assertEqual(result.logits.shape		,     (self.model_tester.batch_size, self.model_tester.num_labels)       )







					def 	_lowerCamelCase						(  self       ):
									__a    ,  __a	:  Optional[Any]         =					self.model_tester.prepare_config_and_inputs_for_common()
									__a	:  Dict         =					3
									__a	:  Tuple         =					'''single_label_classification'''
									__a	:  Tuple         =					input_dict['''input_ids''']
									__a	:  List[str]         =					input_ids.ne(1       ).to(_UpperCAmelCase       )
									__a	:  Any         =					ids_tensor([self.model_tester.batch_size]		,     self.model_tester.type_sequence_label_size       )
									__a	:  List[Any]         =					OpenLlamaForSequenceClassification(_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Tuple         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.assertEqual(result.logits.shape		,     (self.model_tester.batch_size, self.model_tester.num_labels)       )







					def 	_lowerCamelCase						(  self       ):
									__a    ,  __a	:  List[str]         =					self.model_tester.prepare_config_and_inputs_for_common()
									__a	:  int         =					3
									__a	:  Optional[int]         =					'''multi_label_classification'''
									__a	:  Optional[int]         =					input_dict['''input_ids''']
									__a	:  str         =					input_ids.ne(1       ).to(_UpperCAmelCase       )
									__a	:  Tuple         =					ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]		,     self.model_tester.type_sequence_label_size       ).to(torch.float       )
									__a	:  str         =					OpenLlamaForSequenceClassification(_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  int         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.assertEqual(result.logits.shape		,     (self.model_tester.batch_size, self.model_tester.num_labels)       )







					@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''       )
					def 	_lowerCamelCase						(  self       ):
									pass







					@parameterized.expand([('''linear''',), ('''dynamic''',)]       )
					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									__a    ,  __a	:  List[str]         =					self.model_tester.prepare_config_and_inputs_for_common()
									__a	:  List[str]         =					ids_tensor([1, 10]		,     config.vocab_size       )
									__a	:  str         =					ids_tensor([1, int(config.max_position_embeddings * 1.5       )]		,     config.vocab_size       )

									set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
									__a	:  Dict         =					OpenLlamaModel(_UpperCAmelCase       )
									original_model.to(_UpperCAmelCase       )
									original_model.eval()
									__a	:  List[Any]         =					original_model(_UpperCAmelCase       ).last_hidden_state
									__a	:  Optional[int]         =					original_model(_UpperCAmelCase       ).last_hidden_state

									set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
									__a	:  str         =					{'''type''': scaling_type, '''factor''': 1_0.0}
									__a	:  Any         =					OpenLlamaModel(_UpperCAmelCase       )
									scaled_model.to(_UpperCAmelCase       )
									scaled_model.eval()
									__a	:  Optional[Any]         =					scaled_model(_UpperCAmelCase       ).last_hidden_state
									__a	:  Union[str, Any]         =					scaled_model(_UpperCAmelCase       ).last_hidden_state

									# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
									# maximum sequence length, so the outputs for the short input should match.
									if scaling_type == "dynamic":
													self.assertTrue(torch.allclose(_UpperCAmelCase		,     _UpperCAmelCase		,     atol=1e-5       )       )
									else:
													self.assertFalse(torch.allclose(_UpperCAmelCase		,     _UpperCAmelCase		,     atol=1e-5       )       )

									# The output should be different for long inputs
									self.assertFalse(torch.allclose(_UpperCAmelCase		,     _UpperCAmelCase		,     atol=1e-5       )       )






"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


A	     =					{
    '''configuration_mvp''': ['''MVP_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MvpConfig''', '''MvpOnnxConfig'''],
    '''tokenization_mvp''': ['''MvpTokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A	     =					['''MvpTokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A	     =					[
		    '''MVP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''MvpForCausalLM''',
		    '''MvpForConditionalGeneration''',
		    '''MvpForQuestionAnswering''',
		    '''MvpForSequenceClassification''',
		    '''MvpModel''',
		    '''MvpPreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_mvp import MVP_PRETRAINED_CONFIG_ARCHIVE_MAP, MvpConfig, MvpOnnxConfig
		from .tokenization_mvp import MvpTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_mvp_fast import MvpTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mvp import (
				    MVP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    MvpForCausalLM,
				    MvpForConditionalGeneration,
				    MvpForQuestionAnswering,
				    MvpForSequenceClassification,
				    MvpModel,
				    MvpPreTrainedModel,
				)

else:
		import sys

		A	     =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)