
'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase    =		logging.get_logger(__name__)

_lowercase    =		{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}




class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






						'''simple docstring'''






						_lowercase		:				Optional[Any]									=       '''open-llama'''




						def __init__( self   ,       _lowercase=100_000   ,       _lowercase=4_096   ,       _lowercase=11_008   ,       _lowercase=32   ,       _lowercase=32   ,       _lowercase="silu"   ,       _lowercase=2_048   ,       _lowercase=0.02   ,       _lowercase=1e-6   ,       _lowercase=True   ,       _lowercase=0   ,       _lowercase=1   ,       _lowercase=2   ,       _lowercase=False   ,       _lowercase=True   ,       _lowercase=0.1   ,       _lowercase=0.1   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=None   ,       **_lowercase   ,       ):

												"""simple docstring"""



												_lowerCAmelCase  						=			vocab_size
												_lowerCAmelCase  						=			max_position_embeddings
												_lowerCAmelCase  						=			hidden_size
												_lowerCAmelCase  						=			intermediate_size
												_lowerCAmelCase  						=			num_hidden_layers
												_lowerCAmelCase  						=			num_attention_heads
												_lowerCAmelCase  						=			hidden_act
												_lowerCAmelCase  						=			initializer_range
												_lowerCAmelCase  						=			rms_norm_eps
												_lowerCAmelCase  						=			use_cache
												_lowerCAmelCase  						=			kwargs.pop(
												    """use_memorry_efficient_attention"""   ,       _snake_case				)
												_lowerCAmelCase  						=			hidden_dropout_prob
												_lowerCAmelCase  						=			attention_dropout_prob
												_lowerCAmelCase  						=			use_stable_embedding
												_lowerCAmelCase  						=			shared_input_output_embedding
												_lowerCAmelCase  						=			rope_scaling
												self._rope_scaling_validation()

												super().__init__(
												    pad_token_id=_snake_case   ,       bos_token_id=_snake_case   ,       eos_token_id=_snake_case   ,       tie_word_embeddings=_snake_case   ,       **_snake_case   ,       )




						def  _lowercase     ( self				):

												"""simple docstring"""



												if self.rope_scaling is None:
																		return

												if not isinstance(self.rope_scaling   ,       _snake_case				) or len(self.rope_scaling				) != 2:
																		raise ValueError(
																		    """`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, """
																		    F'got {self.rope_scaling}'				)
												_lowerCAmelCase  						=			self.rope_scaling.get("""type"""   ,       _snake_case				)
												_lowerCAmelCase  						=			self.rope_scaling.get("""factor"""   ,       _snake_case				)
												if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																		raise ValueError(
																		    F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'				)
												if rope_scaling_factor is None or not isinstance(_snake_case   ,       _snake_case				) or rope_scaling_factor <= 1.0:
																		raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'				)


'''simple docstring'''



from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TFConvBertForMaskedLM,
						    TFConvBertForMultipleChoice,
						    TFConvBertForQuestionAnswering,
						    TFConvBertForSequenceClassification,
						    TFConvBertForTokenClassification,
						    TFConvBertModel,
						)




class   UpperCAmelCase_					:






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase=13   ,       _lowercase=7   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=99   ,       _lowercase=32   ,       _lowercase=2   ,       _lowercase=4   ,       _lowercase=37   ,       _lowercase="gelu"   ,       _lowercase=0.1   ,       _lowercase=0.1   ,       _lowercase=512   ,       _lowercase=16   ,       _lowercase=2   ,       _lowercase=0.02   ,       _lowercase=3   ,       _lowercase=4   ,       _lowercase=None   ,       ):

												"""simple docstring"""



												_lowerCAmelCase  						=			parent
												_lowerCAmelCase  						=			13
												_lowerCAmelCase  						=			7
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			99
												_lowerCAmelCase  						=			384
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			4
												_lowerCAmelCase  						=			37
												_lowerCAmelCase  						=			"""gelu"""
												_lowerCAmelCase  						=			0.1
												_lowerCAmelCase  						=			0.1
												_lowerCAmelCase  						=			512
												_lowerCAmelCase  						=			16
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			0.02
												_lowerCAmelCase  						=			3
												_lowerCAmelCase  						=			4
												_lowerCAmelCase  						=			128
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			9
												_lowerCAmelCase  						=			1
												_lowerCAmelCase  						=			None




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.vocab_size				)

												_lowerCAmelCase  						=			None
												if self.use_input_mask:
																		_lowerCAmelCase  						=			random_attention_mask([self.batch_size, self.seq_length]				)

												_lowerCAmelCase  						=			None
												if self.use_token_type_ids:
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.type_vocab_size				)

												_lowerCAmelCase  						=			None
												_lowerCAmelCase  						=			None
												_lowerCAmelCase  						=			None
												if self.use_labels:
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size]   ,       self.type_sequence_label_size				)
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.num_labels				)
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size]   ,       self.num_choices				)

												_lowerCAmelCase  						=			ConvBertConfig(
												    vocab_size=self.vocab_size   ,       hidden_size=self.hidden_size   ,       num_hidden_layers=self.num_hidden_layers   ,       num_attention_heads=self.num_attention_heads   ,       intermediate_size=self.intermediate_size   ,       hidden_act=self.hidden_act   ,       hidden_dropout_prob=self.hidden_dropout_prob   ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,       max_position_embeddings=self.max_position_embeddings   ,       type_vocab_size=self.type_vocab_size   ,       initializer_range=self.initializer_range   ,       return_dict=_lowercase   ,       )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel(config=_lowercase				)
												_lowerCAmelCase  						=			{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

												_lowerCAmelCase  						=			[input_ids, input_mask]
												_lowerCAmelCase  						=			model(_lowercase				)

												_lowerCAmelCase  						=			model(_lowercase				)

												self.parent.assertEqual(result.last_hidden_state.shape   ,       (self.batch_size, self.seq_length, self.hidden_size)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertForMaskedLM(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.seq_length, self.vocab_size)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_labels
												_lowerCAmelCase  						=			TFConvBertForSequenceClassification(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.num_labels)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_choices
												_lowerCAmelCase  						=			TFConvBertForMultipleChoice(config=_lowercase				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			{
												    """input_ids""": multiple_choice_inputs_ids,
												    """attention_mask""": multiple_choice_input_mask,
												    """token_type_ids""": multiple_choice_token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.num_choices)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_labels
												_lowerCAmelCase  						=			TFConvBertForTokenClassification(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.seq_length, self.num_labels)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertForQuestionAnswering(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.start_logits.shape   ,       (self.batch_size, self.seq_length)				)
												self.parent.assertEqual(result.end_logits.shape   ,       (self.batch_size, self.seq_length)				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.prepare_config_and_inputs()
												(
												    (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   
												)  						=			config_and_inputs
												_lowerCAmelCase  						=			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict






@require_tf
class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE	,							_SCREAMING_SNAKE_CASE	,							unittest.TestCase			):






						'''simple docstring'''






						_lowercase		:				Union[str, Any]									=       (
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						_lowercase		:				str									=       (
						    {
						        '''feature-extraction''': TFConvBertModel,
						        '''fill-mask''': TFConvBertForMaskedLM,
						        '''question-answering''': TFConvBertForQuestionAnswering,
						        '''text-classification''': TFConvBertForSequenceClassification,
						        '''token-classification''': TFConvBertForTokenClassification,
						        '''zero-shot''': TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						_lowercase		:				Optional[Any]									=       False
						_lowercase		:				Dict									=       False
						_lowercase		:				Any									=       False




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModelTester(self				)
												_lowerCAmelCase  						=			ConfigTester(self   ,       config_class=_lowercase   ,       hidden_size=37				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												self.config_tester.run_common_tests()




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*_lowercase				)




						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase						,   _lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True

												if hasattr(_lowercase   ,       """use_cache"""				):
																		_lowerCAmelCase  						=			True

												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """encoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)

												for model_class in self.all_model_classes:
																		_lowerCAmelCase  						=			self._prepare_for_class(_lowercase   ,       _lowercase				)
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			len(model(_lowercase				)				)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																								model.save_pretrained(_lowercase   ,       saved_model=_lowercase				)
																								_lowerCAmelCase  						=			os.path.join(_lowercase   ,       """saved_model"""   ,       """1"""				)
																								_lowerCAmelCase  						=			tf.keras.models.load_model(_lowercase				)
																								_lowerCAmelCase  						=			model(_lowercase				)

																								if self.is_encoder_decoder:
																														_lowerCAmelCase  						=			outputs["""encoder_hidden_states"""]
																														_lowerCAmelCase  						=			outputs["""encoder_attentions"""]
																								else:
																														_lowerCAmelCase  						=			outputs["""hidden_states"""]
																														_lowerCAmelCase  						=			outputs["""attentions"""]

																								self.assertEqual(len(_lowercase				)   ,       _lowercase				)

																								_lowerCAmelCase  						=			getattr(
																								    self.model_tester   ,       """expected_num_hidden_layers"""   ,       self.model_tester.num_hidden_layers + 1				)

																								self.assertEqual(len(_lowercase				)   ,       _lowercase				)
																								self.assertListEqual(
																								    list(output_hidden_states[0].shape[-2:]				)   ,       [self.model_tester.seq_length, self.model_tester.hidden_size]   ,       )

																								self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																								self.assertListEqual(
																								    list(output_attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,       )




						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
												self.assertIsNotNone(_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase						,   _lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """decoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """encoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)

												def check_decoder_attentions_output(_lowercase				):
																		_lowerCAmelCase  						=			len(_lowercase				)
																		self.assertEqual(out_len % 2   ,       0				)
																		_lowerCAmelCase  						=			outputs.decoder_attentions
																		self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																		self.assertListEqual(
																		    list(decoder_attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]   ,       )

												def check_encoder_attentions_output(_lowercase				):
																		_lowerCAmelCase  						=			[
																		    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																		]
																		self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																		self.assertListEqual(
																		    list(attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,       )

												for model_class in self.all_model_classes:
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			False
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																		_lowerCAmelCase  						=			len(_lowercase				)
																		self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)

																		if self.is_encoder_decoder:
																								_lowerCAmelCase  						=			model_class(_lowercase				)
																								_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																								self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																								check_decoder_attentions_output(_lowercase				)

																		# Check that output attentions can also be changed via the config
																		del inputs_dict["output_attentions"]
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																		self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)

																		# Check attention is always last and order is fine
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)

																		self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)   ,       len(_lowercase				)				)
																		self.assertEqual(model.config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)






@require_tf
class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
												_lowerCAmelCase  						=			tf.constant([[0, 1, 2, 3, 4, 5]]				)
												_lowerCAmelCase  						=			model(_lowercase				)[0]

												_lowerCAmelCase  						=			[1, 6, 768]
												self.assertEqual(output.shape   ,       _lowercase				)

												_lowerCAmelCase  						=			tf.constant(
												    [
												        [
												            [-0.0347_5493, -0.468_6034, -0.3063_8832],
												            [0.2263_7248, -0.2698_8646, -0.742_3424],
												            [0.1032_4868, -0.4501_3508, -0.5828_0784],
												        ]
												    ]				)
												tf.debugging.assert_near(output[:, :3, :3]   ,       _lowercase   ,       atol=1e-4				)
