



"""simple docstring"""


import argparse
import random

import joblib
import numpy as np
import torch
from igf.igf import (
    SecondaryLearner,
    collect_objective_set,
    compute_perplexity,
    generate_datasets,
    load_gpta,
    recopy_gpta,
    set_seed,
    train_secondary_learner,
)
from torch.utils.data import DataLoader, RandomSampler

from transformers import GPTaLMHeadModel

def 			_lowerCAmelCase				(       lowercase_=32		,     lowercase_=10		,     lowercase_=100		,     lowercase_=1026		,     lowercase_=True		,     lowercase_="data/tokenized_stories_train_wikitext103.jbl"		,     lowercase_="igf_context_pairs.jbl"		,     ):
						set_seed(3						)
						# generate train_data and objective_set
						UpperCAmelCase				,   UpperCAmelCase					=			generate_datasets(
						    lowercase_		,     lowercase_		,     number=lowercase_		,     min_len=1026		,     trim=lowercase_						)
						# keeps model same across runs
						set_seed(4						)
						# model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights
						# can we train on GPU?
						UpperCAmelCase					=			torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'						)

						# load pretrained model
						UpperCAmelCase					=			load_gpta('gpt2'						).to(lowercase_						)
						print('computing perplexity on objective set'						)
						UpperCAmelCase					=			compute_perplexity(lowercase_		,     lowercase_		,     lowercase_						).item()
						print('perplexity on objective set:'		,     lowercase_						)

						# collect igf pairs and save to file demo.jbl
						collect_objective_set(lowercase_		,     lowercase_		,     lowercase_		,     lowercase_		,     lowercase_		,     lowercase_		,     lowercase_		,     lowercase_						)

						# clean up, delete model and data we don't need anymore
						del model, train_data, objective_set
						torch.cuda.empty_cache()

def 			_lowerCAmelCase				(       lowercase_		,     lowercase_=15		,     lowercase_=128		,     lowercase_=100		,     lowercase_="igf_model.pt"		,     ):
						set_seed(42						)

						# Load pre-trained model
						UpperCAmelCase					=			GPTaLMHeadModel.from_pretrained('gpt2'						)

						# Initialize secondary learner to use embedding weights of model
						UpperCAmelCase					=			SecondaryLearner(lowercase_						)

						# Train secondary learner
						UpperCAmelCase					=			train_secondary_learner(
						    lowercase_		,     lowercase_		,     max_epochs=lowercase_		,     batch_size=lowercase_		,     eval_freq=100		,     igf_model_path=lowercase_		,     )

						del model, secondary_learner_train_data
						torch.cuda.empty_cache()

						return secondary_learner

def 			_lowerCAmelCase				(       lowercase_		,     lowercase_		,     lowercase_		,     lowercase_=32		,     lowercase_=1000		,     lowercase_=16		,     lowercase_=1.0		,     lowercase_=recopy_gpta		,     lowercase_=None		,     lowercase_=10		,     lowercase_="gpt2_finetuned.pt"		,     ):
						UpperCAmelCase					=			torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'						)
						UpperCAmelCase					=			RandomSampler(lowercase_						)
						UpperCAmelCase					=			DataLoader(lowercase_		,     sampler=lowercase_						)

						UpperCAmelCase					=			max_steps // (len(lowercase_						)) + 1
						UpperCAmelCase					=			0
						UpperCAmelCase					=			torch.zeros((1, context_len)		,     dtype=torch.long		,     device=lowercase_						)
						UpperCAmelCase				,   UpperCAmelCase				,   UpperCAmelCase					=			recopy_model(lowercase_		,     lowercase_		,     lowercase_						)

						model.train()
						if secondary_learner is not None:
												secondary_learner.to(lowercase_						)
												secondary_learner.eval()
						UpperCAmelCase					=			[]
						UpperCAmelCase					=			0

						UpperCAmelCase					=			[]
						UpperCAmelCase					=			[]

						# Compute the performance of the transformer model at the beginning
						UpperCAmelCase					=			compute_perplexity(lowercase_		,     lowercase_		,     lowercase_						)
						test_perps.append(lowercase_						)
						print('Test perplexity, step'		,     lowercase_		,     ':'		,     lowercase_						)
						for epoch in range(int(lowercase_						)						):
												for step, example in enumerate(lowercase_						):
																		torch.cuda.empty_cache()
																		UpperCAmelCase					=			random.randint(0		,     example.size(2						) - context_len - 1						)
																		UpperCAmelCase					=			example[0, 0, start : start + context_len]
																		lm_optimizer.zero_grad()
																		UpperCAmelCase					=			model(lowercase_		,     labels=lowercase_						)
																		UpperCAmelCase					=			True

																		if secondary_learner is not None:
																								UpperCAmelCase					=			secondary_learner.forward(
																								    torch.tensor(lowercase_		,     dtype=torch.long		,     device=lowercase_						).unsqueeze(0						)						)[0].item()
																								observed_qs.append(float(lowercase_						)						)

																								# Here we implement the simple non-constant threshold for the predicted IG(X) value
																								# We will decay the selectivity of our secondary learner filter from
																								# 1 standard deviation above average to 1 below average after 10 batches.

																								if global_step == 10:
																														UpperCAmelCase					=			-1
																								if predicted_q < threshold:
																														UpperCAmelCase					=			False

            # If we passed the filter, add the context to the batch!
																		if do_backprop:
																								contexts.append(np.array(context.cpu()						)						)
																								UpperCAmelCase					=			outputs[0]
																								lm_loss.backward()
																								examples += 1

																		del outputs

																		# Once the batch is filled with enough contexts, backprop on the batch.
																		if examples == batch_size:
																								torch.cuda.empty_cache()
																								UpperCAmelCase					=			0
																								# Do LM backprop
																								torch.nn.utils.clip_grad_norm_(model.parameters()		,     3.0						)
																								lm_optimizer.step()
																								lm_scheduler.step()  # Update learning rate schedule
																								global_step += 1
																								# Compute the performance of the transformer model at this batch
																								if global_step % eval_interval == 0:
																														UpperCAmelCase					=			compute_perplexity(lowercase_		,     lowercase_		,     lowercase_						)
																														test_perps.append(lowercase_						)

																														print('Test perplexity, step'		,     lowercase_		,     ':'		,     lowercase_						)
            # Break out of the loop after 60 batches
																		if max_steps > 0 and global_step > 60:
																								break
												if max_steps > 0 and global_step > 60:
																		break

    # save finetuned transformer model
						torch.save(model.state_dict()		,     lowercase_						)
						torch.cuda.empty_cache()
						# Do some cleaning up so we can reinitialize for the next run of this function
						del lm_optimizer
						del lm_scheduler
						return model

def 			_lowerCAmelCase				(       ):
						UpperCAmelCase					=			argparse.ArgumentParser(description='Fine-tune a transformer model with IGF on a language modeling task'						)

						# Required parameters
						parser.add_argument(
						    '--data_dir'		,     default=lowercase_		,     type=lowercase_		,     required=lowercase_		,     help='The input data dir. Should contain data files for WikiText.'		,     )
						parser.add_argument(
						    '--model_name_or_path'		,     default=lowercase_		,     type=lowercase_		,     required=lowercase_		,     help='Path to pretrained model or model identifier from huggingface.co/models'		,     )
						parser.add_argument(
						    '--data_file'		,     type=lowercase_		,     default=lowercase_		,     help=(
						        'A jbl file containing tokenized data which can be split as objective dataset, '
						        'train_dataset and test_dataset.'
						    )		,     )

						parser.add_argument(
						    '--igf_data_file'		,     type=lowercase_		,     default=lowercase_		,     help='A jbl file containing the context and information gain pairs to train secondary learner.'		,     )

						parser.add_argument(
						    '--output_dir'		,     default=lowercase_		,     type=lowercase_		,     required=lowercase_		,     help='The output directory where the final fine-tuned model is stored.'		,     )

						parser.add_argument(
						    '--tokenizer_name'		,     default=lowercase_		,     type=lowercase_		,     help='Pretrained tokenizer name or path if not the same as model_name'		,     )
						parser.add_argument('--seed'		,     type=lowercase_		,     default=lowercase_		,     help='A seed for reproducible training.'						)

						parser.add_argument(
						    '--context_len'		,     default=32		,     type=lowercase_		,     help=(
						        'The maximum total input sequence length after tokenization. Sequences longer '
						        'than this will be truncated, sequences shorter will be padded.'
						    )		,     )

						parser.add_argument(
						    '--size_objective_set'		,     default=100		,     type=lowercase_		,     help='number of articles that are long enough to be used as our objective set'		,     )
						parser.add_argument(
						    '--eval_freq'		,     default=100		,     type=lowercase_		,     help='secondary model evaluation is triggered at eval_freq'						)

						parser.add_argument('--max_steps'		,     default=1000		,     type=lowercase_		,     help='To calculate training epochs'						)

						parser.add_argument(
						    '--secondary_learner_batch_size'		,     default=128		,     type=lowercase_		,     help='batch size of training data for secondary learner'		,     )

						parser.add_argument(
						    '--batch_size'		,     default=16		,     type=lowercase_		,     help='batch size of training data of language model(gpt2) '						)

						parser.add_argument(
						    '--eval_interval'		,     default=10		,     type=lowercase_		,     help=(
						        'decay the selectivity of our secondary learner filter from'
						        '1 standard deviation above average to 1 below average after 10 batches'
						    )		,     )

						parser.add_argument(
						    '--number'		,     default=100		,     type=lowercase_		,     help='The number of examples split to be used as objective_set/test_data'						)

						parser.add_argument(
						    '--min_len'		,     default=1026		,     type=lowercase_		,     help='The minimum length of the article to be used as objective set'						)

						parser.add_argument(
						    '--secondary_learner_max_epochs'		,     default=15		,     type=lowercase_		,     help='number of epochs to train secondary learner'						)

						parser.add_argument('--trim'		,     default=lowercase_		,     type=lowercase_		,     help='truncate the example if it exceeds context length'						)

						parser.add_argument(
						    '--threshold'		,     default=1.0		,     type=lowercase_		,     help=(
						        'The threshold value used by secondary learner to filter the train_data and allow only'
						        ' informative data as input to the model'
						    )		,     )

						parser.add_argument('--finetuned_model_name'		,     default='gpt2_finetuned.pt'		,     type=lowercase_		,     help='finetuned_model_name'						)

						parser.add_argument(
						    '--recopy_model'		,     default=lowercase_		,     type=lowercase_		,     help='Reset the model to the original pretrained GPT-2 weights after each iteration'		,     )

						# function calls
						# Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner
						generate_n_pairs(
						    context_len=32		,     max_steps=10		,     size_objective_set=100		,     min_len=1026		,     trim=lowercase_		,     data_file='data/tokenized_stories_train_wikitext103.jbl'		,     igf_data_file='igf_context_pairs.jbl'		,     )

						# Load train data for secondary learner
						UpperCAmelCase					=			joblib.load('data/IGF_values.jbl'						)

						# Train secondary learner
						UpperCAmelCase					=			training_secondary_learner(
						    lowercase_		,     secondary_learner_max_epochs=15		,     secondary_learner_batch_size=128		,     eval_freq=100		,     igf_model_path='igf_model.pt'		,     )

						# load pretrained gpt2 model
						UpperCAmelCase					=			GPTaLMHeadModel.from_pretrained('gpt2'						)
						set_seed(42						)

						# Generate train and test data to train and evaluate gpt2 model
						UpperCAmelCase				,   UpperCAmelCase					=			generate_datasets(
						    context_len=32		,     file='data/tokenized_stories_train_wikitext103.jbl'		,     number=100		,     min_len=1026		,     trim=lowercase_						)

						# fine-tuning of the gpt2 model using igf (Information Gain Filtration)
						finetune(
						    lowercase_		,     lowercase_		,     lowercase_		,     context_len=32		,     max_steps=1000		,     batch_size=16		,     threshold=1.0		,     recopy_model=lowercase_		,     secondary_learner=lowercase_		,     eval_interval=10		,     finetuned_model_name='gpt2_finetuned.pt'		,     )


if __name__ == "__main__":
				main()










"""simple docstring"""


import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    DebertaVaForMaskedLM,
				    DebertaVaForMultipleChoice,
				    DebertaVaForQuestionAnswering,
				    DebertaVaForSequenceClassification,
				    DebertaVaForTokenClassification,
				    DebertaVaModel,
				)
				from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						def __init__( self			:Any       ,  lowercase_			:Optional[Any]       ,  lowercase_			:int=13       ,  lowercase_			:Optional[Any]=7       ,  lowercase_			:List[str]=True       ,  lowercase_			:Dict=True       ,  lowercase_			:str=True       ,  lowercase_			:Optional[Any]=True       ,  lowercase_			:Dict=99       ,  lowercase_			:int=32       ,  lowercase_			:str=5       ,  lowercase_			:Dict=4       ,  lowercase_			:Tuple=37       ,  lowercase_			:Dict="gelu"       ,  lowercase_			:List[str]=0.1       ,  lowercase_			:int=0.1       ,  lowercase_			:Any=5_12       ,  lowercase_			:Optional[Any]=16       ,  lowercase_			:Optional[int]=2       ,  lowercase_			:Union[str, Any]=0.02       ,  lowercase_			:Dict=False       ,  lowercase_			:Tuple=True       ,  lowercase_			:Optional[Any]="None"       ,  lowercase_			:int=3       ,  lowercase_			:Tuple=4       ,  lowercase_			:Optional[int]=None       ,  )    ->      Tuple:
												UpperCAmelCase					=			parent
												UpperCAmelCase					=			batch_size
												UpperCAmelCase					=			seq_length
												UpperCAmelCase					=			is_training
												UpperCAmelCase					=			use_input_mask
												UpperCAmelCase					=			use_token_type_ids
												UpperCAmelCase					=			use_labels
												UpperCAmelCase					=			vocab_size
												UpperCAmelCase					=			hidden_size
												UpperCAmelCase					=			num_hidden_layers
												UpperCAmelCase					=			num_attention_heads
												UpperCAmelCase					=			intermediate_size
												UpperCAmelCase					=			hidden_act
												UpperCAmelCase					=			hidden_dropout_prob
												UpperCAmelCase					=			attention_probs_dropout_prob
												UpperCAmelCase					=			max_position_embeddings
												UpperCAmelCase					=			type_vocab_size
												UpperCAmelCase					=			type_sequence_label_size
												UpperCAmelCase					=			initializer_range
												UpperCAmelCase					=			num_labels
												UpperCAmelCase					=			num_choices
												UpperCAmelCase					=			relative_attention
												UpperCAmelCase					=			position_biased_input
												UpperCAmelCase					=			pos_att_type
												UpperCAmelCase					=			scope






						def      UpperCAmelCase__					( self			:Any )    ->      Tuple:
												UpperCAmelCase					=			ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size )

												UpperCAmelCase					=			None
												if self.use_input_mask:
																		UpperCAmelCase					=			ids_tensor([self.batch_size, self.seq_length]       ,  vocab_size=2 )

												UpperCAmelCase					=			None
												if self.use_token_type_ids:
																		UpperCAmelCase					=			ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size )

												UpperCAmelCase					=			None
												UpperCAmelCase					=			None
												UpperCAmelCase					=			None
												if self.use_labels:
																		UpperCAmelCase					=			ids_tensor([self.batch_size]       ,  self.type_sequence_label_size )
																		UpperCAmelCase					=			ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels )
																		UpperCAmelCase					=			ids_tensor([self.batch_size]       ,  self.num_choices )

												UpperCAmelCase					=			self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def      UpperCAmelCase__					( self			:Tuple )    ->      Tuple:
												return DebertaVaConfig(
												    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  initializer_range=self.initializer_range       ,  relative_attention=self.relative_attention       ,  position_biased_input=self.position_biased_input       ,  pos_att_type=self.pos_att_type       ,  )






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:str )    ->      List[str]:
												self.parent.assertListEqual(list(result.loss.size() )       ,  [] )






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:str       ,  lowercase_			:Tuple       ,  lowercase_			:str       ,  lowercase_			:int       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:List[str]       ,  lowercase_			:Optional[int] )    ->      Optional[int]:
												UpperCAmelCase					=			DebertaVaModel(config=lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			model(lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_ )[0]
												UpperCAmelCase					=			model(lowercase_       ,  token_type_ids=lowercase_ )[0]
												UpperCAmelCase					=			model(lowercase_ )[0]

												self.parent.assertListEqual(list(sequence_output.size() )       ,  [self.batch_size, self.seq_length, self.hidden_size] )






						def      UpperCAmelCase__					( self			:Optional[Any]       ,  lowercase_			:Dict       ,  lowercase_			:List[str]       ,  lowercase_			:Any       ,  lowercase_			:List[str]       ,  lowercase_			:Tuple       ,  lowercase_			:List[Any]       ,  lowercase_			:int )    ->      Any:
												UpperCAmelCase					=			DebertaVaForMaskedLM(config=lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			model(lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_       ,  labels=lowercase_ )

												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size) )






						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  lowercase_			:Any       ,  lowercase_			:List[Any]       ,  lowercase_			:Any       ,  lowercase_			:Dict       ,  lowercase_			:str       ,  lowercase_			:List[Any]       ,  lowercase_			:Dict )    ->      Union[str, Any]:
												UpperCAmelCase					=			self.num_labels
												UpperCAmelCase					=			DebertaVaForSequenceClassification(lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			model(lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_       ,  labels=lowercase_ )
												self.parent.assertListEqual(list(result.logits.size() )       ,  [self.batch_size, self.num_labels] )
												self.check_loss_output(lowercase_ )






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Dict       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Any       ,  lowercase_			:Optional[int]       ,  lowercase_			:Optional[Any]       ,  lowercase_			:Any )    ->      List[Any]:
												UpperCAmelCase					=			self.num_labels
												UpperCAmelCase					=			DebertaVaForTokenClassification(config=lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			model(lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_       ,  labels=lowercase_ )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels) )






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:Any       ,  lowercase_			:List[Any]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Dict       ,  lowercase_			:List[Any]       ,  lowercase_			:Optional[int] )    ->      Dict:
												UpperCAmelCase					=			DebertaVaForQuestionAnswering(config=lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			model(
												    lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_       ,  start_positions=lowercase_       ,  end_positions=lowercase_       ,  )
												self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length) )
												self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length) )






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:str       ,  lowercase_			:List[Any]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:List[Any]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Any       ,  lowercase_			:Any )    ->      List[Any]:
												UpperCAmelCase					=			DebertaVaForMultipleChoice(config=lowercase_ )
												model.to(lowercase_ )
												model.eval()
												UpperCAmelCase					=			input_ids.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												UpperCAmelCase					=			token_type_ids.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												UpperCAmelCase					=			input_mask.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												UpperCAmelCase					=			model(
												    lowercase_       ,  attention_mask=lowercase_       ,  token_type_ids=lowercase_       ,  labels=lowercase_       ,  )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_choices) )






						def      UpperCAmelCase__					( self			:Dict )    ->      int:
												UpperCAmelCase					=			self.prepare_config_and_inputs()
												(
												    (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   (
												    UpperCAmelCase
												)				,   
												)					=			config_and_inputs
												UpperCAmelCase					=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
												return config, inputs_dict







@require_torch
class 				A_      (					SCREAMING_SNAKE_CASE_     ,       SCREAMING_SNAKE_CASE_     ,       unittest.TestCase    ):



						"""simple docstring"""
						__UpperCamelCase		    =							(
						    (
						        DebertaVaModel,
						        DebertaVaForMaskedLM,
						        DebertaVaForSequenceClassification,
						        DebertaVaForTokenClassification,
						        DebertaVaForQuestionAnswering,
						        DebertaVaForMultipleChoice,
						    )
						    if is_torch_available()
						    else ()
						)
						__UpperCamelCase		    =							(
						    {
						        """feature-extraction""": DebertaVaModel,
						        """fill-mask""": DebertaVaForMaskedLM,
						        """question-answering""": DebertaVaForQuestionAnswering,
						        """text-classification""": DebertaVaForSequenceClassification,
						        """token-classification""": DebertaVaForTokenClassification,
						        """zero-shot""": DebertaVaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)

						__UpperCamelCase		    =							True
						__UpperCamelCase		    =							False
						__UpperCamelCase		    =							False
						__UpperCamelCase		    =							False
						__UpperCamelCase		    =							False






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Optional[int]:
												UpperCAmelCase					=			DebertaVaModelTester(self )
												UpperCAmelCase					=			ConfigTester(self       ,  config_class=lowercase_       ,  hidden_size=37 )






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Optional[Any]:
												self.config_tester.run_common_tests()






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Tuple:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_model(*lowercase_ )






						def      UpperCAmelCase__					( self			:Any )    ->      List[str]:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_for_sequence_classification(*lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      List[Any]:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_for_masked_lm(*lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Optional[int]:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_for_question_answering(*lowercase_ )






						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      Tuple:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_for_token_classification(*lowercase_ )






						def      UpperCAmelCase__					( self			:Any )    ->      Union[str, Any]:
												UpperCAmelCase					=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_deberta_for_multiple_choice(*lowercase_ )






						@slow
						def      UpperCAmelCase__					( self			:Any )    ->      Optional[int]:
												for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		UpperCAmelCase					=			DebertaVaModel.from_pretrained(lowercase_ )
																		self.assertIsNotNone(lowercase_ )







@require_torch
@require_sentencepiece
@require_tokenizers
class 				A_      (					unittest.TestCase    ):



						"""simple docstring"""






						@unittest.skip(reason='Model not available yet' )
						def      UpperCAmelCase__					( self			:str )    ->      Tuple:
												pass






						@slow
						def      UpperCAmelCase__					( self			:List[Any] )    ->      Any:
												UpperCAmelCase					=			DebertaVaModel.from_pretrained('microsoft/deberta-v2-xlarge' )

												UpperCAmelCase					=			torch.tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]] )
												UpperCAmelCase					=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
												with torch.no_grad():
																		UpperCAmelCase					=			model(lowercase_       ,  attention_mask=lowercase_ )[0]
												# compare the actual values for a slice.
												UpperCAmelCase					=			torch.tensor(
												    [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]] )
												self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,  lowercase_       ,  atol=1E-4 )       ,  f"""{output[:, 1:4, 1:4]}""" )





