





'''simple docstring'''




import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      DistilBertForMaskedLM,
      DistilBertForMultipleChoice,
      DistilBertForQuestionAnswering,
      DistilBertForSequenceClassification,
      DistilBertForTokenClassification,
      DistilBertModel,
  )


class 			SCREAMING_SNAKE_CASE   (a__							):


     def __init__(						self						,     _UpperCAmelCase						,     _UpperCAmelCase=13						,     _UpperCAmelCase=7						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=False						,     _UpperCAmelCase=True						,     _UpperCAmelCase=99						,     _UpperCAmelCase=32						,     _UpperCAmelCase=5						,     _UpperCAmelCase=4						,     _UpperCAmelCase=37						,     _UpperCAmelCase="gelu"						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=512						,     _UpperCAmelCase=16						,     _UpperCAmelCase=2						,     _UpperCAmelCase=0.02						,     _UpperCAmelCase=3						,     _UpperCAmelCase=4						,     _UpperCAmelCase=None						,     ):

          '''simple docstring'''






          __A						:			Optional[Any]								=	parent
          __A						:			str								=	batch_size
          __A						:			Optional[Any]								=	seq_length
          __A						:			int								=	is_training
          __A						:			List[str]								=	use_input_mask
          __A						:			Any								=	use_token_type_ids
          __A						:			List[Any]								=	use_labels
          __A						:			int								=	vocab_size
          __A						:			Tuple								=	hidden_size
          __A						:			List[str]								=	num_hidden_layers
          __A						:			str								=	num_attention_heads
          __A						:			int								=	intermediate_size
          __A						:			int								=	hidden_act
          __A						:			Union[str, Any]								=	hidden_dropout_prob
          __A						:			Optional[int]								=	attention_probs_dropout_prob
          __A						:			List[Any]								=	max_position_embeddings
          __A						:			List[Any]								=	type_vocab_size
          __A						:			int								=	type_sequence_label_size
          __A						:			Tuple								=	initializer_range
          __A						:			List[Any]								=	num_labels
          __A						:			List[str]								=	num_choices
          __A						:			int								=	scope


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			str								=	ids_tensor([self.batch_size, self.seq_length]						,     self.vocab_size)

          __A						:			int								=	None
          if self.use_input_mask:
               __A						:			Any								=	random_attention_mask([self.batch_size, self.seq_length])

          __A						:			Dict								=	None
          __A						:			Dict								=	None
          __A						:			Dict								=	None
          if self.use_labels:
               __A						:			List[str]								=	ids_tensor([self.batch_size]						,     self.type_sequence_label_size)
               __A						:			Tuple								=	ids_tensor([self.batch_size, self.seq_length]						,     self.num_labels)
               __A						:			Any								=	ids_tensor([self.batch_size]						,     self.num_choices)

          __A						:			Tuple								=	self.get_config()

          return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return DistilBertConfig(
              vocab_size=self.vocab_size						,     dim=self.hidden_size						,     n_layers=self.num_hidden_layers						,     n_heads=self.num_attention_heads						,     hidden_dim=self.intermediate_size						,     hidden_act=self.hidden_act						,     dropout=self.hidden_dropout_prob						,     attention_dropout=self.attention_probs_dropout_prob						,     max_position_embeddings=self.max_position_embeddings						,     initializer_range=self.initializer_range						,     )


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Optional[Any]								=	DistilBertModel(config=_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()
          __A						:			Optional[int]								=	model(_UpperCAmelCase						,     _UpperCAmelCase)
          __A						:			Optional[int]								=	model(_UpperCAmelCase)
          self.parent.assertEqual(result.last_hidden_state.shape						,     (self.batch_size, self.seq_length, self.hidden_size))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Any								=	DistilBertForMaskedLM(config=_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()
          __A						:			List[str]								=	model(_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     labels=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.seq_length, self.vocab_size))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Dict								=	DistilBertForQuestionAnswering(config=_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()
          __A						:			List[Any]								=	model(
              _UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     start_positions=_UpperCAmelCase						,     end_positions=_UpperCAmelCase)
          self.parent.assertEqual(result.start_logits.shape						,     (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape						,     (self.batch_size, self.seq_length))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Tuple								=	self.num_labels
          __A						:			List[str]								=	DistilBertForSequenceClassification(_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()
          __A						:			Any								=	model(_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     labels=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Tuple								=	self.num_labels
          __A						:			Any								=	DistilBertForTokenClassification(config=_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()

          __A						:			str								=	model(_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     labels=_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.seq_length, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			List[str]								=	self.num_choices
          __A						:			Tuple								=	DistilBertForMultipleChoice(config=_UpperCAmelCase)
          model.to(_UpperCAmelCase)
          model.eval()
          __A						:			Any								=	input_ids.unsqueeze(1).expand(-1						,     self.num_choices						,     -1).contiguous()
          __A						:			str								=	input_mask.unsqueeze(1).expand(-1						,     self.num_choices						,     -1).contiguous()
          __A						:			Optional[int]								=	model(
              _UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     labels=_UpperCAmelCase						,     )
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.num_choices))


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	self.prepare_config_and_inputs()
          ((__A)			,(__A)			,(__A)			,(__A)			,(__A)			,(__A))						:			int								=	config_and_inputs
          __A						:			List[str]								=	{'input_ids': input_ids, 'attention_mask': input_mask}
          return config, inputs_dict




@require_torch
class 			SCREAMING_SNAKE_CASE   (a__ ,		a__ ,		unittest.TestCase							):
     lowerCAmelCase		      =	(
         (
             DistilBertModel,
             DistilBertForMaskedLM,
             DistilBertForMultipleChoice,
             DistilBertForQuestionAnswering,
             DistilBertForSequenceClassification,
             DistilBertForTokenClassification,
         )
         if is_torch_available()
         else None
     )
     lowerCAmelCase		      =	(
         {
             '''feature-extraction''': DistilBertModel,
             '''fill-mask''': DistilBertForMaskedLM,
             '''question-answering''': DistilBertForQuestionAnswering,
             '''text-classification''': DistilBertForSequenceClassification,
             '''token-classification''': DistilBertForTokenClassification,
             '''zero-shot''': DistilBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     lowerCAmelCase		      =	True
     lowerCAmelCase		      =	True
     lowerCAmelCase		      =	True
     lowerCAmelCase		      =	True


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	DistilBertModelTester(self)
          __A						:			List[Any]								=	ConfigTester(self						,     config_class=_UpperCAmelCase						,     dim=37)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.config_tester.run_common_tests()


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_model(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_for_masked_lm(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_for_question_answering(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_for_sequence_classification(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_for_token_classification(*_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_distilbert_for_multiple_choice(*_UpperCAmelCase)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               __A						:			int								=	DistilBertModel.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)


     @slow
     @require_torch_gpu
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A			,__A						:			Dict								=	self.model_tester.prepare_config_and_inputs_for_common()
          for model_class in self.all_model_classes:
               # BertForMultipleChoice behaves incorrectly in JIT environments.
               if model_class == DistilBertForMultipleChoice:
                    return

               __A						:			Dict								=	True
               __A						:			Optional[Any]								=	model_class(config=_UpperCAmelCase)

               __A						:			Any								=	self._prepare_for_class(_UpperCAmelCase						,     _UpperCAmelCase)
               __A						:			str								=	torch.jit.trace(
                   _UpperCAmelCase						,     (inputs_dict['input_ids'].to('cpu'), inputs_dict['attention_mask'].to('cpu')))

               with tempfile.TemporaryDirectory() as tmp:
                    torch.jit.save(_UpperCAmelCase						,     os.path.join(_UpperCAmelCase						,     'traced_model.pt'))
                    __A						:			List[Any]								=	torch.jit.load(os.path.join(_UpperCAmelCase						,     'traced_model.pt')						,     map_location=_UpperCAmelCase)
                    loaded(inputs_dict['input_ids'].to(_UpperCAmelCase)						,     inputs_dict['attention_mask'].to(_UpperCAmelCase))




@require_torch
class 			SCREAMING_SNAKE_CASE   (unittest.TestCase							):


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	DistilBertModel.from_pretrained('distilbert-base-uncased')
          __A						:			List[str]								=	torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
          __A						:			Optional[int]								=	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
          with torch.no_grad():
               __A						:			str								=	model(_UpperCAmelCase						,     attention_mask=_UpperCAmelCase)[0]
          __A						:			List[Any]								=	torch.Size((1, 11, 768))
          self.assertEqual(output.shape						,     _UpperCAmelCase)
          __A						:			List[str]								=	torch.tensor(
              [[[-0.1639, 0.3299, 0.1648], [-0.1746, 0.3289, 0.1710], [-0.1884, 0.3357, 0.1810]]])

          self.assertTrue(torch.allclose(output[:, 1:4, 1:4]						,     _UpperCAmelCase						,     atol=1e-4))






'''simple docstring'''




import argparse
from copy import deepcopy

import numpy as np
from datasets import ClassLabel, DatasetDict, load_dataset
from evaluate import load

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    set_seed,
)






def 	_lowerCAmelCase      (  )					->					Union[str, Any]:
     __A						:			Optional[int]								=	argparse.ArgumentParser()
     parser.add_argument('--model_ckpt'			,							type=__snake_case			,							default='microsoft/unixcoder-base-nine'      )
     parser.add_argument('--num_epochs'			,							type=__snake_case			,							default=5      )
     parser.add_argument('--batch_size'			,							type=__snake_case			,							default=6      )
     parser.add_argument('--gradient_accumulation_steps'			,							type=__snake_case			,							default=1      )
     parser.add_argument('--freeze'			,							type=__snake_case			,							default=__snake_case      )
     parser.add_argument('--learning_rate'			,							type=__snake_case			,							default=5e-4      )
     parser.add_argument('--seed'			,							type=__snake_case			,							default=0      )
     parser.add_argument('--lr_scheduler_type'			,							type=__snake_case			,							default='cosine'      )
     parser.add_argument('--num_warmup_steps'			,							type=__snake_case			,							default=10      )
     parser.add_argument('--weight_decay'			,							type=__snake_case			,							default=0.01      )
     parser.add_argument('--output_dir'			,							type=__snake_case			,							default='./results'      )
     return parser.parse_args()


lowercase__   :				Tuple      =   load('''accuracy''')






def 	_lowerCAmelCase      (  __snake_case  :			int      )					->					Any:
     __A			,__A						:			List[Any]								=	eval_pred
     __A						:			Dict								=	np.argmax(__snake_case			,							axis=1      )
     return metric.compute(predictions=__snake_case			,							references=__snake_case      )


class 			SCREAMING_SNAKE_CASE   (a__							):


     def __init__(						self						,     _UpperCAmelCase):

          '''simple docstring'''






          super().__init__()
          __A						:			int								=	trainer


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     **_UpperCAmelCase):

          '''simple docstring'''






          if control.should_evaluate:
               __A						:			str								=	deepcopy(_UpperCAmelCase)
               self._trainer.evaluate(eval_dataset=self._trainer.train_dataset						,     metric_key_prefix='train')
               return control_copy






def 	_lowerCAmelCase      (  )					->					str:
     __A						:			List[Any]								=	get_args()
     set_seed(args.seed      )

     __A						:			Union[str, Any]								=	load_dataset('codeparrot/codecomplex'			,							split='train'      )
     __A						:			Optional[int]								=	dataset.train_test_split(test_size=0.2      )
     __A						:			Union[str, Any]								=	train_test['test'].train_test_split(test_size=0.5      )
     __A						:			Optional[Any]								=	DatasetDict(
         {
             'train': train_test['train'],
             'test': test_validation['train'],
             'valid': test_validation['test'],
         }      )

     print('Loading tokenizer and model'      )
     __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(args.model_ckpt      )
     __A						:			Tuple								=	tokenizer.eos_token
     __A						:			Dict								=	AutoModelForSequenceClassification.from_pretrained(args.model_ckpt			,							num_labels=7      )
     __A						:			Optional[int]								=	model.config.eos_token_id

     if args.freeze:
          for param in model.roberta.parameters():
               __A						:			Optional[Any]								=	False

     __A						:			Dict								=	ClassLabel(num_classes=7			,							names=list(set(train_test_validation['train']['complexity']      )      )      )

     def tokenize(__snake_case  :			Optional[Any]      ):
          __A						:			Optional[Any]								=	tokenizer(example['src']			,							truncation=__snake_case			,							max_length=10_24      )
          __A						:			str								=	labels.straint(example['complexity']      )
          return {
              "input_ids": inputs["input_ids"],
              "attention_mask": inputs["attention_mask"],
              "label": label,
          }

     __A						:			str								=	train_test_validation.map(
         __snake_case			,							batched=__snake_case			,							remove_columns=train_test_validation['train'].column_names			,							)
     __A						:			str								=	DataCollatorWithPadding(tokenizer=__snake_case      )

     __A						:			str								=	TrainingArguments(
         output_dir=args.output_dir			,							learning_rate=args.learning_rate			,							lr_scheduler_type=args.lr_scheduler_type			,							evaluation_strategy='epoch'			,							save_strategy='epoch'			,							logging_strategy='epoch'			,							per_device_train_batch_size=args.batch_size			,							per_device_eval_batch_size=args.batch_size			,							num_train_epochs=args.num_epochs			,							gradient_accumulation_steps=args.gradient_accumulation_steps			,							weight_decay=0.01			,							metric_for_best_model='accuracy'			,							run_name='complexity-java'			,							report_to='wandb'			,							)

     __A						:			Tuple								=	Trainer(
         model=__snake_case			,							args=__snake_case			,							train_dataset=tokenized_datasets['train']			,							eval_dataset=tokenized_datasets['valid']			,							tokenizer=__snake_case			,							data_collator=__snake_case			,							compute_metrics=__snake_case			,							)

     print('Training...'      )
     trainer.add_callback(CustomCallback(__snake_case      )      )
     trainer.train()


if __name__ == "__main__":
  main()