





import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


_snake_case           =			{
    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
    "gpt2": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}


def _A       (   __magic_name__	):
   assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
   assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
   if args.mlm:
      assert os.path.isfile(args.token_counts	)
      assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
   else:
      assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

   assert args.teacher_type == args.student_type or (
       args.student_type == "distilbert" and args.teacher_type == "bert"
   )
   assert os.path.isfile(args.student_config	)
   if args.student_pretrained_weights is not None:
      assert os.path.isfile(args.student_pretrained_weights	)

   if args.freeze_token_type_embds:
      assert args.student_type in ["roberta"]

   assert args.alpha_ce >= 0.0
   assert args.alpha_mlm >= 0.0
   assert args.alpha_clm >= 0.0
   assert args.alpha_mse >= 0.0
   assert args.alpha_cos >= 0.0
   assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0


def _A       (   __magic_name__  , __magic_name__	):
   if args.student_type == "roberta":
      lowercase__			       =  False
   elif args.student_type == "gpt2":
      lowercase__			       =  False


def _A       (   __magic_name__  , __magic_name__	):
   if args.student_type == "roberta":
      lowercase__			       =  False


def _A       (   ):
   lowercase__			       =  argparse.ArgumentParser(description="Training"	)
   parser.add_argument("--force"  , action="store_true"  , help="Overwrite dump_path if it already exists."	)

   parser.add_argument(
       "--dump_path"  , type=__magic_name__  , required=__magic_name__  , help="The output directory (log, checkpoints, parameters, etc.)"	)
   parser.add_argument(
       "--data_file"  , type=__magic_name__  , required=__magic_name__  , help="The binarized file (tokenized + tokens_to_ids) and grouped by sequence."  , )

   parser.add_argument(
       "--student_type"  , type=__magic_name__  , choices=["distilbert", "roberta", "gpt2"]  , required=__magic_name__  , help="The student type (DistilBERT, RoBERTa)."  , )
   parser.add_argument("--student_config"  , type=__magic_name__  , required=__magic_name__  , help="Path to the student configuration."	)
   parser.add_argument(
       "--student_pretrained_weights"  , default=__magic_name__  , type=__magic_name__  , help="Load student initialization checkpoint."	)

   parser.add_argument(
       "--teacher_type"  , choices=["bert", "roberta", "gpt2"]  , required=__magic_name__  , help="Teacher type (BERT, RoBERTa)."	)
   parser.add_argument("--teacher_name"  , type=__magic_name__  , required=__magic_name__  , help="The teacher model."	)

   parser.add_argument("--temperature"  , default=2.0  , type=__magic_name__  , help="Temperature for the softmax temperature."	)
   parser.add_argument(
       "--alpha_ce"  , default=0.5  , type=__magic_name__  , help="Linear weight for the distillation loss. Must be >=0."	)
   parser.add_argument(
       "--alpha_mlm"  , default=0.0  , type=__magic_name__  , help="Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."  , )
   parser.add_argument("--alpha_clm"  , default=0.5  , type=__magic_name__  , help="Linear weight for the CLM loss. Must be >=0."	)
   parser.add_argument("--alpha_mse"  , default=0.0  , type=__magic_name__  , help="Linear weight of the MSE loss. Must be >=0."	)
   parser.add_argument(
       "--alpha_cos"  , default=0.0  , type=__magic_name__  , help="Linear weight of the cosine embedding loss. Must be >=0."	)

   parser.add_argument(
       "--mlm"  , action="store_true"  , help="The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM."	)
   parser.add_argument(
       "--mlm_mask_prop"  , default=0.15  , type=__magic_name__  , help="Proportion of tokens for which we need to make a prediction."  , )
   parser.add_argument("--word_mask"  , default=0.8  , type=__magic_name__  , help="Proportion of tokens to mask out."	)
   parser.add_argument("--word_keep"  , default=0.1  , type=__magic_name__  , help="Proportion of tokens to keep."	)
   parser.add_argument("--word_rand"  , default=0.1  , type=__magic_name__  , help="Proportion of tokens to randomly replace."	)
   parser.add_argument(
       "--mlm_smoothing"  , default=0.7  , type=__magic_name__  , help="Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."  , )
   parser.add_argument("--token_counts"  , type=__magic_name__  , help="The token counts in the data_file for MLM."	)

   parser.add_argument(
       "--restrict_ce_to_mask"  , action="store_true"  , help="If true, compute the distillation loss only the [MLM] prediction distribution."  , )
   parser.add_argument(
       "--freeze_pos_embs"  , action="store_true"  , help="Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."  , )
   parser.add_argument(
       "--freeze_token_type_embds"  , action="store_true"  , help="Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."  , )

   parser.add_argument("--n_epoch"  , type=__magic_name__  , default=3  , help="Number of pass on the whole dataset."	)
   parser.add_argument("--batch_size"  , type=__magic_name__  , default=5  , help="Batch size (for each process)."	)
   parser.add_argument(
       "--group_by_size"  , action="store_false"  , help="If true, group sequences that have similar length into the same batch. Default is true."  , )

   parser.add_argument(
       "--gradient_accumulation_steps"  , type=__magic_name__  , default=50  , help="Gradient accumulation for larger training batches."  , )
   parser.add_argument("--warmup_prop"  , default=0.05  , type=__magic_name__  , help="Linear warmup proportion."	)
   parser.add_argument("--weight_decay"  , default=0.0  , type=__magic_name__  , help="Weight decay if we apply some."	)
   parser.add_argument("--learning_rate"  , default=5e-4  , type=__magic_name__  , help="The initial learning rate for Adam."	)
   parser.add_argument("--adam_epsilon"  , default=1e-6  , type=__magic_name__  , help="Epsilon for Adam optimizer."	)
   parser.add_argument("--max_grad_norm"  , default=5.0  , type=__magic_name__  , help="Max gradient norm."	)
   parser.add_argument("--initializer_range"  , default=0.02  , type=__magic_name__  , help="Random initialization range."	)

   parser.add_argument(
       "--fp16"  , action="store_true"  , help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"  , )
   parser.add_argument(
       "--fp16_opt_level"  , type=__magic_name__  , default="O1"  , help=(
           "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
           "See details at https://nvidia.github.io/apex/amp.html"
       )  , )
   parser.add_argument("--n_gpu"  , type=__magic_name__  , default=1  , help="Number of GPUs in the node."	)
   parser.add_argument("--local_rank"  , type=__magic_name__  , default=-1  , help="Distributed training - Local rank"	)
   parser.add_argument("--seed"  , type=__magic_name__  , default=56  , help="Random seed"	)

   parser.add_argument("--log_interval"  , type=__magic_name__  , default=500  , help="Tensorboard logging interval."	)
   parser.add_argument("--checkpoint_interval"  , type=__magic_name__  , default=4000  , help="Checkpoint interval."	)
   lowercase__			       =  parser.parse_args()
   sanity_checks(__magic_name__	)

   # ARGS #
   init_gpu_params(__magic_name__	)
   set_seed(__magic_name__	)
   if args.is_master:
      if os.path.exists(args.dump_path	):
         if not args.force:
            raise ValueError(
                f'''Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'''
                " itUse `--force` if you want to overwrite it"	)
         else:
            shutil.rmtree(args.dump_path	)

      if not os.path.exists(args.dump_path	):
         os.makedirs(args.dump_path	)
      logger.info(f'''Experiment will be dumped and logged in {args.dump_path}'''	)

      # SAVE PARAMS #
      logger.info(f'''Param: {args}'''	)
      with open(os.path.join(args.dump_path  , "parameters.json"	)  , "w"	) as f:
         json.dump(vars(__magic_name__	)  , __magic_name__  , indent=4	)
      git_log(args.dump_path	)

   lowercase__			       =  MODEL_CLASSES[args.student_type]
   lowercase__			       =  MODEL_CLASSES[args.teacher_type]

   # TOKENIZER #
   lowercase__			       =  teacher_tokenizer_class.from_pretrained(args.teacher_name	)
   lowercase__			       =  {}
   for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
      lowercase__			       =  tokenizer.all_special_tokens.index(__magic_name__	)
      lowercase__			       =  tokenizer.all_special_ids[idx]
   logger.info(f'''Special tokens {special_tok_ids}'''	)
   lowercase__			       =  special_tok_ids
   lowercase__			       =  tokenizer.max_model_input_sizes[args.teacher_name]

   # DATA LOADER #
   logger.info(f'''Loading data from {args.data_file}'''	)
   with open(args.data_file  , "rb"	) as fp:
      lowercase__			       =  pickle.load(__magic_name__	)

   if args.mlm:
      logger.info(f'''Loading token counts from {args.token_counts} (already pre-computed)'''	)
      with open(args.token_counts  , "rb"	) as fp:
         lowercase__			       =  pickle.load(__magic_name__	)

      lowercase__			       =  np.maximum(__magic_name__  , 1	) ** -args.mlm_smoothing
      for idx in special_tok_ids.values():
         lowercase__			       =  0.0  # do not predict special tokens
      lowercase__			       =  torch.from_numpy(__magic_name__	)
   else:
      lowercase__			       =  None

   lowercase__			       =  LmSeqsDataset(params=__magic_name__  , data=__magic_name__	)
   logger.info("Data loader created."	)

   # STUDENT #
   logger.info(f'''Loading student config from {args.student_config}'''	)
   lowercase__			       =  student_config_class.from_pretrained(args.student_config	)
   lowercase__			       =  True

   if args.student_pretrained_weights is not None:
      logger.info(f'''Loading pretrained weights from {args.student_pretrained_weights}'''	)
      lowercase__			       =  student_model_class.from_pretrained(args.student_pretrained_weights  , config=__magic_name__	)
   else:
      lowercase__			       =  student_model_class(__magic_name__	)

   if args.n_gpu > 0:
      student.to(f'''cuda:{args.local_rank}'''	)
   logger.info("Student loaded."	)

   # TEACHER #
   lowercase__			       =  teacher_model_class.from_pretrained(args.teacher_name  , output_hidden_states=__magic_name__	)
   if args.n_gpu > 0:
      teacher.to(f'''cuda:{args.local_rank}'''	)
   logger.info(f'''Teacher loaded from {args.teacher_name}.'''	)

   # FREEZING #
   if args.freeze_pos_embs:
      freeze_pos_embeddings(__magic_name__  , __magic_name__	)
   if args.freeze_token_type_embds:
      freeze_token_type_embeddings(__magic_name__  , __magic_name__	)

   # SANITY CHECKS #
   assert student.config.vocab_size == teacher.config.vocab_size
   assert student.config.hidden_size == teacher.config.hidden_size
   assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
   if args.mlm:
      assert token_probs.size(0	) == stu_architecture_config.vocab_size

   # DISTILLER #
   torch.cuda.empty_cache()
   lowercase__			       =  Distiller(
       params=__magic_name__  , dataset=__magic_name__  , token_probs=__magic_name__  , student=__magic_name__  , teacher=__magic_name__	)
   distiller.train()
   logger.info("Let's go get some drinks."	)


if __name__ == "__main__":
  main()













from abc import ABC, abstractmethod
from typing import Optional, Union

from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
from ..utils.typing import NestedDataStructureLike, PathLike





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :str   ,							_lowercase  :Optional[NestedDataStructureLike[PathLike]] = None   ,							_lowercase  :Optional[NamedSplit] = None   ,							_lowercase  :Optional[Features] = None   ,							_lowercase  :str = None   ,							_lowercase  :bool = False   ,							_lowercase  :bool = False   ,							_lowercase  :Optional[int] = None   ,							**_lowercase  :Tuple   ,							):






     '''simple docstring'''





     lowercase__			       =  path_or_paths
     lowercase__			       =  split if split or isinstance(_lowercase   ,							_lowercase ) else "train"
     lowercase__			       =  features
     lowercase__			       =  cache_dir
     lowercase__			       =  keep_in_memory
     lowercase__			       =  streaming
     lowercase__			       =  num_proc
     lowercase__			       =  kwargs







  @abstractmethod
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     pass





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :List[Any]   ,							_lowercase  :Optional[Features] = None   ,							_lowercase  :str = None   ,							_lowercase  :bool = False   ,							_lowercase  :bool = False   ,							_lowercase  :Optional[int] = None   ,							**_lowercase  :Optional[int]   ,							):






     '''simple docstring'''





     lowercase__			       =  features
     lowercase__			       =  cache_dir
     lowercase__			       =  keep_in_memory
     lowercase__			       =  streaming
     lowercase__			       =  num_proc
     lowercase__			       =  kwargs







  @abstractmethod
  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     pass






