


import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
       lowercase_	   =     argparse.ArgumentParser(
           description=(
               'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
               ' Distillation'
           )
       )
       parser.add_argument('--model_type', default='bert', choices=['bert'])
       parser.add_argument('--model_name', default='bert-base-uncased', type=str)
       parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
       parser.add_argument('--vocab_transform', action='store_true')
       lowercase_	   =     parser.parse_args()

       if args.model_type == "bert":
              lowercase_	   =     BertForMaskedLM.from_pretrained(args.model_name)
              lowercase_	   =     'bert'
       else:
              raise ValueError('args.model_type should be "bert".')

       lowercase_	   =     model.state_dict()
       lowercase_	   =     {}

       for w in ["word_embeddings", "position_embeddings"]:
              lowercase_	   =     state_dict[F"""{prefix}.embeddings.{w}.weight"""]
       for w in ["weight", "bias"]:
              lowercase_	   =     state_dict[F"""{prefix}.embeddings.LayerNorm.{w}"""]

       lowercase_	   =     0
       for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
              for w in ["weight", "bias"]:
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
                     ]
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
                     ]
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
                     ]

                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
                     ]
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
                     ]

                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
                     ]
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
                     ]
                     lowercase_	   =     state_dict[
                         F"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
                     ]
              std_idx += 1

       lowercase_	   =     state_dict['cls.predictions.decoder.weight']
       lowercase_	   =     state_dict['cls.predictions.bias']
       if args.vocab_transform:
              for w in ["weight", "bias"]:
                     lowercase_	   =     state_dict[F"""cls.predictions.transform.dense.{w}"""]
                     lowercase_	   =     state_dict[F"""cls.predictions.transform.LayerNorm.{w}"""]

       print(F"""N layers selected for distillation: {std_idx}""")
       print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

       print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
       torch.save(compressed_sd, args.dump_checkpoint)





import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowercase_	   =     5_0_0_0_0_0

lowercase_   ,lowercase_	   =     os.path.split(__file__)
lowercase_	   =     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))
@get_duration
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Optional[int]		 =						dataset.map(**SCREAMING_SNAKE_CASE__						)
@get_duration
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Tuple		 =						dataset.filter(**SCREAMING_SNAKE_CASE__						)
def 				UpperCamelCase__					(  ):
 __lowerCamelCase		:							str		 =						{'num examples': SPEED_TEST_N_EXAMPLES}
 with tempfile.TemporaryDirectory() as tmp_dir:
  __lowerCamelCase		:							Any		 =						datasets.Features({'text': datasets.Value('string'						), 'numbers': datasets.Value('float32'						)}						)
  __lowerCamelCase		:							Any		 =						generate_example_dataset(
      os.path.join(SCREAMING_SNAKE_CASE__      ,	'dataset.arrow'						)      ,	SCREAMING_SNAKE_CASE__      ,	num_examples=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							List[Any]		 =						transformers.AutoTokenizer.from_pretrained('bert-base-cased'      ,	use_fast=SCREAMING_SNAKE_CASE__						)

  def tokenize(SCREAMING_SNAKE_CASE__						):
   return tokenizer(examples['text']						)

  __lowerCamelCase		:							str		 =						map(SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							int		 =						map(SCREAMING_SNAKE_CASE__      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[int]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='numpy'						):
   __lowerCamelCase		:							Union[str, Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='pandas'						):
   __lowerCamelCase		:							Any		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='torch'      ,	columns='numbers'						):
   __lowerCamelCase		:							List[str]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='tensorflow'      ,	columns='numbers'						):
   __lowerCamelCase		:							List[Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=SCREAMING_SNAKE_CASE__      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[int]		 =						filter(SCREAMING_SNAKE_CASE__						)

  # Activate later when tokenizer support batched inputs
  # with dataset.formatted_as(type='numpy'):
  #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

 with open(SCREAMING_SNAKE_CASE__      ,	'wb'						) as f:
  f.write(json.dumps(SCREAMING_SNAKE_CASE__						).encode('utf-8'						)						)


if __name__ == "__main__":  # useful to run the profiler
       benchmark_map_filter()

