






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowercase_	   =     1_6
lowercase_	   =     3_2
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__ = 16						):
 __lowerCamelCase		:							Tuple		 =						AutoTokenizer.from_pretrained('bert-base-cased'						)
 __lowerCamelCase		:							Optional[int]		 =						load_dataset('glue'      ,	'mrpc'						)

 def tokenize_function(SCREAMING_SNAKE_CASE__						):
  # max_length=None => use the model max length (it's actually the default)
  __lowerCamelCase		:							Optional[int]		 =						tokenizer(examples['sentence1']      ,	examples['sentence2']      ,	truncation=SCREAMING_SNAKE_CASE__      ,	max_length=SCREAMING_SNAKE_CASE__						)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  __lowerCamelCase		:							Any		 =						datasets.map(
      SCREAMING_SNAKE_CASE__      ,	batched=SCREAMING_SNAKE_CASE__      ,	remove_columns=['idx', 'sentence1', 'sentence2']      ,	)

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 __lowerCamelCase		:							List[Any]		 =						tokenized_datasets.rename_column('label'      ,	'labels'						)

 def collate_fn(SCREAMING_SNAKE_CASE__						):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  __lowerCamelCase		:							Dict		 =						128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   __lowerCamelCase		:							Tuple		 =						16
  elif accelerator.mixed_precision != "no":
   __lowerCamelCase		:							Any		 =						8
  else:
   __lowerCamelCase		:							Tuple		 =						None

  return tokenizer.pad(
      SCREAMING_SNAKE_CASE__      ,	padding='longest'      ,	max_length=SCREAMING_SNAKE_CASE__      ,	pad_to_multiple_of=SCREAMING_SNAKE_CASE__      ,	return_tensors='pt'      ,	)

 # Instantiate dataloaders.
 __lowerCamelCase		:							Optional[Any]		 =						DataLoader(
     tokenized_datasets['train']      ,	shuffle=SCREAMING_SNAKE_CASE__      ,	collate_fn=SCREAMING_SNAKE_CASE__      ,	batch_size=SCREAMING_SNAKE_CASE__						)
 __lowerCamelCase		:							List[str]		 =						DataLoader(
     tokenized_datasets['validation']      ,	shuffle=SCREAMING_SNAKE_CASE__      ,	collate_fn=SCREAMING_SNAKE_CASE__      ,	batch_size=SCREAMING_SNAKE_CASE__						)

 return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
       from accelerate.test_utils.training import mocked_dataloaders

       lowercase_	   =     mocked_dataloaders  # noqa: F811
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 # For testing only
 if os.environ.get('TESTING_MOCKED_DATALOADERS'      ,	SCREAMING_SNAKE_CASE__						) == "1":
  __lowerCamelCase		:							str		 =						2
 # New Code #
 __lowerCamelCase		:							List[str]		 =						int(args.gradient_accumulation_steps						)
 # Initialize accelerator
 __lowerCamelCase		:							str		 =						Accelerator(
     cpu=args.cpu      ,	mixed_precision=args.mixed_precision      ,	gradient_accumulation_steps=SCREAMING_SNAKE_CASE__						)
 if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
  raise NotImplementedError(
      'Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`'						)
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 __lowerCamelCase		:							Dict		 =						config['lr']
 __lowerCamelCase		:							Union[str, Any]		 =						int(config['num_epochs']						)
 __lowerCamelCase		:							int		 =						int(config['seed']						)
 __lowerCamelCase		:							str		 =						int(config['batch_size']						)

 __lowerCamelCase		:							str		 =						evaluate.load('glue'      ,	'mrpc'						)

 set_seed(SCREAMING_SNAKE_CASE__						)
 __lowerCamelCase		:							Optional[int]		 =						get_dataloaders(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)
 # Instantiate the model (we build the model here so that the seed also control new weights initialization)
 __lowerCamelCase		:							str		 =						AutoModelForSequenceClassification.from_pretrained('bert-base-cased'      ,	return_dict=SCREAMING_SNAKE_CASE__						)

 # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
 # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
 # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
 __lowerCamelCase		:							Optional[int]		 =						model.to(accelerator.device						)

 # Instantiate optimizer
 __lowerCamelCase		:							Tuple		 =						AdamW(params=model.parameters()      ,	lr=SCREAMING_SNAKE_CASE__						)

 # Instantiate scheduler
 __lowerCamelCase		:							Union[str, Any]		 =						get_linear_schedule_with_warmup(
     optimizer=SCREAMING_SNAKE_CASE__      ,	num_warmup_steps=100      ,	num_training_steps=(len(SCREAMING_SNAKE_CASE__						) * num_epochs)      ,	)

 # Prepare everything
 # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
 # prepare method.
 __lowerCamelCase		:							str		 =						accelerator.prepare(
     SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)

 # Now we train the model
 for epoch in range(SCREAMING_SNAKE_CASE__						):
  model.train()
  for step, batch in enumerate(SCREAMING_SNAKE_CASE__						):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device						)
   # New code #
   # We use the new `accumulate` context manager to perform gradient accumulation
   # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
   with accelerator.accumulate(SCREAMING_SNAKE_CASE__						):
    __lowerCamelCase		:							int		 =						model(**SCREAMING_SNAKE_CASE__						)
    __lowerCamelCase		:							Tuple		 =						output.loss
    accelerator.backward(SCREAMING_SNAKE_CASE__						)
    optimizer.step()
    lr_scheduler.step()
    optimizer.zero_grad()

  model.eval()
  for step, batch in enumerate(SCREAMING_SNAKE_CASE__						):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device						)
   with torch.no_grad():
    __lowerCamelCase		:							Optional[Any]		 =						model(**SCREAMING_SNAKE_CASE__						)
   __lowerCamelCase		:							Any		 =						outputs.logits.argmax(dim=-1						)
   __lowerCamelCase		:							str		 =						accelerator.gather_for_metrics((predictions, batch['labels'])						)
   metric.add_batch(
       predictions=SCREAMING_SNAKE_CASE__      ,	references=SCREAMING_SNAKE_CASE__      ,	)

  __lowerCamelCase		:							Union[str, Any]		 =						metric.compute()
  # Use accelerator.print to print only on the main process.
  accelerator.print(f'epoch {epoch}:'      ,	SCREAMING_SNAKE_CASE__						)
def 				UpperCamelCase__					(  ):
 __lowerCamelCase		:							Optional[Any]		 =						argparse.ArgumentParser(description='Simple example of training script.'						)
 parser.add_argument(
     '--mixed_precision'      ,	type=SCREAMING_SNAKE_CASE__      ,	default=SCREAMING_SNAKE_CASE__      ,	choices=['no', 'fp16', 'bf16', 'fp8']      ,	help='Whether to use mixed precision. Choose'
     'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
     'and an Nvidia Ampere GPU.'      ,	)
 # New Code #
 parser.add_argument(
     '--gradient_accumulation_steps'      ,	type=SCREAMING_SNAKE_CASE__      ,	default=1      ,	help='The number of minibatches to be ran before gradients are accumulated.'      ,	)
 parser.add_argument('--cpu'      ,	action='store_true'      ,	help='If passed, will train on the CPU.'						)
 __lowerCamelCase		:							Dict		 =						parser.parse_args()
 __lowerCamelCase		:							List[str]		 =						{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
 training_function(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)


if __name__ == "__main__":
       main()





import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowercase_	   =     5_0_0_0_0_0

lowercase_   ,lowercase_	   =     os.path.split(__file__)
lowercase_	   =     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))
@get_duration
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Optional[int]		 =						dataset.map(**SCREAMING_SNAKE_CASE__						)
@get_duration
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Tuple		 =						dataset.filter(**SCREAMING_SNAKE_CASE__						)
def 				UpperCamelCase__					(  ):
 __lowerCamelCase		:							str		 =						{'num examples': SPEED_TEST_N_EXAMPLES}
 with tempfile.TemporaryDirectory() as tmp_dir:
  __lowerCamelCase		:							Any		 =						datasets.Features({'text': datasets.Value('string'						), 'numbers': datasets.Value('float32'						)}						)
  __lowerCamelCase		:							Any		 =						generate_example_dataset(
      os.path.join(SCREAMING_SNAKE_CASE__      ,	'dataset.arrow'						)      ,	SCREAMING_SNAKE_CASE__      ,	num_examples=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							List[Any]		 =						transformers.AutoTokenizer.from_pretrained('bert-base-cased'      ,	use_fast=SCREAMING_SNAKE_CASE__						)

  def tokenize(SCREAMING_SNAKE_CASE__						):
   return tokenizer(examples['text']						)

  __lowerCamelCase		:							str		 =						map(SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							int		 =						map(SCREAMING_SNAKE_CASE__      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[int]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='numpy'						):
   __lowerCamelCase		:							Union[str, Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='pandas'						):
   __lowerCamelCase		:							Any		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='torch'      ,	columns='numbers'						):
   __lowerCamelCase		:							List[str]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  with dataset.formatted_as(type='tensorflow'      ,	columns='numbers'						):
   __lowerCamelCase		:							List[Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=lambda SCREAMING_SNAKE_CASE__						: None      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[Any]		 =						map(SCREAMING_SNAKE_CASE__      ,	function=SCREAMING_SNAKE_CASE__      ,	batched=SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							Optional[int]		 =						filter(SCREAMING_SNAKE_CASE__						)

  # Activate later when tokenizer support batched inputs
  # with dataset.formatted_as(type='numpy'):
  #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

 with open(SCREAMING_SNAKE_CASE__      ,	'wb'						) as f:
  f.write(json.dumps(SCREAMING_SNAKE_CASE__						).encode('utf-8'						)						)


if __name__ == "__main__":  # useful to run the profiler
       benchmark_map_filter()

