




import math
import os
from copy import deepcopy

import datasets
import evaluate
import torch
import transformers
from datasets import load_dataset
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer

from accelerate import Accelerator
from accelerate.test_utils import RegressionDataset, RegressionModel
from accelerate.utils import is_tpu_available, set_seed


lowercase__				:      int	   = "true"

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase=82		, __UpperCamelCase=16)      ->					Union[str, Any]:
     set_seed(42)
     a     =				RegressionModel()
     a     =				deepcopy(SCREAMING_SNAKE_CASE_)
     a     =				RegressionDataset(length=SCREAMING_SNAKE_CASE_)
     a     =				DataLoader(SCREAMING_SNAKE_CASE_		, batch_size=SCREAMING_SNAKE_CASE_)
     model.to(accelerator.device)
     a  ,				a     =				accelerator.prepare(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
     return model, ddp_model, dataloader

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase=False)      ->					Union[str, Any]:
     a     =				AutoTokenizer.from_pretrained("hf-internal-testing/mrpc-bert-base-cased")
     a     =				load_dataset("glue"		, "mrpc"		, split="validation")

     def tokenize_function(__UpperCamelCase):
          a     =				tokenizer(examples["sentence1"]		, examples["sentence2"]		, truncation=SCREAMING_SNAKE_CASE_		, max_length=SCREAMING_SNAKE_CASE_)
          return outputs

     with accelerator.main_process_first():
          a     =				dataset.map(
              SCREAMING_SNAKE_CASE_		, batched=SCREAMING_SNAKE_CASE_		, remove_columns=["idx", "sentence1", "sentence2"]		, )

     a     =				tokenized_datasets.rename_column("label"		, "labels")

     def collate_fn(__UpperCamelCase):
          if use_longest:
               return tokenizer.pad(SCREAMING_SNAKE_CASE_		, padding="longest"		, return_tensors="pt")
          return tokenizer.pad(SCREAMING_SNAKE_CASE_		, padding="max_length"		, max_length=1_28		, return_tensors="pt")

     return DataLoader(SCREAMING_SNAKE_CASE_		, shuffle=SCREAMING_SNAKE_CASE_		, collate_fn=SCREAMING_SNAKE_CASE_		, batch_size=16)

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase)      ->					Dict:
     a     =				Accelerator(dispatch_batches=SCREAMING_SNAKE_CASE_		, split_batches=SCREAMING_SNAKE_CASE_)
     a     =				get_dataloader(SCREAMING_SNAKE_CASE_		, not dispatch_batches)
     a     =				AutoModelForSequenceClassification.from_pretrained(
         "hf-internal-testing/mrpc-bert-base-cased"		, return_dict=SCREAMING_SNAKE_CASE_)
     a  ,				a     =				accelerator.prepare(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
     return {"ddp": [ddp_model, ddp_dataloader, "cuda:0"], "no": [model, dataloader, accelerator.device]}, accelerator

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					Dict:
     a     =				[]
     for batch in dataloader:
          a  ,				a     =				batch.values()
          with torch.no_grad():
               a     =				model(SCREAMING_SNAKE_CASE_)
               a  ,				a     =				accelerator.gather_for_metrics((logit, target))
               logits_and_targets.append((logit, target))
     a  ,				a     =				[], []
     for logit, targ in logits_and_targets:
          logits.append(SCREAMING_SNAKE_CASE_)
          targs.append(SCREAMING_SNAKE_CASE_)
     a  ,				a     =				torch.cat(SCREAMING_SNAKE_CASE_), torch.cat(SCREAMING_SNAKE_CASE_)
     return logits, targs

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase=82		, __UpperCamelCase=False		, __UpperCamelCase=False		, __UpperCamelCase=16)      ->					List[str]:
     a  ,				a  ,				a     =				get_basic_setup(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
     a  ,				a     =				generate_predictions(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
     assert (
         len(SCREAMING_SNAKE_CASE_) == num_samples
     ), f'''Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(SCREAMING_SNAKE_CASE_)}'''

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase = False		, __UpperCamelCase = False)      ->					Optional[int]:
     a     =				evaluate.load("glue"		, "mrpc")
     a  ,				a     =				get_mrpc_setup(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
     # First do baseline
     a  ,				a  ,				a     =				setup["no"]
     model.to(SCREAMING_SNAKE_CASE_)
     model.eval()
     for batch in dataloader:
          batch.to(SCREAMING_SNAKE_CASE_)
          with torch.inference_mode():
               a     =				model(**SCREAMING_SNAKE_CASE_)
          a     =				outputs.logits.argmax(dim=-1)
          metric.add_batch(predictions=SCREAMING_SNAKE_CASE_		, references=batch["labels"])
     a     =				metric.compute()

     # Then do distributed
     a  ,				a  ,				a     =				setup["ddp"]
     model.eval()
     for batch in dataloader:
          with torch.inference_mode():
               a     =				model(**SCREAMING_SNAKE_CASE_)
          a     =				outputs.logits.argmax(dim=-1)
          a     =				batch["labels"]
          a  ,				a     =				accelerator.gather_for_metrics((preds, references))
          metric.add_batch(predictions=SCREAMING_SNAKE_CASE_		, references=SCREAMING_SNAKE_CASE_)
     a     =				metric.compute()

     for key in "accuracy f1".split():
          assert math.isclose(
              baseline[key]		, distributed[key]), f'''Baseline and Distributed are not the same for key {key}:\n\tBaseline: {baseline[key]}\n\tDistributed: {distributed[key]}\n'''

def  SCREAMING_SNAKE_CASE							(       )      ->					Dict:
     a     =				Accelerator(split_batches=SCREAMING_SNAKE_CASE_		, dispatch_batches=SCREAMING_SNAKE_CASE_)
     if accelerator.is_local_main_process:
          datasets.utils.logging.set_verbosity_warning()
          transformers.utils.logging.set_verbosity_warning()
     else:
          datasets.utils.logging.set_verbosity_error()
          transformers.utils.logging.set_verbosity_error()
     # These are a bit slower so they should only be ran on the GPU or TPU
     if torch.cuda.is_available() or is_tpu_available():
          if accelerator.is_local_main_process:
               print("**Testing gather_for_metrics**")
          for split_batches in [True, False]:
               for dispatch_batches in [True, False]:
                    if accelerator.is_local_main_process:
                         print(f'''With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`''')
                    test_mrpc(SCREAMING_SNAKE_CASE_		, SCREAMING_SNAKE_CASE_)
                    accelerator.state._reset_state()
     if accelerator.is_local_main_process:
          print("**Test torch metrics**")
     for split_batches in [True, False]:
          for dispatch_batches in [True, False]:
               a     =				Accelerator(split_batches=SCREAMING_SNAKE_CASE_		, dispatch_batches=SCREAMING_SNAKE_CASE_)
               if accelerator.is_local_main_process:
                    print(f'''With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`, length=99''')
               test_torch_metrics(SCREAMING_SNAKE_CASE_		, 99)
               accelerator.state._reset_state()
     if accelerator.is_local_main_process:
          print("**Test last batch is not dropped when perfectly divisible**")
     a     =				Accelerator()
     test_torch_metrics(SCREAMING_SNAKE_CASE_		, 5_12)
     accelerator.state._reset_state()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase)      ->					Union[str, Any]:
     # For xla_spawn (TPUs)
     main()


if __name__ == "__main__":
   main()







import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowercase__				:      Optional[int]	   = os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(F'{bindir}/../../examples/pytorch/translation'):
   from run_translation import main  # noqa


set_seed(42)
lowercase__				:      List[str]	   = "sshleifer/student_marian_en_ro_6_1"
lowercase__				:      List[Any]	   = "sshleifer/tiny-mbart"




@require_torch
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self  ,		A=False  ,		A=None  ,		A=True  ,		A=True  ,		A=True  ,		A=True  ,		)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=1  ,		max_len=12  ,		model_name=A  ,		num_train_epochs=1  ,		distributed=A  ,		extra_args_str=A  ,		predict_with_generate=A  ,		do_train=A  ,		do_eval=A  ,		do_predict=A  ,		)
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history

          if not do_eval:
               return

          a     =				[log for log in logs if "eval_loss" in log.keys()]

          a     =				eval_metrics[0]
          if predict_with_generate:
               assert "eval_bleu" in first_step_stats

               a     =				eval_metrics[-1]
               assert isinstance(last_step_stats["eval_bleu"]  ,		A			)
               assert not math.isnan(float(last_step_stats["eval_loss"]			)			), "eval_loss must not be `nan`"







     @require_torch_non_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          self.run_seqaseq_quick()







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple --fp16"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2"  ,		predict_with_generate=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          self.run_seqaseq_quick(
              distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2 --fp16"  ,		predict_with_generate=A			)







     @require_apex
     @require_torch_gpu
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)
          # test 2nd time - was getting eval_loss': nan'
          # to reproduce the problem set distributed=False
          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)







     @parameterized.expand(["base", "low", "high", "mixed"]			)
     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self  ,		A			)  ->   Dict:



          '''simple docstring'''



          a     =				{
              # test with the default log_level - should be info and thus log info once
              "base": {"extra_args_str": "", "n_matches": 1},
              # test with low log_level and log_level_replica - should be noisy on all processes
              # now the info string should appear twice on 2 processes
              "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
              # test with high log_level and low log_level_replica
              # now the info string should appear once only on the replica
              "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
              # test with high log_level and log_level_replica - should be quiet on all processes
              "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
          }

          a     =				experiments[experiment_id]
          a     =				{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
          a     =				"Running training"
          with CaptureStderr() as cl:
               self.run_seqaseq_quick(**A  ,		extra_args_str=data["extra_args_str"]			)
          a     =				len(re.findall(A  ,		cl.err			)			)
          self.assertEqual(A  ,		data["n_matches"]			)







     @slow
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=2  ,		max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=10  ,		distributed=A  ,		)

          # Check metrics
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history
          a     =				[log for log in logs if "eval_loss" in log.keys()]
          a     =				eval_metrics[0]
          a     =				eval_metrics[-1]

          assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
          assert isinstance(last_step_stats["eval_bleu"]  ,		A			)

          # test if do_predict saves generations and metrics
          a     =				os.listdir(A			)
          a     =				{os.path.basename(A			) for p in contents}
          assert "generated_predictions.txt" in contents
          assert "predict_results.json" in contents







     @slow
     @require_bitsandbytes
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from transformers.training_args import OptimizerNames

          def train_and_return_metrics(A			) -> Tuple[int, float]:
               a     =				"--skip_memory_metrics 0"

               a     =				self.run_trainer(
                   max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=1  ,		optim=A  ,		distributed=A  ,		extra_args_str=A  ,		do_eval=A  ,		do_predict=A  ,		n_gpus_to_use=1  ,		)

               # Check metrics
               a     =				TrainerState.load_from_json(Path(A  ,		"trainer_state.json"			)			).log_history
               a     =				int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20			)
               a     =				int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20			)

               a     =				logs[0]["train_loss"]
               return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value			)
          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value			)

          a     =				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

          a     =				gpu_peak_mem_orig + gpu_alloc_mem_orig
          a     =				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
          a     =				gpu_total_mem_orig - gpu_total_mem_bnb

          # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
          # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
          # in 2 bytes and the diff in optim memory usage is derived as so:
          #
          # - normal 25*8=~200MB (8 bytes per param)
          # - bnb    25*2= ~50MB (2 bytes per param)
          #
          # Thus we should expect ~150MB total memory saved.
          #
          # Peak memory should be the same - the total should be different by about that same margin
          #
          # After leaving a small margin to accommodate for differences between gpus let's check
          # that we have at least 120MB in savings
          a     =				120

          # uncomment the following if this test starts failing - requires py38 for a new print feature
          # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
          # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
          # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
          # print(f"{gpu_alloc_mem_diff=}MB")
          # print(f"{gpu_peak_mem_diff=}MB")
          # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
          # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and'''
              F''' gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB'''  ,		)

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and'''
              F''' gpu_total_mem_bnb={gpu_total_mem_bnb}MB'''  ,		)

          self.assertEqual(
              A  ,		A  ,		F'''loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}'''			)







     def       lowerCAmelCase_    (     self  ,		A  ,		A  ,		A  ,		A = 3e-3  ,		A = "adafactor"  ,		A = False  ,		A = None  ,		A = 0  ,		A = True  ,		A = True  ,		A = True  ,		A = True  ,		A = None  ,		)  ->   Tuple:



          '''simple docstring'''



          a     =				self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
          a     =				self.get_auto_remove_tmp_dir()
          a     =				F'''
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(A			)}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(A			)}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        '''.split()

          a     =				F'''
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(A			)}
        '''.split()

          a     =				"\n            --do_predict\n        ".split()

          a     =				[]
          if do_train:
               args += args_train

          if do_eval:
               args += args_eval

          if do_predict:
               args += args_predict

          if predict_with_generate:
               args += "--predict_with_generate".split()

          if do_train:
               if optim == "adafactor":
                    args += "--adafactor".split()
               else:
                    args += F'''--optim {optim}'''.split()

          if extra_args_str is not None:
               args += extra_args_str.split()

          if distributed:
               if n_gpus_to_use is None:
                    a     =				get_gpu_count()
               a     =				get_torch_dist_unique_port()
               a     =				F'''
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            '''.split()
               a     =				[sys.executable] + distributed_args + args
               # keep for quick debug
               # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
               execute_subprocess_async(A  ,		env=self.get_env()			)
          else:
               a     =				["run_translation.py"] + args
               with patch.object(A  ,		"argv"  ,		A			):
                    main()

          return output_dir

