




#
# This a `torch.distributed` diagnostics script that checks that all GPUs in the cluster (one or
# many nodes) can talk to each other via nccl and allocate gpu memory.
#
# To run first adjust the number of processes and nodes:
#
# python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# You may need to add --master_addr $MASTER_ADDR --master_port $MASTER_PORT if using a custom addr:port
#
# You can also use the rdzv API: --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT --rdzv_backend c10d
#
# use torch.distributed.launch instead of torch.distributed.run for torch < 1.9
#
# If you get a hanging in `barrier` calls you have some network issues, you may try to debug this with:
#
# NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# which should tell you what's going on behind the scenes.
#
#
# This script can be run via `srun` in the SLURM environment as well. Here is a SLURM script that
# runs on 2 nodes of 4 gpus per node:
#
# #SBATCH --job-name=test-nodes        # name
# #SBATCH --nodes=2                    # nodes
# #SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
# #SBATCH --cpus-per-task=10           # number of cores per tasks
# #SBATCH --gres=gpu:4                 # number of gpus
# #SBATCH --time 0:05:00               # maximum execution time (HH:MM:SS)
# #SBATCH --output=%x-%j.out           # output file name
#
# GPUS_PER_NODE=4
# MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
# MASTER_PORT=6000
#
# srun --jobid $SLURM_JOBID bash -c 'python -m torch.distributed.run \
# --nproc_per_node $GPUS_PER_NODE --nnodes $SLURM_NNODES --node_rank $SLURM_PROCID \
# --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
# torch-distributed-gpu-test.py'
#

import fcntl
import os
import socket

import torch
import torch.distributed as dist

def  SCREAMING_SNAKE_CASE							(       *__UpperCamelCase)      ->					int:
     with open(__UpperCamelCase		, "r") as fh:
          fcntl.flock(__UpperCamelCase		, fcntl.LOCK_EX)
          try:
               print(*__UpperCamelCase)
          finally:
               fcntl.flock(__UpperCamelCase		, fcntl.LOCK_UN)


lowercase__				:      List[Any]	   = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local_rank)
lowercase__				:      Optional[int]	   = torch.device("cuda", local_rank)
lowercase__				:      int	   = socket.gethostname()

lowercase__				:      Union[str, Any]	   = F'[{hostname}-{local_rank}]'

try:
   # test distributed
   dist.init_process_group("nccl")
   dist.all_reduce(torch.ones(1).to(device), op=dist.ReduceOp.SUM)
   dist.barrier()

   # test cuda is available and can allocate memory
   torch.cuda.is_available()
   torch.ones(1).cuda(local_rank)

   # global rank
   lowercase__				:      Union[str, Any]	   = dist.get_rank()
   lowercase__				:      Optional[Any]	   = dist.get_world_size()

   printflock(F'{gpu} is OK (global rank: {rank}/{world_size})')

   dist.barrier()
   if rank == 0:
      printflock(F'pt={torch.__version__}, cuda={torch.version.cuda}, nccl={torch.cuda.nccl.version()}')

except Exception:
   printflock(F'{gpu} is broken')
   raise







import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowercase__				:      Optional[int]	   = os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(F'{bindir}/../../examples/pytorch/translation'):
   from run_translation import main  # noqa


set_seed(42)
lowercase__				:      List[str]	   = "sshleifer/student_marian_en_ro_6_1"
lowercase__				:      List[Any]	   = "sshleifer/tiny-mbart"




@require_torch
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self  ,		A=False  ,		A=None  ,		A=True  ,		A=True  ,		A=True  ,		A=True  ,		)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=1  ,		max_len=12  ,		model_name=A  ,		num_train_epochs=1  ,		distributed=A  ,		extra_args_str=A  ,		predict_with_generate=A  ,		do_train=A  ,		do_eval=A  ,		do_predict=A  ,		)
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history

          if not do_eval:
               return

          a     =				[log for log in logs if "eval_loss" in log.keys()]

          a     =				eval_metrics[0]
          if predict_with_generate:
               assert "eval_bleu" in first_step_stats

               a     =				eval_metrics[-1]
               assert isinstance(last_step_stats["eval_bleu"]  ,		A			)
               assert not math.isnan(float(last_step_stats["eval_loss"]			)			), "eval_loss must not be `nan`"







     @require_torch_non_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          self.run_seqaseq_quick()







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple --fp16"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2"  ,		predict_with_generate=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          self.run_seqaseq_quick(
              distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2 --fp16"  ,		predict_with_generate=A			)







     @require_apex
     @require_torch_gpu
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)
          # test 2nd time - was getting eval_loss': nan'
          # to reproduce the problem set distributed=False
          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)







     @parameterized.expand(["base", "low", "high", "mixed"]			)
     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self  ,		A			)  ->   Dict:



          '''simple docstring'''



          a     =				{
              # test with the default log_level - should be info and thus log info once
              "base": {"extra_args_str": "", "n_matches": 1},
              # test with low log_level and log_level_replica - should be noisy on all processes
              # now the info string should appear twice on 2 processes
              "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
              # test with high log_level and low log_level_replica
              # now the info string should appear once only on the replica
              "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
              # test with high log_level and log_level_replica - should be quiet on all processes
              "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
          }

          a     =				experiments[experiment_id]
          a     =				{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
          a     =				"Running training"
          with CaptureStderr() as cl:
               self.run_seqaseq_quick(**A  ,		extra_args_str=data["extra_args_str"]			)
          a     =				len(re.findall(A  ,		cl.err			)			)
          self.assertEqual(A  ,		data["n_matches"]			)







     @slow
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=2  ,		max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=10  ,		distributed=A  ,		)

          # Check metrics
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history
          a     =				[log for log in logs if "eval_loss" in log.keys()]
          a     =				eval_metrics[0]
          a     =				eval_metrics[-1]

          assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
          assert isinstance(last_step_stats["eval_bleu"]  ,		A			)

          # test if do_predict saves generations and metrics
          a     =				os.listdir(A			)
          a     =				{os.path.basename(A			) for p in contents}
          assert "generated_predictions.txt" in contents
          assert "predict_results.json" in contents







     @slow
     @require_bitsandbytes
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from transformers.training_args import OptimizerNames

          def train_and_return_metrics(A			) -> Tuple[int, float]:
               a     =				"--skip_memory_metrics 0"

               a     =				self.run_trainer(
                   max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=1  ,		optim=A  ,		distributed=A  ,		extra_args_str=A  ,		do_eval=A  ,		do_predict=A  ,		n_gpus_to_use=1  ,		)

               # Check metrics
               a     =				TrainerState.load_from_json(Path(A  ,		"trainer_state.json"			)			).log_history
               a     =				int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20			)
               a     =				int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20			)

               a     =				logs[0]["train_loss"]
               return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value			)
          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value			)

          a     =				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

          a     =				gpu_peak_mem_orig + gpu_alloc_mem_orig
          a     =				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
          a     =				gpu_total_mem_orig - gpu_total_mem_bnb

          # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
          # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
          # in 2 bytes and the diff in optim memory usage is derived as so:
          #
          # - normal 25*8=~200MB (8 bytes per param)
          # - bnb    25*2= ~50MB (2 bytes per param)
          #
          # Thus we should expect ~150MB total memory saved.
          #
          # Peak memory should be the same - the total should be different by about that same margin
          #
          # After leaving a small margin to accommodate for differences between gpus let's check
          # that we have at least 120MB in savings
          a     =				120

          # uncomment the following if this test starts failing - requires py38 for a new print feature
          # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
          # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
          # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
          # print(f"{gpu_alloc_mem_diff=}MB")
          # print(f"{gpu_peak_mem_diff=}MB")
          # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
          # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and'''
              F''' gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB'''  ,		)

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and'''
              F''' gpu_total_mem_bnb={gpu_total_mem_bnb}MB'''  ,		)

          self.assertEqual(
              A  ,		A  ,		F'''loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}'''			)







     def       lowerCAmelCase_    (     self  ,		A  ,		A  ,		A  ,		A = 3e-3  ,		A = "adafactor"  ,		A = False  ,		A = None  ,		A = 0  ,		A = True  ,		A = True  ,		A = True  ,		A = True  ,		A = None  ,		)  ->   Tuple:



          '''simple docstring'''



          a     =				self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
          a     =				self.get_auto_remove_tmp_dir()
          a     =				F'''
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(A			)}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(A			)}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        '''.split()

          a     =				F'''
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(A			)}
        '''.split()

          a     =				"\n            --do_predict\n        ".split()

          a     =				[]
          if do_train:
               args += args_train

          if do_eval:
               args += args_eval

          if do_predict:
               args += args_predict

          if predict_with_generate:
               args += "--predict_with_generate".split()

          if do_train:
               if optim == "adafactor":
                    args += "--adafactor".split()
               else:
                    args += F'''--optim {optim}'''.split()

          if extra_args_str is not None:
               args += extra_args_str.split()

          if distributed:
               if n_gpus_to_use is None:
                    a     =				get_gpu_count()
               a     =				get_torch_dist_unique_port()
               a     =				F'''
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            '''.split()
               a     =				[sys.executable] + distributed_args + args
               # keep for quick debug
               # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
               execute_subprocess_async(A  ,		env=self.get_env()			)
          else:
               a     =				["run_translation.py"] + args
               with patch.object(A  ,		"argv"  ,		A			):
                    main()

          return output_dir

