




import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowercase__				:      str	   = "bert-base-cased"
lowercase__				:      str	   = "fp16"
lowercase__				:      int	   = "bf16"
lowercase__				:      Tuple	   = [FPaa, BFaa]




@require_fsdp
@require_cuda
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          super().setUp()

          a     =				dict(
              ACCELERATE_USE_FSDP="true"  ,		MASTER_ADDR="localhost"  ,		MASTER_PORT="10999"  ,		RANK="0"  ,		LOCAL_RANK="0"  ,		WORLD_SIZE="1"  ,		)







     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

          for i, strategy in enumerate(A			):
               a     =				self.dist_env.copy()
               a     =				F'''{i + 1}'''
               a     =				strategy
               with mockenv_context(**A			):
                    a     =				FullyShardedDataParallelPlugin()
                    self.assertEqual(fsdp_plugin.sharding_strategy  ,		ShardingStrategy(i + 1			)			)







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

          for i, prefetch_policy in enumerate(A			):
               a     =				self.dist_env.copy()
               a     =				prefetch_policy
               with mockenv_context(**A			):
                    a     =				FullyShardedDataParallelPlugin()
                    if prefetch_policy == "NO_PREFETCH":
                         self.assertIsNone(fsdp_plugin.backward_prefetch			)
                    else:
                         self.assertEqual(fsdp_plugin.backward_prefetch  ,		BackwardPrefetch(i + 1			)			)







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

          for i, state_dict_type in enumerate(A			):
               a     =				self.dist_env.copy()
               a     =				state_dict_type
               with mockenv_context(**A			):
                    a     =				FullyShardedDataParallelPlugin()
                    self.assertEqual(fsdp_plugin.state_dict_type  ,		StateDictType(i + 1			)			)
                    if state_dict_type == "FULL_STATE_DICT":
                         self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu			)
                         self.assertTrue(fsdp_plugin.state_dict_config.ranka_only			)







     def       lowerCAmelCase_    (     self			)  ->   List[str]:



          '''simple docstring'''



          a     =				AutoModel.from_pretrained(A			)
          for policy in FSDP_AUTO_WRAP_POLICY:
               a     =				self.dist_env.copy()
               a     =				policy
               if policy == "TRANSFORMER_BASED_WRAP":
                    a     =				"BertLayer"
               elif policy == "SIZE_BASED_WRAP":
                    a     =				"2000"
               with mockenv_context(**A			):
                    a     =				FullyShardedDataParallelPlugin()
                    fsdp_plugin.set_auto_wrap_policy(A			)
                    if policy == "NO_WRAP":
                         self.assertIsNone(fsdp_plugin.auto_wrap_policy			)
                    else:
                         self.assertIsNotNone(fsdp_plugin.auto_wrap_policy			)

          a     =				self.dist_env.copy()
          a     =				"TRANSFORMER_BASED_WRAP"
          a     =				"T5Layer"
          with mockenv_context(**A			):
               a     =				FullyShardedDataParallelPlugin()
               with self.assertRaises(A			) as cm:
                    fsdp_plugin.set_auto_wrap_policy(A			)
               self.assertTrue("Could not find the transformer layer class to wrap in the model." in str(cm.exception			)			)

          a     =				self.dist_env.copy()
          a     =				"SIZE_BASED_WRAP"
          a     =				"0"
          with mockenv_context(**A			):
               a     =				FullyShardedDataParallelPlugin()
               fsdp_plugin.set_auto_wrap_policy(A			)
               self.assertIsNone(fsdp_plugin.auto_wrap_policy			)







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
          from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

          for mp_dtype in dtypes:
               a     =				self.dist_env.copy()
               a     =				mp_dtype
               with mockenv_context(**A			):
                    a     =				Accelerator()
                    if mp_dtype == "fp16":
                         a     =				torch.floataa
                    elif mp_dtype == "bf16":
                         a     =				torch.bfloataa
                    a     =				MixedPrecision(param_dtype=A  ,		reduce_dtype=A  ,		buffer_dtype=A			)
                    self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy  ,		A			)
                    if mp_dtype == FPaa:
                         self.assertTrue(isinstance(accelerator.scaler  ,		A			)			)
                    elif mp_dtype == BFaa:
                         self.assertIsNone(accelerator.scaler			)
                    AcceleratorState._reset_state(A			)







     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

          for flag in [True, False]:
               a     =				self.dist_env.copy()
               a     =				str(A			).lower()
               with mockenv_context(**A			):
                    a     =				FullyShardedDataParallelPlugin()
                    self.assertEqual(fsdp_plugin.cpu_offload  ,		CPUOffload(offload_params=A			)			)







@require_fsdp
@require_multi_gpu
@slow
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          super().setUp()
          a     =				0.8_2
          a     =				[
              "fsdp_shard_grad_op_transformer_based_wrap",
              "fsdp_full_shard_transformer_based_wrap",
          ]
          a     =				{
              "multi_gpu_fp16": 3200,
              "fsdp_shard_grad_op_transformer_based_wrap_fp16": 2000,
              "fsdp_full_shard_transformer_based_wrap_fp16": 1900,
              # Disabling below test as it overwhelms the RAM memory usage
              # on CI self-hosted runner leading to tests getting killed.
              # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
          }
          a     =				160
          a     =				160

          a     =				inspect.getfile(accelerate.test_utils			)
          a     =				os.path.sep.join(mod_file.split(os.path.sep			)[:-1] + ["scripts", "external_deps"]			)







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				os.path.join(self.test_scripts_folder  ,		"test_performance.py"			)
          a     =				["accelerate", "launch", "--num_processes=2", "--num_machines=1", "--machine_rank=0", "--use_fsdp"]
          for config in self.performance_configs:
               a     =				cmd.copy()
               for i, strategy in enumerate(A			):
                    if strategy.lower() in config:
                         cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''			)
                         break

               if "fp32" in config:
                    cmd_config.append("--mixed_precision=no"			)
               else:
                    cmd_config.append("--mixed_precision=fp16"			)

               if "cpu_offload" in config:
                    cmd_config.append("--fsdp_offload_params=True"			)

               for policy in FSDP_AUTO_WRAP_POLICY:
                    if policy.lower() in config:
                         cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''			)
                         break

               if policy == "TRANSFORMER_BASED_WRAP":
                    cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer"			)
               elif policy == "SIZE_BASED_WRAP":
                    cmd_config.append("--fsdp_min_num_params=2000"			)

               cmd_config.extend(
                   [
                       self.test_file_path,
                       F'''--output_dir={self.tmpdir}''',
                       F'''--performance_lower_bound={self.performance_lower_bound}''',
                   ]			)
               with patch_environment(omp_num_threads=1			):
                    execute_subprocess_async(A  ,		env=os.environ.copy()			)







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				os.path.join(self.test_scripts_folder  ,		"test_checkpointing.py"			)
          a     =				[
              "accelerate",
              "launch",
              "--num_processes=2",
              "--num_machines=1",
              "--machine_rank=0",
              "--use_fsdp",
              "--mixed_precision=fp16",
              "--fsdp_transformer_layer_cls_to_wrap=BertLayer",
          ]

          for i, strategy in enumerate(A			):
               a     =				cmd.copy()
               cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''			)
               if strategy != "FULL_SHARD":
                    continue
               a     =				len(A			)
               for state_dict_type in FSDP_STATE_DICT_TYPE:
                    a     =				cmd_config[:state_dict_config_index]
                    cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''			)
                    cmd_config.extend(
                        [
                            self.test_file_path,
                            F'''--output_dir={self.tmpdir}''',
                            "--partial_train_epoch=1",
                        ]			)
                    with patch_environment(omp_num_threads=1			):
                         execute_subprocess_async(A  ,		env=os.environ.copy()			)

                    a     =				cmd_config[:-1]
                    a     =				os.path.join(self.tmpdir  ,		"epoch_0"			)
                    cmd_config.extend(
                        [
                            F'''--resume_from_checkpoint={resume_from_checkpoint}''',
                        ]			)
                    with patch_environment(omp_num_threads=1			):
                         execute_subprocess_async(A  ,		env=os.environ.copy()			)







     def       lowerCAmelCase_    (     self			)  ->   List[str]:



          '''simple docstring'''



          a     =				os.path.join(self.test_scripts_folder  ,		"test_peak_memory_usage.py"			)
          a     =				[
              "accelerate",
              "launch",
              "--num_processes=2",
              "--num_machines=1",
              "--machine_rank=0",
          ]
          for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
               a     =				cmd.copy()
               if "fp16" in spec:
                    cmd_config.extend(["--mixed_precision=fp16"]			)
               else:
                    cmd_config.extend(["--mixed_precision=no"]			)

               if "multi_gpu" in spec:
                    continue
               else:
                    cmd_config.extend(["--use_fsdp"]			)
                    for i, strategy in enumerate(A			):
                         if strategy.lower() in spec:
                              cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''			)
                              break

                    if "cpu_offload" in spec:
                         cmd_config.append("--fsdp_offload_params=True"			)

                    for policy in FSDP_AUTO_WRAP_POLICY:
                         if policy.lower() in spec:
                              cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''			)
                              break

                    if policy == "TRANSFORMER_BASED_WRAP":
                         cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer"			)
                    elif policy == "SIZE_BASED_WRAP":
                         cmd_config.append("--fsdp_min_num_params=2000"			)

               cmd_config.extend(
                   [
                       self.test_file_path,
                       F'''--output_dir={self.tmpdir}''',
                       F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
                       F'''--n_train={self.n_train}''',
                       F'''--n_val={self.n_val}''',
                   ]			)
               with patch_environment(omp_num_threads=1			):
                    execute_subprocess_async(A  ,		env=os.environ.copy()			)







import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowercase__				:      Optional[int]	   = os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(F'{bindir}/../../examples/pytorch/translation'):
   from run_translation import main  # noqa


set_seed(42)
lowercase__				:      List[str]	   = "sshleifer/student_marian_en_ro_6_1"
lowercase__				:      List[Any]	   = "sshleifer/tiny-mbart"




@require_torch
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self  ,		A=False  ,		A=None  ,		A=True  ,		A=True  ,		A=True  ,		A=True  ,		)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=1  ,		max_len=12  ,		model_name=A  ,		num_train_epochs=1  ,		distributed=A  ,		extra_args_str=A  ,		predict_with_generate=A  ,		do_train=A  ,		do_eval=A  ,		do_predict=A  ,		)
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history

          if not do_eval:
               return

          a     =				[log for log in logs if "eval_loss" in log.keys()]

          a     =				eval_metrics[0]
          if predict_with_generate:
               assert "eval_bleu" in first_step_stats

               a     =				eval_metrics[-1]
               assert isinstance(last_step_stats["eval_bleu"]  ,		A			)
               assert not math.isnan(float(last_step_stats["eval_loss"]			)			), "eval_loss must not be `nan`"







     @require_torch_non_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          self.run_seqaseq_quick()







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple --fp16"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2"  ,		predict_with_generate=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          self.run_seqaseq_quick(
              distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2 --fp16"  ,		predict_with_generate=A			)







     @require_apex
     @require_torch_gpu
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)
          # test 2nd time - was getting eval_loss': nan'
          # to reproduce the problem set distributed=False
          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)







     @parameterized.expand(["base", "low", "high", "mixed"]			)
     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self  ,		A			)  ->   Dict:



          '''simple docstring'''



          a     =				{
              # test with the default log_level - should be info and thus log info once
              "base": {"extra_args_str": "", "n_matches": 1},
              # test with low log_level and log_level_replica - should be noisy on all processes
              # now the info string should appear twice on 2 processes
              "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
              # test with high log_level and low log_level_replica
              # now the info string should appear once only on the replica
              "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
              # test with high log_level and log_level_replica - should be quiet on all processes
              "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
          }

          a     =				experiments[experiment_id]
          a     =				{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
          a     =				"Running training"
          with CaptureStderr() as cl:
               self.run_seqaseq_quick(**A  ,		extra_args_str=data["extra_args_str"]			)
          a     =				len(re.findall(A  ,		cl.err			)			)
          self.assertEqual(A  ,		data["n_matches"]			)







     @slow
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=2  ,		max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=10  ,		distributed=A  ,		)

          # Check metrics
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history
          a     =				[log for log in logs if "eval_loss" in log.keys()]
          a     =				eval_metrics[0]
          a     =				eval_metrics[-1]

          assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
          assert isinstance(last_step_stats["eval_bleu"]  ,		A			)

          # test if do_predict saves generations and metrics
          a     =				os.listdir(A			)
          a     =				{os.path.basename(A			) for p in contents}
          assert "generated_predictions.txt" in contents
          assert "predict_results.json" in contents







     @slow
     @require_bitsandbytes
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from transformers.training_args import OptimizerNames

          def train_and_return_metrics(A			) -> Tuple[int, float]:
               a     =				"--skip_memory_metrics 0"

               a     =				self.run_trainer(
                   max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=1  ,		optim=A  ,		distributed=A  ,		extra_args_str=A  ,		do_eval=A  ,		do_predict=A  ,		n_gpus_to_use=1  ,		)

               # Check metrics
               a     =				TrainerState.load_from_json(Path(A  ,		"trainer_state.json"			)			).log_history
               a     =				int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20			)
               a     =				int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20			)

               a     =				logs[0]["train_loss"]
               return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value			)
          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value			)

          a     =				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

          a     =				gpu_peak_mem_orig + gpu_alloc_mem_orig
          a     =				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
          a     =				gpu_total_mem_orig - gpu_total_mem_bnb

          # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
          # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
          # in 2 bytes and the diff in optim memory usage is derived as so:
          #
          # - normal 25*8=~200MB (8 bytes per param)
          # - bnb    25*2= ~50MB (2 bytes per param)
          #
          # Thus we should expect ~150MB total memory saved.
          #
          # Peak memory should be the same - the total should be different by about that same margin
          #
          # After leaving a small margin to accommodate for differences between gpus let's check
          # that we have at least 120MB in savings
          a     =				120

          # uncomment the following if this test starts failing - requires py38 for a new print feature
          # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
          # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
          # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
          # print(f"{gpu_alloc_mem_diff=}MB")
          # print(f"{gpu_peak_mem_diff=}MB")
          # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
          # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and'''
              F''' gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB'''  ,		)

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and'''
              F''' gpu_total_mem_bnb={gpu_total_mem_bnb}MB'''  ,		)

          self.assertEqual(
              A  ,		A  ,		F'''loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}'''			)







     def       lowerCAmelCase_    (     self  ,		A  ,		A  ,		A  ,		A = 3e-3  ,		A = "adafactor"  ,		A = False  ,		A = None  ,		A = 0  ,		A = True  ,		A = True  ,		A = True  ,		A = True  ,		A = None  ,		)  ->   Tuple:



          '''simple docstring'''



          a     =				self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
          a     =				self.get_auto_remove_tmp_dir()
          a     =				F'''
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(A			)}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(A			)}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        '''.split()

          a     =				F'''
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(A			)}
        '''.split()

          a     =				"\n            --do_predict\n        ".split()

          a     =				[]
          if do_train:
               args += args_train

          if do_eval:
               args += args_eval

          if do_predict:
               args += args_predict

          if predict_with_generate:
               args += "--predict_with_generate".split()

          if do_train:
               if optim == "adafactor":
                    args += "--adafactor".split()
               else:
                    args += F'''--optim {optim}'''.split()

          if extra_args_str is not None:
               args += extra_args_str.split()

          if distributed:
               if n_gpus_to_use is None:
                    a     =				get_gpu_count()
               a     =				get_torch_dist_unique_port()
               a     =				F'''
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            '''.split()
               a     =				[sys.executable] + distributed_args + args
               # keep for quick debug
               # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
               execute_subprocess_async(A  ,		env=self.get_env()			)
          else:
               a     =				["run_translation.py"] + args
               with patch.object(A  ,		"argv"  ,		A			):
                    main()

          return output_dir

