




import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
   from transformers import SpeechaTextFeatureExtractor

lowercase__				:      Dict	   = random.Random()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase=1.0		, __UpperCamelCase=None		, __UpperCamelCase=None)      ->					str:
     if rng is None:
          a     =				global_rng

     a     =				[]
     for batch_idx in range(shape[0]):
          values.append([])
          for _ in range(shape[1]):
               values[-1].append(rng.random() * scale)

     return values




@require_torch
@require_torchaudio
class       a__	( unittest.TestCase				):







     def __init__(     self  ,		A  ,		A=7  ,		A=400  ,		A=2000  ,		A=24  ,		A=24  ,		A=0.0  ,		A=16000  ,		A=True  ,		A=True  ,		)  ->   str:



          '''simple docstring'''



          a     =				parent
          a     =				batch_size
          a     =				min_seq_length
          a     =				max_seq_length
          a     =				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
          a     =				feature_size
          a     =				num_mel_bins
          a     =				padding_value
          a     =				sampling_rate
          a     =				return_attention_mask
          a     =				do_normalize







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          return {
              "feature_size": self.feature_size,
              "num_mel_bins": self.num_mel_bins,
              "padding_value": self.padding_value,
              "sampling_rate": self.sampling_rate,
              "return_attention_mask": self.return_attention_mask,
              "do_normalize": self.do_normalize,
          }







     def       lowerCAmelCase_    (     self  ,		A=False  ,		A=False			)  ->   Union[str, Any]:



          '''simple docstring'''



          def _flatten(A			):
               return list(itertools.chain(*A			)			)

          if equal_length:
               a     =				[floats_list((self.max_seq_length, self.feature_size)			) for _ in range(self.batch_size			)]
          else:
               # make sure that inputs increase in size
               a     =				[
                   floats_list((x, self.feature_size)			)
                   for x in range(self.min_seq_length  ,		self.max_seq_length  ,		self.seq_length_diff			)
               ]
          if numpify:
               a     =				[np.asarray(A			) for x in speech_inputs]
          return speech_inputs




@require_torch
@require_torchaudio
class       a__	( UpperCamelCase__       , unittest.TestCase				):
     a		:      Any		 =     SpeechaTextFeatureExtractor if is_speech_available() else None







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				SpeechaTextFeatureExtractionTester(self			)







     def       lowerCAmelCase_    (     self  ,		A			)  ->   Any:



          '''simple docstring'''



          self.assertTrue(np.all(np.mean(A  ,		axis=0			) < 1e-3			)			)
          self.assertTrue(np.all(np.abs(np.var(A  ,		axis=0			) - 1			) < 1e-3			)			)







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          # create three inputs of length 800, 1000, and 1200
          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]
          a     =				[np.asarray(A			) for speech_input in speech_inputs]

          # Test feature size
          a     =				feature_extractor(A  ,		padding=A  ,		return_tensors="np"			).input_features
          self.assertTrue(input_features.ndim == 3			)
          self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size			)

          # Test not batched input
          a     =				feature_extractor(speech_inputs[0]  ,		return_tensors="np"			).input_features
          a     =				feature_extractor(np_speech_inputs[0]  ,		return_tensors="np"			).input_features
          self.assertTrue(np.allclose(A  ,		A  ,		atol=1e-3			)			)

          # Test batched
          a     =				feature_extractor(A  ,		return_tensors="np"			).input_features
          a     =				feature_extractor(A  ,		return_tensors="np"			).input_features
          for enc_seq_a, enc_seq_a in zip(A  ,		A			):
               self.assertTrue(np.allclose(A  ,		A  ,		atol=1e-3			)			)

          # Test 2-D numpy arrays are batched.
          a     =				[floats_list((1, x)			)[0] for x in (800, 800, 800)]
          a     =				np.asarray(A			)
          a     =				feature_extractor(A  ,		return_tensors="np"			).input_features
          a     =				feature_extractor(A  ,		return_tensors="np"			).input_features
          for enc_seq_a, enc_seq_a in zip(A  ,		A			):
               self.assertTrue(np.allclose(A  ,		A  ,		atol=1e-3			)			)







     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]

          a     =				["longest", "max_length", "do_not_pad"]
          a     =				[None, 16, None]
          for max_length, padding in zip(A  ,		A			):
               a     =				feature_extractor(
                   A  ,		padding=A  ,		max_length=A  ,		return_attention_mask=A			)
               a     =				inputs.input_features
               a     =				inputs.attention_mask
               a     =				[np.sum(A			) for x in attention_mask]

               self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]			)
               self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]			)
               self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]			)







     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]

          a     =				["longest", "max_length", "do_not_pad"]
          a     =				[None, 16, None]
          for max_length, padding in zip(A  ,		A			):
               a     =				feature_extractor(
                   A  ,		max_length=A  ,		padding=A  ,		return_tensors="np"  ,		return_attention_mask=A			)
               a     =				inputs.input_features
               a     =				inputs.attention_mask
               a     =				[np.sum(A			) for x in attention_mask]

               self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]			)
               self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6			)
               self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]			)
               self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6			)
               self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]			)







     def       lowerCAmelCase_    (     self			)  ->   int:



          '''simple docstring'''



          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]
          a     =				feature_extractor(
              A  ,		padding="max_length"  ,		max_length=4  ,		truncation=A  ,		return_tensors="np"  ,		return_attention_mask=A  ,		)
          a     =				inputs.input_features
          a     =				inputs.attention_mask
          a     =				np.sum(attention_mask == 1  ,		axis=1			)

          self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
          self._check_zero_mean_unit_variance(input_features[1]			)
          self._check_zero_mean_unit_variance(input_features[2]			)







     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]
          a     =				feature_extractor(
              A  ,		padding="longest"  ,		max_length=4  ,		truncation=A  ,		return_tensors="np"  ,		return_attention_mask=A  ,		)
          a     =				inputs.input_features
          a     =				inputs.attention_mask
          a     =				np.sum(attention_mask == 1  ,		axis=1			)

          self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
          self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]			)
          self._check_zero_mean_unit_variance(input_features[2]			)

          # make sure that if max_length < longest -> then pad to max_length
          self.assertEqual(input_features.shape  ,		(3, 4, 24)			)

          a     =				[floats_list((1, x)			)[0] for x in range(800  ,		1400  ,		200			)]
          a     =				feature_extractor(
              A  ,		padding="longest"  ,		max_length=16  ,		truncation=A  ,		return_tensors="np"  ,		return_attention_mask=A  ,		)
          a     =				inputs.input_features
          a     =				inputs.attention_mask
          a     =				np.sum(attention_mask == 1  ,		axis=1			)

          self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
          self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]			)
          self._check_zero_mean_unit_variance(input_features[2]			)

          # make sure that if max_length < longest -> then pad to max_length
          self.assertEqual(input_features.shape  ,		(3, 6, 24)			)







     def       lowerCAmelCase_    (     self			)  ->   Optional[Any]:



          '''simple docstring'''



          import torch

          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				np.random.rand(100  ,		32			).astype(np.floataa			)
          a     =				np_speech_inputs.tolist()

          for inputs in [py_speech_inputs, np_speech_inputs]:
               a     =				feature_extractor.pad([{"input_features": inputs}]  ,		return_tensors="np"			)
               self.assertTrue(np_processed.input_features.dtype == np.floataa			)
               a     =				feature_extractor.pad([{"input_features": inputs}]  ,		return_tensors="pt"			)
               self.assertTrue(pt_processed.input_features.dtype == torch.floataa			)







     def       lowerCAmelCase_    (     self  ,		A			)  ->   Dict:



          '''simple docstring'''



          from datasets import load_dataset

          a     =				load_dataset("hf-internal-testing/librispeech_asr_dummy"  ,		"clean"  ,		split="validation"			)
          # automatic decoding with librispeech
          a     =				ds.sort("id"			).select(range(A			)			)[:num_samples]["audio"]

          return [x["array"] for x in speech_samples]







     def       lowerCAmelCase_    (     self			)  ->   Optional[Any]:



          '''simple docstring'''



          a     =				np.array([
              -1.5_7_4_5, -1.7_7_1_3, -1.7_0_2_0, -1.6_0_6_9, -1.2_2_5_0, -1.1_1_0_5, -0.9_0_7_2, -0.8_2_4_1,
              -1.2_3_1_0, -0.8_0_9_8, -0.3_3_2_0, -0.4_1_0_1, -0.7_9_8_5, -0.4_9_9_6, -0.8_2_1_3, -0.9_1_2_8,
              -1.0_4_2_0, -1.1_2_8_6, -1.0_4_4_0, -0.7_9_9_9, -0.8_4_0_5, -1.2_2_7_5, -1.5_4_4_3, -1.4_6_2_5,
          ]			)
          # fmt: on

          a     =				self._load_datasamples(1			)
          a     =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
          a     =				feature_extractor(A  ,		return_tensors="pt"			).input_features
          self.assertEquals(input_features.shape  ,		(1, 584, 24)			)
          self.assertTrue(np.allclose(input_features[0, 0, :30]  ,		A  ,		atol=1e-4			)			)







import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowercase__				:      Optional[int]	   = os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(F'{bindir}/../../examples/pytorch/translation'):
   from run_translation import main  # noqa


set_seed(42)
lowercase__				:      List[str]	   = "sshleifer/student_marian_en_ro_6_1"
lowercase__				:      List[Any]	   = "sshleifer/tiny-mbart"




@require_torch
class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self  ,		A=False  ,		A=None  ,		A=True  ,		A=True  ,		A=True  ,		A=True  ,		)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=1  ,		max_len=12  ,		model_name=A  ,		num_train_epochs=1  ,		distributed=A  ,		extra_args_str=A  ,		predict_with_generate=A  ,		do_train=A  ,		do_eval=A  ,		do_predict=A  ,		)
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history

          if not do_eval:
               return

          a     =				[log for log in logs if "eval_loss" in log.keys()]

          a     =				eval_metrics[0]
          if predict_with_generate:
               assert "eval_bleu" in first_step_stats

               a     =				eval_metrics[-1]
               assert isinstance(last_step_stats["eval_bleu"]  ,		A			)
               assert not math.isnan(float(last_step_stats["eval_loss"]			)			), "eval_loss must not be `nan`"







     @require_torch_non_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          self.run_seqaseq_quick()







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp simple --fp16"			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2"  ,		predict_with_generate=A			)







     @unittest.skip("Requires an update of the env running those tests"			)
     @require_torch_multi_gpu
     @require_fairscale
     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          self.run_seqaseq_quick(
              distributed=A  ,		extra_args_str="--sharded_ddp zero_dp_2 --fp16"  ,		predict_with_generate=A			)







     @require_apex
     @require_torch_gpu
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)
          # test 2nd time - was getting eval_loss': nan'
          # to reproduce the problem set distributed=False
          self.run_seqaseq_quick(distributed=A  ,		extra_args_str="--fp16 --fp16_backend=apex"			)







     @parameterized.expand(["base", "low", "high", "mixed"]			)
     @require_torch_multi_gpu
     def       lowerCAmelCase_    (     self  ,		A			)  ->   Dict:



          '''simple docstring'''



          a     =				{
              # test with the default log_level - should be info and thus log info once
              "base": {"extra_args_str": "", "n_matches": 1},
              # test with low log_level and log_level_replica - should be noisy on all processes
              # now the info string should appear twice on 2 processes
              "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
              # test with high log_level and low log_level_replica
              # now the info string should appear once only on the replica
              "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
              # test with high log_level and log_level_replica - should be quiet on all processes
              "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
          }

          a     =				experiments[experiment_id]
          a     =				{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
          a     =				"Running training"
          with CaptureStderr() as cl:
               self.run_seqaseq_quick(**A  ,		extra_args_str=data["extra_args_str"]			)
          a     =				len(re.findall(A  ,		cl.err			)			)
          self.assertEqual(A  ,		data["n_matches"]			)







     @slow
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.run_trainer(
              eval_steps=2  ,		max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=10  ,		distributed=A  ,		)

          # Check metrics
          a     =				TrainerState.load_from_json(os.path.join(A  ,		"trainer_state.json"			)			).log_history
          a     =				[log for log in logs if "eval_loss" in log.keys()]
          a     =				eval_metrics[0]
          a     =				eval_metrics[-1]

          assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
          assert isinstance(last_step_stats["eval_bleu"]  ,		A			)

          # test if do_predict saves generations and metrics
          a     =				os.listdir(A			)
          a     =				{os.path.basename(A			) for p in contents}
          assert "generated_predictions.txt" in contents
          assert "predict_results.json" in contents







     @slow
     @require_bitsandbytes
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          from transformers.training_args import OptimizerNames

          def train_and_return_metrics(A			) -> Tuple[int, float]:
               a     =				"--skip_memory_metrics 0"

               a     =				self.run_trainer(
                   max_len=128  ,		model_name=A  ,		learning_rate=3e-4  ,		num_train_epochs=1  ,		optim=A  ,		distributed=A  ,		extra_args_str=A  ,		do_eval=A  ,		do_predict=A  ,		n_gpus_to_use=1  ,		)

               # Check metrics
               a     =				TrainerState.load_from_json(Path(A  ,		"trainer_state.json"			)			).log_history
               a     =				int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20			)
               a     =				int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20			)

               a     =				logs[0]["train_loss"]
               return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value			)
          a  ,				a  ,				a     =				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value			)

          a     =				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

          a     =				gpu_peak_mem_orig + gpu_alloc_mem_orig
          a     =				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
          a     =				gpu_total_mem_orig - gpu_total_mem_bnb

          # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
          # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
          # in 2 bytes and the diff in optim memory usage is derived as so:
          #
          # - normal 25*8=~200MB (8 bytes per param)
          # - bnb    25*2= ~50MB (2 bytes per param)
          #
          # Thus we should expect ~150MB total memory saved.
          #
          # Peak memory should be the same - the total should be different by about that same margin
          #
          # After leaving a small margin to accommodate for differences between gpus let's check
          # that we have at least 120MB in savings
          a     =				120

          # uncomment the following if this test starts failing - requires py38 for a new print feature
          # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
          # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
          # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
          # print(f"{gpu_alloc_mem_diff=}MB")
          # print(f"{gpu_peak_mem_diff=}MB")
          # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
          # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and'''
              F''' gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB'''  ,		)

          self.assertGreater(
              A  ,		A  ,		"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
              F''' a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and'''
              F''' gpu_total_mem_bnb={gpu_total_mem_bnb}MB'''  ,		)

          self.assertEqual(
              A  ,		A  ,		F'''loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}'''			)







     def       lowerCAmelCase_    (     self  ,		A  ,		A  ,		A  ,		A = 3e-3  ,		A = "adafactor"  ,		A = False  ,		A = None  ,		A = 0  ,		A = True  ,		A = True  ,		A = True  ,		A = True  ,		A = None  ,		)  ->   Tuple:



          '''simple docstring'''



          a     =				self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
          a     =				self.get_auto_remove_tmp_dir()
          a     =				F'''
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(A			)}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(A			)}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        '''.split()

          a     =				F'''
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(A			)}
        '''.split()

          a     =				"\n            --do_predict\n        ".split()

          a     =				[]
          if do_train:
               args += args_train

          if do_eval:
               args += args_eval

          if do_predict:
               args += args_predict

          if predict_with_generate:
               args += "--predict_with_generate".split()

          if do_train:
               if optim == "adafactor":
                    args += "--adafactor".split()
               else:
                    args += F'''--optim {optim}'''.split()

          if extra_args_str is not None:
               args += extra_args_str.split()

          if distributed:
               if n_gpus_to_use is None:
                    a     =				get_gpu_count()
               a     =				get_torch_dist_unique_port()
               a     =				F'''
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            '''.split()
               a     =				[sys.executable] + distributed_args + args
               # keep for quick debug
               # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
               execute_subprocess_async(A  ,		env=self.get_env()			)
          else:
               a     =				["run_translation.py"] + args
               with patch.object(A  ,		"argv"  ,		A			):
                    main()

          return output_dir

