


"""simple docstring"""
import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment







def   _a			(   _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE=()			,	_SCREAMING_SNAKE_CASE=None			,	_SCREAMING_SNAKE_CASE="no"			,	_SCREAMING_SNAKE_CASE="29500"			)      ->			Any:
							snake_case_          =						False
							snake_case_          =						False
							if any(key.startswith("""KAGGLE"""			) for key in os.environ.keys()			):
														snake_case_          =						True
							elif "IPython" in sys.modules:
														snake_case_          =						"""google.colab""" in str(sys.modules["""IPython"""].get_ipython()			)

							try:
														snake_case_          =						PrecisionType(mixed_precision.lower()			)
							except ValueError:
														raise ValueError(
														    f"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."""			)

							if (in_colab or in_kaggle) and (os.environ.get("""TPU_NAME"""			,	_UpperCamelCase			) is not None):
														# TPU launch
														import torch_xla.distributed.xla_multiprocessing as xmp

														if len(AcceleratorState._shared_state			) > 0:
																					raise ValueError(
																					    """To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside """
																					    """your training function. Restart your notebook and make sure no cells initializes an """
																					    """`Accelerator`."""			)
														if num_processes is None:
																					snake_case_          =						8

														snake_case_          =						PrepareForLaunch(_UpperCamelCase			,	distributed_type="""TPU"""			)
														print(f"""Launching a training on {num_processes} TPU cores."""			)
														xmp.spawn(_UpperCamelCase			,	args=_UpperCamelCase			,	nprocs=_UpperCamelCase			,	start_method="""fork"""			)
							elif in_colab:
														# No need for a distributed launch otherwise as it's either CPU or one GPU.
														if torch.cuda.is_available():
																					print("""Launching training on one GPU."""			)
														else:
																					print("""Launching training on one CPU."""			)
														function(*_UpperCamelCase			)
							else:
														if num_processes is None:
																					raise ValueError(
																					    """You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."""			)

														if num_processes > 1:
																					# Multi-GPU launch
																					from torch.multiprocessing import start_processes
																					from torch.multiprocessing.spawn import ProcessRaisedException

																					if len(AcceleratorState._shared_state			) > 0:
																												raise ValueError(
																												    """To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized """
																												    """inside your training function. Restart your notebook and make sure no cells initializes an """
																												    """`Accelerator`."""			)

																					if torch.cuda.is_initialized():
																												raise ValueError(
																												    """To launch a multi-GPU training from your notebook, you need to avoid running any instruction """
																												    """using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA """
																												    """function."""			)

																					# torch.distributed will expect a few environment variable to be here. We set the ones common to each
																					# process here (the other ones will be set be the launcher).
																					with patch_environment(
																					    world_size=_UpperCamelCase			,	master_addr="""127.0.01"""			,	master_port=_UpperCamelCase			,	mixed_precision=_UpperCamelCase			):
																												snake_case_          =						PrepareForLaunch(_UpperCamelCase			,	distributed_type="""MULTI_GPU"""			)
																												print(f"""Launching training on {num_processes} GPUs."""			)
																												try:
																																			start_processes(_UpperCamelCase			,	args=_UpperCamelCase			,	nprocs=_UpperCamelCase			,	start_method="""fork"""			)
																												except ProcessRaisedException as e:
																																			if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																																										raise RuntimeError(
																																										    """CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. """
																																										    """This likely stems from an outside import causing issues once the `notebook_launcher()` is called. """
																																										    """Please review your imports and test them when running the `notebook_launcher()` to identify """
																																										    """which one is problematic."""			) from e

														else:
																					# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
																					if is_mps_available():
																												snake_case_          =						"""1"""
																												print("""Launching training on MPS."""			)
																					elif torch.cuda.is_available():
																												print("""Launching training on one GPU."""			)
																					else:
																												print("""Launching training on CPU."""			)
																					function(*_UpperCamelCase			)







def   _a			(   _SCREAMING_SNAKE_CASE			,	_SCREAMING_SNAKE_CASE=()			,	_SCREAMING_SNAKE_CASE=2			)      ->			Union[str, Any]:
							from torch.multiprocessing import start_processes

							with tempfile.NamedTemporaryFile() as tmp_file:
														# torch.distributed will expect a few environment variable to be here. We set the ones common to each
														# process here (the other ones will be set be the launcher).
														with patch_environment(
														    world_size=_UpperCamelCase			,	master_addr="""127.0.01"""			,	master_port="""29500"""			,	accelerate_mixed_precision="""no"""			,	accelerate_debug_rdv_file=tmp_file.name			,	accelerate_use_cpu="""yes"""			,	):
																					snake_case_          =						PrepareForLaunch(_UpperCamelCase			,	debug=_UpperCamelCase			)
																					start_processes(_UpperCamelCase			,	args=_UpperCamelCase			,	nprocs=_UpperCamelCase			,	start_method="""fork"""			)







"""simple docstring"""
import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

__SCREAMING_SNAKE_CASE							:      Tuple						=      [
    'cross_validation.py',
    'gradient_accumulation.py',
    'local_sgd.py',
    'multi_process_metrics.py',
    'memory.py',
    'automatic_gradient_accumulation.py',
    'fsdp_with_peak_mem_tracking.py',
    'deepspeed_with_config_support.py',
    'megatron_lm_gpt_pretraining.py',
]





class        __A       (unittest.TestCase):





							'''simple docstring'''

							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: bool						,    UpperCAmelCase_							: str = None						,    UpperCAmelCase_							: list = None    )   ->List[Any]:



														"""simple docstring"""



														snake_case_          =						None
														snake_case_          =						os.path.abspath(os.path.join("""examples"""						,    """by_feature"""    )    )
														snake_case_          =						os.path.abspath("""examples"""    )
														for item in os.listdir(UpperCAmelCase_    ):
																					if item not in EXCLUDE_EXAMPLES:
																												snake_case_          =						os.path.join(UpperCAmelCase_						,    UpperCAmelCase_    )
																												if os.path.isfile(UpperCAmelCase_    ) and ".py" in item_path:
																																			with self.subTest(
																																			    tested_script=UpperCAmelCase_						,    feature_script=UpperCAmelCase_						,    tested_section="""main()""" if parser_only else """training_function()"""						,    ):
																																										snake_case_          =						compare_against_test(
																																										    os.path.join(UpperCAmelCase_						,    UpperCAmelCase_    )						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )
																																										snake_case_          =						"""\n""".join(UpperCAmelCase_    )
																																										if special_strings is not None:
																																																	for string in special_strings:
																																																								snake_case_          =						diff.replace(UpperCAmelCase_						,    """"""    )
																																										self.assertEqual(UpperCAmelCase_						,    """"""    )



							def 						lowerCAmelCase    (			self							: Tuple    )   ->List[Any]:



														"""simple docstring"""



														self.one_complete_example("""complete_nlp_example.py"""						,    UpperCAmelCase_    )
														self.one_complete_example("""complete_nlp_example.py"""						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Tuple    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						os.path.abspath(os.path.join("""examples"""						,    """cv_example.py"""    )    )
														snake_case_          =						[
														    """ """ * 16 + """{\n\n""",
														    """ """ * 20 + """\"accuracy\": eval_metric[\"accuracy\"],\n\n""",
														    """ """ * 20 + """\"f1\": eval_metric[\"f1\"],\n\n""",
														    """ """ * 20 + """\"train_loss\": total_loss.item() / len(train_dataloader),\n\n""",
														    """ """ * 20 + """\"epoch\": epoch,\n\n""",
														    """ """ * 16 + """},\n\n""",
														    """ """ * 16 + """step=epoch,\n""",
														    """ """ * 12,
														    """ """ * 8 + """for step, batch in enumerate(active_dataloader):\n""",
														]
														self.one_complete_example("""complete_cv_example.py"""						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )
														self.one_complete_example("""complete_cv_example.py"""						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )





@mock.patch.dict(os.environ  ,						{"""TESTING_MOCKED_DATALOADERS""": """1"""})
class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						str							 =      False



							@classmethod
							def 						lowerCAmelCase    (			cls							: Any    )   ->List[str]:



														"""simple docstring"""



														super().setUpClass()
														snake_case_          =						tempfile.mkdtemp()
														snake_case_          =						os.path.join(cls._tmpdir						,    """default_config.yml"""    )

														write_basic_config(save_location=cls.configPath    )
														snake_case_          =						["""accelerate""", """launch""", """--config_file""", cls.configPath]



							@classmethod
							def 						lowerCAmelCase    (			cls							: List[str]    )   ->int:



														"""simple docstring"""



														super().tearDownClass()
														shutil.rmtree(cls._tmpdir    )



							def 						lowerCAmelCase    (			self							: List[str]    )   ->Optional[Any]:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps epoch
        --output_dir {self.tmpdir}
        """.split()
														run_command(self._launch_args + testargs    )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir						,    """epoch_0"""    )    )    )



							def 						lowerCAmelCase    (			self							: Any    )   ->Optional[Any]:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps 1
        --output_dir {self.tmpdir}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs    )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir						,    """step_2"""    )    )    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->str:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir						,    "epoch_0"    )}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
														self.assertNotIn("""epoch 0:"""						,    UpperCAmelCase_    )
														self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir						,    "step_2"    )}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
														if torch.cuda.is_available():
																					snake_case_          =						torch.cuda.device_count()
														else:
																					snake_case_          =						1
														if num_processes > 1:
																					self.assertNotIn("""epoch 0:"""						,    UpperCAmelCase_    )
																					self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )
														else:
																					self.assertIn("""epoch 0:"""						,    UpperCAmelCase_    )
																					self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )



							@slow
							def 						lowerCAmelCase    (			self							: Dict    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						"""
        examples/by_feature/cross_validation.py
        --num_folds 2
        """.split()
														with mock.patch.dict(os.environ						,    {"""TESTING_MOCKED_DATALOADERS""": """0"""}    ):
																					snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
																					snake_case_          =						re.findall("""({.+})"""						,    UpperCAmelCase_    )
																					snake_case_          =						[r for r in results if """accuracy""" in r][-1]
																					snake_case_          =						ast.literal_eval(UpperCAmelCase_    )
																					self.assertGreaterEqual(results["""accuracy"""]						,    0.75    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->int:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/multi_process_metrics.py"""]
														run_command(self._launch_args + testargs    )



							@require_trackers
							@mock.patch.dict(os.environ						,    {"""WANDB_MODE""": """offline"""}    )
							def 						lowerCAmelCase    (			self							: Dict    )   ->Optional[Any]:



														"""simple docstring"""



														with tempfile.TemporaryDirectory() as tmpdir:
																					snake_case_          =						F"""
            examples/by_feature/tracking.py
            --with_tracking
            --project_dir {tmpdir}
            """.split()
																					run_command(self._launch_args + testargs    )
																					self.assertTrue(os.path.exists(os.path.join(UpperCAmelCase_						,    """tracking"""    )    )    )



							def 						lowerCAmelCase    (			self							: List[str]    )   ->Union[str, Any]:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/gradient_accumulation.py"""]
														run_command(self._launch_args + testargs    )



							def 						lowerCAmelCase    (			self							: str    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/local_sgd.py"""]
														run_command(self._launch_args + testargs    )



