



import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the experiment tracking capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

UpperCAmelCase_							:      str      =  16
UpperCAmelCase_							:      Optional[Any]      =  32
def  UpperCamelCase						(       _A			:				Accelerator			,							_A			:				int = 16    )->       Dict:





						"""simple docstring"""





						A__							=       AutoTokenizer.from_pretrained("bert-base-cased"    )
						A__							=       load_dataset("glue"			,							"mrpc"    )

						def tokenize_function(_A			:				Any    ):
												# max_length=None => use the model max length (it's actually the default)
												A__							=       tokenizer(examples["sentence1"]			,							examples["sentence2"]			,							truncation=_A			,							max_length=_A    )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												A__							=       datasets.map(
												    _A			,							batched=_A			,							remove_columns=["idx", "sentence1", "sentence2"]			,							)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						A__							=       tokenized_datasets.rename_column("label"			,							"labels"    )

						def collate_fn(_A			:				List[Any]    ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												A__							=       128 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		A__							=       16
												elif accelerator.mixed_precision != "no":
																		A__							=       8
												else:
																		A__							=       None

												return tokenizer.pad(
												    _A			,							padding="longest"			,							max_length=_A			,							pad_to_multiple_of=_A			,							return_tensors="pt"			,							)

						# Instantiate dataloaders.
						A__							=       DataLoader(
						    tokenized_datasets["train"]			,							shuffle=_A			,							collate_fn=_A			,							batch_size=_A    )
						A__							=       DataLoader(
						    tokenized_datasets["validation"]			,							shuffle=_A			,							collate_fn=_A			,							batch_size=_A    )

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						UpperCAmelCase_							:      Tuple      =  mocked_dataloaders  # noqa: F811
def  UpperCamelCase						(       _A			:				Tuple			,							_A			:				int    )->       Any:





						"""simple docstring"""





						if os.environ.get("TESTING_MOCKED_DATALOADERS"			,							_A    ) == "1":
												A__							=       2
						# Initialize Accelerator

						# New Code #
						# We pass in "all" to `log_with` to grab all available trackers in the environment
						# Note: If using a custom `Tracker` class, should be passed in here such as:
						# >>> log_with = ["all", MyCustomTrackerClassInstance()]
						if args.with_tracking:
												A__							=       Accelerator(
												    cpu=args.cpu			,							mixed_precision=args.mixed_precision			,							log_with="all"			,							project_dir=args.project_dir    )
						else:
												A__							=       Accelerator(cpu=args.cpu			,							mixed_precision=args.mixed_precision    )
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						A__							=       config["lr"]
						A__							=       int(config["num_epochs"]    )
						A__							=       int(config["seed"]    )
						A__							=       int(config["batch_size"]    )
						set_seed(_A    )

						A__						,       A__							=       get_dataloaders(_A			,							_A    )
						A__							=       evaluate.load("glue"			,							"mrpc"    )

						# If the batch size is too big we use gradient accumulation
						A__							=       1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												A__							=       batch_size // MAX_GPU_BATCH_SIZE
												A__							=       MAX_GPU_BATCH_SIZE

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						A__							=       AutoModelForSequenceClassification.from_pretrained("bert-base-cased"			,							return_dict=_A    )

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						A__							=       model.to(accelerator.device    )

						# Instantiate optimizer
						A__							=       AdamW(params=model.parameters()			,							lr=_A    )

						# Instantiate scheduler
						A__							=       get_linear_schedule_with_warmup(
						    optimizer=_A			,							num_warmup_steps=100			,							num_training_steps=(len(_A    ) * num_epochs) // gradient_accumulation_steps			,							)

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						A__						,       A__						,       A__						,       A__						,       A__							=       accelerator.prepare(
						    _A			,							_A			,							_A			,							_A			,							_A    )

						# New Code #
						# We need to initialize the trackers we use. Overall configurations can also be stored
						if args.with_tracking:
												A__							=       os.path.split(_A    )[-1].split("."    )[0]
												accelerator.init_trackers(_A			,							_A    )

						# Now we train the model
						for epoch in range(_A    ):
												model.train()
												# New Code #
												# For our tracking example, we will log the total loss of each epoch
												if args.with_tracking:
																		A__							=       0
												for step, batch in enumerate(_A    ):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device    )
																		A__							=       model(**_A    )
																		A__							=       outputs.loss
																		# New Code #
																		if args.with_tracking:
																								total_loss += loss.detach().float()
																		A__							=       loss / gradient_accumulation_steps
																		accelerator.backward(_A    )
																		if step % gradient_accumulation_steps == 0:
																								optimizer.step()
																								lr_scheduler.step()
																								optimizer.zero_grad()

												model.eval()
												for step, batch in enumerate(_A    ):
																		# We could avoid this line since we set the accelerator with `device_placement=True` (the default).
																		batch.to(accelerator.device    )
																		with torch.no_grad():
																								A__							=       model(**_A    )
																		A__							=       outputs.logits.argmax(dim=-1    )
																		A__						,       A__							=       accelerator.gather_for_metrics((predictions, batch["labels"])    )
																		metric.add_batch(
																		    predictions=_A			,							references=_A			,							)

												A__							=       metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f"""epoch {epoch}:"""			,							_A    )

												# New Code #
												# To actually log, we call `Accelerator.log`
												# The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        "accuracy": eval_metric["accuracy"],
																		        "f1": eval_metric["f1"],
																		        "train_loss": total_loss.item() / len(_A    ),
																		        "epoch": epoch,
																		    }			,							step=_A			,							)

    # New Code #
    # When a run is finished, you should call `accelerator.end_training()`
    # to close all of the open trackers
						if args.with_tracking:
												accelerator.end_training()
def  UpperCamelCase						(       )->       int:





						"""simple docstring"""





						A__							=       argparse.ArgumentParser(description="Simple example of training script."    )
						parser.add_argument(
						    "--mixed_precision"			,							type=_A			,							default=_A			,							choices=["no", "fp16", "bf16", "fp8"]			,							help="Whether to use mixed precision. Choose"
						    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
						    "and an Nvidia Ampere GPU."			,							)
						parser.add_argument("--cpu"			,							action="store_true"			,							help="If passed, will train on the CPU."    )
						parser.add_argument(
						    "--with_tracking"			,							action="store_true"			,							help="Whether to load in all available experiment trackers from the environment and use them for logging."			,							)
						parser.add_argument(
						    "--project_dir"			,							type=_A			,							default="logs"			,							help="Location on where to store experiment tracking logs` and relevent project information"			,							)
						A__							=       parser.parse_args()
						A__							=       {"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
						training_function(_A			,							_A    )


if __name__ == "__main__":
						main()










import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionAttendAndExcitePipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_numpy, skip_mps, slow
from diffusers.utils.testing_utils import require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


UpperCAmelCase_							:      Dict      =  False






@skip_mps
class      UpperCamelCase    (						_UpperCAmelCase       ,    _UpperCAmelCase       ,    _UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   Optional[Any]				=	StableDiffusionAttendAndExcitePipeline
	lowerCAmelCase	:   Optional[Any]				=	False
	lowerCAmelCase	:   Any				=	TEXT_TO_IMAGE_PARAMS
	lowerCAmelCase	:   Dict				=	TEXT_TO_IMAGE_BATCH_PARAMS.union({"""token_indices"""}     )
	lowerCAmelCase	:   List[str]				=	TEXT_TO_IMAGE_IMAGE_PARAMS
	lowerCAmelCase	:   List[Any]				=	TEXT_TO_IMAGE_IMAGE_PARAMS



	@classmethod
	def 			__A							(       cls		):
							super().setUpClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	@classmethod
	def 			__A							(       cls		):
							super().tearDownClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	def 			__A							(       self		):
							torch.manual_seed(0		)
							A__							=       UNetaDConditionModel(
							    block_out_channels=(32, 64)			,	layers_per_block=1			,	sample_size=32			,	in_channels=4			,	out_channels=4			,	down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")			,	up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")			,	cross_attention_dim=32			,	attention_head_dim=(2, 4)			,	use_linear_projection=UpperCAmelCase__			,	)
							A__							=       DDIMScheduler(
							    beta_start=0.00_085			,	beta_end=0.012			,	beta_schedule="scaled_linear"			,	clip_sample=UpperCAmelCase__			,	set_alpha_to_one=UpperCAmelCase__			,	)
							torch.manual_seed(0		)
							A__							=       AutoencoderKL(
							    block_out_channels=[32, 64]			,	in_channels=3			,	out_channels=3			,	down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]			,	up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]			,	latent_channels=4			,	sample_size=128			,	)
							torch.manual_seed(0		)
							A__							=       CLIPTextConfig(
							    bos_token_id=0			,	eos_token_id=2			,	hidden_size=32			,	intermediate_size=37			,	layer_norm_eps=1e-0_5			,	num_attention_heads=4			,	num_hidden_layers=5			,	pad_token_id=1			,	vocab_size=1_000			,	hidden_act="gelu"			,	projection_dim=512			,	)
							A__							=       CLIPTextModel(UpperCAmelCase__		)
							A__							=       CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"		)

							A__							=       {
							    "unet": unet,
							    "scheduler": scheduler,
							    "vae": vae,
							    "text_encoder": text_encoder,
							    "tokenizer": tokenizer,
							    "safety_checker": None,
							    "feature_extractor": None,
							}

							return components



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__=0		):
							if str(UpperCAmelCase__		).startswith("mps"		):
													A__							=       torch.manual_seed(UpperCAmelCase__		)
							else:
													A__							=       torch.Generator(device=UpperCAmelCase__		).manual_seed(UpperCAmelCase__		)
							A__							=       A__							=       {
							    "prompt": "a cat and a frog",
							    "token_indices": [2, 5],
							    "generator": generator,
							    "num_inference_steps": 1,
							    "guidance_scale": 6.0,
							    "output_type": "numpy",
							    "max_iter_to_alter": 2,
							    "thresholds": {0: 0.7},
							}
							return inputs



	def 			__A							(       self		):
							A__							=       "cpu"

							A__							=       self.get_dummy_components()
							A__							=       self.pipeline_class(**UpperCAmelCase__		)
							pipe.to(UpperCAmelCase__		)
							pipe.set_progress_bar_config(disable=UpperCAmelCase__		)

							A__							=       self.get_dummy_inputs(UpperCAmelCase__		)
							A__							=       pipe(**UpperCAmelCase__		).images
							A__							=       image[0, -3:, -3:, -1]

							self.assertEqual(image.shape			,	(1, 64, 64, 3)		)
							A__							=       np.array(
							    [0.63_905_364, 0.62_897_307, 0.48_599_017, 0.5_133_624, 0.5_550_048, 0.45_769_516, 0.50_326_973, 0.5_023_139, 0.45_384_496]		)
							A__							=       np.abs(image_slice.flatten() - expected_slice		).max()
							self.assertLessEqual(UpperCAmelCase__			,	1e-3		)



	def 			__A							(       self		):
							super().test_cpu_offload_forward_pass(expected_max_diff=5e-4		)



	def 			__A							(       self		):
							# NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches
							self._test_inference_batch_consistent(batch_sizes=[1, 2]		)



	def 			__A							(       self		):
							self._test_inference_batch_single_identical(batch_size=2			,	expected_max_diff=7e-4		)



	def 			__A							(       self		):
							super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3		)



	def 			__A							(       self		):
							super().test_pt_np_pil_outputs_equivalent(expected_max_diff=5e-4		)



	def 			__A							(       self		):
							super().test_save_load_local(expected_max_difference=5e-4		)



	def 			__A							(       self		):
							super().test_save_load_optional_components(expected_max_difference=4e-4		)






@require_torch_gpu
@slow
class      UpperCamelCase    (						unittest.TestCase     ):



	@classmethod
	def 			__A							(       cls		):
							super().setUpClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	@classmethod
	def 			__A							(       cls		):
							super().tearDownClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	def 			__A							(       self		):
							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()



	def 			__A							(       self		):
							A__							=       torch.manual_seed(51		)

							A__							=       StableDiffusionAttendAndExcitePipeline.from_pretrained(
							    "CompVis/stable-diffusion-v1-4"			,	safety_checker=UpperCAmelCase__			,	torch_dtype=torch.floataa		)
							pipe.to("cuda"		)

							A__							=       "a painting of an elephant with glasses"
							A__							=       [5, 7]

							A__							=       pipe(
							    prompt=UpperCAmelCase__			,	token_indices=UpperCAmelCase__			,	guidance_scale=7.5			,	generator=UpperCAmelCase__			,	num_inference_steps=5			,	max_iter_to_alter=5			,	output_type="numpy"			,	).images[0]

							A__							=       load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/attend-and-excite/elephant_glasses.npy"		)
							assert np.abs((expected_image - image).max()		) < 5e-1





