



import argparse
import json
import os

import fairseq
import torch
from torch import nn

from transformers import (
    SpeechaTextaConfig,
    SpeechaTextaForCausalLM,
    SpeechaTextaTokenizer,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
UpperCAmelCase_							:      Union[str, Any]      =  logging.get_logger(__name__)

UpperCAmelCase_							:      Union[str, Any]      =  {
    "post_extract_proj": "feature_projection.projection",
    "encoder.pos_conv.0": "encoder.pos_conv_embed.conv",
    "self_attn.k_proj": "encoder.layers.*.attention.k_proj",
    "self_attn.v_proj": "encoder.layers.*.attention.v_proj",
    "self_attn.q_proj": "encoder.layers.*.attention.q_proj",
    "self_attn.out_proj": "encoder.layers.*.attention.out_proj",
    "self_attn_layer_norm": "encoder.layers.*.layer_norm",
    "fc1": "encoder.layers.*.feed_forward.intermediate_dense",
    "fc2": "encoder.layers.*.feed_forward.output_dense",
    "final_layer_norm": "encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "encoder.layer_norm",
    "w2v_model.layer_norm": "feature_projection.layer_norm",
    "quantizer.weight_proj": "quantizer.weight_proj",
    "quantizer.vars": "quantizer.codevectors",
    "project_q": "project_q",
    "final_proj": "project_hid",
    "w2v_encoder.proj": "lm_head",
    "mask_emb": "masked_spec_embed",
}
UpperCAmelCase_							:      Optional[int]      =  [
    "lm_head",
    "quantizer.weight_proj",
    "quantizer.codevectors",
    "project_q",
    "project_hid",
]
def  UpperCamelCase						(       _A			:				Any			,							_A			:				List[str]			,							_A			:				int			,							_A			:				Optional[int]			,							_A			:				Any    )->       Any:





						"""simple docstring"""





						for attribute in key.split("."    ):
												A__							=       getattr(_A			,							_A    )

						if weight_type is not None:
												A__							=       getattr(_A			,							_A    ).shape
						else:
												A__							=       hf_pointer.shape

						assert hf_shape == value.shape, (
						    f"""Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be"""
						    f""" {value.shape} for {full_name}"""
						)

						if weight_type == "weight":
												A__							=       value
						elif weight_type == "weight_g":
												A__							=       value
						elif weight_type == "weight_v":
												A__							=       value
						elif weight_type == "bias":
												A__							=       value
						else:
												A__							=       value

						logger.info(f"""{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}."""    )
def  UpperCamelCase						(       _A			:				Union[str, Any]			,							_A			:				List[Any]    )->       int:





						"""simple docstring"""





						A__							=       []
						A__							=       fairseq_model.state_dict()

						A__							=       hf_model.feature_extractor

						# if encoder has different dim to decoder -> use proj_weight
						A__							=       None

						for name, value in fairseq_dict.items():
												A__							=       False
												if "conv_layers" in name:
																		load_conv_layer(
																		    _A			,							_A			,							_A			,							_A			,							hf_model.config.feat_extract_norm == "group"			,							)
																		A__							=       True
												elif name.split("."    )[0] == "proj":
																		A__							=       fairseq_model.proj
																		A__							=       True
												else:
																		for key, mapped_key in MAPPING.items():
																								if key in name or key.split("w2v_model."    )[-1] == name.split("."    )[0]:
																														A__							=       True
																														if "*" in mapped_key:
																																				A__							=       name.split(_A    )[0].split("."    )[-2]
																																				A__							=       mapped_key.replace("*"			,							_A    )
																														if "weight_g" in name:
																																				A__							=       "weight_g"
																														elif "weight_v" in name:
																																				A__							=       "weight_v"
																														elif "bias" in name:
																																				A__							=       "bias"
																														elif "weight" in name:
																																				A__							=       "weight"
																														else:
																																				A__							=       None
																														set_recursively(_A			,							_A			,							_A			,							_A			,							_A    )
																								continue
												if not is_used:
																		unused_weights.append(_A    )

						logger.warning(f"""Unused weights: {unused_weights}"""    )

						return proj_weight
def  UpperCamelCase						(       _A			:				Union[str, Any]			,							_A			:				Union[str, Any]			,							_A			:				Optional[Any]			,							_A			:				Optional[int]			,							_A			:				Optional[Any]    )->       List[Any]:





						"""simple docstring"""





						A__							=       full_name.split("conv_layers."    )[-1]
						A__							=       name.split("."    )
						A__							=       int(items[0]    )
						A__							=       int(items[1]    )

						if type_id == 0:
												if "bias" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
																		    f"""{full_name} has size {value.shape}, but"""
																		    f""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""
																		)
																		A__							=       value
																		logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""    )
												elif "weight" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
																		    f"""{full_name} has size {value.shape}, but"""
																		    f""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""
																		)
																		A__							=       value
																		logger.info(f"""Feat extract conv layer {layer_id} was initialized from {full_name}."""    )
						elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
												if "bias" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
																		    f"""{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was"""
																		    " found."
																		)
																		A__							=       value
																		logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""    )
												elif "weight" in name:
																		assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
																		    f"""{full_name} has size {value.shape}, but"""
																		    f""" {feature_extractor[layer_id].layer_norm.weight.data.shape} was found."""
																		)
																		A__							=       value
																		logger.info(f"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""    )
						else:
												unused_weights.append(_A    )
def  UpperCamelCase						(       _A			:				Tuple    )->       Optional[int]:





						"""simple docstring"""





						A__						,       A__							=       emb.weight.shape
						A__							=       nn.Linear(_A			,							_A			,							bias=_A    )
						A__							=       emb.weight.data
						return lin_layer
def  UpperCamelCase						(       _A			:				Any    )->       Dict:





						"""simple docstring"""





						with open(_A			,							"r"			,							encoding="utf-8"    ) as f:
												A__							=       f.readlines()
												A__							=       [line.split(" "    )[0] for line in lines]

						A__							=       len(_A    )

						A__							=       {
						    "<s>": 0,
						    "<pad>": 1,
						    "</s>": 2,
						    "<unk>": 3,
						}

						vocab_dict.update(dict(zip(_A			,							range(4			,							num_words + 4    )    )    )    )
						return vocab_dict
@torch.no_grad()
def  UpperCamelCase						(       _A			:				Optional[int]			,							_A			:				Optional[int]			,							_A			:				Union[str, Any]			,							_A			:				List[str]			,							_A			:				Any			,							_A			:				List[str]			,							_A			:				str			,							)->       int:





						"""simple docstring"""





						A__							=       WavaVecaConfig.from_pretrained(_A    )
						A__							=       SpeechaTextaConfig.from_pretrained(
						    _A			,							vocab_size=_A			,							decoder_layers=_A			,							do_stable_layer_norm=_A    )

						A__							=       WavaVecaFeatureExtractor(
						    feature_size=1			,							sampling_rate=16000			,							padding_value=0			,							do_normalize=_A			,							return_attention_mask=_A			,							)

						A__						,       A__						,       A__							=       fairseq.checkpoint_utils.load_model_ensemble_and_task(
						    [checkpoint_path]			,							arg_overrides={"data": "/".join(dict_path.split("/"    )[:-1]    )}    )
						A__							=       model[0].eval()

						# set weights for wav2vec2 encoder
						A__							=       WavaVecaModel(_A    )
						A__							=       recursively_load_weights_wavaveca(model.encoder			,							_A    )

						A__							=       SpeechaTextaForCausalLM(_A    )
						A__						,       A__							=       hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict()			,							strict=_A    )

						# set output linear layer
						unexpected_keys.remove("embed_out"    )
						A__							=       nn.Parameter(model.decoder.embed_out.detach()    )

						# layer norm is init to identity matrix so leaving it is fine
						logger.warning(f"""The following keys are missing when loading the decoder weights: {missing_keys}"""    )
						logger.warning(f"""The following keys are unexpected when loading the decoder weights: {unexpected_keys}"""    )

						A__							=       SpeechEncoderDecoderModel(encoder=_A			,							decoder=_A    )
						A__							=       False

						# add projection layer
						A__							=       nn.Parameter(projection_layer.weight    )
						A__							=       nn.Parameter(projection_layer.bias    )

						A__							=       create_vocab_dict(_A    )

						with open(os.path.join(_A			,							"vocab.json"    )			,							"w"    ) as fp:
												json.dump(_A			,							_A    )

						A__							=       SpeechaTextaTokenizer(os.path.join(_A			,							"vocab.json"    )    )
						tokenizer.save_pretrained(_A    )

						A__							=       hf_wavavec.config.to_dict()
						A__							=       tokenizer.pad_token_id
						A__							=       tokenizer.bos_token_id
						A__							=       tokenizer.eos_token_id
						A__							=       "speech_to_text_2"
						A__							=       "wav2vec2"

						A__							=       SpeechEncoderDecoderConfig.from_dict(_A    )

						hf_wavavec.save_pretrained(_A    )
						feature_extractor.save_pretrained(_A    )


if __name__ == "__main__":
						UpperCAmelCase_							:      Optional[int]      =  argparse.ArgumentParser()
						parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
						parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint")
						parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model")
						parser.add_argument(
						    "--encoder_config_path",
						    default="facebook/wav2vec2-large-lv60",
						    type=str,
						    help="Path to hf encoder wav2vec2 checkpoint config",
						)
						parser.add_argument(
						    "--decoder_config_path",
						    default="facebook/s2t-small-mustc-en-fr-st",
						    type=str,
						    help="Path to hf decoder s2t checkpoint config",
						)
						parser.add_argument("--vocab_size", default=10_224, type=int, help="Vocab size of decoder")
						parser.add_argument("--num_decoder_layers", default=7, type=int, help="Number of decoder layers")

						UpperCAmelCase_							:      Dict      =  parser.parse_args()
						convert_wavaveca_checkpoint(
						    args.checkpoint_path,
						    args.pytorch_dump_folder_path,
						    args.dict_path,
						    encoder_config_path=args.encoder_config_path,
						    decoder_config_path=args.decoder_config_path,
						    vocab_size=args.vocab_size,
						    num_decoder_layers=args.num_decoder_layers,
						)










import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionAttendAndExcitePipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_numpy, skip_mps, slow
from diffusers.utils.testing_utils import require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


UpperCAmelCase_							:      Dict      =  False






@skip_mps
class      UpperCamelCase    (						_UpperCAmelCase       ,    _UpperCAmelCase       ,    _UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   Optional[Any]				=	StableDiffusionAttendAndExcitePipeline
	lowerCAmelCase	:   Optional[Any]				=	False
	lowerCAmelCase	:   Any				=	TEXT_TO_IMAGE_PARAMS
	lowerCAmelCase	:   Dict				=	TEXT_TO_IMAGE_BATCH_PARAMS.union({"""token_indices"""}     )
	lowerCAmelCase	:   List[str]				=	TEXT_TO_IMAGE_IMAGE_PARAMS
	lowerCAmelCase	:   List[Any]				=	TEXT_TO_IMAGE_IMAGE_PARAMS



	@classmethod
	def 			__A							(       cls		):
							super().setUpClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	@classmethod
	def 			__A							(       cls		):
							super().tearDownClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	def 			__A							(       self		):
							torch.manual_seed(0		)
							A__							=       UNetaDConditionModel(
							    block_out_channels=(32, 64)			,	layers_per_block=1			,	sample_size=32			,	in_channels=4			,	out_channels=4			,	down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")			,	up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")			,	cross_attention_dim=32			,	attention_head_dim=(2, 4)			,	use_linear_projection=UpperCAmelCase__			,	)
							A__							=       DDIMScheduler(
							    beta_start=0.00_085			,	beta_end=0.012			,	beta_schedule="scaled_linear"			,	clip_sample=UpperCAmelCase__			,	set_alpha_to_one=UpperCAmelCase__			,	)
							torch.manual_seed(0		)
							A__							=       AutoencoderKL(
							    block_out_channels=[32, 64]			,	in_channels=3			,	out_channels=3			,	down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]			,	up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]			,	latent_channels=4			,	sample_size=128			,	)
							torch.manual_seed(0		)
							A__							=       CLIPTextConfig(
							    bos_token_id=0			,	eos_token_id=2			,	hidden_size=32			,	intermediate_size=37			,	layer_norm_eps=1e-0_5			,	num_attention_heads=4			,	num_hidden_layers=5			,	pad_token_id=1			,	vocab_size=1_000			,	hidden_act="gelu"			,	projection_dim=512			,	)
							A__							=       CLIPTextModel(UpperCAmelCase__		)
							A__							=       CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"		)

							A__							=       {
							    "unet": unet,
							    "scheduler": scheduler,
							    "vae": vae,
							    "text_encoder": text_encoder,
							    "tokenizer": tokenizer,
							    "safety_checker": None,
							    "feature_extractor": None,
							}

							return components



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__=0		):
							if str(UpperCAmelCase__		).startswith("mps"		):
													A__							=       torch.manual_seed(UpperCAmelCase__		)
							else:
													A__							=       torch.Generator(device=UpperCAmelCase__		).manual_seed(UpperCAmelCase__		)
							A__							=       A__							=       {
							    "prompt": "a cat and a frog",
							    "token_indices": [2, 5],
							    "generator": generator,
							    "num_inference_steps": 1,
							    "guidance_scale": 6.0,
							    "output_type": "numpy",
							    "max_iter_to_alter": 2,
							    "thresholds": {0: 0.7},
							}
							return inputs



	def 			__A							(       self		):
							A__							=       "cpu"

							A__							=       self.get_dummy_components()
							A__							=       self.pipeline_class(**UpperCAmelCase__		)
							pipe.to(UpperCAmelCase__		)
							pipe.set_progress_bar_config(disable=UpperCAmelCase__		)

							A__							=       self.get_dummy_inputs(UpperCAmelCase__		)
							A__							=       pipe(**UpperCAmelCase__		).images
							A__							=       image[0, -3:, -3:, -1]

							self.assertEqual(image.shape			,	(1, 64, 64, 3)		)
							A__							=       np.array(
							    [0.63_905_364, 0.62_897_307, 0.48_599_017, 0.5_133_624, 0.5_550_048, 0.45_769_516, 0.50_326_973, 0.5_023_139, 0.45_384_496]		)
							A__							=       np.abs(image_slice.flatten() - expected_slice		).max()
							self.assertLessEqual(UpperCAmelCase__			,	1e-3		)



	def 			__A							(       self		):
							super().test_cpu_offload_forward_pass(expected_max_diff=5e-4		)



	def 			__A							(       self		):
							# NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches
							self._test_inference_batch_consistent(batch_sizes=[1, 2]		)



	def 			__A							(       self		):
							self._test_inference_batch_single_identical(batch_size=2			,	expected_max_diff=7e-4		)



	def 			__A							(       self		):
							super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3		)



	def 			__A							(       self		):
							super().test_pt_np_pil_outputs_equivalent(expected_max_diff=5e-4		)



	def 			__A							(       self		):
							super().test_save_load_local(expected_max_difference=5e-4		)



	def 			__A							(       self		):
							super().test_save_load_optional_components(expected_max_difference=4e-4		)






@require_torch_gpu
@slow
class      UpperCamelCase    (						unittest.TestCase     ):



	@classmethod
	def 			__A							(       cls		):
							super().setUpClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	@classmethod
	def 			__A							(       cls		):
							super().tearDownClass()
							torch.use_deterministic_algorithms(UpperCAmelCase__		)



	def 			__A							(       self		):
							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()



	def 			__A							(       self		):
							A__							=       torch.manual_seed(51		)

							A__							=       StableDiffusionAttendAndExcitePipeline.from_pretrained(
							    "CompVis/stable-diffusion-v1-4"			,	safety_checker=UpperCAmelCase__			,	torch_dtype=torch.floataa		)
							pipe.to("cuda"		)

							A__							=       "a painting of an elephant with glasses"
							A__							=       [5, 7]

							A__							=       pipe(
							    prompt=UpperCAmelCase__			,	token_indices=UpperCAmelCase__			,	guidance_scale=7.5			,	generator=UpperCAmelCase__			,	num_inference_steps=5			,	max_iter_to_alter=5			,	output_type="numpy"			,	).images[0]

							A__							=       load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/attend-and-excite/elephant_glasses.npy"		)
							assert np.abs((expected_image - image).max()		) < 5e-1





