


import argparse

import torch

from transformers import (
    SpeechTaConfig,
    SpeechTaFeatureExtractor,
    SpeechTaForSpeechToSpeech,
    SpeechTaForSpeechToText,
    SpeechTaForTextToSpeech,
    SpeechTaProcessor,
    SpeechTaTokenizer,
    logging,
)
from transformers.tokenization_utils import AddedToken


logging.set_verbosity_info()
lowercase_	   =     logging.get_logger('transformers.models.speecht5')

lowercase_	   =     {
    'speech_encoder_prenet.layer_norm': 'speecht5.encoder.prenet.feature_projection.layer_norm',
    'speech_encoder_prenet.post_extract_proj': 'speecht5.encoder.prenet.feature_projection.projection',
    'speech_encoder_prenet.pos_conv.0': 'speecht5.encoder.prenet.pos_conv_embed.conv',
    'speech_encoder_prenet.mask_emb': 'speecht5.encoder.prenet.masked_spec_embed',
}
lowercase_	   =     {
    'text_encoder_prenet.encoder_prenet.0': 'speecht5.encoder.prenet.embed_tokens',
    'text_encoder_prenet.encoder_prenet.1.alpha': 'speecht5.encoder.prenet.encode_positions.alpha',
}
lowercase_	   =     {
    'speech_decoder_prenet.decoder_prenet.0.0.prenet.0.0': 'speecht5.decoder.prenet.layers.0',
    'speech_decoder_prenet.decoder_prenet.0.0.prenet.1.0': 'speecht5.decoder.prenet.layers.1',
    'speech_decoder_prenet.decoder_prenet.0.1': 'speecht5.decoder.prenet.final_layer',
    'speech_decoder_prenet.decoder_prenet.1.alpha': 'speecht5.decoder.prenet.encode_positions.alpha',
    'speech_decoder_prenet.spkembs_layer.0': 'speecht5.decoder.prenet.speaker_embeds_layer',
}
lowercase_	   =     {
    'speech_decoder_postnet.feat_out': 'speech_decoder_postnet.feat_out',
    'speech_decoder_postnet.prob_out': 'speech_decoder_postnet.prob_out',
    'speech_decoder_postnet.postnet.postnet.0.0': 'speech_decoder_postnet.layers.0.conv',
    'speech_decoder_postnet.postnet.postnet.0.1': 'speech_decoder_postnet.layers.0.batch_norm',
    'speech_decoder_postnet.postnet.postnet.1.0': 'speech_decoder_postnet.layers.1.conv',
    'speech_decoder_postnet.postnet.postnet.1.1': 'speech_decoder_postnet.layers.1.batch_norm',
    'speech_decoder_postnet.postnet.postnet.2.0': 'speech_decoder_postnet.layers.2.conv',
    'speech_decoder_postnet.postnet.postnet.2.1': 'speech_decoder_postnet.layers.2.batch_norm',
    'speech_decoder_postnet.postnet.postnet.3.0': 'speech_decoder_postnet.layers.3.conv',
    'speech_decoder_postnet.postnet.postnet.3.1': 'speech_decoder_postnet.layers.3.batch_norm',
    'speech_decoder_postnet.postnet.postnet.4.0': 'speech_decoder_postnet.layers.4.conv',
    'speech_decoder_postnet.postnet.postnet.4.1': 'speech_decoder_postnet.layers.4.batch_norm',
}
lowercase_	   =     {
    'text_decoder_prenet.embed_tokens': 'speecht5.decoder.prenet.embed_tokens',
}
lowercase_	   =     {
    'text_decoder_postnet.output_projection': 'text_decoder_postnet.lm_head',
}
lowercase_	   =     {
    'encoder.layers.*.self_attn.k_proj': 'speecht5.encoder.wrapped_encoder.layers.*.attention.k_proj',
    'encoder.layers.*.self_attn.v_proj': 'speecht5.encoder.wrapped_encoder.layers.*.attention.v_proj',
    'encoder.layers.*.self_attn.q_proj': 'speecht5.encoder.wrapped_encoder.layers.*.attention.q_proj',
    'encoder.layers.*.self_attn.out_proj': 'speecht5.encoder.wrapped_encoder.layers.*.attention.out_proj',
    'encoder.layers.*.self_attn_layer_norm': 'speecht5.encoder.wrapped_encoder.layers.*.layer_norm',
    'encoder.layers.*.fc1': 'speecht5.encoder.wrapped_encoder.layers.*.feed_forward.intermediate_dense',
    'encoder.layers.*.fc2': 'speecht5.encoder.wrapped_encoder.layers.*.feed_forward.output_dense',
    'encoder.layers.*.final_layer_norm': 'speecht5.encoder.wrapped_encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'speecht5.encoder.wrapped_encoder.layer_norm',
    'encoder.pos_emb.pe_k': 'speecht5.encoder.wrapped_encoder.embed_positions.pe_k',
}
lowercase_	   =     {
    'decoder.layers.*.self_attn.k_proj': 'speecht5.decoder.wrapped_decoder.layers.*.self_attn.k_proj',
    'decoder.layers.*.self_attn.v_proj': 'speecht5.decoder.wrapped_decoder.layers.*.self_attn.v_proj',
    'decoder.layers.*.self_attn.q_proj': 'speecht5.decoder.wrapped_decoder.layers.*.self_attn.q_proj',
    'decoder.layers.*.self_attn.out_proj': 'speecht5.decoder.wrapped_decoder.layers.*.self_attn.out_proj',
    'decoder.layers.*.self_attn_layer_norm': 'speecht5.decoder.wrapped_decoder.layers.*.self_attn_layer_norm',
    'decoder.layers.*.encoder_attn.k_proj': 'speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.k_proj',
    'decoder.layers.*.encoder_attn.v_proj': 'speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.v_proj',
    'decoder.layers.*.encoder_attn.q_proj': 'speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.q_proj',
    'decoder.layers.*.encoder_attn.out_proj': 'speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.out_proj',
    'decoder.layers.*.encoder_attn_layer_norm': 'speecht5.decoder.wrapped_decoder.layers.*.encoder_attn_layer_norm',
    'decoder.layers.*.fc1': 'speecht5.decoder.wrapped_decoder.layers.*.feed_forward.intermediate_dense',
    'decoder.layers.*.fc2': 'speecht5.decoder.wrapped_decoder.layers.*.feed_forward.output_dense',
    'decoder.layers.*.final_layer_norm': 'speecht5.decoder.wrapped_decoder.layers.*.final_layer_norm',
}
lowercase_	   =     {
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_TEXT_DECODER_PRENET,
    **MAPPING_TEXT_DECODER_POSTNET,
}
lowercase_	   =     {
    **MAPPING_TEXT_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
lowercase_	   =     {
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
lowercase_	   =     []
lowercase_	   =     [
    'encoder.version',
    'encoder.layers.*.norm_k.weight',
    'encoder.layers.*.norm_k.bias',
    'decoder.version',
    'decoder.layers.*.norm_k.weight',
    'decoder.layers.*.norm_k.bias',
    'decoder.pos_emb.pe_k',
    'speech_encoder_prenet.embed_positions._float_tensor',
    'text_decoder_prenet.embed_positions._float_tensor',
]
lowercase_	   =     IGNORE_KEYS + [
    'encoder.proj',
    'text_encoder_prenet.*',
    'speech_decoder_prenet.*',
    'speech_decoder_postnet.*',
]
lowercase_	   =     IGNORE_KEYS + [
    'encoder.proj',
    'speech_encoder_prenet.*',
    'text_decoder_prenet.*',
    'text_decoder_postnet.*',
]
lowercase_	   =     IGNORE_KEYS + [
    'encoder.proj',
    'text_encoder_prenet.*',
    'text_decoder_prenet.*',
    'text_decoder_postnet.*',
]
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 for attribute in key.split('.'						):
  __lowerCamelCase		:							str		 =						getattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)

 if weight_type is not None:
  __lowerCamelCase		:							str		 =						getattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						).shape
 else:
  __lowerCamelCase		:							Any		 =						hf_pointer.shape

 if hf_shape != value.shape:
  raise ValueError(
      f'Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'
      f' {value.shape} for {full_name}'						)

 if weight_type == "weight":
  __lowerCamelCase		:							str		 =						value
 elif weight_type == "weight_g":
  __lowerCamelCase		:							Any		 =						value
 elif weight_type == "weight_v":
  __lowerCamelCase		:							int		 =						value
 elif weight_type == "bias":
  __lowerCamelCase		:							List[Any]		 =						value
 elif weight_type == "running_mean":
  __lowerCamelCase		:							List[str]		 =						value
 elif weight_type == "running_var":
  __lowerCamelCase		:							str		 =						value
 elif weight_type == "num_batches_tracked":
  __lowerCamelCase		:							str		 =						value
 else:
  __lowerCamelCase		:							Optional[Any]		 =						value

 logger.info(f'{key + ("." + weight_type if weight_type is not None else "")} was initialized from {full_name}.'						)
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 for key in ignore_keys:
  if key.endswith('.*'						):
   if name.startswith(key[:-1]						):
    return True
  elif ".*." in key:
   __lowerCamelCase ,					__lowerCamelCase		:							str		 =						key.split('.*.'						)
   if prefix in name and suffix in name:
    return True
  elif key in name:
   return True
 return False
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Tuple		 =						[]

 if task == "s2t":
  __lowerCamelCase		:							Optional[Any]		 =						hf_model.speechta.encoder.prenet.feature_encoder
  __lowerCamelCase		:							Optional[Any]		 =						MAPPING_S2T
  __lowerCamelCase		:							Optional[int]		 =						IGNORE_KEYS_S2T
 elif task == "t2s":
  __lowerCamelCase		:							List[str]		 =						None
  __lowerCamelCase		:							Optional[int]		 =						MAPPING_T2S
  __lowerCamelCase		:							List[str]		 =						IGNORE_KEYS_T2S
 elif task == "s2s":
  __lowerCamelCase		:							Any		 =						hf_model.speechta.encoder.prenet.feature_encoder
  __lowerCamelCase		:							Optional[Any]		 =						MAPPING_S2S
  __lowerCamelCase		:							Union[str, Any]		 =						IGNORE_KEYS_S2S
 else:
  raise ValueError(f'Unsupported task: {task}'						)

 for name, value in fairseq_dict.items():
  if should_ignore(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
   logger.info(f'{name} was ignored'						)
   continue

  __lowerCamelCase		:							Optional[int]		 =						False
  if "conv_layers" in name:
   load_conv_layer(
       SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	hf_model.config.feat_extract_norm == 'group'      ,	)
   __lowerCamelCase		:							Any		 =						True
  else:
   for key, mapped_key in MAPPING.items():
    # mapped_key = "speecht5." + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key

    if "*" in key:
     __lowerCamelCase ,					__lowerCamelCase		:							List[str]		 =						key.split('.*.'						)
     if prefix in name and suffix in name:
      __lowerCamelCase		:							Optional[Any]		 =						suffix

                # if key in name or key.split("w2v_model.")[-1] == name.split(".")[0]:
    if key in name:
     __lowerCamelCase		:							Optional[int]		 =						True
     if "*" in mapped_key:
      __lowerCamelCase		:							List[str]		 =						name.split(SCREAMING_SNAKE_CASE__						)[0].split('.'						)[-2]
      __lowerCamelCase		:							Any		 =						mapped_key.replace('*'      ,	SCREAMING_SNAKE_CASE__						)
     if "weight_g" in name:
      __lowerCamelCase		:							List[Any]		 =						'weight_g'
     elif "weight_v" in name:
      __lowerCamelCase		:							Dict		 =						'weight_v'
     elif "bias" in name:
      __lowerCamelCase		:							Any		 =						'bias'
     elif "weight" in name:
      __lowerCamelCase		:							List[Any]		 =						'weight'
     elif "running_mean" in name:
      __lowerCamelCase		:							Union[str, Any]		 =						'running_mean'
     elif "running_var" in name:
      __lowerCamelCase		:							Optional[int]		 =						'running_var'
     elif "num_batches_tracked" in name:
      __lowerCamelCase		:							Optional[Any]		 =						'num_batches_tracked'
     else:
      __lowerCamelCase		:							Dict		 =						None
     set_recursively(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)
    continue
  if not is_used:
   unused_weights.append(SCREAMING_SNAKE_CASE__						)

 logger.warning(f'Unused weights: {unused_weights}'						)
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Dict		 =						full_name.split('conv_layers.'						)[-1]
 __lowerCamelCase		:							List[Any]		 =						name.split('.'						)
 __lowerCamelCase		:							str		 =						int(items[0]						)
 __lowerCamelCase		:							List[str]		 =						int(items[1]						)

 if type_id == 0:
  if "bias" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
    raise ValueError(
        f'{full_name} has size {value.shape}, but'
        f' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'						)
   __lowerCamelCase		:							Dict		 =						value
   logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'						)
  elif "weight" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
    raise ValueError(
        f'{full_name} has size {value.shape}, but'
        f' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'						)
   __lowerCamelCase		:							Union[str, Any]		 =						value
   logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'						)
 elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
  if "bias" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
    raise ValueError(
        f'{full_name} has size {value.shape}, but'
        f' {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.'						)
   __lowerCamelCase		:							Union[str, Any]		 =						value
   logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'						)
  elif "weight" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
    raise ValueError(
        f'{full_name} has size {value.shape}, but'
        f' {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.'						)
   __lowerCamelCase		:							int		 =						value
   logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'						)
 else:
  unused_weights.append(SCREAMING_SNAKE_CASE__						)
@torch.no_grad()
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__=None      ,	SCREAMING_SNAKE_CASE__=None      ,	SCREAMING_SNAKE_CASE__=None      ,	):
 if config_path is not None:
  __lowerCamelCase		:							Optional[int]		 =						SpeechTaConfig.from_pretrained(SCREAMING_SNAKE_CASE__						)
 else:
  __lowerCamelCase		:							Optional[int]		 =						SpeechTaConfig()

 if task == "s2t":
  __lowerCamelCase		:							Union[str, Any]		 =						config.max_text_positions
  __lowerCamelCase		:							List[Any]		 =						SpeechTaForSpeechToText(SCREAMING_SNAKE_CASE__						)
 elif task == "t2s":
  __lowerCamelCase		:							Tuple		 =						1_876
  __lowerCamelCase		:							int		 =						600
  __lowerCamelCase		:							Union[str, Any]		 =						config.max_speech_positions
  __lowerCamelCase		:							Optional[Any]		 =						SpeechTaForTextToSpeech(SCREAMING_SNAKE_CASE__						)
 elif task == "s2s":
  __lowerCamelCase		:							Optional[int]		 =						1_876
  __lowerCamelCase		:							Union[str, Any]		 =						config.max_speech_positions
  __lowerCamelCase		:							str		 =						SpeechTaForSpeechToSpeech(SCREAMING_SNAKE_CASE__						)
 else:
  raise ValueError(f'Unknown task name: {task}'						)

 if vocab_path:
  __lowerCamelCase		:							List[str]		 =						SpeechTaTokenizer(SCREAMING_SNAKE_CASE__      ,	model_max_length=config.max_text_positions						)

  # Mask token behaves like a normal word, i.e. include the space before it
  __lowerCamelCase		:							Optional[int]		 =						AddedToken('<mask>'      ,	lstrip=SCREAMING_SNAKE_CASE__      ,	rstrip=SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							Any		 =						mask_token
  tokenizer.add_special_tokens({'mask_token': mask_token}						)
  tokenizer.add_tokens(['<ctc_blank>']						)

 __lowerCamelCase		:							int		 =						SpeechTaFeatureExtractor()
 __lowerCamelCase		:							Dict		 =						SpeechTaProcessor(tokenizer=SCREAMING_SNAKE_CASE__      ,	feature_extractor=SCREAMING_SNAKE_CASE__						)
 processor.save_pretrained(SCREAMING_SNAKE_CASE__						)

 __lowerCamelCase		:							Optional[int]		 =						torch.load(SCREAMING_SNAKE_CASE__						)
 recursively_load_weights(fairseq_checkpoint['model']      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)

 model.save_pretrained(SCREAMING_SNAKE_CASE__						)

 if repo_id:
  print('Pushing to the hub...'						)
  processor.push_to_hub(SCREAMING_SNAKE_CASE__						)
  model.push_to_hub(SCREAMING_SNAKE_CASE__						)


if __name__ == "__main__":
       lowercase_	   =     argparse.ArgumentParser()
       parser.add_argument(
           '--task',
           default='s2t',
           type=str,
           help='Type of the SpeechT5 model you\'d like to convert. Should be one of \'s2t\', \'t2s\', \'s2s\'.',
       )
       parser.add_argument('--checkpoint_path', required=True, default=None, type=str, help='Path to fairseq checkpoint')
       parser.add_argument('--vocab_path', default=None, type=str, help='Path to SentencePiece model')
       parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
       parser.add_argument(
           '--pytorch_dump_folder_path', required=True, default=None, type=str, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--push_to_hub', default=None, type=str, help='Where to upload the converted model on the 🤗 hub.'
       )

       lowercase_	   =     parser.parse_args()
       convert_speechta_checkpoint(
           args.task,
           args.checkpoint_path,
           args.pytorch_dump_folder_path,
           args.config_path,
           args.vocab_path,
           args.push_to_hub,
       )





import inspect
from typing import Optional, Union

import numpy as np
import PIL
import torch
from torch.nn import functional as F
from torchvision import transforms
from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    DPMSolverMultistepScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
from diffusers.utils import (
    PIL_INTERPOLATION,
    randn_tensor,
)
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 if isinstance(SCREAMING_SNAKE_CASE__      ,	torch.Tensor						):
  return image
 elif isinstance(SCREAMING_SNAKE_CASE__      ,	PIL.Image.Image						):
  __lowerCamelCase		:							Union[str, Any]		 =						[image]

 if isinstance(image[0]      ,	PIL.Image.Image						):
  __lowerCamelCase		:							Any		 =						[np.array(i.resize((w, h)      ,	resample=PIL_INTERPOLATION['lanczos']						)						)[None, :] for i in image]
  __lowerCamelCase		:							Optional[int]		 =						np.concatenate(SCREAMING_SNAKE_CASE__      ,	axis=0						)
  __lowerCamelCase		:							str		 =						np.array(SCREAMING_SNAKE_CASE__						).astype(np.floataa						) / 255.0
  __lowerCamelCase		:							List[str]		 =						image.transpose(0      ,	3      ,	1      ,	2						)
  __lowerCamelCase		:							Union[str, Any]		 =						2.0 * image - 1.0
  __lowerCamelCase		:							Tuple		 =						torch.from_numpy(SCREAMING_SNAKE_CASE__						)
 elif isinstance(image[0]      ,	torch.Tensor						):
  __lowerCamelCase		:							str		 =						torch.cat(SCREAMING_SNAKE_CASE__      ,	dim=0						)
 return image
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__=0.9_995						):
 if not isinstance(SCREAMING_SNAKE_CASE__      ,	np.ndarray						):
  __lowerCamelCase		:							List[str]		 =						True
  __lowerCamelCase		:							str		 =						va.device
  __lowerCamelCase		:							int		 =						va.cpu().numpy()
  __lowerCamelCase		:							List[str]		 =						va.cpu().numpy()

 __lowerCamelCase		:							str		 =						np.sum(va * va / (np.linalg.norm(SCREAMING_SNAKE_CASE__						) * np.linalg.norm(SCREAMING_SNAKE_CASE__						))						)
 if np.abs(SCREAMING_SNAKE_CASE__						) > DOT_THRESHOLD:
  __lowerCamelCase		:							Union[str, Any]		 =						(1 - t) * va + t * va
 else:
  __lowerCamelCase		:							List[Any]		 =						np.arccos(SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							Dict		 =						np.sin(SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							str		 =						theta_a * t
  __lowerCamelCase		:							List[Any]		 =						np.sin(SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							str		 =						np.sin(theta_a - theta_t						) / sin_theta_a
  __lowerCamelCase		:							List[Any]		 =						sin_theta_t / sin_theta_a
  __lowerCamelCase		:							Union[str, Any]		 =						sa * va + sa * va

 if inputs_are_torch:
  __lowerCamelCase		:							str		 =						torch.from_numpy(SCREAMING_SNAKE_CASE__						).to(SCREAMING_SNAKE_CASE__						)

 return va
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							List[Any]		 =						F.normalize(SCREAMING_SNAKE_CASE__      ,	dim=-1						)
 __lowerCamelCase		:							Union[str, Any]		 =						F.normalize(SCREAMING_SNAKE_CASE__      ,	dim=-1						)
 return (x - y).norm(dim=-1						).div(2						).arcsin().pow(2						).mul(2						)
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 for param in model.parameters():
  __lowerCamelCase		:							Any		 =						value



class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      def __init__(    self:	Any						,				a:	AutoencoderKL						,				a:	CLIPTextModel						,				a:	CLIPModel						,				a:	CLIPTokenizer						,				a:	UNetaDConditionModel						,				a:	Union[PNDMScheduler, LMSDiscreteScheduler, DDIMScheduler, DPMSolverMultistepScheduler]						,				a:	CLIPFeatureExtractor						,				a:	Union[str, Any]=None						,				a:	Union[str, Any]=None						,				a:	Union[str, Any]=None						,				):
       super().__init__()
       self.register_modules(
           vae=a						,				text_encoder=a						,				clip_model=a						,				tokenizer=a						,				unet=a						,				scheduler=a						,				feature_extractor=a						,				coca_model=a						,				coca_tokenizer=a						,				coca_transform=a						,				)
       __lowerCamelCase		:							Tuple		 =						(
           feature_extractor.size
           if isinstance(feature_extractor.size						,				a    )
           else feature_extractor.size['shortest_edge']
       )
       __lowerCamelCase		:							List[Any]		 =						transforms.Normalize(mean=feature_extractor.image_mean						,				std=feature_extractor.image_std    )
       set_requires_grad(self.text_encoder						,				a    )
       set_requires_grad(self.clip_model						,				a    )




      def 					_snake_case     (    self:	Optional[Any]						,				a:	Optional[Union[str, int]] = "auto"    ):
       if slice_size == "auto":
        # half the attention head size is usually a good trade-off between
        # speed and memory
        __lowerCamelCase		:							Any		 =						self.unet.config.attention_head_dim // 2
       self.unet.set_attention_slice(a    )




      def 					_snake_case     (    self:	Dict    ):
       self.enable_attention_slicing(a    )




      def 					_snake_case     (    self:	Optional[Any]    ):
       set_requires_grad(self.vae						,				a    )




      def 					_snake_case     (    self:	List[Any]    ):
       set_requires_grad(self.vae						,				a    )




      def 					_snake_case     (    self:	int    ):
       set_requires_grad(self.unet						,				a    )




      def 					_snake_case     (    self:	int    ):
       set_requires_grad(self.unet						,				a    )




      def 					_snake_case     (    self:	Optional[Any]						,				a:	Union[str, Any]						,				a:	List[str]						,				a:	List[Any]    ):
       # get the original timestep using init_timestep
       __lowerCamelCase		:							List[Any]		 =						min(int(num_inference_steps * strength    )						,				a    )

       __lowerCamelCase		:							str		 =						max(num_inference_steps - init_timestep						,				0    )
       __lowerCamelCase		:							List[Any]		 =						self.scheduler.timesteps[t_start:]

       return timesteps, num_inference_steps - t_start




      def 					_snake_case     (    self:	Union[str, Any]						,				a:	Optional[Any]						,				a:	Any						,				a:	Optional[int]						,				a:	Optional[Any]						,				a:	Union[str, Any]						,				a:	List[str]=None    ):
       if not isinstance(a						,				torch.Tensor    ):
        raise ValueError(F'`image` has to be of type `torch.Tensor` but is {type(a    )}'    )

       __lowerCamelCase		:							Union[str, Any]		 =						image.to(device=a						,				dtype=a    )

       if isinstance(a						,				a    ):
        __lowerCamelCase		:							str		 =						[
            self.vae.encode(image[i : i + 1]    ).latent_dist.sample(generator[i]    ) for i in range(a    )
        ]
        __lowerCamelCase		:							Tuple		 =						torch.cat(a						,				dim=0    )
       else:
        __lowerCamelCase		:							List[Any]		 =						self.vae.encode(a    ).latent_dist.sample(a    )

       # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
       __lowerCamelCase		:							List[str]		 =						0.1_8_2_1_5 * init_latents
       __lowerCamelCase		:							Union[str, Any]		 =						init_latents.repeat_interleave(a						,				dim=0    )

       __lowerCamelCase		:							Optional[int]		 =						randn_tensor(init_latents.shape						,				generator=a						,				device=a						,				dtype=a    )

       # get latents
       __lowerCamelCase		:							Union[str, Any]		 =						self.scheduler.add_noise(a						,				a						,				a    )
       __lowerCamelCase		:							int		 =						init_latents

       return latents




      def 					_snake_case     (    self:	Optional[int]						,				a:	Any    ):
       __lowerCamelCase		:							List[Any]		 =						self.coca_transform(a    ).unsqueeze(0    )
       with torch.no_grad(), torch.cuda.amp.autocast():
        __lowerCamelCase		:							Any		 =						self.coca_model.generate(transformed_image.to(device=self.device						,				dtype=self.coca_model.dtype    )    )
       __lowerCamelCase		:							str		 =						self.coca_tokenizer.decode(generated[0].cpu().numpy()    )
       return generated.split('<end_of_text>'    )[0].replace('<start_of_text>'						,				''    ).rstrip(' .,'    )




      def 					_snake_case     (    self:	Any						,				a:	Tuple						,				a:	Tuple    ):
       __lowerCamelCase		:							Dict		 =						self.feature_extractor.preprocess(a    )
       __lowerCamelCase		:							Dict		 =						torch.from_numpy(clip_image_input['pixel_values'][0]    ).unsqueeze(0    ).to(self.device    ).half()
       __lowerCamelCase		:							List[str]		 =						self.clip_model.get_image_features(a    )
       __lowerCamelCase		:							Optional[Any]		 =						image_embeddings_clip / image_embeddings_clip.norm(p=2						,				dim=-1						,				keepdim=a    )
       __lowerCamelCase		:							Tuple		 =						image_embeddings_clip.repeat_interleave(a						,				dim=0    )
       return image_embeddings_clip




      @torch.enable_grad()
      def 					_snake_case     (    self:	str						,				a:	str						,				a:	int						,				a:	List[Any]						,				a:	str						,				a:	List[Any]						,				a:	Dict						,				a:	int						,				):
       __lowerCamelCase		:							Optional[Any]		 =						latents.detach().requires_grad_()

       __lowerCamelCase		:							str		 =						self.scheduler.scale_model_input(a						,				a    )

       # predict the noise residual
       __lowerCamelCase		:							Optional[int]		 =						self.unet(a						,				a						,				encoder_hidden_states=a    ).sample

       if isinstance(self.scheduler						,				(PNDMScheduler, DDIMScheduler, DPMSolverMultistepScheduler)    ):
        __lowerCamelCase		:							str		 =						self.scheduler.alphas_cumprod[timestep]
        __lowerCamelCase		:							Dict		 =						1 - alpha_prod_t
        # compute predicted original sample from predicted noise also called
        # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        __lowerCamelCase		:							Optional[int]		 =						(latents - beta_prod_t ** 0.5 * noise_pred) / alpha_prod_t ** 0.5

        __lowerCamelCase		:							Optional[int]		 =						torch.sqrt(a    )
        __lowerCamelCase		:							int		 =						pred_original_sample * (fac) + latents * (1 - fac)
       elif isinstance(self.scheduler						,				a    ):
        __lowerCamelCase		:							str		 =						self.scheduler.sigmas[index]
        __lowerCamelCase		:							List[Any]		 =						latents - sigma * noise_pred
       else:
        raise ValueError(F'scheduler type {type(self.scheduler    )} not supported'    )

       # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
       __lowerCamelCase		:							Optional[int]		 =						1 / 0.1_8_2_1_5 * sample
       __lowerCamelCase		:							Optional[Any]		 =						self.vae.decode(a    ).sample
       __lowerCamelCase		:							Tuple		 =						(image / 2 + 0.5).clamp(0						,				1    )

       __lowerCamelCase		:							Any		 =						transforms.Resize(self.feature_extractor_size    )(a    )
       __lowerCamelCase		:							Union[str, Any]		 =						self.normalize(a    ).to(latents.dtype    )

       __lowerCamelCase		:							Tuple		 =						self.clip_model.get_image_features(a    )
       __lowerCamelCase		:							List[str]		 =						image_embeddings_clip / image_embeddings_clip.norm(p=2						,				dim=-1						,				keepdim=a    )

       __lowerCamelCase		:							List[str]		 =						spherical_dist_loss(a						,				a    ).mean() * clip_guidance_scale

       __lowerCamelCase		:							Tuple		 =						-torch.autograd.grad(a						,				a    )[0]

       if isinstance(self.scheduler						,				a    ):
        __lowerCamelCase		:							Optional[int]		 =						latents.detach() + grads * (sigma**2)
        __lowerCamelCase		:							List[Any]		 =						noise_pred_original
       else:
        __lowerCamelCase		:							str		 =						noise_pred_original - torch.sqrt(a    ) * grads
       return noise_pred, latents




      @torch.no_grad()
      def __call__(    self:	Any						,				a:	Union[torch.FloatTensor, PIL.Image.Image]						,				a:	Union[torch.FloatTensor, PIL.Image.Image]						,				a:	Optional[str] = None						,				a:	Optional[str] = None						,				a:	Optional[int] = 512						,				a:	Optional[int] = 512						,				a:	float = 0.6						,				a:	Optional[int] = 50						,				a:	Optional[float] = 7.5						,				a:	Optional[int] = 1						,				a:	float = 0.0						,				a:	Optional[float] = 100						,				a:	Optional[torch.Generator] = None						,				a:	Optional[str] = "pil"						,				a:	bool = True						,				a:	float = 0.8						,				a:	float = 0.1						,				a:	float = 0.1						,				):
       if isinstance(a						,				a    ) and len(a    ) != batch_size:
        raise ValueError(F'You have passed {batch_size} batch_size, but only {len(a    )} generators.'    )

       if height % 8 != 0 or width % 8 != 0:
        raise ValueError(F'`height` and `width` have to be divisible by 8 but are {height} and {width}.'    )

       if isinstance(a						,				torch.Generator    ) and batch_size > 1:
        __lowerCamelCase		:							List[Any]		 =						[generator] + [None] * (batch_size - 1)

       __lowerCamelCase		:							Dict		 =						[
           ('model', self.coca_model is None),
           ('tokenizer', self.coca_tokenizer is None),
           ('transform', self.coca_transform is None),
       ]
       __lowerCamelCase		:							Any		 =						[x[0] for x in coca_is_none if x[1]]
       __lowerCamelCase		:							str		 =						', '.join(a    )
       # generate prompts with coca model if prompt is None
       if content_prompt is None:
        if len(a    ):
         raise ValueError(
             F'Content prompt is None and CoCa [{coca_is_none_str}] is None.'
             F'Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'    )
        __lowerCamelCase		:							Any		 =						self.get_image_description(a    )
       if style_prompt is None:
        if len(a    ):
         raise ValueError(
             F'Style prompt is None and CoCa [{coca_is_none_str}] is None.'
             F' Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'    )
        __lowerCamelCase		:							Tuple		 =						self.get_image_description(a    )

       # get prompt text embeddings for content and style
       __lowerCamelCase		:							int		 =						self.tokenizer(
           a						,				padding='max_length'						,				max_length=self.tokenizer.model_max_length						,				truncation=a						,				return_tensors='pt'						,				)
       __lowerCamelCase		:							Dict		 =						self.text_encoder(content_text_input.input_ids.to(self.device    )    )[0]

       __lowerCamelCase		:							Union[str, Any]		 =						self.tokenizer(
           a						,				padding='max_length'						,				max_length=self.tokenizer.model_max_length						,				truncation=a						,				return_tensors='pt'						,				)
       __lowerCamelCase		:							Any		 =						self.text_encoder(style_text_input.input_ids.to(self.device    )    )[0]

       __lowerCamelCase		:							List[Any]		 =						slerp(a						,				a						,				a    )

       # duplicate text embeddings for each generation per prompt
       __lowerCamelCase		:							Any		 =						text_embeddings.repeat_interleave(a						,				dim=0    )

       # set timesteps
       __lowerCamelCase		:							List[Any]		 =						'offset' in set(inspect.signature(self.scheduler.set_timesteps    ).parameters.keys()    )
       __lowerCamelCase		:							Union[str, Any]		 =						{}
       if accepts_offset:
        __lowerCamelCase		:							Dict		 =						1

       self.scheduler.set_timesteps(a						,				**a    )
       # Some schedulers like PNDM have timesteps as arrays
       # It's more optimized to move all timesteps to correct device beforehand
       self.scheduler.timesteps.to(self.device    )

       __lowerCamelCase ,					__lowerCamelCase		:							Dict		 =						self.get_timesteps(a						,				a						,				self.device    )
       __lowerCamelCase		:							Tuple		 =						timesteps[:1].repeat(a    )

       # Preprocess image
       __lowerCamelCase		:							Any		 =						preprocess(a						,				a						,				a    )
       __lowerCamelCase		:							str		 =						self.prepare_latents(
           a						,				a						,				a						,				text_embeddings.dtype						,				self.device						,				a    )

       __lowerCamelCase		:							Dict		 =						preprocess(a						,				a						,				a    )
       __lowerCamelCase		:							Optional[int]		 =						self.prepare_latents(
           a						,				a						,				a						,				text_embeddings.dtype						,				self.device						,				a    )

       __lowerCamelCase		:							int		 =						slerp(a						,				a						,				a    )

       if clip_guidance_scale > 0:
        __lowerCamelCase		:							List[str]		 =						self.get_clip_image_embeddings(a						,				a    )
        __lowerCamelCase		:							Union[str, Any]		 =						self.get_clip_image_embeddings(a						,				a    )
        __lowerCamelCase		:							Union[str, Any]		 =						slerp(
            a						,				a						,				a    )

       # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
       # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
       # corresponds to doing no classifier free guidance.
       __lowerCamelCase		:							Tuple		 =						guidance_scale > 1.0
       # get unconditional embeddings for classifier free guidance
       if do_classifier_free_guidance:
        __lowerCamelCase		:							Optional[int]		 =						content_text_input.input_ids.shape[-1]
        __lowerCamelCase		:							int		 =						self.tokenizer(['']						,				padding='max_length'						,				max_length=a						,				return_tensors='pt'    )
        __lowerCamelCase		:							List[Any]		 =						self.text_encoder(uncond_input.input_ids.to(self.device    )    )[0]
        # duplicate unconditional embeddings for each generation per prompt
        __lowerCamelCase		:							List[Any]		 =						uncond_embeddings.repeat_interleave(a						,				dim=0    )

        # For classifier free guidance, we need to do two forward passes.
        # Here we concatenate the unconditional and text embeddings into a single batch
        # to avoid doing two forward passes
        __lowerCamelCase		:							int		 =						torch.cat([uncond_embeddings, text_embeddings]    )

       # get the initial random noise unless the user supplied it

       # Unlike in other pipelines, latents need to be generated in the target device
       # for 1-to-1 results reproducibility with the CompVis implementation.
       # However this currently doesn't work in `mps`.
       __lowerCamelCase		:							str		 =						(batch_size, self.unet.config.in_channels, height // 8, width // 8)
       __lowerCamelCase		:							List[str]		 =						text_embeddings.dtype
       if latents is None:
        if self.device.type == "mps":
         # randn does not work reproducibly on mps
         __lowerCamelCase		:							Tuple		 =						torch.randn(a						,				generator=a						,				device='cpu'						,				dtype=a    ).to(
             self.device    )
        else:
         __lowerCamelCase		:							List[Any]		 =						torch.randn(a						,				generator=a						,				device=self.device						,				dtype=a    )
       else:
        if latents.shape != latents_shape:
         raise ValueError(F'Unexpected latents shape, got {latents.shape}, expected {latents_shape}'    )
        __lowerCamelCase		:							List[str]		 =						latents.to(self.device    )

       # scale the initial noise by the standard deviation required by the scheduler
       __lowerCamelCase		:							str		 =						latents * self.scheduler.init_noise_sigma

       # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
       # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
       # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
       # and should be between [0, 1]
       __lowerCamelCase		:							int		 =						'eta' in set(inspect.signature(self.scheduler.step    ).parameters.keys()    )
       __lowerCamelCase		:							Dict		 =						{}
       if accepts_eta:
        __lowerCamelCase		:							List[str]		 =						eta

       # check if the scheduler accepts generator
       __lowerCamelCase		:							Optional[int]		 =						'generator' in set(inspect.signature(self.scheduler.step    ).parameters.keys()    )
       if accepts_generator:
        __lowerCamelCase		:							Optional[Any]		 =						generator

       with self.progress_bar(total=a    ):
        for i, t in enumerate(a    ):
         # expand the latents if we are doing classifier free guidance
         __lowerCamelCase		:							Optional[int]		 =						torch.cat([latents] * 2    ) if do_classifier_free_guidance else latents
         __lowerCamelCase		:							Union[str, Any]		 =						self.scheduler.scale_model_input(a						,				a    )

         # predict the noise residual
         __lowerCamelCase		:							Tuple		 =						self.unet(a						,				a						,				encoder_hidden_states=a    ).sample

         # perform classifier free guidance
         if do_classifier_free_guidance:
          __lowerCamelCase ,					__lowerCamelCase		:							str		 =						noise_pred.chunk(2    )
          __lowerCamelCase		:							List[str]		 =						noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

         # perform clip guidance
         if clip_guidance_scale > 0:
          __lowerCamelCase		:							str		 =						(
              text_embeddings.chunk(2    )[1] if do_classifier_free_guidance else text_embeddings
          )
          __lowerCamelCase ,					__lowerCamelCase		:							int		 =						self.cond_fn(
              a						,				a						,				a						,				a						,				a						,				a						,				a						,				)

         # compute the previous noisy sample x_t -> x_t-1
         __lowerCamelCase		:							Tuple		 =						self.scheduler.step(a						,				a						,				a						,				**a    ).prev_sample

        # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
       __lowerCamelCase		:							List[Any]		 =						1 / 0.1_8_2_1_5 * latents
       __lowerCamelCase		:							Union[str, Any]		 =						self.vae.decode(a    ).sample

       __lowerCamelCase		:							Optional[int]		 =						(image / 2 + 0.5).clamp(0						,				1    )
       __lowerCamelCase		:							Any		 =						image.cpu().permute(0						,				2						,				3						,				1    ).numpy()

       if output_type == "pil":
        __lowerCamelCase		:							Union[str, Any]		 =						self.numpy_to_pil(a    )

       if not return_dict:
        return (image, None)

       return StableDiffusionPipelineOutput(images=a						,				nsfw_content_detected=a    )

