import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD



def      __SCREAMING_SNAKE_CASE  ( )							-> Tuple:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__      							=				"""https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"""
  SCREAMING_SNAKE_CASE__      							=				Image.open(requests.get(__UpperCamelCase			,					stream=__UpperCamelCase     ).raw     ).convert("""RGB"""     )

  return image



def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	Dict     )							-> Dict:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__      							=				[]
  # fmt: off

  # vision encoder
  rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""")     )
  rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""")     )
  rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""")     )
  rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""")     )
  rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""")     )
  rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""")     )

  for i in range(config.vision_config.num_hidden_layers     ):
    rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.weight""", f"""vision_model.encoder.layers.{i}.layer_norm1.weight""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.bias""", f"""vision_model.encoder.layers.{i}.layer_norm1.bias""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.weight""", f"""vision_model.encoder.layers.{i}.layer_norm2.weight""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.bias""", f"""vision_model.encoder.layers.{i}.layer_norm2.bias""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.attn.qkv.weight""", f"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.weight""", f"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",)     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.bias""", f"""vision_model.encoder.layers.{i}.self_attn.projection.bias""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc1.weight""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc1.bias""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc2.weight""")     )
    rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc2.bias""")     )

  # QFormer
  rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.embeddings.layernorm.weight""")     )
  rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.embeddings.layernorm.bias""")     )

  # fmt: on
  return rename_keys



def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	Union[str, Any]			,					__UpperCamelCase   :	List[Any]			,					__UpperCamelCase   :	List[str]     )							-> Dict:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__      							=				dct.pop(__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				val



def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	Dict			,					__UpperCamelCase   :	Tuple     )							-> Union[str, Any]:







  """simple docstring"""





  for i in range(config.vision_config.num_hidden_layers     ):
    # read in original q and v biases
    SCREAMING_SNAKE_CASE__      							=				state_dict.pop(f"""visual_encoder.blocks.{i}.attn.q_bias"""     )
    SCREAMING_SNAKE_CASE__      							=				state_dict.pop(f"""visual_encoder.blocks.{i}.attn.v_bias"""     )

    # next, set bias in the state dict
    SCREAMING_SNAKE_CASE__      							=				torch.cat((q_bias, torch.zeros_like(__UpperCamelCase			,					requires_grad=__UpperCamelCase     ), v_bias)     )
    SCREAMING_SNAKE_CASE__      							=				qkv_bias



def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	List[Any]     )							-> Any:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__      							=				3_64 if """coco""" in model_name else 2_24
  SCREAMING_SNAKE_CASE__      							=				InstructBlipVisionConfig(image_size=__UpperCamelCase     ).to_dict()

  # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
  # seems like flan-T5 models don't have bos_token_id properly set?
  if "t5-xl" in model_name:
    SCREAMING_SNAKE_CASE__      							=				TaConfig.from_pretrained("""google/flan-t5-xl"""			,					dense_act_fn="""gelu"""			,					bos_token_id=1     ).to_dict()
  elif "t5-xxl" in model_name:
    SCREAMING_SNAKE_CASE__      							=				TaConfig.from_pretrained("""google/flan-t5-xxl"""			,					dense_act_fn="""gelu"""			,					bos_token_id=1     ).to_dict()
  elif "vicuna-7b" in model_name:
    SCREAMING_SNAKE_CASE__      							=				LlamaConfig.from_pretrained("""decapoda-research/llama-7b-hf"""			,					vocab_size=3_20_01     ).to_dict()
  elif "vicuna-13b" in model_name:
    SCREAMING_SNAKE_CASE__      							=				LlamaConfig.from_pretrained("""decapoda-research/llama-13b-hf"""			,					vocab_size=3_20_01     ).to_dict()
  else:
    raise ValueError("""Model name not supported"""     )

  # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
  SCREAMING_SNAKE_CASE__      							=				InstructBlipQFormerConfig(vocab_size=3_05_23     ).to_dict()
  SCREAMING_SNAKE_CASE__      							=				InstructBlipConfig(vision_config=__UpperCamelCase			,					text_config=__UpperCamelCase			,					qformer_config=__UpperCamelCase     )

  return config, image_size



@torch.no_grad()
def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	Union[str, Any]			,					__UpperCamelCase   :	Tuple=None			,					__UpperCamelCase   :	int=False     )							-> Union[str, Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__      							=				AutoTokenizer.from_pretrained("""bert-base-uncased"""			,					truncation_side="""left"""     )
  qformer_tokenizer.add_special_tokens({"""bos_token""": """[DEC]"""}     )

  if "t5" in model_name:
    SCREAMING_SNAKE_CASE__      							=				TaTokenizerFast.from_pretrained("""google/flan-t5-xl"""			,					truncation_side="""left"""     )
  elif "vicuna" in model_name:
    # the following was used in the original implementation:
    # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
    # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
    # tokenizer.add_special_tokens({"bos_token": "</s>"})
    # tokenizer.add_special_tokens({"eos_token": "</s>"})
    # tokenizer.add_special_tokens({"unk_token": "</s>"})
    SCREAMING_SNAKE_CASE__      							=				LlamaTokenizerFast.from_pretrained(
        """huggyllama/llama-7b"""			,					truncation_side="""left"""			,					bos_token="""</s>"""			,					unk_token="""</s>"""     )
    tokenizer.add_special_tokens({"""pad_token""": """[PAD]"""}     )

  SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__      							=				get_blipa_config(__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				InstructBlipForConditionalGeneration(__UpperCamelCase     ).eval()

  SCREAMING_SNAKE_CASE__      							=				{
      """instructblip-vicuna-7b""": ("""blip2_vicuna_instruct""", """vicuna7b"""),
      """instructblip-vicuna-13b""": ("""blip2_vicuna_instruct""", """vicuna13b"""),
      """instructblip-flan-t5-xl""": ("""blip2_t5_instruct""", """flant5xl"""),
      """instructblip-flan-t5-xxl""": ("""blip2_t5_instruct""", """flant5xxl"""),
  }

  SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__      							=				model_name_to_original[model_name]

  # load original model
  print("""Loading original model..."""     )
  SCREAMING_SNAKE_CASE__      							=				"""cuda:1""" if torch.cuda.is_available() else """cpu"""
  SCREAMING_SNAKE_CASE__      							=				"""cuda:2""" if torch.cuda.is_available() else """cpu"""
  SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__      							=				load_model_and_preprocess(
      name=__UpperCamelCase			,					model_type=__UpperCamelCase			,					is_eval=__UpperCamelCase			,					device=__UpperCamelCase     )
  original_model.eval()
  print("""Done!"""     )

  # update state dict keys
  SCREAMING_SNAKE_CASE__      							=				original_model.state_dict()
  SCREAMING_SNAKE_CASE__      							=				create_rename_keys(__UpperCamelCase     )
  for src, dest in rename_keys:
    rename_key(__UpperCamelCase			,					__UpperCamelCase			,					__UpperCamelCase     )

  # some keys can be renamed efficiently
  for key, val in state_dict.copy().items():
    SCREAMING_SNAKE_CASE__      							=				state_dict.pop(__UpperCamelCase     )
    if key.startswith("""Qformer.bert"""     ):
      SCREAMING_SNAKE_CASE__      							=				key.replace("""Qformer.bert"""			,					"""qformer"""     )
    if "attention.self" in key:
      SCREAMING_SNAKE_CASE__      							=				key.replace("""self"""			,					"""attention"""     )
    if "llm_proj" in key:
      SCREAMING_SNAKE_CASE__      							=				key.replace("""llm_proj"""			,					"""language_projection"""     )
    if "t5_proj" in key:
      SCREAMING_SNAKE_CASE__      							=				key.replace("""t5_proj"""			,					"""language_projection"""     )
    if key.startswith("""llm_model"""     ):
      SCREAMING_SNAKE_CASE__      							=				key.replace("""llm_model"""			,					"""language_model"""     )
    if key.startswith("""t5"""     ):
      SCREAMING_SNAKE_CASE__      							=				key.replace("""t5"""			,					"""language"""     )
    SCREAMING_SNAKE_CASE__      							=				val

  # read in qv biases
  read_in_q_v_bias(__UpperCamelCase			,					__UpperCamelCase     )

  # note: weights get loaded in torch.float32 by default
  hf_model.load_state_dict(__UpperCamelCase			,					strict=__UpperCamelCase     )

  SCREAMING_SNAKE_CASE__      							=				load_demo_image()
  SCREAMING_SNAKE_CASE__      							=				"""What is unusual about this image?"""

  # create processor
  SCREAMING_SNAKE_CASE__      							=				BlipImageProcessor(
      size={"""height""": image_size, """width""": image_size}			,					image_mean=__UpperCamelCase			,					image_std=__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				InstructBlipProcessor(
      image_processor=__UpperCamelCase			,					tokenizer=__UpperCamelCase			,					qformer_tokenizer=__UpperCamelCase			,					)
  SCREAMING_SNAKE_CASE__      							=				processor(images=__UpperCamelCase			,					text=__UpperCamelCase			,					return_tensors="""pt"""     ).to(__UpperCamelCase     )

  # make sure processor creates exact same pixel values
  SCREAMING_SNAKE_CASE__      							=				vis_processors["""eval"""](__UpperCamelCase     ).unsqueeze(0     ).to(__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				inputs.pixel_values
  assert torch.allclose(original_pixel_values.to(pixel_values.device     )			,					__UpperCamelCase     )

  original_model.to(__UpperCamelCase     )
  hf_model.to(__UpperCamelCase     )
  with torch.no_grad():
    if "vicuna" in model_name:
      SCREAMING_SNAKE_CASE__      							=				original_model({"""image""": original_pixel_values, """text_input""": [prompt]}     ).logits
      SCREAMING_SNAKE_CASE__      							=				hf_model(**__UpperCamelCase     ).logits
    else:
      SCREAMING_SNAKE_CASE__      							=				original_model(
          {"""image""": original_pixel_values, """text_input""": [prompt], """text_output""": ["""\n"""]}     ).logits
      SCREAMING_SNAKE_CASE__      							=				tokenizer("""\n"""			,					return_tensors="""pt"""     ).input_ids.to(__UpperCamelCase     )
      SCREAMING_SNAKE_CASE__      							=				label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id			,					-1_00     )
      SCREAMING_SNAKE_CASE__      							=				hf_model(**__UpperCamelCase			,					labels=__UpperCamelCase     ).logits

  print("""First values of original logits:"""			,					original_logits[0, :3, :3]     )
  print("""First values of HF logits:"""			,					logits[0, :3, :3]     )

  # assert values
  assert original_logits.shape == logits.shape
  SCREAMING_SNAKE_CASE__      							=				1E-4 if """vicuna""" in model_name else 1E-5
  assert torch.allclose(original_logits.to(logits.device     )			,					__UpperCamelCase			,					atol=__UpperCamelCase     )
  print("""Looks ok!"""     )

  print("""Generating with original model..."""     )
  SCREAMING_SNAKE_CASE__      							=				original_model.generate({"""image""": original_pixel_values, """prompt""": prompt}			,					num_beams=5     )

  # important: we need to cast the weights of the HF model to the appropriate type
  print("""Generating with HF model..."""     )
  SCREAMING_SNAKE_CASE__      							=				hf_model.generate(
      **__UpperCamelCase			,					do_sample=__UpperCamelCase			,					num_beams=5			,					max_length=2_56			,					min_length=1			,					top_p=0.9			,					repetition_penalty=1.5			,					length_penalty=1.0			,					temperature=1			,					)
  if "vicuna" in model_name:
    # convert output id 0 to 2 (eos_token_id)
    # TODO add this in the generate method?
    SCREAMING_SNAKE_CASE__      							=				2
  print("""Original generation:"""			,					__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				processor.batch_decode(__UpperCamelCase			,					skip_special_tokens=__UpperCamelCase     )
  SCREAMING_SNAKE_CASE__      							=				[text.strip() for text in output_text]
  print("""HF generation:"""			,					__UpperCamelCase     )

  if pytorch_dump_folder_path is not None:
    processor.save_pretrained(__UpperCamelCase     )
    hf_model.save_pretrained(__UpperCamelCase     )

  if push_to_hub:
    processor.push_to_hub(f"""Salesforce/{model_name}"""     )
    hf_model.push_to_hub(f"""Salesforce/{model_name}"""     )


if __name__ == "__main__":
   __lowerCamelCase      :			int							       =		argparse.ArgumentParser()
   __lowerCamelCase      :			Optional[int]							       =		[
       '''instructblip-vicuna-7b''',
       '''instructblip-vicuna-13b''',
       '''instructblip-flan-t5-xl''',
       '''instructblip-flan-t5-xxl''',
   ]
   parser.add_argument(
       '''--model_name''',
       default='''instructblip-flan-t5-xl''',
       choices=choices,
       type=str,
       help='''Path to hf config.json of model to convert''',
   )
   parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
   parser.add_argument(
       '''--push_to_hub''',
       action='''store_true''',
       help='''Whether to push the model and processor to the hub after converting''',
   )

   __lowerCamelCase      :			str							       =		parser.parse_args()

   convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)

import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()







class __snake_case    (						unittest.TestCase		):




   def      __a						(   self      :    str				):



     """simple docstring"""






     super().tearDown()
     gc.collect()
     torch.cuda.empty_cache()




   @property
   def      __a						(   self      :    str				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				1
     SCREAMING_SNAKE_CASE__      							=				3
     SCREAMING_SNAKE_CASE__      							=				(32, 32)

     SCREAMING_SNAKE_CASE__      							=				floats_tensor((batch_size, num_channels) + sizes	,     rng=random.Random(0				)				).to(_lowercase				)
     return image




   @property
   def      __a						(   self      :    Dict				):



     """simple docstring"""






     torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				UNetaDConditionModel(
         block_out_channels=(32, 32, 64)	,     layers_per_block=2	,     sample_size=32	,     in_channels=7	,     out_channels=4	,     down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""", """CrossAttnDownBlock2D""")	,     up_block_types=("""CrossAttnUpBlock2D""", """CrossAttnUpBlock2D""", """UpBlock2D""")	,     cross_attention_dim=32	,     attention_head_dim=8	,     use_linear_projection=_lowercase	,     only_cross_attention=(True, True, False)	,     num_class_embeds=1_00	,     )
     return model




   @property
   def      __a						(   self      :    Optional[int]				):



     """simple docstring"""






     torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				AutoencoderKL(
         block_out_channels=[32, 32, 64]	,     in_channels=3	,     out_channels=3	,     down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D""", """DownEncoderBlock2D"""]	,     up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D""", """UpDecoderBlock2D"""]	,     latent_channels=4	,     )
     return model




   @property
   def      __a						(   self      :    Dict				):



     """simple docstring"""






     torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				CLIPTextConfig(
         bos_token_id=0	,     eos_token_id=2	,     hidden_size=32	,     intermediate_size=37	,     layer_norm_eps=1E-05	,     num_attention_heads=4	,     num_hidden_layers=5	,     pad_token_id=1	,     vocab_size=10_00	,     hidden_act="""gelu"""	,     projection_dim=5_12	,     )
     return CLIPTextModel(_lowercase				)




   def      __a						(   self      :    Dict				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""cpu"""  # ensure determinism for the device-dependent torch.Generator
     SCREAMING_SNAKE_CASE__      							=				self.dummy_cond_unet_upscale
     SCREAMING_SNAKE_CASE__      							=				DDPMScheduler()
     SCREAMING_SNAKE_CASE__      							=				DDIMScheduler(prediction_type="""v_prediction"""				)
     SCREAMING_SNAKE_CASE__      							=				self.dummy_vae
     SCREAMING_SNAKE_CASE__      							=				self.dummy_text_encoder
     SCREAMING_SNAKE_CASE__      							=				CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""				)

     SCREAMING_SNAKE_CASE__      							=				self.dummy_image.cpu().permute(0	,     2	,     3	,     1				)[0]
     SCREAMING_SNAKE_CASE__      							=				Image.fromarray(np.uinta(_lowercase				)				).convert("""RGB"""				).resize((64, 64)				)

     # make sure here that pndm scheduler skips prk
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline(
         unet=_lowercase	,     low_res_scheduler=_lowercase	,     scheduler=_lowercase	,     vae=_lowercase	,     text_encoder=_lowercase	,     tokenizer=_lowercase	,     max_noise_level=3_50	,     )
     SCREAMING_SNAKE_CASE__      							=				sd_pipe.to(_lowercase				)
     sd_pipe.set_progress_bar_config(disable=_lowercase				)

     SCREAMING_SNAKE_CASE__      							=				"""A painting of a squirrel eating a burger"""
     SCREAMING_SNAKE_CASE__      							=				torch.Generator(device=_lowercase				).manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				sd_pipe(
         [prompt]	,     image=_lowercase	,     generator=_lowercase	,     guidance_scale=6.0	,     noise_level=20	,     num_inference_steps=2	,     output_type="""np"""	,     )

     SCREAMING_SNAKE_CASE__      							=				output.images

     SCREAMING_SNAKE_CASE__      							=				torch.Generator(device=_lowercase				).manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				sd_pipe(
         [prompt]	,     image=_lowercase	,     generator=_lowercase	,     guidance_scale=6.0	,     noise_level=20	,     num_inference_steps=2	,     output_type="""np"""	,     return_dict=_lowercase	,     )[0]

     SCREAMING_SNAKE_CASE__      							=				image[0, -3:, -3:, -1]
     SCREAMING_SNAKE_CASE__      							=				image_from_tuple[0, -3:, -3:, -1]

     SCREAMING_SNAKE_CASE__      							=				low_res_image.size[0] * 4
     assert image.shape == (1, expected_height_width, expected_height_width, 3)
     SCREAMING_SNAKE_CASE__      							=				np.array([0.31_13, 0.39_10, 0.42_72, 0.48_59, 0.50_61, 0.46_52, 0.53_62, 0.57_15, 0.56_61]				)

     assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2
     assert np.abs(image_from_tuple_slice.flatten() - expected_slice				).max() < 1E-2




   def      __a						(   self      :    Dict				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""cpu"""  # ensure determinism for the device-dependent torch.Generator
     SCREAMING_SNAKE_CASE__      							=				self.dummy_cond_unet_upscale
     SCREAMING_SNAKE_CASE__      							=				DDPMScheduler()
     SCREAMING_SNAKE_CASE__      							=				DDIMScheduler(prediction_type="""v_prediction"""				)
     SCREAMING_SNAKE_CASE__      							=				self.dummy_vae
     SCREAMING_SNAKE_CASE__      							=				self.dummy_text_encoder
     SCREAMING_SNAKE_CASE__      							=				CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""				)

     SCREAMING_SNAKE_CASE__      							=				self.dummy_image.cpu().permute(0	,     2	,     3	,     1				)[0]
     SCREAMING_SNAKE_CASE__      							=				Image.fromarray(np.uinta(_lowercase				)				).convert("""RGB"""				).resize((64, 64)				)

     # make sure here that pndm scheduler skips prk
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline(
         unet=_lowercase	,     low_res_scheduler=_lowercase	,     scheduler=_lowercase	,     vae=_lowercase	,     text_encoder=_lowercase	,     tokenizer=_lowercase	,     max_noise_level=3_50	,     )
     SCREAMING_SNAKE_CASE__      							=				sd_pipe.to(_lowercase				)
     sd_pipe.set_progress_bar_config(disable=_lowercase				)

     SCREAMING_SNAKE_CASE__      							=				"""A painting of a squirrel eating a burger"""
     SCREAMING_SNAKE_CASE__      							=				sd_pipe(
         2 * [prompt]	,     image=2 * [low_res_image]	,     guidance_scale=6.0	,     noise_level=20	,     num_inference_steps=2	,     output_type="""np"""	,     )
     SCREAMING_SNAKE_CASE__      							=				output.images
     assert image.shape[0] == 2

     SCREAMING_SNAKE_CASE__      							=				torch.Generator(device=_lowercase				).manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				sd_pipe(
         [prompt]	,     image=_lowercase	,     generator=_lowercase	,     num_images_per_prompt=2	,     guidance_scale=6.0	,     noise_level=20	,     num_inference_steps=2	,     output_type="""np"""	,     )
     SCREAMING_SNAKE_CASE__      							=				output.images
     assert image.shape[0] == 2




   @unittest.skipIf(torch_device != """cuda"""	,     """This test requires a GPU"""				)
   def      __a						(   self      :    str				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.dummy_cond_unet_upscale
     SCREAMING_SNAKE_CASE__      							=				DDPMScheduler()
     SCREAMING_SNAKE_CASE__      							=				DDIMScheduler(prediction_type="""v_prediction"""				)
     SCREAMING_SNAKE_CASE__      							=				self.dummy_vae
     SCREAMING_SNAKE_CASE__      							=				self.dummy_text_encoder
     SCREAMING_SNAKE_CASE__      							=				CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""				)

     SCREAMING_SNAKE_CASE__      							=				self.dummy_image.cpu().permute(0	,     2	,     3	,     1				)[0]
     SCREAMING_SNAKE_CASE__      							=				Image.fromarray(np.uinta(_lowercase				)				).convert("""RGB"""				).resize((64, 64)				)

     # put models in fp16, except vae as it overflows in fp16
     SCREAMING_SNAKE_CASE__      							=				unet.half()
     SCREAMING_SNAKE_CASE__      							=				text_encoder.half()

     # make sure here that pndm scheduler skips prk
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline(
         unet=_lowercase	,     low_res_scheduler=_lowercase	,     scheduler=_lowercase	,     vae=_lowercase	,     text_encoder=_lowercase	,     tokenizer=_lowercase	,     max_noise_level=3_50	,     )
     SCREAMING_SNAKE_CASE__      							=				sd_pipe.to(_lowercase				)
     sd_pipe.set_progress_bar_config(disable=_lowercase				)

     SCREAMING_SNAKE_CASE__      							=				"""A painting of a squirrel eating a burger"""
     SCREAMING_SNAKE_CASE__      							=				torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				sd_pipe(
         [prompt]	,     image=_lowercase	,     generator=_lowercase	,     num_inference_steps=2	,     output_type="""np"""	,     ).images

     SCREAMING_SNAKE_CASE__      							=				low_res_image.size[0] * 4
     assert image.shape == (1, expected_height_width, expected_height_width, 3)







@slow
@require_torch_gpu
class __snake_case    (						unittest.TestCase		):




   def      __a						(   self      :    Optional[int]				):



     """simple docstring"""






     super().tearDown()
     gc.collect()
     torch.cuda.empty_cache()




   def      __a						(   self      :    str				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				load_image(
         """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
         """/sd2-upscale/low_res_cat.png"""				)
     SCREAMING_SNAKE_CASE__      							=				load_numpy(
         """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"""
         """/upsampled_cat.npy"""				)

     SCREAMING_SNAKE_CASE__      							=				"""stabilityai/stable-diffusion-x4-upscaler"""
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline.from_pretrained(_lowercase				)
     pipe.to(_lowercase				)
     pipe.set_progress_bar_config(disable=_lowercase				)
     pipe.enable_attention_slicing()

     SCREAMING_SNAKE_CASE__      							=				"""a cat sitting on a park bench"""

     SCREAMING_SNAKE_CASE__      							=				torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				pipe(
         prompt=_lowercase	,     image=_lowercase	,     generator=_lowercase	,     output_type="""np"""	,     )
     SCREAMING_SNAKE_CASE__      							=				output.images[0]

     assert image.shape == (5_12, 5_12, 3)
     assert np.abs(expected_image - image				).max() < 1E-3




   def      __a						(   self      :    List[Any]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				load_image(
         """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
         """/sd2-upscale/low_res_cat.png"""				)
     SCREAMING_SNAKE_CASE__      							=				load_numpy(
         """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale"""
         """/upsampled_cat_fp16.npy"""				)

     SCREAMING_SNAKE_CASE__      							=				"""stabilityai/stable-diffusion-x4-upscaler"""
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline.from_pretrained(
         _lowercase	,     torch_dtype=torch.floataa	,     )
     pipe.to(_lowercase				)
     pipe.set_progress_bar_config(disable=_lowercase				)
     pipe.enable_attention_slicing()

     SCREAMING_SNAKE_CASE__      							=				"""a cat sitting on a park bench"""

     SCREAMING_SNAKE_CASE__      							=				torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				pipe(
         prompt=_lowercase	,     image=_lowercase	,     generator=_lowercase	,     output_type="""np"""	,     )
     SCREAMING_SNAKE_CASE__      							=				output.images[0]

     assert image.shape == (5_12, 5_12, 3)
     assert np.abs(expected_image - image				).max() < 5E-1




   def      __a						(   self      :    Any				):



     """simple docstring"""






     torch.cuda.empty_cache()
     torch.cuda.reset_max_memory_allocated()
     torch.cuda.reset_peak_memory_stats()

     SCREAMING_SNAKE_CASE__      							=				load_image(
         """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
         """/sd2-upscale/low_res_cat.png"""				)

     SCREAMING_SNAKE_CASE__      							=				"""stabilityai/stable-diffusion-x4-upscaler"""
     SCREAMING_SNAKE_CASE__      							=				StableDiffusionUpscalePipeline.from_pretrained(
         _lowercase	,     torch_dtype=torch.floataa	,     )
     pipe.to(_lowercase				)
     pipe.set_progress_bar_config(disable=_lowercase				)
     pipe.enable_attention_slicing(1				)
     pipe.enable_sequential_cpu_offload()

     SCREAMING_SNAKE_CASE__      							=				"""a cat sitting on a park bench"""

     SCREAMING_SNAKE_CASE__      							=				torch.manual_seed(0				)
     SCREAMING_SNAKE_CASE__      							=				pipe(
         prompt=_lowercase	,     image=_lowercase	,     generator=_lowercase	,     num_inference_steps=5	,     output_type="""np"""	,     )

     SCREAMING_SNAKE_CASE__      							=				torch.cuda.max_memory_allocated()
     # make sure that less than 2.9 GB is allocated
     assert mem_bytes < 2.9 * 10**9
