





"""simple docstring"""




import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD


def _A       (   ):
   lowercase__			       =  "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"
   lowercase__			       =  Image.open(requests.get(__magic_name__  , stream=__magic_name__	).raw	).convert("RGB"	)

   return image


def _A       (   __magic_name__	):
   lowercase__			       =  []
   # fmt: off

   # vision encoder
   rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding")	)
   rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding")	)
   rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight")	)
   rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias")	)
   rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight")	)
   rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias")	)

   for i in range(config.vision_config.num_hidden_layers	):
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.weight''', f'''vision_model.encoder.layers.{i}.layer_norm1.weight''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.bias''', f'''vision_model.encoder.layers.{i}.layer_norm1.bias''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.weight''', f'''vision_model.encoder.layers.{i}.layer_norm2.weight''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.bias''', f'''vision_model.encoder.layers.{i}.layer_norm2.bias''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.qkv.weight''', f'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.weight''', f'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.bias''', f'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')	)
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')	)

   # QFormer
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.layernorm.weight")	)
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.layernorm.bias")	)

   # fmt: on
   return rename_keys


def _A       (   __magic_name__  , __magic_name__  , __magic_name__	):
   lowercase__			       =  dct.pop(__magic_name__	)
   lowercase__			       =  val


def _A       (   __magic_name__  , __magic_name__	):






   for i in range(config.vision_config.num_hidden_layers	):
      # read in original q and v biases
      lowercase__			       =  state_dict.pop(f'''visual_encoder.blocks.{i}.attn.q_bias'''	)
      lowercase__			       =  state_dict.pop(f'''visual_encoder.blocks.{i}.attn.v_bias'''	)

      # next, set bias in the state dict
      lowercase__			       =  torch.cat((q_bias, torch.zeros_like(__magic_name__  , requires_grad=__magic_name__	), v_bias)	)
      lowercase__			       =  qkv_bias


def _A       (   __magic_name__  , __magic_name__	):
   lowercase__			       =  364 if "coco" in model_name else 224
   lowercase__			       =  BlipaVisionConfig(image_size=__magic_name__	).to_dict()

   # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
   # seems like flan-T5 models don't have bos_token_id properly set?
   if "opt-2.7b" in model_name:
      lowercase__			       =  OPTConfig.from_pretrained("facebook/opt-2.7b"  , eos_token_id=__magic_name__	).to_dict()
   elif "opt-6.7b" in model_name:
      lowercase__			       =  OPTConfig.from_pretrained("facebook/opt-6.7b"  , eos_token_id=__magic_name__	).to_dict()
   elif "t5-xl" in model_name:
      lowercase__			       =  TaConfig.from_pretrained("google/flan-t5-xl"  , dense_act_fn="gelu"  , bos_token_id=1	).to_dict()
   elif "t5-xxl" in model_name:
      lowercase__			       =  TaConfig.from_pretrained("google/flan-t5-xxl"  , dense_act_fn="gelu"  , bos_token_id=1	).to_dict()

   lowercase__			       =  BlipaConfig(vision_config=__magic_name__  , text_config=__magic_name__	)

   return config, image_size


@torch.no_grad()
def _A       (   __magic_name__  , __magic_name__=None  , __magic_name__=False	):
   lowercase__			       =  (
       AutoTokenizer.from_pretrained("facebook/opt-2.7b"	)
       if "opt" in model_name
       else AutoTokenizer.from_pretrained("google/flan-t5-xl"	)
   )
   lowercase__			       =  tokenizer("\n"  , add_special_tokens=__magic_name__	).input_ids[0]
   lowercase__       ,							lowercase__			       =  get_blipa_config(__magic_name__  , eos_token_id=__magic_name__	)

   lowercase__			       =  BlipaForConditionalGeneration(__magic_name__	).eval()

   lowercase__			       =  {
       "blip2-opt-2.7b": ("blip2_opt", "pretrain_opt2.7b"),
       "blip2-opt-6.7b": ("blip2_opt", "pretrain_opt6.7b"),
       "blip2-opt-2.7b-coco": ("blip2_opt", "caption_coco_opt2.7b"),
       "blip2-opt-6.7b-coco": ("blip2_opt", "caption_coco_opt6.7b"),
       "blip2-flan-t5-xl": ("blip2_t5", "pretrain_flant5xl"),
       "blip2-flan-t5-xl-coco": ("blip2_t5", "caption_coco_flant5xl"),
       "blip2-flan-t5-xxl": ("blip2_t5", "pretrain_flant5xxl"),
   }

   lowercase__       ,							lowercase__			       =  model_name_to_original[model_name]

   # load original model
   print("Loading original model..."	)
   lowercase__			       =  "cuda" if torch.cuda.is_available() else "cpu"
   lowercase__       ,							lowercase__       ,							lowercase__			       =  load_model_and_preprocess(
       name=__magic_name__  , model_type=__magic_name__  , is_eval=__magic_name__  , device=__magic_name__	)
   original_model.eval()
   print("Done!"	)

   # update state dict keys
   lowercase__			       =  original_model.state_dict()
   lowercase__			       =  create_rename_keys(__magic_name__	)
   for src, dest in rename_keys:
      rename_key(__magic_name__  , __magic_name__  , __magic_name__	)

   # some keys can be renamed efficiently
   for key, val in state_dict.copy().items():
      lowercase__			       =  state_dict.pop(__magic_name__	)
      if key.startswith("Qformer.bert"	):
         lowercase__			       =  key.replace("Qformer.bert"  , "qformer"	)
      if "attention.self" in key:
         lowercase__			       =  key.replace("self"  , "attention"	)
      if "opt_proj" in key:
         lowercase__			       =  key.replace("opt_proj"  , "language_projection"	)
      if "t5_proj" in key:
         lowercase__			       =  key.replace("t5_proj"  , "language_projection"	)
      if key.startswith("opt"	):
         lowercase__			       =  key.replace("opt"  , "language"	)
      if key.startswith("t5"	):
         lowercase__			       =  key.replace("t5"  , "language"	)
      lowercase__			       =  val

   # read in qv biases
   read_in_q_v_bias(__magic_name__  , __magic_name__	)

   lowercase__       ,							lowercase__			       =  hf_model.load_state_dict(__magic_name__  , strict=__magic_name__	)
   assert len(__magic_name__	) == 0
   assert unexpected_keys == ["qformer.embeddings.position_ids"]

   lowercase__			       =  load_demo_image()
   lowercase__			       =  vis_processors["eval"](__magic_name__	).unsqueeze(0	).to(__magic_name__	)
   lowercase__			       =  tokenizer(["\n"]  , return_tensors="pt"	).input_ids.to(__magic_name__	)

   # create processor
   lowercase__			       =  BlipImageProcessor(
       size={"height": image_size, "width": image_size}  , image_mean=__magic_name__  , image_std=__magic_name__	)
   lowercase__			       =  BlipaProcessor(image_processor=__magic_name__  , tokenizer=__magic_name__	)
   lowercase__			       =  processor(images=__magic_name__  , return_tensors="pt"	).pixel_values.to(__magic_name__	)

   # make sure processor creates exact same pixel values
   assert torch.allclose(__magic_name__  , __magic_name__	)

   original_model.to(__magic_name__	)
   hf_model.to(__magic_name__	)
   with torch.no_grad():
      if "opt" in model_name:
         lowercase__			       =  original_model({"image": original_pixel_values, "text_input": [""]}	).logits
         lowercase__			       =  hf_model(__magic_name__  , __magic_name__	).logits
      else:
         lowercase__			       =  original_model(
             {"image": original_pixel_values, "text_input": ["\n"], "text_output": ["\n"]}	).logits
         lowercase__			       =  input_ids.masked_fill(input_ids == tokenizer.pad_token_id  , -100	)
         lowercase__			       =  hf_model(__magic_name__  , __magic_name__  , labels=__magic_name__	).logits

   assert original_logits.shape == logits.shape
   print("First values of original logits:"  , original_logits[0, :3, :3]	)
   print("First values of HF logits:"  , logits[0, :3, :3]	)

   # assert values
   if model_name == "blip2-flan-t5-xl":
      lowercase__			       =  torch.tensor(
          [[-41.5_850, -4.4_440, -8.9_922], [-47.4_322, -5.9_143, -1.7_340]]  , device=__magic_name__	)
      assert torch.allclose(logits[0, :3, :3]  , __magic_name__  , atol=1e-4	)
   elif model_name == "blip2-flan-t5-xl-coco":
      lowercase__			       =  torch.tensor(
          [[-57.0_109, -9.8_967, -12.6_280], [-68.6_578, -12.7_191, -10.5_065]]  , device=__magic_name__	)
   else:
      # cast to same type
      lowercase__			       =  logits.dtype
      assert torch.allclose(original_logits.to(__magic_name__	)  , __magic_name__  , atol=1e-2	)
   print("Looks ok!"	)

   print("Generating a caption..."	)
   lowercase__			       =  ""
   lowercase__			       =  tokenizer(__magic_name__  , return_tensors="pt"	).input_ids.to(__magic_name__	)

   lowercase__			       =  original_model.generate({"image": original_pixel_values}	)
   lowercase__			       =  hf_model.generate(
       __magic_name__  , __magic_name__  , do_sample=__magic_name__  , num_beams=5  , max_length=30  , min_length=1  , top_p=0.9  , repetition_penalty=1.0  , length_penalty=1.0  , temperature=1  , )
   print("Original generation:"  , __magic_name__	)
   lowercase__			       =  input_ids.shape[1]
   lowercase__			       =  processor.batch_decode(outputs[:, prompt_length:]  , skip_special_tokens=__magic_name__	)
   lowercase__			       =  [text.strip() for text in output_text]
   print("HF generation:"  , __magic_name__	)

   if pytorch_dump_folder_path is not None:
      processor.save_pretrained(__magic_name__	)
      hf_model.save_pretrained(__magic_name__	)

   if push_to_hub:
      processor.push_to_hub(f'''nielsr/{model_name}'''	)
      hf_model.push_to_hub(f'''nielsr/{model_name}'''	)


if __name__ == "__main__":
  _snake_case           =			argparse.ArgumentParser()
  _snake_case           =			[
      "blip2-opt-2.7b",
      "blip2-opt-6.7b",
      "blip2-opt-2.7b-coco",
      "blip2-opt-6.7b-coco",
      "blip2-flan-t5-xl",
      "blip2-flan-t5-xl-coco",
      "blip2-flan-t5-xxl",
  ]
  parser.add_argument(
      """--model_name""",
      default="""blip2-opt-2.7b""",
      choices=choices,
      type=str,
      help="""Path to hf config.json of model to convert""",
  )
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  parser.add_argument(
      """--push_to_hub""",
      action="""store_true""",
      help="""Whether to push the model and processor to the hub after converting""",
  )

  _snake_case           =			parser.parse_args()

  convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)













import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
  from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm


def _A       (   __magic_name__	):
   if is_torch_version("<"  , "2.0.0"	) or not hasattr(__magic_name__  , "_dynamo"	):
      return False
   return isinstance(__magic_name__  , torch._dynamo.eval_frame.OptimizedModule	)


def _A       (   __magic_name__  , __magic_name__ = True	):
   lowercase__			       =  (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

   lowercase__			       =  is_compiled_module(__magic_name__	)
   if is_compiled:
      lowercase__			       =  model
      lowercase__			       =  model._orig_mod

   if is_deepspeed_available():
      options += (DeepSpeedEngine,)

   while isinstance(__magic_name__  , __magic_name__	):
      lowercase__			       =  model.module

   if not keep_fpaa_wrapper:
      lowercase__			       =  getattr(__magic_name__  , "forward"	)
      lowercase__			       =  model.__dict__.pop("_original_forward"  , __magic_name__	)
      if original_forward is not None:
         while hasattr(__magic_name__  , "__wrapped__"	):
            lowercase__			       =  forward.__wrapped__
            if forward == original_forward:
               break
         lowercase__			       =  forward
      if getattr(__magic_name__  , "_converted_to_transformer_engine"  , __magic_name__	):
         convert_model(__magic_name__  , to_transformer_engine=__magic_name__	)

   if is_compiled:
      lowercase__			       =  model
      lowercase__			       =  compiled_model

   return model


def _A       (   ):
   PartialState().wait_for_everyone()


def _A       (   __magic_name__  , __magic_name__	):
   if PartialState().distributed_type == DistributedType.TPU:
      xm.save(__magic_name__  , __magic_name__	)
   elif PartialState().local_process_index == 0:
      torch.save(__magic_name__  , __magic_name__	)


@contextmanager
def _A       (   **__magic_name__	):
   for key, value in kwargs.items():
      lowercase__			       =  str(__magic_name__	)

   yield

   for key in kwargs:
      if key.upper() in os.environ:
         del os.environ[key.upper()]


def _A       (   __magic_name__	):
   if not hasattr(__magic_name__  , "__qualname__"	) and not hasattr(__magic_name__  , "__name__"	):
      lowercase__			       =  getattr(__magic_name__  , "__class__"  , __magic_name__	)
   if hasattr(__magic_name__  , "__qualname__"	):
      return obj.__qualname__
   if hasattr(__magic_name__  , "__name__"	):
      return obj.__name__
   return str(__magic_name__	)


def _A       (   __magic_name__  , __magic_name__	):
   for key, value in source.items():
      if isinstance(__magic_name__  , __magic_name__	):
         lowercase__			       =  destination.setdefault(__magic_name__  , {}	)
         merge_dicts(__magic_name__  , __magic_name__	)
      else:
         lowercase__			       =  value

   return destination







def _A       (   __magic_name__ = None	):
   if port is None:
      lowercase__			       =  2_9500
   with socket.socket(socket.AF_INET  , socket.SOCK_STREAM	) as s:
      return s.connect_ex(("localhost", port)	) == 0






