

import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD

def      snake_case__      (   ):



     '''simple docstring'''





     lowercase__   :       Tuple				=	'https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg'
     lowercase__   :       Any				=	Image.open(requests.get(a__     ,					stream=a__      ).raw      ).convert('RGB'      )

     return image

def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Optional[Any]      ):



     '''simple docstring'''





     lowercase__   :       Tuple				=	[]
     # fmt: off

     # vision encoder
     rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding')      )
     rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding')      )
     rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight')      )
     rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias')      )
     rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight')      )
     rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias')      )

     for i in range(config.vision_config.num_hidden_layers      ):
          rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.weight""", f"""vision_model.encoder.layers.{i}.layer_norm1.weight""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.bias""", f"""vision_model.encoder.layers.{i}.layer_norm1.bias""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.weight""", f"""vision_model.encoder.layers.{i}.layer_norm2.weight""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.bias""", f"""vision_model.encoder.layers.{i}.layer_norm2.bias""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.attn.qkv.weight""", f"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.weight""", f"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",)      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.bias""", f"""vision_model.encoder.layers.{i}.self_attn.projection.bias""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc1.weight""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc1.bias""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc2.weight""")      )
          rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc2.bias""")      )

     # QFormer
     rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.embeddings.layernorm.weight')      )
     rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.embeddings.layernorm.bias')      )

     # fmt: on
     return rename_keys

def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   List[str]     ,					SCREAMING_SNAKE_CASE_		:   Optional[Any]     ,					SCREAMING_SNAKE_CASE_		:   Optional[int]      ):



     '''simple docstring'''





     lowercase__   :       Optional[int]				=	dct.pop(a__      )
     lowercase__   :       Union[str, Any]				=	val

def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Any     ,					SCREAMING_SNAKE_CASE_		:   List[str]      ):



     '''simple docstring'''





     for i in range(config.vision_config.num_hidden_layers      ):
          # read in original q and v biases
          lowercase__   :       List[str]				=	state_dict.pop(f"""visual_encoder.blocks.{i}.attn.q_bias"""      )
          lowercase__   :       List[Any]				=	state_dict.pop(f"""visual_encoder.blocks.{i}.attn.v_bias"""      )

          # next, set bias in the state dict
          lowercase__   :       str				=	torch.cat((q_bias, torch.zeros_like(a__     ,					requires_grad=a__      ), v_bias)      )
          lowercase__   :       Union[str, Any]				=	qkv_bias

def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   int      ):



     '''simple docstring'''





     lowercase__   :       List[str]				=	364 if 'coco' in model_name else 224
     lowercase__   :       str				=	InstructBlipVisionConfig(image_size=a__      ).to_dict()

     # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
     # seems like flan-T5 models don't have bos_token_id properly set?
     if "t5-xl" in model_name:
          lowercase__   :       Union[str, Any]				=	TaConfig.from_pretrained('google/flan-t5-xl'     ,					dense_act_fn='gelu'     ,					bos_token_id=1      ).to_dict()
     elif "t5-xxl" in model_name:
          lowercase__   :       Any				=	TaConfig.from_pretrained('google/flan-t5-xxl'     ,					dense_act_fn='gelu'     ,					bos_token_id=1      ).to_dict()
     elif "vicuna-7b" in model_name:
          lowercase__   :       Dict				=	LlamaConfig.from_pretrained('decapoda-research/llama-7b-hf'     ,					vocab_size=32_001      ).to_dict()
     elif "vicuna-13b" in model_name:
          lowercase__   :       Any				=	LlamaConfig.from_pretrained('decapoda-research/llama-13b-hf'     ,					vocab_size=32_001      ).to_dict()
     else:
          raise ValueError('Model name not supported'      )

     # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
     lowercase__   :       Dict				=	InstructBlipQFormerConfig(vocab_size=30_523      ).to_dict()
     lowercase__   :       Union[str, Any]				=	InstructBlipConfig(vision_config=a__     ,					text_config=a__     ,					qformer_config=a__      )

     return config, image_size

@torch.no_grad()
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   str     ,					SCREAMING_SNAKE_CASE_		:   Optional[Any]=None     ,					SCREAMING_SNAKE_CASE_		:   Tuple=False      ):



     '''simple docstring'''





     lowercase__   :       Any				=	AutoTokenizer.from_pretrained('bert-base-uncased'     ,					truncation_side='left'      )
     qformer_tokenizer.add_special_tokens({'bos_token': '[DEC]'}      )

     if "t5" in model_name:
          lowercase__   :       Any				=	TaTokenizerFast.from_pretrained('google/flan-t5-xl'     ,					truncation_side='left'      )
     elif "vicuna" in model_name:
          # the following was used in the original implementation:
          # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
          # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
          # tokenizer.add_special_tokens({"bos_token": "</s>"})
          # tokenizer.add_special_tokens({"eos_token": "</s>"})
          # tokenizer.add_special_tokens({"unk_token": "</s>"})
          lowercase__   :       Dict				=	LlamaTokenizerFast.from_pretrained(
              'huggyllama/llama-7b'     ,					truncation_side='left'     ,					bos_token='</s>'     ,					unk_token='</s>'      )
          tokenizer.add_special_tokens({'pad_token': '[PAD]'}      )

     lowercase__				,			lowercase__   :       Optional[int]				=	get_blipa_config(a__      )
     lowercase__   :       Optional[Any]				=	InstructBlipForConditionalGeneration(a__      ).eval()

     lowercase__   :       Optional[Any]				=	{
         'instructblip-vicuna-7b': ('blip2_vicuna_instruct', 'vicuna7b'),
         'instructblip-vicuna-13b': ('blip2_vicuna_instruct', 'vicuna13b'),
         'instructblip-flan-t5-xl': ('blip2_t5_instruct', 'flant5xl'),
         'instructblip-flan-t5-xxl': ('blip2_t5_instruct', 'flant5xxl'),
     }

     lowercase__				,			lowercase__   :       str				=	model_name_to_original[model_name]

     # load original model
     print('Loading original model...'      )
     lowercase__   :       int				=	'cuda:1' if torch.cuda.is_available() else 'cpu'
     lowercase__   :       Dict				=	'cuda:2' if torch.cuda.is_available() else 'cpu'
     lowercase__				,			lowercase__				,			lowercase__   :       int				=	load_model_and_preprocess(
         name=a__     ,					model_type=a__     ,					is_eval=a__     ,					device=a__      )
     original_model.eval()
     print('Done!'      )

     # update state dict keys
     lowercase__   :       List[str]				=	original_model.state_dict()
     lowercase__   :       Dict				=	create_rename_keys(a__      )
     for src, dest in rename_keys:
          rename_key(a__     ,					a__     ,					a__      )

     # some keys can be renamed efficiently
     for key, val in state_dict.copy().items():
          lowercase__   :       int				=	state_dict.pop(a__      )
          if key.startswith('Qformer.bert'      ):
               lowercase__   :       str				=	key.replace('Qformer.bert'     ,					'qformer'      )
          if "attention.self" in key:
               lowercase__   :       List[str]				=	key.replace('self'     ,					'attention'      )
          if "llm_proj" in key:
               lowercase__   :       List[Any]				=	key.replace('llm_proj'     ,					'language_projection'      )
          if "t5_proj" in key:
               lowercase__   :       int				=	key.replace('t5_proj'     ,					'language_projection'      )
          if key.startswith('llm_model'      ):
               lowercase__   :       Optional[Any]				=	key.replace('llm_model'     ,					'language_model'      )
          if key.startswith('t5'      ):
               lowercase__   :       Dict				=	key.replace('t5'     ,					'language'      )
          lowercase__   :       Any				=	val

     # read in qv biases
     read_in_q_v_bias(a__     ,					a__      )

     # note: weights get loaded in torch.float32 by default
     hf_model.load_state_dict(a__     ,					strict=a__      )

     lowercase__   :       int				=	load_demo_image()
     lowercase__   :       Union[str, Any]				=	'What is unusual about this image?'

     # create processor
     lowercase__   :       List[str]				=	BlipImageProcessor(
         size={'height': image_size, 'width': image_size}     ,					image_mean=a__     ,					image_std=a__      )
     lowercase__   :       str				=	InstructBlipProcessor(
         image_processor=a__     ,					tokenizer=a__     ,					qformer_tokenizer=a__     ,					)
     lowercase__   :       List[Any]				=	processor(images=a__     ,					text=a__     ,					return_tensors='pt'      ).to(a__      )

     # make sure processor creates exact same pixel values
     lowercase__   :       Optional[Any]				=	vis_processors['eval'](a__      ).unsqueeze(0      ).to(a__      )
     lowercase__   :       Tuple				=	inputs.pixel_values
     assert torch.allclose(original_pixel_values.to(pixel_values.device      )     ,					a__      )

     original_model.to(a__      )
     hf_model.to(a__      )
     with torch.no_grad():
          if "vicuna" in model_name:
               lowercase__   :       Tuple				=	original_model({'image': original_pixel_values, 'text_input': [prompt]}      ).logits
               lowercase__   :       int				=	hf_model(**a__      ).logits
          else:
               lowercase__   :       Optional[int]				=	original_model(
                   {'image': original_pixel_values, 'text_input': [prompt], 'text_output': ['\n']}      ).logits
               lowercase__   :       Union[str, Any]				=	tokenizer('\n'     ,					return_tensors='pt'      ).input_ids.to(a__      )
               lowercase__   :       Optional[int]				=	label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id     ,					-100      )
               lowercase__   :       List[Any]				=	hf_model(**a__     ,					labels=a__      ).logits

     print('First values of original logits:'     ,					original_logits[0, :3, :3]      )
     print('First values of HF logits:'     ,					logits[0, :3, :3]      )

     # assert values
     assert original_logits.shape == logits.shape
     lowercase__   :       List[str]				=	1E-4 if 'vicuna' in model_name else 1E-5
     assert torch.allclose(original_logits.to(logits.device      )     ,					a__     ,					atol=a__      )
     print('Looks ok!'      )

     print('Generating with original model...'      )
     lowercase__   :       Tuple				=	original_model.generate({'image': original_pixel_values, 'prompt': prompt}     ,					num_beams=5      )

     # important: we need to cast the weights of the HF model to the appropriate type
     print('Generating with HF model...'      )
     lowercase__   :       List[str]				=	hf_model.generate(
         **a__     ,					do_sample=a__     ,					num_beams=5     ,					max_length=256     ,					min_length=1     ,					top_p=0.9     ,					repetition_penalty=1.5     ,					length_penalty=1.0     ,					temperature=1     ,					)
     if "vicuna" in model_name:
          # convert output id 0 to 2 (eos_token_id)
          # TODO add this in the generate method?
          lowercase__   :       List[str]				=	2
     print('Original generation:'     ,					a__      )
     lowercase__   :       str				=	processor.batch_decode(a__     ,					skip_special_tokens=a__      )
     lowercase__   :       str				=	[text.strip() for text in output_text]
     print('HF generation:'     ,					a__      )

     if pytorch_dump_folder_path is not None:
          processor.save_pretrained(a__      )
          hf_model.save_pretrained(a__      )

     if push_to_hub:
          processor.push_to_hub(f"""Salesforce/{model_name}"""      )
          hf_model.push_to_hub(f"""Salesforce/{model_name}"""      )


if __name__ == "__main__":
       snake_case_        =			argparse.ArgumentParser()
       snake_case_        =			[
           '''instructblip-vicuna-7b''',
           '''instructblip-vicuna-13b''',
           '''instructblip-flan-t5-xl''',
           '''instructblip-flan-t5-xxl''',
       ]
       parser.add_argument(
           '''--model_name''',
           default='''instructblip-flan-t5-xl''',
           choices=choices,
           type=str,
           help='''Path to hf config.json of model to convert''',
       )
       parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
       parser.add_argument(
           '''--push_to_hub''',
           action='''store_true''',
           help='''Whether to push the model and processor to the hub after converting''',
       )

       snake_case_        =			parser.parse_args()

       convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)




from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader


class 				SCREAMING_SNAKE_CASE__ (__snake_case					):





    def __init__(				self						,       a						,       a = None						,       a = None						,       a = True						,       a = None						,       a = False						,       a = None						,       a = True						,       a = "arrow"						,       **a						,       ):
         super().__init__(
             split=a						,       features=a						,       cache_dir=a						,       keep_in_memory=a						,       streaming=a						,       **a						,       )
         lowercase__   :       Optional[int]				=	load_from_cache_file
         lowercase__   :       Optional[int]				=	file_format
         lowercase__   :       int				=	Spark(
             df=a						,       features=a						,       cache_dir=a						,       working_dir=a						,       **a						,       )





    def 				snake_case_    (				self):
         if self.streaming:
              return self.builder.as_streaming_dataset(split=self.split)
         lowercase__   :       Dict				=	None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
         self.builder.download_and_prepare(
             download_mode=a						,       file_format=self._file_format						,       )
         return self.builder.as_dataset(split=self.split)

