






'''simple docstring'''
import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a_					=	logging.get_logger(__name__)





def 	_a(					UpperCamelCase__			:  Tuple,		UpperCamelCase__			:  Any=False,		UpperCamelCase__			:  int=False,		UpperCamelCase__			:  Optional[Any]=False							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Any													=[]
      for i in range(config.num_hidden_layers							):
            # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
            rename_keys.append((f"transformer.blocks.{i}.norm1.weight", f"vilt.encoder.layer.{i}.layernorm_before.weight")							)
            rename_keys.append((f"transformer.blocks.{i}.norm1.bias", f"vilt.encoder.layer.{i}.layernorm_before.bias")							)
            rename_keys.append(
                (f"transformer.blocks.{i}.attn.proj.weight", f"vilt.encoder.layer.{i}.attention.output.dense.weight")							)
            rename_keys.append(
                (f"transformer.blocks.{i}.attn.proj.bias", f"vilt.encoder.layer.{i}.attention.output.dense.bias")							)
            rename_keys.append((f"transformer.blocks.{i}.norm2.weight", f"vilt.encoder.layer.{i}.layernorm_after.weight")							)
            rename_keys.append((f"transformer.blocks.{i}.norm2.bias", f"vilt.encoder.layer.{i}.layernorm_after.bias")							)
            rename_keys.append(
                (f"transformer.blocks.{i}.mlp.fc1.weight", f"vilt.encoder.layer.{i}.intermediate.dense.weight")							)
            rename_keys.append((f"transformer.blocks.{i}.mlp.fc1.bias", f"vilt.encoder.layer.{i}.intermediate.dense.bias")							)
            rename_keys.append((f"transformer.blocks.{i}.mlp.fc2.weight", f"vilt.encoder.layer.{i}.output.dense.weight")							)
            rename_keys.append((f"transformer.blocks.{i}.mlp.fc2.bias", f"vilt.encoder.layer.{i}.output.dense.bias")							)

      # embeddings
      rename_keys.extend(
          [
              # text embeddings
              ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
              (
                  '''text_embeddings.position_embeddings.weight''',
                  '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
              ),
              ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
              (
                  '''text_embeddings.token_type_embeddings.weight''',
                  '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
              ),
              ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
              ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
              # patch embeddings
              ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
              ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
              ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
              ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
              # token type embeddings
              ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
          ]							)

      # final layernorm + pooler
      rename_keys.extend(
          [
              ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
              ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
              ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
              ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
          ]							)

      # classifier head(s)
      if vqa_model:
            # classification head
            rename_keys.extend(
                [
                    ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
                    ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
                    ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
                    ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
                    ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
                    ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
                ]							)
      elif nlvr_model:
            # classification head
            rename_keys.extend(
                [
                    ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
                    ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
                    ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
                    ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
                    ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
                    ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
                ]							)
      else:
            pass

      return rename_keys





def 	_a(					UpperCamelCase__			:  int,		UpperCamelCase__			:  Union[str, Any]							):




      '''simple docstring'''






      for i in range(config.num_hidden_layers							):
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													='''vilt.'''
            # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
            SCREAMING_SNAKE_CASE__       :			List[Any]													=state_dict.pop(f"transformer.blocks.{i}.attn.qkv.weight"							)
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=state_dict.pop(f"transformer.blocks.{i}.attn.qkv.bias"							)
            # next, add query, keys and values (in that order) to the state dict
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=in_proj_weight[
                : config.hidden_size, :
            ]
            SCREAMING_SNAKE_CASE__       :			Dict													=in_proj_bias[: config.hidden_size]
            SCREAMING_SNAKE_CASE__       :			List[str]													=in_proj_weight[
                config.hidden_size : config.hidden_size * 2, :
            ]
            SCREAMING_SNAKE_CASE__       :			Dict													=in_proj_bias[
                config.hidden_size : config.hidden_size * 2
            ]
            SCREAMING_SNAKE_CASE__       :			str													=in_proj_weight[
                -config.hidden_size :, :
            ]
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=in_proj_bias[-config.hidden_size :]





def 	_a(					UpperCamelCase__			:  List[Any]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=['''head.weight''', '''head.bias''']
      for k in ignore_keys:
            state_dict.pop(UpperCamelCase__,		UpperCamelCase__							)





def 	_a(					UpperCamelCase__			:  Any,		UpperCamelCase__			:  Union[str, Any],		UpperCamelCase__			:  List[str]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			int													=dct.pop(UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=val





@torch.no_grad()
def 	_a(					UpperCamelCase__			:  Dict,		UpperCamelCase__			:  Union[str, Any]							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Dict													=ViltConfig(image_size=3_8_4,		patch_size=3_2,		tie_word_embeddings=UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__       :			str													=False
      SCREAMING_SNAKE_CASE__       :			str													=False
      SCREAMING_SNAKE_CASE__       :			List[Any]													=False
      SCREAMING_SNAKE_CASE__       :			List[str]													=False
      if "vqa" in checkpoint_url:
            SCREAMING_SNAKE_CASE__       :			List[str]													=True
            SCREAMING_SNAKE_CASE__       :			Tuple													=3_1_2_9
            SCREAMING_SNAKE_CASE__       :			Optional[int]													='''huggingface/label-files'''
            SCREAMING_SNAKE_CASE__       :			int													='''vqa2-id2label.json'''
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=json.load(open(hf_hub_download(UpperCamelCase__,		UpperCamelCase__,		repo_type='''dataset'''							),		'''r'''							)							)
            SCREAMING_SNAKE_CASE__       :			Any													={int(UpperCamelCase__							): v for k, v in idalabel.items()}
            SCREAMING_SNAKE_CASE__       :			int													=idalabel
            SCREAMING_SNAKE_CASE__       :			int													={v: k for k, v in idalabel.items()}
            SCREAMING_SNAKE_CASE__       :			List[str]													=ViltForQuestionAnswering(UpperCamelCase__							)
      elif "nlvr" in checkpoint_url:
            SCREAMING_SNAKE_CASE__       :			int													=True
            SCREAMING_SNAKE_CASE__       :			Tuple													=2
            SCREAMING_SNAKE_CASE__       :			Any													={0: '''False''', 1: '''True'''}
            SCREAMING_SNAKE_CASE__       :			Optional[int]													={v: k for k, v in config.idalabel.items()}
            SCREAMING_SNAKE_CASE__       :			List[Any]													=3
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=ViltForImagesAndTextClassification(UpperCamelCase__							)
      elif "irtr" in checkpoint_url:
            SCREAMING_SNAKE_CASE__       :			Any													=True
            SCREAMING_SNAKE_CASE__       :			Dict													=ViltForImageAndTextRetrieval(UpperCamelCase__							)
      elif "mlm_itm" in checkpoint_url:
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=True
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=ViltForMaskedLM(UpperCamelCase__							)
      else:
            raise ValueError('''Unknown model type'''							)

      # load state_dict of original model, remove and rename some keys
      SCREAMING_SNAKE_CASE__       :			int													=torch.hub.load_state_dict_from_url(UpperCamelCase__,		map_location='''cpu'''							)['''state_dict''']
      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=create_rename_keys(UpperCamelCase__,		UpperCamelCase__,		UpperCamelCase__,		UpperCamelCase__							)
      for src, dest in rename_keys:
            rename_key(UpperCamelCase__,		UpperCamelCase__,		UpperCamelCase__							)
      read_in_q_k_v(UpperCamelCase__,		UpperCamelCase__							)
      if mlm_model or irtr_model:
            SCREAMING_SNAKE_CASE__       :			List[str]													=['''itm_score.fc.weight''', '''itm_score.fc.bias''']
            for k in ignore_keys:
                  state_dict.pop(UpperCamelCase__,		UpperCamelCase__							)

    # load state dict into HuggingFace model
      model.eval()
      if mlm_model:
            SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			str													=model.load_state_dict(UpperCamelCase__,		strict=UpperCamelCase__							)
            assert missing_keys == ["mlm_score.decoder.bias"]
      else:
            model.load_state_dict(UpperCamelCase__							)

      # Define processor
      SCREAMING_SNAKE_CASE__       :			Optional[int]													=ViltImageProcessor(size=3_8_4							)
      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=BertTokenizer.from_pretrained('''bert-base-uncased'''							)
      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=ViltProcessor(UpperCamelCase__,		UpperCamelCase__							)

      # Forward pass on example inputs (image + text)
      if nlvr_model:
            SCREAMING_SNAKE_CASE__       :			List[Any]													=Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg''',		stream=UpperCamelCase__							).raw							)
            SCREAMING_SNAKE_CASE__       :			Tuple													=Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg''',		stream=UpperCamelCase__							).raw							)
            SCREAMING_SNAKE_CASE__       :			List[Any]													=(
                '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
                ''' standing.'''
            )
            SCREAMING_SNAKE_CASE__       :			int													=processor(UpperCamelCase__,		UpperCamelCase__,		return_tensors='''pt'''							)
            SCREAMING_SNAKE_CASE__       :			Tuple													=processor(UpperCamelCase__,		UpperCamelCase__,		return_tensors='''pt'''							)
            SCREAMING_SNAKE_CASE__       :			Any													=model(
                input_ids=encoding_a.input_ids,		pixel_values=encoding_a.pixel_values,		pixel_values_a=encoding_a.pixel_values,		)
      else:
            SCREAMING_SNAKE_CASE__       :			Tuple													=Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg''',		stream=UpperCamelCase__							).raw							)
            if mlm_model:
                  SCREAMING_SNAKE_CASE__       :			Optional[Any]													='''a bunch of [MASK] laying on a [MASK].'''
            else:
                  SCREAMING_SNAKE_CASE__       :			Dict													='''How many cats are there?'''
            SCREAMING_SNAKE_CASE__       :			Any													=processor(UpperCamelCase__,		UpperCamelCase__,		return_tensors='''pt'''							)
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=model(**UpperCamelCase__							)

      # Verify outputs
      if mlm_model:
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=torch.Size([1, 1_1, 3_0_5_2_2]							)
            SCREAMING_SNAKE_CASE__       :			List[Any]													=torch.tensor([-1_2.5_0_6_1, -1_2.5_1_2_3, -1_2.5_1_7_4]							)
            assert outputs.logits.shape == expected_shape
            assert torch.allclose(outputs.logits[0, 0, :3],		UpperCamelCase__,		atol=1e-4							)

            # verify masked token prediction equals "cats"
            SCREAMING_SNAKE_CASE__       :			str													=outputs.logits[0, 4, :].argmax(-1							).item()
            assert tokenizer.decode([predicted_id]							) == "cats"
      elif vqa_model:
            SCREAMING_SNAKE_CASE__       :			List[Any]													=torch.Size([1, 3_1_2_9]							)
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=torch.tensor([-1_5.9_4_9_5, -1_8.1_4_7_2, -1_0.3_0_4_1]							)
            assert torch.allclose(outputs.logits[0, :3],		UpperCamelCase__,		atol=1e-4							)
            assert outputs.logits.shape == expected_shape
            assert torch.allclose(outputs.logits[0, 0, :3],		UpperCamelCase__,		atol=1e-4							)

            # verify vqa prediction equals "2"
            SCREAMING_SNAKE_CASE__       :			str													=outputs.logits.argmax(-1							).item()
            assert model.config.idalabel[predicted_idx] == "2"
      elif nlvr_model:
            SCREAMING_SNAKE_CASE__       :			Tuple													=torch.Size([1, 2]							)
            SCREAMING_SNAKE_CASE__       :			str													=torch.tensor([-2.8_7_2_1, 2.1_2_9_1]							)
            assert torch.allclose(outputs.logits[0, :3],		UpperCamelCase__,		atol=1e-4							)
            assert outputs.logits.shape == expected_shape

      Path(UpperCamelCase__							).mkdir(exist_ok=UpperCamelCase__							)
      print(f"Saving model and processor to {pytorch_dump_folder_path}"							)
      model.save_pretrained(UpperCamelCase__							)
      processor.save_pretrained(UpperCamelCase__							)


if __name__ == "__main__":
  a_					=	argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '--checkpoint_url',
      default='https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt',
      type=str,
      help='URL of the checkpoint you\'d like to convert.',
  )
  parser.add_argument(
      '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
  )

  a_					=	parser.parse_args()
  convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a_					=	{
    'configuration_instructblip': [
        'INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'InstructBlipConfig',
        'InstructBlipQFormerConfig',
        'InstructBlipVisionConfig',
    ],
    'processing_instructblip': ['InstructBlipProcessor'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  a_					=	[
      'INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
      'InstructBlipQFormerModel',
      'InstructBlipPreTrainedModel',
      'InstructBlipForConditionalGeneration',
      'InstructBlipVisionModel',
  ]

if TYPE_CHECKING:
  from .configuration_instructblip import (
      INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
      InstructBlipConfig,
      InstructBlipQFormerConfig,
      InstructBlipVisionConfig,
  )
  from .processing_instructblip import InstructBlipProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_instructblip import (
        INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
        InstructBlipForConditionalGeneration,
        InstructBlipPreTrainedModel,
        InstructBlipQFormerModel,
        InstructBlipVisionModel,
    )

else:
  import sys

  a_					=	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)