


import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowercase_	   =     [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

lowercase_	   =     [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Any		 =						{
     'word_embeddings.weight': 'word_embeddings.weight',
     'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
     'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
     'weight': 'ln_f.weight',
     'bias': 'ln_f.bias',
 }

 if key in layer_rename_map:
  return layer_rename_map[key]

 # Handle transformer blocks
 __lowerCamelCase		:							Any		 =						int(re.match(r'.*layer_(\d*).*'      ,	SCREAMING_SNAKE_CASE__						)[1]						)
 layer_number -= 3
 return f'h.{layer_number}.' + key
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 if dtype == torch.bool:
  return 1 / 8
 __lowerCamelCase		:							Optional[int]		 =						re.search(r'[^\d](\d+)$'      ,	str(SCREAMING_SNAKE_CASE__						)						)
 if bit_search is None:
  raise ValueError(f'`dtype` is not a valid dtype: {dtype}.'						)
 __lowerCamelCase		:							int		 =						int(bit_search.groups()[0]						)
 return bit_size // 8
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 # Construct model
 if bloom_config_file == "":
  __lowerCamelCase		:							List[str]		 =						BloomConfig()
 else:
  __lowerCamelCase		:							str		 =						BloomConfig.from_json_file(SCREAMING_SNAKE_CASE__						)

 if shard_model:
  __lowerCamelCase		:							int		 =						os.listdir(SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							List[Any]		 =						sorted(filter(lambda SCREAMING_SNAKE_CASE__						: s.startswith('layer'						) and "model_00" in s      ,	SCREAMING_SNAKE_CASE__						)						)

  __lowerCamelCase		:							Dict		 =						{'weight_map': {}, 'metadata': {}}
  __lowerCamelCase		:							Optional[int]		 =						0

  __lowerCamelCase		:							Optional[Any]		 =						None

  __lowerCamelCase		:							Any		 =						BloomConfig()

  for j, file in enumerate(SCREAMING_SNAKE_CASE__						):
   print('Processing file: {}'.format(SCREAMING_SNAKE_CASE__						)						)
   __lowerCamelCase		:							str		 =						None

   for i in range(SCREAMING_SNAKE_CASE__						):
    # load all TP files
    __lowerCamelCase		:							Any		 =						file.replace('model_00'      ,	f'model_0{i}'						)
    __lowerCamelCase		:							List[Any]		 =						torch.load(os.path.join(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)      ,	map_location='cpu'						)

    # Rename keys in the transformers names
    __lowerCamelCase		:							Optional[Any]		 =						list(temp.keys()						)
    for key in keys:
     __lowerCamelCase		:							Optional[Any]		 =						temp.pop(SCREAMING_SNAKE_CASE__						)

    if tensors is None:
     __lowerCamelCase		:							str		 =						temp
    else:
     for key in tensors.keys():
      if any(key.endswith(SCREAMING_SNAKE_CASE__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
       # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       __lowerCamelCase		:							List[Any]		 =						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN						) else 0
       # We concatenate these weights accross TP ranks
       __lowerCamelCase		:							Union[str, Any]		 =						torch.cat([tensors[key], temp[key]]      ,	dim=SCREAMING_SNAKE_CASE__						)

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(SCREAMING_SNAKE_CASE__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
     __lowerCamelCase		:							Optional[int]		 =						tensors[key] / pretraining_tp
   torch.save(
       SCREAMING_SNAKE_CASE__      ,	os.path.join(
           SCREAMING_SNAKE_CASE__      ,	'pytorch_model_{}-of-{}.bin'.format(str(j + 1						).zfill(5						)      ,	str(len(SCREAMING_SNAKE_CASE__						)						).zfill(5						)						)      ,	)      ,	)

   for key in tensors.keys():
    __lowerCamelCase		:							Union[str, Any]		 =						tensors[key]
    total_size += value.numel() * get_dtype_size(value.dtype						)
    if key not in index_dict["weight_map"]:
     __lowerCamelCase		:							Optional[int]		 =						'pytorch_model_{}-of-{}.bin'.format(
         str(j + 1						).zfill(5						)      ,	str(len(SCREAMING_SNAKE_CASE__						)						).zfill(5						)						)

  __lowerCamelCase		:							Optional[Any]		 =						BloomConfig()
  __lowerCamelCase		:							Any		 =						pytorch_dump_folder_path + '/' + CONFIG_NAME
  __lowerCamelCase		:							List[Any]		 =						total_size
  with open(SCREAMING_SNAKE_CASE__      ,	'w'      ,	encoding='utf-8'						) as f:
   f.write(config.to_json_string()						)
  with open(os.path.join(SCREAMING_SNAKE_CASE__      ,	WEIGHTS_NAME + '.index.json'						)      ,	'w'      ,	encoding='utf-8'						) as f:
   __lowerCamelCase		:							int		 =						json.dumps(SCREAMING_SNAKE_CASE__      ,	indent=2      ,	sort_keys=SCREAMING_SNAKE_CASE__						) + '\n'
   f.write(SCREAMING_SNAKE_CASE__						)
 else:
  __lowerCamelCase		:							Optional[Any]		 =						BloomModel(SCREAMING_SNAKE_CASE__						)

  __lowerCamelCase		:							List[str]		 =						os.listdir(SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							str		 =						sorted(filter(lambda SCREAMING_SNAKE_CASE__						: s.startswith('layer'						) and "model_00" in s      ,	SCREAMING_SNAKE_CASE__						)						)

  __lowerCamelCase		:							Tuple		 =						None
  for i, file in enumerate(SCREAMING_SNAKE_CASE__						):
   __lowerCamelCase		:							Optional[Any]		 =						None
   for i in range(SCREAMING_SNAKE_CASE__						):
    # load all TP files
    __lowerCamelCase		:							Optional[int]		 =						file.replace('model_00'      ,	f'model_0{i}'						)
    __lowerCamelCase		:							int		 =						torch.load(os.path.join(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)      ,	map_location='cpu'						)

    # Rename keys in the transformers names
    __lowerCamelCase		:							Any		 =						list(temp.keys()						)
    for key in keys:
     __lowerCamelCase		:							Dict		 =						temp.pop(SCREAMING_SNAKE_CASE__						)

    if tensors is None:
     __lowerCamelCase		:							Tuple		 =						temp
    else:
     for key in tensors.keys():
      # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
      if any(key.endswith(SCREAMING_SNAKE_CASE__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       __lowerCamelCase		:							Any		 =						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN						) else 0
       # We concatenate these weights accross TP ranks
       __lowerCamelCase		:							List[str]		 =						torch.cat([tensors[key], temp[key]]      ,	dim=SCREAMING_SNAKE_CASE__						)

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(SCREAMING_SNAKE_CASE__						) for end in WEIGHTS_TO_AVERAGE_ENDSWITH						):
     __lowerCamelCase		:							Dict		 =						tensors[key] / pretraining_tp

   __lowerCamelCase		:							Any		 =						model.load_state_dict(SCREAMING_SNAKE_CASE__      ,	strict=SCREAMING_SNAKE_CASE__						)
   assert not other_keys.unexpected_keys, f'The keys {other_keys.unexpected_keys} are unexpected'
   if missing_keys is None:
    __lowerCamelCase		:							Any		 =						set(other_keys.missing_keys						)
   else:
    __lowerCamelCase		:							List[Any]		 =						missing_keys.intersection(set(other_keys.missing_keys						)						)

  assert not missing_keys, f'The keys {missing_keys} are missing'

  # Save pytorch-model
  os.makedirs(SCREAMING_SNAKE_CASE__      ,	exist_ok=SCREAMING_SNAKE_CASE__						)
  __lowerCamelCase		:							Optional[Any]		 =						pytorch_dump_folder_path + '/' + WEIGHTS_NAME
  __lowerCamelCase		:							Any		 =						pytorch_dump_folder_path + '/' + CONFIG_NAME
  print(f'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'						)
  if config.torch_dtype is not None:
   __lowerCamelCase		:							Optional[int]		 =						model.to(config.torch_dtype						)
  torch.save(model.state_dict()      ,	SCREAMING_SNAKE_CASE__						)
  print(f'Save configuration file to {pytorch_config_dump_path}'						)
  with open(SCREAMING_SNAKE_CASE__      ,	'w'      ,	encoding='utf-8'						) as f:
   f.write(config.to_json_string()						)


if __name__ == "__main__":
       lowercase_	   =     argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--bloom_checkpoint_path',
           default=None,
           type=str,
           required=True,
           help='Path to the Megatron-LM checkpoint path.',
       )
       parser.add_argument(
           '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--bloom_config_file',
           default='',
           type=str,
           help=(
               'An optional config json file corresponding to the pre-trained model. \n'
               'This specifies the model architecture.'
           ),
       )
       parser.add_argument(
           '--shard_model',
           action='store_true',
           help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
       )
       parser.add_argument(
           '--pretraining_tp',
           default=4,
           type=int,
           help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
       )
       lowercase_	   =     parser.parse_args()
       convert_bloom_checkpoint_to_pytorch(
           args.bloom_checkpoint_path,
           args.bloom_config_file,
           args.pytorch_dump_folder_path,
           args.shard_model,
           args.pretraining_tp,
       )





import copy
import os
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Union


if TYPE_CHECKING:
       from ...processing_utils import ProcessorMixin
       from ...utils import TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowercase_	   =     logging.get_logger(__name__)

lowercase_	   =     {
    'google/owlvit-base-patch32': 'https://huggingface.co/google/owlvit-base-patch32/resolve/main/config.json',
    'google/owlvit-base-patch16': 'https://huggingface.co/google/owlvit-base-patch16/resolve/main/config.json',
    'google/owlvit-large-patch14': 'https://huggingface.co/google/owlvit-large-patch14/resolve/main/config.json',
}


class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      __snake_case 	=       """owlvit_text_model"""




      def __init__(    self:	Optional[int]						,				a:	Dict=4_9408						,				a:	Optional[Any]=512						,				a:	Dict=2048						,				a:	Optional[Any]=12						,				a:	Tuple=8						,				a:	Union[str, Any]=16						,				a:	str="quick_gelu"						,				a:	List[Any]=1e-5						,				a:	Dict=0.0						,				a:	Optional[int]=0.0_2						,				a:	Dict=1.0						,				a:	Any=0						,				a:	Union[str, Any]=4_9406						,				a:	Any=4_9407						,				**a:	Dict						,				):
       super().__init__(pad_token_id=a						,				bos_token_id=a						,				eos_token_id=a						,				**a    )

       __lowerCamelCase		:							List[Any]		 =						vocab_size
       __lowerCamelCase		:							int		 =						hidden_size
       __lowerCamelCase		:							Optional[Any]		 =						intermediate_size
       __lowerCamelCase		:							List[Any]		 =						num_hidden_layers
       __lowerCamelCase		:							Any		 =						num_attention_heads
       __lowerCamelCase		:							Union[str, Any]		 =						max_position_embeddings
       __lowerCamelCase		:							List[Any]		 =						hidden_act
       __lowerCamelCase		:							List[str]		 =						layer_norm_eps
       __lowerCamelCase		:							Tuple		 =						attention_dropout
       __lowerCamelCase		:							Optional[int]		 =						initializer_range
       __lowerCamelCase		:							Tuple		 =						initializer_factor




      @classmethod
      def 					_snake_case     (    cls:	Dict						,				a:	Union[str, os.PathLike]						,				**a:	Optional[int]    ):
       cls._set_token_in_kwargs(a    )

       __lowerCamelCase ,					__lowerCamelCase		:							Dict		 =						cls.get_config_dict(a						,				**a    )

       # get the text config dict if we are loading from OwlViTConfig
       if config_dict.get('model_type'    ) == "owlvit":
        __lowerCamelCase		:							Optional[Any]		 =						config_dict['text_config']

       if "model_type" in config_dict and hasattr(cls						,				'model_type'    ) and config_dict["model_type"] != cls.model_type:
        logger.warning(
            F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
            F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'    )

       return cls.from_dict(a						,				**a    )



class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      __snake_case 	=       """owlvit_vision_model"""




      def __init__(    self:	int						,				a:	Tuple=768						,				a:	int=3072						,				a:	List[str]=12						,				a:	Optional[Any]=12						,				a:	Optional[int]=3						,				a:	Optional[int]=768						,				a:	Optional[Any]=32						,				a:	Optional[int]="quick_gelu"						,				a:	Union[str, Any]=1e-5						,				a:	Union[str, Any]=0.0						,				a:	Union[str, Any]=0.0_2						,				a:	int=1.0						,				**a:	Union[str, Any]						,				):
       super().__init__(**a    )

       __lowerCamelCase		:							str		 =						hidden_size
       __lowerCamelCase		:							Tuple		 =						intermediate_size
       __lowerCamelCase		:							Dict		 =						num_hidden_layers
       __lowerCamelCase		:							Optional[Any]		 =						num_attention_heads
       __lowerCamelCase		:							int		 =						num_channels
       __lowerCamelCase		:							Optional[Any]		 =						image_size
       __lowerCamelCase		:							Tuple		 =						patch_size
       __lowerCamelCase		:							List[str]		 =						hidden_act
       __lowerCamelCase		:							Tuple		 =						layer_norm_eps
       __lowerCamelCase		:							List[Any]		 =						attention_dropout
       __lowerCamelCase		:							Tuple		 =						initializer_range
       __lowerCamelCase		:							List[Any]		 =						initializer_factor




      @classmethod
      def 					_snake_case     (    cls:	Optional[int]						,				a:	Union[str, os.PathLike]						,				**a:	int    ):
       cls._set_token_in_kwargs(a    )

       __lowerCamelCase ,					__lowerCamelCase		:							Dict		 =						cls.get_config_dict(a						,				**a    )

       # get the vision config dict if we are loading from OwlViTConfig
       if config_dict.get('model_type'    ) == "owlvit":
        __lowerCamelCase		:							Any		 =						config_dict['vision_config']

       if "model_type" in config_dict and hasattr(cls						,				'model_type'    ) and config_dict["model_type"] != cls.model_type:
        logger.warning(
            F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
            F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'    )

       return cls.from_dict(a						,				**a    )



class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      __snake_case 	=       """owlvit"""
      __snake_case 	=       True




      def __init__(    self:	Dict						,				a:	int=None						,				a:	str=None						,				a:	Tuple=512						,				a:	Tuple=2.6_5_9_2						,				a:	int=True						,				**a:	int						,				):
       super().__init__(**a    )

       if text_config is None:
        __lowerCamelCase		:							List[str]		 =						{}
        logger.info('text_config is None. Initializing the OwlViTTextConfig with default values.'    )

       if vision_config is None:
        __lowerCamelCase		:							str		 =						{}
        logger.info('vision_config is None. initializing the OwlViTVisionConfig with default values.'    )

       __lowerCamelCase		:							List[Any]		 =						OwlViTTextConfig(**a    )
       __lowerCamelCase		:							str		 =						OwlViTVisionConfig(**a    )

       __lowerCamelCase		:							Union[str, Any]		 =						projection_dim
       __lowerCamelCase		:							Tuple		 =						logit_scale_init_value
       __lowerCamelCase		:							Dict		 =						return_dict
       __lowerCamelCase		:							Tuple		 =						1.0




      @classmethod
      def 					_snake_case     (    cls:	str						,				a:	Union[str, os.PathLike]						,				**a:	List[Any]    ):
       cls._set_token_in_kwargs(a    )

       __lowerCamelCase ,					__lowerCamelCase		:							List[Any]		 =						cls.get_config_dict(a						,				**a    )

       if "model_type" in config_dict and hasattr(cls						,				'model_type'    ) and config_dict["model_type"] != cls.model_type:
        logger.warning(
            F'You are using a model of type {config_dict["model_type"]} to instantiate a model of type '
            F'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'    )

       return cls.from_dict(a						,				**a    )




      @classmethod
      def 					_snake_case     (    cls:	Tuple						,				a:	Dict						,				a:	Dict						,				**a:	str    ):
       __lowerCamelCase		:							List[str]		 =						{}
       __lowerCamelCase		:							List[str]		 =						text_config
       __lowerCamelCase		:							Optional[int]		 =						vision_config

       return cls.from_dict(a						,				**a    )




      def 					_snake_case     (    self:	Optional[int]    ):
       __lowerCamelCase		:							Optional[Any]		 =						copy.deepcopy(self.__dict__    )
       __lowerCamelCase		:							List[Any]		 =						self.text_config.to_dict()
       __lowerCamelCase		:							List[str]		 =						self.vision_config.to_dict()
       __lowerCamelCase		:							Optional[Any]		 =						self.__class__.model_type
       return output



class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      @property
      def 					_snake_case     (    self:	str    ):
       return OrderedDict(
           [
               ('input_ids', {0: 'batch', 1: 'sequence'}),
               ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
               ('attention_mask', {0: 'batch', 1: 'sequence'}),
           ]    )




      @property
      def 					_snake_case     (    self:	Dict    ):
       return OrderedDict(
           [
               ('logits_per_image', {0: 'batch'}),
               ('logits_per_text', {0: 'batch'}),
               ('text_embeds', {0: 'batch'}),
               ('image_embeds', {0: 'batch'}),
           ]    )




      @property
      def 					_snake_case     (    self:	int    ):
       return 1e-4




      def 					_snake_case     (    self:	Any						,				a:	"ProcessorMixin"						,				a:	int = -1						,				a:	int = -1						,				a:	Optional["TensorType"] = None						,				):
       __lowerCamelCase		:							List[str]		 =						super().generate_dummy_inputs(
           processor.tokenizer						,				batch_size=a						,				seq_length=a						,				framework=a    )
       __lowerCamelCase		:							int		 =						super().generate_dummy_inputs(
           processor.image_processor						,				batch_size=a						,				framework=a    )
       return {**text_input_dict, **image_input_dict}




      @property
      def 					_snake_case     (    self:	int    ):
       return 14

