


'''simple docstring'''







# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig






def 				lowerCAmelCase__	(    lowerCamelCase					:      Any  ,lowerCamelCase					:      Tuple  ,lowerCamelCase					:      Union[str, Any]=0       ):
       # Format the message.
       if name is None:
              _A					:     Union[str, Any]     = None
       else:
              _A					:     Tuple     = '.' * max(0  ,spaces - 2       ) + '# {:' + str(50 - spaces       ) + 's}'
              _A					:     Dict     = fmt.format(lowerCamelCase       )

       # Print and recurse (if needed).
       if isinstance(lowerCamelCase  ,lowerCamelCase       ):
              if msg is not None:
                     print(lowerCamelCase       )
              for k in val.keys():
                     recursive_print(lowerCamelCase  ,val[k]  ,spaces + 2       )
       elif isinstance(lowerCamelCase  ,torch.Tensor       ):
              print(lowerCamelCase  ,':'  ,val.size()       )
       else:
              print(lowerCamelCase  ,':'  ,lowerCamelCase       )






def 				lowerCAmelCase__	(    lowerCamelCase					:      Tuple  ,lowerCamelCase					:      Any  ,lowerCamelCase					:      List[Any]  ,lowerCamelCase					:      Optional[Any]  ,lowerCamelCase					:      str       ):
       # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
       # for compatibility with later versions of NVIDIA Megatron-LM.
       # The inverse operation is performed inside Megatron-LM to read checkpoints:
       # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
       # If param is the weight tensor of the self-attention block, the returned tensor
       # will have to be transposed one more time to be read by HuggingFace GPT2.
       _A					:     int     = param.size()
       if checkpoint_version == 1.0:
              # version 1.0 stores [num_heads * hidden_size * num_splits, :]
              _A					:     List[str]     = (num_heads, hidden_size, num_splits) + input_shape[1:]
              _A					:     Optional[Any]     = param.view(*lowerCamelCase       )
              _A					:     Optional[int]     = param.transpose(0  ,2       )
              _A					:     Union[str, Any]     = param.transpose(1  ,2       ).contiguous()
       elif checkpoint_version >= 2.0:
              # other versions store [num_heads * num_splits * hidden_size, :]
              _A					:     int     = (num_heads, num_splits, hidden_size) + input_shape[1:]
              _A					:     Optional[int]     = param.view(*lowerCamelCase       )
              _A					:     Dict     = param.transpose(0  ,1       ).contiguous()
       _A					:     Tuple     = param.view(*lowerCamelCase       )
       return param






def 				lowerCAmelCase__	(    lowerCamelCase					:      List[str]  ,lowerCamelCase					:      List[Any]  ,lowerCamelCase					:      Optional[Any]       ):
       # The converted output model.
       _A					:     str     = {}

       # old versions did not store training args
       _A					:     Optional[int]     = input_state_dict.get('args'  ,lowerCamelCase       )
       if ds_args is not None:
              # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
              # from pprint import pprint
              # pprint(vars(ds_args))

              _A					:     Tuple     = ds_args.padded_vocab_size
              _A					:     Union[str, Any]     = ds_args.max_position_embeddings
              _A					:     str     = ds_args.hidden_size
              _A					:     str     = ds_args.num_layers
              _A					:     int     = ds_args.num_attention_heads
              _A					:     str     = ds_args.ffn_hidden_size
              # pprint(config)

       # The number of heads.
       _A					:     List[Any]     = config.n_head
       # The hidden_size per head.
       _A					:     Any     = config.n_embd // config.n_head
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
              _A					:     List[Any]     = input_state_dict['checkpoint_version']
       else:
              _A					:     Optional[Any]     = 0.0

       # The model.
       _A					:     List[str]     = input_state_dict['model']
       # The language model.
       _A					:     Optional[Any]     = model['language_model']
       # The embeddings.
       _A					:     Any     = lm['embedding']

       # The word embeddings.
       _A					:     str     = embeddings['word_embeddings']['weight']
       # Truncate the embedding table to vocab_size rows.
       _A					:     str     = word_embeddings[: config.vocab_size, :]
       _A					:     Any     = word_embeddings

       # The position embeddings.
       _A					:     List[Any]     = embeddings['position_embeddings']['weight']
       # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
       _A					:     Union[str, Any]     = pos_embeddings.size(0       )
       if n_positions != config.n_positions:
              raise ValueError(
                  F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'       )
       # Store the position embeddings.
       _A					:     int     = pos_embeddings

       # The transformer.
       _A					:     Any     = lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

       # The regex to extract layer names.
       _A					:     List[str]     = re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'       )

       # The simple map of names for "automated" rules.
       _A					:     int     = {
           'attention.dense': '.attn.c_proj.',
           'self_attention.dense': '.attn.c_proj.',
           'mlp.dense_h_to_4h': '.mlp.c_fc.',
           'mlp.dense_4h_to_h': '.mlp.c_proj.',
       }

       # Extract the layers.
       for key, val in transformer.items():
              # Match the name.
              _A					:     Union[str, Any]     = layer_re.match(lowerCamelCase       )

              # Stop if that's not a layer
              if m is None:
                     break

              # The index of the layer.
              _A					:     Tuple     = int(m.group(1       )       )
              # The name of the operation.
              _A					:     List[Any]     = m.group(2       )
              # Is it a weight or a bias?
              _A					:     List[Any]     = m.group(3       )

              # The name of the layer.
              _A					:     int     = F'transformer.h.{layer_idx}'

              # For layernorm(s), simply store the layer norm.
              if op_name.endswith('layernorm'       ):
                     _A					:     Dict     = 'ln_1' if op_name.startswith('input'       ) else 'ln_2'
                     _A					:     List[str]     = val

              # Transpose the QKV matrix.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "weight":
                     # Insert a tensor of 1x1xDxD bias.
                     _A					:     str     = torch.tril(torch.ones((n_positions, n_positions)  ,dtype=torch.floataa       )       ).view(
                         1  ,1  ,lowerCamelCase  ,lowerCamelCase       )
                     _A					:     Tuple     = causal_mask

                     # Insert a "dummy" tensor for masked_bias.
                     _A					:     int     = torch.tensor(-1E4  ,dtype=torch.floataa       )
                     _A					:     Union[str, Any]     = masked_bias

                     _A					:     int     = fix_query_key_value_ordering(lowerCamelCase  ,lowerCamelCase  ,3  ,lowerCamelCase  ,lowerCamelCase       )
                     # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                     _A					:     Optional[Any]     = out_val.transpose(0  ,1       ).contiguous()
                     # Store.
                     _A					:     int     = out_val

              # Transpose the bias.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "bias":
                     _A					:     List[str]     = fix_query_key_value_ordering(lowerCamelCase  ,lowerCamelCase  ,3  ,lowerCamelCase  ,lowerCamelCase       )
                     # Store. No change of shape.
                     _A					:     str     = out_val

              # Transpose the weights.
              elif weight_or_bias == "weight":
                     _A					:     Union[str, Any]     = megatron_to_transformers[op_name]
                     _A					:     List[Any]     = val.transpose(0  ,1       )

              # Copy the bias.
              elif weight_or_bias == "bias":
                     _A					:     Dict     = megatron_to_transformers[op_name]
                     _A					:     Dict     = val

    # DEBUG.
       assert config.n_layer == layer_idx + 1

       # The final layernorm.
       _A					:     str     = transformer['final_layernorm.weight']
       _A					:     Dict     = transformer['final_layernorm.bias']

       # For LM head, transformers' wants the matrix to weight embeddings.
       _A					:     List[str]     = word_embeddings

       # It should be done!
       return output_state_dict






def 				lowerCAmelCase__	(    ):
       # Create the argument parser.
       _A					:     Optional[Any]     = argparse.ArgumentParser()
       parser.add_argument('--print-checkpoint-structure'  ,action='store_true'       )
       parser.add_argument(
           'path_to_checkpoint'  ,type=lowerCamelCase  ,help='Path to the checkpoint file (.zip archive or direct .pt file)'  ,)
       parser.add_argument(
           '--config_file'  ,default=''  ,type=lowerCamelCase  ,help='An optional config json file describing the pre-trained model.'  ,)
       _A					:     str     = parser.parse_args()

       # Extract the basename.
       _A					:     str     = os.path.dirname(args.path_to_checkpoint       )

       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'       )
       if args.path_to_checkpoint.endswith('.zip'       ):
              with zipfile.ZipFile(args.path_to_checkpoint  ,'r'       ) as checkpoint:
                     with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'       ) as pytorch_dict:
                            _A					:     List[str]     = torch.load(lowerCamelCase  ,map_location='cpu'       )
       else:
              _A					:     Tuple     = torch.load(args.path_to_checkpoint  ,map_location='cpu'       )

       _A					:     Optional[int]     = input_state_dict.get('args'  ,lowerCamelCase       )

       # Read the config, or default to the model released by NVIDIA.
       if args.config_file == "":
              if ds_args is not None:
                     if ds_args.bias_gelu_fusion:
                            _A					:     Optional[int]     = 'gelu_fast'
                     elif ds_args.openai_gelu:
                            _A					:     Optional[int]     = 'gelu_new'
                     else:
                            _A					:     str     = 'gelu'
              else:
                     # in the very early days this used to be "gelu_new"
                     _A					:     Union[str, Any]     = 'gelu_new'

              # Spell out all parameters in case the defaults change.
              _A					:     Dict     = GPTaConfig(
                  vocab_size=50257  ,n_positions=1024  ,n_embd=1024  ,n_layer=24  ,n_head=16  ,n_inner=4096  ,activation_function=lowerCamelCase  ,resid_pdrop=0.1  ,embd_pdrop=0.1  ,attn_pdrop=0.1  ,layer_norm_epsilon=1E-5  ,initializer_range=0.02  ,summary_type='cls_index'  ,summary_use_proj=lowerCamelCase  ,summary_activation=lowerCamelCase  ,summary_proj_to_labels=lowerCamelCase  ,summary_first_dropout=0.1  ,scale_attn_weights=lowerCamelCase  ,use_cache=lowerCamelCase  ,bos_token_id=50256  ,eos_token_id=50256  ,)
       else:
              _A					:     Optional[int]     = GPTaConfig.from_json_file(args.config_file       )

       _A					:     str     = ['GPT2LMHeadModel']

       # Convert.
       print('Converting'       )
       _A					:     Optional[Any]     = convert_megatron_checkpoint(lowerCamelCase  ,lowerCamelCase  ,lowerCamelCase       )

       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
              recursive_print(lowerCamelCase  ,lowerCamelCase       )

       # Add tokenizer class info to config
       # see https://github.com/huggingface/transformers/issues/13906)
       if ds_args is not None:
              _A					:     str     = ds_args.tokenizer_type
              if tokenizer_type == "GPT2BPETokenizer":
                     _A					:     Any     = 'gpt2'
              elif tokenizer_type == "PretrainedFromHF":
                     _A					:     Union[str, Any]     = ds_args.tokenizer_name_or_path
              else:
                     raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}'       )
       else:
              _A					:     Tuple     = 'gpt2'

       _A					:     Union[str, Any]     = AutoTokenizer.from_pretrained(lowerCamelCase       )
       _A					:     int     = type(lowerCamelCase       ).__name__
       _A					:     int     = tokenizer_class

       # Store the config to file.
       print('Saving config'       )
       config.save_pretrained(lowerCamelCase       )

       # Save tokenizer based on args
       print(F'Adding {tokenizer_class} tokenizer files'       )
       tokenizer.save_pretrained(lowerCamelCase       )

       # Store the state_dict to file.
       _A					:     Optional[Any]     = os.path.join(lowerCamelCase  ,'pytorch_model.bin'       )
       print(F'Saving checkpoint to "{output_checkpoint_file}"'       )
       torch.save(lowerCamelCase  ,lowerCamelCase       )


####################################################################################################

if __name__ == "__main__":
   main()

####################################################################################################










'''simple docstring'''







import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyVaaPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()


class 			__lowerCamelCase			(	a_    , unittest.TestCase	):
    """simple docstring"""
    a													=   KandinskyVaaPriorPipeline
    a													=   ["prompt"]
    a													=   ["prompt", "negative_prompt"]
    a													=   [
        "num_images_per_prompt",
        "generator",
        "num_inference_steps",
        "latents",
        "negative_prompt",
        "guidance_scale",
        "output_type",
        "return_dict",
    ]
    a													=   False


    @property
    def        A   (  self      :							List[str]):
           return 32


    @property
    def        A   (  self      :							List[Any]):
           return 32


    @property
    def        A   (  self      :							Dict):
           return self.time_input_dim


    @property
    def        A   (  self      :							Tuple):
           return self.time_input_dim * 4


    @property
    def        A   (  self      :							Optional[int]):
           return 100


    @property
    def        A   (  self      :							Dict):
           _A					:     Tuple     = CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip')
           return tokenizer


    @property
    def        A   (  self      :							Optional[Any]):
           torch.manual_seed(0)
           _A					:     Any     = CLIPTextConfig(
               bos_token_id=0				,							eos_token_id=2				,							hidden_size=self.text_embedder_hidden_size				,							projection_dim=self.text_embedder_hidden_size				,							intermediate_size=37				,							layer_norm_eps=1e-05				,							num_attention_heads=4				,							num_hidden_layers=5				,							pad_token_id=1				,							vocab_size=1000				,							)
           return CLIPTextModelWithProjection(SCREAMING_SNAKE_CASE)


    @property
    def        A   (  self      :							List[Any]):
           torch.manual_seed(0)

           _A					:     Optional[Any]     = {
               'num_attention_heads': 2,
               'attention_head_dim': 12,
               'embedding_dim': self.text_embedder_hidden_size,
               'num_layers': 1,
           }

           _A					:     Any     = PriorTransformer(**SCREAMING_SNAKE_CASE)
           # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
           _A					:     str     = nn.Parameter(torch.ones(model.clip_std.shape))
           return model


    @property
    def        A   (  self      :							List[str]):
           torch.manual_seed(0)
           _A					:     List[Any]     = CLIPVisionConfig(
               hidden_size=self.text_embedder_hidden_size				,							image_size=224				,							projection_dim=self.text_embedder_hidden_size				,							intermediate_size=37				,							num_attention_heads=4				,							num_channels=3				,							num_hidden_layers=5				,							patch_size=14				,							)

           _A					:     Union[str, Any]     = CLIPVisionModelWithProjection(SCREAMING_SNAKE_CASE)
           return model


    @property
    def        A   (  self      :							int):
           _A					:     Optional[Any]     = CLIPImageProcessor(
               crop_size=224				,							do_center_crop=SCREAMING_SNAKE_CASE				,							do_normalize=SCREAMING_SNAKE_CASE				,							do_resize=SCREAMING_SNAKE_CASE				,							image_mean=[0.4814_5466, 0.457_8275, 0.4082_1073]				,							image_std=[0.2686_2954, 0.2613_0258, 0.2757_7711]				,							resample=3				,							size=224				,							)

           return image_processor


    def        A   (  self      :							Optional[Any]):
           _A					:     Optional[int]     = self.dummy_prior
           _A					:     Dict     = self.dummy_image_encoder
           _A					:     Dict     = self.dummy_text_encoder
           _A					:     str     = self.dummy_tokenizer
           _A					:     Optional[Any]     = self.dummy_image_processor

           _A					:     Optional[Any]     = UnCLIPScheduler(
               variance_type='fixed_small_log'				,							prediction_type='sample'				,							num_train_timesteps=1000				,							clip_sample=SCREAMING_SNAKE_CASE				,							clip_sample_range=10.0				,							)

           _A					:     Dict     = {
               'prior': prior,
               'image_encoder': image_encoder,
               'text_encoder': text_encoder,
               'tokenizer': tokenizer,
               'scheduler': scheduler,
               'image_processor': image_processor,
           }

           return components


    def        A   (  self      :							Tuple				,							SCREAMING_SNAKE_CASE      :							List[Any]				,							SCREAMING_SNAKE_CASE      :							List[str]=0):
           if str(SCREAMING_SNAKE_CASE).startswith('mps'):
                  _A					:     Dict     = torch.manual_seed(SCREAMING_SNAKE_CASE)
           else:
                  _A					:     int     = torch.Generator(device=SCREAMING_SNAKE_CASE).manual_seed(SCREAMING_SNAKE_CASE)
           _A					:     List[Any]     = {
               'prompt': 'horse',
               'generator': generator,
               'guidance_scale': 4.0,
               'num_inference_steps': 2,
               'output_type': 'np',
           }
           return inputs


    def        A   (  self      :							List[Any]):
           _A					:     str     = 'cpu'

           _A					:     Tuple     = self.get_dummy_components()

           _A					:     List[Any]     = self.pipeline_class(**SCREAMING_SNAKE_CASE)
           _A					:     Any     = pipe.to(SCREAMING_SNAKE_CASE)

           pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE)

           _A					:     Dict     = pipe(**self.get_dummy_inputs(SCREAMING_SNAKE_CASE))
           _A					:     str     = output.image_embeds

           _A					:     Optional[int]     = pipe(
               **self.get_dummy_inputs(SCREAMING_SNAKE_CASE)				,							return_dict=SCREAMING_SNAKE_CASE				,							)[0]

           _A					:     Optional[int]     = image[0, -10:]
           _A					:     int     = image_from_tuple[0, -10:]

           assert image.shape == (1, 32)

           _A					:     Dict     = np.array(
               [-0.0532, 1.7120, 0.3656, -1.0852, -0.8946, -1.1756, 0.4348, 0.2482, 0.5146, -0.1156])

           assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
           assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2


    @skip_mps
    def        A   (  self      :							Any):
           _A					:     Tuple     = torch_device == 'cpu'
           _A					:     Optional[int]     = True
           _A					:     Tuple     = False

           self._test_inference_batch_single_identical(
               test_max_difference=SCREAMING_SNAKE_CASE				,							relax_max_difference=SCREAMING_SNAKE_CASE				,							test_mean_pixel_difference=SCREAMING_SNAKE_CASE				,							)


    @skip_mps
    def        A   (  self      :							int):
           _A					:     Tuple     = torch_device == 'cpu'
           _A					:     Optional[Any]     = False

           self._test_attention_slicing_forward_pass(
               test_max_difference=SCREAMING_SNAKE_CASE				,							test_mean_pixel_difference=SCREAMING_SNAKE_CASE				,							)






