




"""simple docstring"""




import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_lowerCamelCase      			=						[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

_lowerCamelCase      			=						[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						List[str]	,					__UpperCamelCase      :						Dict				)  ->							str:
    UpperCAmelCase_										=					{
        '''word_embeddings.weight''': '''word_embeddings.weight''',
        '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
        '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
        '''weight''': '''ln_f.weight''',
        '''bias''': '''ln_f.bias''',
    }

    if key in layer_rename_map:
        return layer_rename_map[key]

    # Handle transformer blocks
    UpperCAmelCase_										=					int(re.match(R'''.*layer_(\d*).*'''	,					snake_case__				)[1]				)
    layer_number -= 3
    return f'h.{layer_number}.' + key




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Tuple				)  ->							int:
    if dtype == torch.bool:
        return 1 / 8
    UpperCAmelCase_										=					re.search(R'''[^\d](\d+)$'''	,					str(snake_case__				)				)
    if bit_search is None:
        raise ValueError(f'`dtype` is not a valid dtype: {dtype}.'				)
    UpperCAmelCase_										=					int(bit_search.groups()[0]				)
    return bit_size // 8




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Dict	,					__UpperCamelCase      :						Any	,					__UpperCamelCase      :						Union[str, Any]	,					__UpperCamelCase      :						Tuple	,					__UpperCamelCase      :						List[Any]				)  ->							List[str]:
    # Construct model
    if bloom_config_file == "":
        UpperCAmelCase_										=					BloomConfig()
    else:
        UpperCAmelCase_										=					BloomConfig.from_json_file(snake_case__				)

    if shard_model:
        UpperCAmelCase_										=					os.listdir(snake_case__				)
        UpperCAmelCase_										=					sorted(filter(lambda __UpperCamelCase				: s.startswith('''layer'''				) and "model_00" in s	,					snake_case__				)				)

        UpperCAmelCase_										=					{'''weight_map''': {}, '''metadata''': {}}
        UpperCAmelCase_										=					0

        UpperCAmelCase_										=					None

        UpperCAmelCase_										=					BloomConfig()

        for j, file in enumerate(snake_case__				):
            print('''Processing file: {}'''.format(snake_case__				)				)
            UpperCAmelCase_										=					None

            for i in range(snake_case__				):
                # load all TP files
                UpperCAmelCase_										=					file.replace('''model_00'''	,					f'model_0{i}'				)
                UpperCAmelCase_										=					torch.load(os.path.join(snake_case__	,					snake_case__				)	,					map_location='''cpu'''				)

                # Rename keys in the transformers names
                UpperCAmelCase_										=					list(temp.keys()				)
                for key in keys:
                    UpperCAmelCase_										=					temp.pop(snake_case__				)

                if tensors is None:
                    UpperCAmelCase_										=					temp
                else:
                    for key in tensors.keys():
                        if any(key.endswith(snake_case__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            UpperCAmelCase_										=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
                            # We concatenate these weights accross TP ranks
                            UpperCAmelCase_										=					torch.cat([tensors[key], temp[key]]	,					dim=snake_case__				)

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(snake_case__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                    UpperCAmelCase_										=					tensors[key] / pretraining_tp
            torch.save(
                snake_case__	,					os.path.join(
                    snake_case__	,					'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1				).zfill(5				)	,					str(len(snake_case__				)				).zfill(5				)				)	,					)	,					)

            for key in tensors.keys():
                UpperCAmelCase_										=					tensors[key]
                total_size += value.numel() * get_dtype_size(value.dtype				)
                if key not in index_dict["weight_map"]:
                    UpperCAmelCase_										=					'''pytorch_model_{}-of-{}.bin'''.format(
                        str(j + 1				).zfill(5				)	,					str(len(snake_case__				)				).zfill(5				)				)

        UpperCAmelCase_										=					BloomConfig()
        UpperCAmelCase_										=					pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        UpperCAmelCase_										=					total_size
        with open(snake_case__	,					'''w'''	,					encoding='''utf-8'''				) as f:
            f.write(config.to_json_string()				)
        with open(os.path.join(snake_case__	,					WEIGHTS_NAME + '''.index.json'''				)	,					'''w'''	,					encoding='''utf-8'''				) as f:
            UpperCAmelCase_										=					json.dumps(snake_case__	,					indent=2	,					sort_keys=snake_case__				) + '''\n'''
            f.write(snake_case__				)
    else:
        UpperCAmelCase_										=					BloomModel(snake_case__				)

        UpperCAmelCase_										=					os.listdir(snake_case__				)
        UpperCAmelCase_										=					sorted(filter(lambda __UpperCamelCase				: s.startswith('''layer'''				) and "model_00" in s	,					snake_case__				)				)

        UpperCAmelCase_										=					None
        for i, file in enumerate(snake_case__				):
            UpperCAmelCase_										=					None
            for i in range(snake_case__				):
                # load all TP files
                UpperCAmelCase_										=					file.replace('''model_00'''	,					f'model_0{i}'				)
                UpperCAmelCase_										=					torch.load(os.path.join(snake_case__	,					snake_case__				)	,					map_location='''cpu'''				)

                # Rename keys in the transformers names
                UpperCAmelCase_										=					list(temp.keys()				)
                for key in keys:
                    UpperCAmelCase_										=					temp.pop(snake_case__				)

                if tensors is None:
                    UpperCAmelCase_										=					temp
                else:
                    for key in tensors.keys():
                        # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                        if any(key.endswith(snake_case__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            UpperCAmelCase_										=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
                            # We concatenate these weights accross TP ranks
                            UpperCAmelCase_										=					torch.cat([tensors[key], temp[key]]	,					dim=snake_case__				)

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(snake_case__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                    UpperCAmelCase_										=					tensors[key] / pretraining_tp

            UpperCAmelCase_										=					model.load_state_dict(snake_case__	,					strict=snake_case__				)
            assert not other_keys.unexpected_keys, f'The keys {other_keys.unexpected_keys} are unexpected'
            if missing_keys is None:
                UpperCAmelCase_										=					set(other_keys.missing_keys				)
            else:
                UpperCAmelCase_										=					missing_keys.intersection(set(other_keys.missing_keys				)				)

        assert not missing_keys, f'The keys {missing_keys} are missing'

        # Save pytorch-model
        os.makedirs(snake_case__	,					exist_ok=snake_case__				)
        UpperCAmelCase_										=					pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
        UpperCAmelCase_										=					pytorch_dump_folder_path + '''/''' + CONFIG_NAME
        print(f'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'				)
        if config.torch_dtype is not None:
            UpperCAmelCase_										=					model.to(config.torch_dtype				)
        torch.save(model.state_dict()	,					snake_case__				)
        print(f'Save configuration file to {pytorch_config_dump_path}'				)
        with open(snake_case__	,					'''w'''	,					encoding='''utf-8'''				) as f:
            f.write(config.to_json_string()				)


if __name__ == "__main__":
 _lowerCamelCase      			=						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--bloom_checkpoint_path',
     default=None,
     type=str,
     required=True,
     help='Path to the Megatron-LM checkpoint path.',
 )
 parser.add_argument(
     '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
 )
 parser.add_argument(
     '--bloom_config_file',
     default='',
     type=str,
     help=(
         'An optional config json file corresponding to the pre-trained model. \n'
         'This specifies the model architecture.'
     ),
 )
 parser.add_argument(
     '--shard_model',
     action='store_true',
     help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
 )
 parser.add_argument(
     '--pretraining_tp',
     default=4,
     type=int,
     help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
 )
 _lowerCamelCase      			=						parser.parse_args()
 convert_bloom_checkpoint_to_pytorch(
     args.bloom_checkpoint_path,
     args.bloom_config_file,
     args.pytorch_dump_folder_path,
     args.shard_model,
     args.pretraining_tp,
 )











import baseaa




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						str				)  ->							bytes:
    return baseaa.baaencode(string.encode('''utf-8'''				)				)




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						bytes				)  ->							str:
    return baseaa.baadecode(__UpperCamelCase				).decode('''utf-8'''				)


if __name__ == "__main__":
 _lowerCamelCase      			=						'Hello World!'
 _lowerCamelCase      			=						baseaa_encode(test)
 print(encoded)

 _lowerCamelCase      			=						baseaa_decode(encoded)
 print(decoded)





