





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_UpperCAmelCase             =							[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

_UpperCAmelCase             =							[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Dict   ,   SCREAMING_SNAKE_CASE		:Optional[Any]   )      ->   List[Any]:
 __lowerCAmelCase					:	Dict      =       {
     """word_embeddings.weight""": """word_embeddings.weight""",
     """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
     """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
     """weight""": """ln_f.weight""",
     """bias""": """ln_f.bias""",
 }

 if key in layer_rename_map:
  return layer_rename_map[key]

 # Handle transformer blocks
 __lowerCAmelCase					:	List[str]      =       int(re.match(r""".*layer_(\d*).*"""   ,   SCREAMING_SNAKE_CASE   )[1]   )
 layer_number -= 3
 return F'''h.{layer_number}.''' + key






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Tuple   )      ->   List[str]:
 if dtype == torch.bool:
  return 1 / 8
 __lowerCAmelCase					:	Union[str, Any]      =       re.search(r"""[^\d](\d+)$"""   ,   str(SCREAMING_SNAKE_CASE   )   )
 if bit_search is None:
  raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''   )
 __lowerCAmelCase					:	Dict      =       int(bit_search.groups()[0]   )
 return bit_size // 8






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Any   ,   SCREAMING_SNAKE_CASE		:List[str]   ,   SCREAMING_SNAKE_CASE		:Optional[int]   ,   SCREAMING_SNAKE_CASE		:List[str]   ,   SCREAMING_SNAKE_CASE		:int   )      ->   Union[str, Any]:
 # Construct model
 if bloom_config_file == "":
  __lowerCAmelCase					:	str      =       BloomConfig()
 else:
  __lowerCAmelCase					:	Optional[int]      =       BloomConfig.from_json_file(SCREAMING_SNAKE_CASE   )

 if shard_model:
  __lowerCAmelCase					:	Tuple      =       os.listdir(SCREAMING_SNAKE_CASE   )
  __lowerCAmelCase					:	int      =       sorted(filter(lambda SCREAMING_SNAKE_CASE   : s.startswith("""layer"""   ) and "model_00" in s   ,   SCREAMING_SNAKE_CASE   )   )

  __lowerCAmelCase					:	Any      =       {"""weight_map""": {}, """metadata""": {}}
  __lowerCAmelCase					:	List[str]      =       0

  __lowerCAmelCase					:	Optional[int]      =       None

  __lowerCAmelCase					:	Dict      =       BloomConfig()

  for j, file in enumerate(SCREAMING_SNAKE_CASE   ):
   print("""Processing file: {}""".format(SCREAMING_SNAKE_CASE   )   )
   __lowerCAmelCase					:	Optional[Any]      =       None

   for i in range(SCREAMING_SNAKE_CASE   ):
    # load all TP files
    __lowerCAmelCase					:	Dict      =       file.replace("""model_00"""   ,   F'''model_0{i}'''   )
    __lowerCAmelCase					:	List[str]      =       torch.load(os.path.join(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )   ,   map_location="""cpu"""   )

    # Rename keys in the transformers names
    __lowerCAmelCase					:	Tuple      =       list(temp.keys()   )
    for key in keys:
     __lowerCAmelCase					:	int      =       temp.pop(SCREAMING_SNAKE_CASE   )

    if tensors is None:
     __lowerCAmelCase					:	Union[str, Any]      =       temp
    else:
     for key in tensors.keys():
      if any(key.endswith(SCREAMING_SNAKE_CASE   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
       # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       __lowerCAmelCase					:	List[Any]      =       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
       # We concatenate these weights accross TP ranks
       __lowerCAmelCase					:	Union[str, Any]      =       torch.cat([tensors[key], temp[key]]   ,   dim=SCREAMING_SNAKE_CASE   )

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(SCREAMING_SNAKE_CASE   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
     __lowerCAmelCase					:	List[Any]      =       tensors[key] / pretraining_tp
   torch.save(
       SCREAMING_SNAKE_CASE   ,   os.path.join(
           SCREAMING_SNAKE_CASE   ,   """pytorch_model_{}-of-{}.bin""".format(str(j + 1   ).zfill(5   )   ,   str(len(SCREAMING_SNAKE_CASE   )   ).zfill(5   )   )   ,   )   ,   )

   for key in tensors.keys():
    __lowerCAmelCase					:	List[Any]      =       tensors[key]
    total_size += value.numel() * get_dtype_size(value.dtype   )
    if key not in index_dict["weight_map"]:
     __lowerCAmelCase					:	Optional[Any]      =       """pytorch_model_{}-of-{}.bin""".format(
         str(j + 1   ).zfill(5   )   ,   str(len(SCREAMING_SNAKE_CASE   )   ).zfill(5   )   )

  __lowerCAmelCase					:	Union[str, Any]      =       BloomConfig()
  __lowerCAmelCase					:	Dict      =       pytorch_dump_folder_path + """/""" + CONFIG_NAME
  __lowerCAmelCase					:	List[Any]      =       total_size
  with open(SCREAMING_SNAKE_CASE   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
   f.write(config.to_json_string()   )
  with open(os.path.join(SCREAMING_SNAKE_CASE   ,   WEIGHTS_NAME + """.index.json"""   )   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
   __lowerCAmelCase					:	Tuple      =       json.dumps(SCREAMING_SNAKE_CASE   ,   indent=2   ,   sort_keys=SCREAMING_SNAKE_CASE   ) + """\n"""
   f.write(SCREAMING_SNAKE_CASE   )
 else:
  __lowerCAmelCase					:	List[str]      =       BloomModel(SCREAMING_SNAKE_CASE   )

  __lowerCAmelCase					:	Optional[int]      =       os.listdir(SCREAMING_SNAKE_CASE   )
  __lowerCAmelCase					:	Optional[Any]      =       sorted(filter(lambda SCREAMING_SNAKE_CASE   : s.startswith("""layer"""   ) and "model_00" in s   ,   SCREAMING_SNAKE_CASE   )   )

  __lowerCAmelCase					:	Tuple      =       None
  for i, file in enumerate(SCREAMING_SNAKE_CASE   ):
   __lowerCAmelCase					:	Union[str, Any]      =       None
   for i in range(SCREAMING_SNAKE_CASE   ):
    # load all TP files
    __lowerCAmelCase					:	Optional[Any]      =       file.replace("""model_00"""   ,   F'''model_0{i}'''   )
    __lowerCAmelCase					:	Optional[int]      =       torch.load(os.path.join(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )   ,   map_location="""cpu"""   )

    # Rename keys in the transformers names
    __lowerCAmelCase					:	Optional[Any]      =       list(temp.keys()   )
    for key in keys:
     __lowerCAmelCase					:	Any      =       temp.pop(SCREAMING_SNAKE_CASE   )

    if tensors is None:
     __lowerCAmelCase					:	List[Any]      =       temp
    else:
     for key in tensors.keys():
      # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
      if any(key.endswith(SCREAMING_SNAKE_CASE   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       __lowerCAmelCase					:	List[Any]      =       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
       # We concatenate these weights accross TP ranks
       __lowerCAmelCase					:	List[Any]      =       torch.cat([tensors[key], temp[key]]   ,   dim=SCREAMING_SNAKE_CASE   )

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(SCREAMING_SNAKE_CASE   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
     __lowerCAmelCase					:	Dict      =       tensors[key] / pretraining_tp

   __lowerCAmelCase					:	Optional[int]      =       model.load_state_dict(SCREAMING_SNAKE_CASE   ,   strict=SCREAMING_SNAKE_CASE   )
   assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
   if missing_keys is None:
    __lowerCAmelCase					:	Union[str, Any]      =       set(other_keys.missing_keys   )
   else:
    __lowerCAmelCase					:	List[str]      =       missing_keys.intersection(set(other_keys.missing_keys   )   )

  assert not missing_keys, F'''The keys {missing_keys} are missing'''

  # Save pytorch-model
  os.makedirs(SCREAMING_SNAKE_CASE   ,   exist_ok=SCREAMING_SNAKE_CASE   )
  __lowerCAmelCase					:	Any      =       pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
  __lowerCAmelCase					:	Dict      =       pytorch_dump_folder_path + """/""" + CONFIG_NAME
  print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''   )
  if config.torch_dtype is not None:
   __lowerCAmelCase					:	List[str]      =       model.to(config.torch_dtype   )
  torch.save(model.state_dict()   ,   SCREAMING_SNAKE_CASE   )
  print(F'''Save configuration file to {pytorch_config_dump_path}'''   )
  with open(SCREAMING_SNAKE_CASE   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
   f.write(config.to_json_string()   )


if __name__ == "__main__":
      _UpperCAmelCase             =							argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--bloom_checkpoint_path',
          default=None,
          type=str,
          required=True,
          help='Path to the Megatron-LM checkpoint path.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
      )
      parser.add_argument(
          '--bloom_config_file',
          default='',
          type=str,
          help=(
              'An optional config json file corresponding to the pre-trained model. \n'
              'This specifies the model architecture.'
          ),
      )
      parser.add_argument(
          '--shard_model',
          action='store_true',
          help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
      )
      parser.add_argument(
          '--pretraining_tp',
          default=4,
          type=int,
          help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
      )
      _UpperCAmelCase             =							parser.parse_args()
      convert_bloom_checkpoint_to_pytorch(
          args.bloom_checkpoint_path,
          args.bloom_config_file,
          args.pytorch_dump_folder_path,
          args.shard_model,
          args.pretraining_tp,
      )






from __future__ import annotations






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:list[list[int]]   )      ->   bool:
 __lowerCAmelCase					:	Tuple      =       len(SCREAMING_SNAKE_CASE   )
 # We need to create solution object to save path.
 __lowerCAmelCase					:	str      =       [[0 for _ in range(SCREAMING_SNAKE_CASE   )] for _ in range(SCREAMING_SNAKE_CASE   )]
 __lowerCAmelCase					:	str      =       run_maze(SCREAMING_SNAKE_CASE   ,   0   ,   0   ,   SCREAMING_SNAKE_CASE   )
 if solved:
  print("""\n""".join(str(SCREAMING_SNAKE_CASE   ) for row in solutions   )   )
 else:
  print("""No solution exists!"""   )
 return solved






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:list[list[int]]   ,   SCREAMING_SNAKE_CASE		:int   ,   SCREAMING_SNAKE_CASE		:int   ,   SCREAMING_SNAKE_CASE		:list[list[int]]   )      ->   bool:
 __lowerCAmelCase					:	List[str]      =       len(SCREAMING_SNAKE_CASE   )
 # Final check point.
 if i == j == (size - 1):
  __lowerCAmelCase					:	str      =       1
  return True

 __lowerCAmelCase					:	Optional[Any]      =       (not i < 0) and (not j < 0)  # Check lower bounds
 __lowerCAmelCase					:	Optional[Any]      =       (i < size) and (j < size)  # Check upper bounds

 if lower_flag and upper_flag:
  # check for already visited and block points.
  __lowerCAmelCase					:	int      =       (not solutions[i][j]) and (not maze[i][j])
  if block_flag:
   # check visited
   __lowerCAmelCase					:	Tuple      =       1

   # check for directions
   if (
       run_maze(SCREAMING_SNAKE_CASE   ,   i + 1   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )
       or run_maze(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   j + 1   ,   SCREAMING_SNAKE_CASE   )
       or run_maze(SCREAMING_SNAKE_CASE   ,   i - 1   ,   SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   )
       or run_maze(SCREAMING_SNAKE_CASE   ,   SCREAMING_SNAKE_CASE   ,   j - 1   ,   SCREAMING_SNAKE_CASE   )
   ):
    return True

   __lowerCAmelCase					:	Tuple      =       0
   return False
 return False


if __name__ == "__main__":
      import doctest

      doctest.testmod()