


'''simple docstring'''
import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__snake_case											=					[
    '''word_embeddings_layernorm.weight''',
    '''word_embeddings_layernorm.bias''',
    '''input_layernorm.weight''',
    '''input_layernorm.bias''',
    '''post_attention_layernorm.weight''',
    '''post_attention_layernorm.bias''',
    '''self_attention.dense.bias''',
    '''mlp.dense_4h_to_h.bias''',
    '''ln_f.weight''',
    '''ln_f.bias''',
]

__snake_case											=					[
    '''mlp.dense_4h_to_h.weight''',
    '''self_attention.dense.weight''',
]


def       a				(      __a			, __a       )  ->     int:



					'''simple docstring'''






					UpperCamelCase__       :List[Any]							 =			{
					    '''word_embeddings.weight''': '''word_embeddings.weight''',
					    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
					    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
					    '''weight''': '''ln_f.weight''',
					    '''bias''': '''ln_f.bias''',
					}

					if key in layer_rename_map:
										return layer_rename_map[key]

					# Handle transformer blocks
					UpperCamelCase__       :Optional[int]							 =			int(re.match(R'''.*layer_(\d*).*'''			, __a       )[1]       )
					layer_number -= 3
					return f'''h.{layer_number}.''' + key


def       a				(      __a       )  ->     int:



					'''simple docstring'''






					if dtype == torch.bool:
										return 1 / 8
					UpperCamelCase__       :Optional[int]							 =			re.search(R'''[^\d](\d+)$'''			, str(__a       )       )
					if bit_search is None:
										raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''       )
					UpperCamelCase__       :Optional[int]							 =			int(bit_search.groups()[0]       )
					return bit_size // 8


def       a				(      __a			, __a			, __a			, __a			, __a       )  ->     Union[str, Any]:



					'''simple docstring'''






					if bloom_config_file == "":
										UpperCamelCase__       :Dict							 =			BloomConfig()
					else:
										UpperCamelCase__       :List[str]							 =			BloomConfig.from_json_file(__a       )

					if shard_model:
										UpperCamelCase__       :int							 =			os.listdir(__a       )
										UpperCamelCase__       :Dict							 =			sorted(filter(lambda __a       : s.startswith('''layer'''       ) and "model_00" in s			, __a       )       )

										UpperCamelCase__       :List[Any]							 =			{'''weight_map''': {}, '''metadata''': {}}
										UpperCamelCase__       :Union[str, Any]							 =			0

										UpperCamelCase__       :Union[str, Any]							 =			None

										UpperCamelCase__       :int							 =			BloomConfig()

										for j, file in enumerate(__a       ):
															print('''Processing file: {}'''.format(__a       )       )
															UpperCamelCase__       :Tuple							 =			None

															for i in range(__a       ):
																				# load all TP files
																				UpperCamelCase__       :int							 =			file.replace('''model_00'''			, f'''model_0{i}'''       )
																				UpperCamelCase__       :List[str]							 =			torch.load(os.path.join(__a			, __a       )			, map_location='''cpu'''       )

																				# Rename keys in the transformers names
																				UpperCamelCase__       :List[Any]							 =			list(temp.keys()       )
																				for key in keys:
																									UpperCamelCase__       :Tuple							 =			temp.pop(__a       )

																				if tensors is None:
																									UpperCamelCase__       :Any							 =			temp
																				else:
																									for key in tensors.keys():
																														if any(key.endswith(__a       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																			# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			UpperCamelCase__       :Dict							 =			1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN       ) else 0
																																			# We concatenate these weights accross TP ranks
																																			UpperCamelCase__       :Optional[int]							 =			torch.cat([tensors[key], temp[key]]			, dim=__a       )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(__a       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																									UpperCamelCase__       :int							 =			tensors[key] / pretraining_tp
															torch.save(
															    __a			, os.path.join(
															        __a			, '''pytorch_model_{}-of-{}.bin'''.format(str(j + 1       ).zfill(5       )			, str(len(__a       )       ).zfill(5       )       )			, )			, )

															for key in tensors.keys():
																				UpperCamelCase__       :Tuple							 =			tensors[key]
																				total_size += value.numel() * get_dtype_size(value.dtype       )
																				if key not in index_dict["weight_map"]:
																									UpperCamelCase__       :Tuple							 =			'''pytorch_model_{}-of-{}.bin'''.format(
																									    str(j + 1       ).zfill(5       )			, str(len(__a       )       ).zfill(5       )       )

										UpperCamelCase__       :Dict							 =			BloomConfig()
										UpperCamelCase__       :List[str]							 =			pytorch_dump_folder_path + '''/''' + CONFIG_NAME
										UpperCamelCase__       :Tuple							 =			total_size
										with open(__a			, '''w'''			, encoding='''utf-8'''       ) as f:
															f.write(config.to_json_string()       )
										with open(os.path.join(__a			, WEIGHTS_NAME + '''.index.json'''       )			, '''w'''			, encoding='''utf-8'''       ) as f:
															UpperCamelCase__       :Dict							 =			json.dumps(__a			, indent=2			, sort_keys=__a       ) + '''\n'''
															f.write(__a       )
					else:
										UpperCamelCase__       :Optional[Any]							 =			BloomModel(__a       )

										UpperCamelCase__       :Dict							 =			os.listdir(__a       )
										UpperCamelCase__       :str							 =			sorted(filter(lambda __a       : s.startswith('''layer'''       ) and "model_00" in s			, __a       )       )

										UpperCamelCase__       :Union[str, Any]							 =			None
										for i, file in enumerate(__a       ):
															UpperCamelCase__       :List[Any]							 =			None
															for i in range(__a       ):
																				# load all TP files
																				UpperCamelCase__       :str							 =			file.replace('''model_00'''			, f'''model_0{i}'''       )
																				UpperCamelCase__       :Optional[int]							 =			torch.load(os.path.join(__a			, __a       )			, map_location='''cpu'''       )

																				# Rename keys in the transformers names
																				UpperCamelCase__       :Optional[int]							 =			list(temp.keys()       )
																				for key in keys:
																									UpperCamelCase__       :Dict							 =			temp.pop(__a       )

																				if tensors is None:
																									UpperCamelCase__       :Dict							 =			temp
																				else:
																									for key in tensors.keys():
																														# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																														if any(key.endswith(__a       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			UpperCamelCase__       :Optional[Any]							 =			1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN       ) else 0
																																			# We concatenate these weights accross TP ranks
																																			UpperCamelCase__       :Dict							 =			torch.cat([tensors[key], temp[key]]			, dim=__a       )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(__a       ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH       ):
																									UpperCamelCase__       :Tuple							 =			tensors[key] / pretraining_tp

															UpperCamelCase__       :Optional[Any]							 =			model.load_state_dict(__a			, strict=__a       )
															assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
															if missing_keys is None:
																				UpperCamelCase__       :Optional[Any]							 =			set(other_keys.missing_keys       )
															else:
																				UpperCamelCase__       :str							 =			missing_keys.intersection(set(other_keys.missing_keys       )       )

										assert not missing_keys, f'''The keys {missing_keys} are missing'''

										# Save pytorch-model
										os.makedirs(__a			, exist_ok=__a       )
										UpperCamelCase__       :Optional[Any]							 =			pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
										UpperCamelCase__       :Tuple							 =			pytorch_dump_folder_path + '''/''' + CONFIG_NAME
										print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''       )
										if config.torch_dtype is not None:
															UpperCamelCase__       :List[Any]							 =			model.to(config.torch_dtype       )
										torch.save(model.state_dict()			, __a       )
										print(f'''Save configuration file to {pytorch_config_dump_path}'''       )
										with open(__a			, '''w'''			, encoding='''utf-8'''       ) as f:
															f.write(config.to_json_string()       )


if __name__ == "__main__":
		__snake_case											=					argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--bloom_checkpoint_path''',
		    default=None,
		    type=str,
		    required=True,
		    help='''Path to the Megatron-LM checkpoint path.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
		)
		parser.add_argument(
		    '''--bloom_config_file''',
		    default='''''',
		    type=str,
		    help=(
		        '''An optional config json file corresponding to the pre-trained model. \n'''
		        '''This specifies the model architecture.'''
		    ),
		)
		parser.add_argument(
		    '''--shard_model''',
		    action='''store_true''',
		    help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
		)
		parser.add_argument(
		    '''--pretraining_tp''',
		    default=4,
		    type=int,
		    help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
		)
		__snake_case											=					parser.parse_args()
		convert_bloom_checkpoint_to_pytorch(
		    args.bloom_checkpoint_path,
		    args.bloom_config_file,
		    args.pytorch_dump_folder_path,
		    args.shard_model,
		    args.pretraining_tp,
		)



'''simple docstring'''
from collections import Counter
from timeit import timeit


def       a				(      __a = ""			, )  ->     bool:



					'''simple docstring'''






					return sum(c % 2 for c in Counter(input_str.replace(''' '''			, ''''''       ).lower()       ).values()       ) < 2


def       a				(      __a = ""       )  ->     bool:



					'''simple docstring'''






					if len(__a       ) == 0:
										return True
					UpperCamelCase__       :List[Any]							 =			input_str.replace(''' '''			, ''''''       ).lower()
					# character_freq_dict: Stores the frequency of every character in the input string
					UpperCamelCase__       :dict[str, int]							 =			{}

					for character in lower_case_input_str:
										UpperCamelCase__       :Optional[int]							 =			character_freq_dict.get(__a			, 0       ) + 1
					UpperCamelCase__       :List[str]							 =			0

					for character_count in character_freq_dict.values():
										if character_count % 2:
															odd_char += 1
					if odd_char > 1:
										return False
					return True


def       a				(      __a = ""       )  ->     None:



					'''simple docstring'''






					print('''\nFor string = '''			, __a			, ''':'''       )
					print(
					    '''> can_string_be_rearranged_as_palindrome_counter()'''			, '''\tans ='''			, can_string_be_rearranged_as_palindrome_counter(__a       )			, '''\ttime ='''			, timeit(
					        '''z.can_string_be_rearranged_as_palindrome_counter(z.check_str)'''			, setup='''import __main__ as z'''			, )			, '''seconds'''			, )
					print(
					    '''> can_string_be_rearranged_as_palindrome()'''			, '''\tans ='''			, can_string_be_rearranged_as_palindrome(__a       )			, '''\ttime ='''			, timeit(
					        '''z.can_string_be_rearranged_as_palindrome(z.check_str)'''			, setup='''import __main__ as z'''			, )			, '''seconds'''			, )


if __name__ == "__main__":
		__snake_case											=					input(
		    '''Enter string to determine if it can be rearranged as a palindrome or not: '''
		).strip()
		benchmark(check_str)
		__snake_case											=					can_string_be_rearranged_as_palindrome_counter(check_str)
		print(F"""{check_str} can {'' if status else 'not '}be rearranged as a palindrome""")