import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
	)
	_UpperCAmelCase            =		None
_UpperCAmelCase            =		{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_UpperCAmelCase            =		{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}




def     UpperCamelCase						(	__lowercase						:			str						,__lowercase						:			Dict=1						,__lowercase						:			List[str]=2_56     ):
		'''simple docstring'''




		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)




def     UpperCamelCase						(	__lowercase						:			Optional[Any]     ):
		'''simple docstring'''




		with open(__lowercase						,'r'     ) as f:
				return json.load(__lowercase     )




def     UpperCamelCase						(	__lowercase						:			str						,__lowercase						:			List[str]     ):
		'''simple docstring'''




		with open(__lowercase						,'w'     ) as f:
				json.dump(__lowercase						,__lowercase     )




def     UpperCamelCase						(	__lowercase						:			Dict						,__lowercase						:			str						,__lowercase						:			Dict						,__lowercase						:			Optional[Any]=True     ):
		'''simple docstring'''




		os.makedirs(__lowercase						,exist_ok=__lowercase     )
		A_		:    str         =				os.path.join(__lowercase						,'tmp'     )
		os.makedirs(__lowercase						,exist_ok=__lowercase     )

		A_		:    Union[str, Any]         =				read_json(os.path.join(__lowercase						,'params.json'     )     )
		A_		:    str         =				NUM_SHARDS[model_size]
		A_		:    Optional[Any]         =				params['n_layers']
		A_		:    int         =				params['n_heads']
		A_		:    List[Any]         =				n_heads // num_shards
		A_		:    Any         =				params['dim']
		A_		:    Tuple         =				dim // n_heads
		A_		:    Optional[int]         =				1_00_00.0
		A_		:    str         =				1.0 / (base ** (torch.arange(0						,__lowercase						,2     ).float() / dims_per_head))

		if "n_kv_heads" in params:
				A_		:    List[Any]         =				params['n_kv_heads']  # for GQA / MQA
				A_		:    List[Any]         =				n_heads_per_shard // num_key_value_heads
				A_		:    Optional[int]         =				dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				A_		:    Any         =				n_heads
				A_		:    Union[str, Any]         =				n_heads_per_shard
				A_		:    str         =				dim

		# permute for sliced rotary
		def permute(__lowercase						:			str						,__lowercase						:			int=n_heads						,__lowercase						:			Optional[Any]=dim						,__lowercase						:			Tuple=dim     ):
				return w.view(__lowercase						,dima // n_heads // 2						,2						,__lowercase     ).transpose(1						,2     ).reshape(__lowercase						,__lowercase     )

		print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''     )
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				A_		:    List[Any]         =				torch.load(os.path.join(__lowercase						,'consolidated.00.pth'     )						,map_location='cpu'     )
		else:
				# Sharded
				A_		:    int         =				[
				    torch.load(os.path.join(__lowercase						,f'''consolidated.{i:02d}.pth'''     )						,map_location='cpu'     )
				    for i in range(__lowercase     )
				]
		A_		:    Tuple         =				0
		A_		:    Optional[int]         =				{'weight_map': {}}
		for layer_i in range(__lowercase     ):
				A_		:    str         =				f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
				if model_size == "7B":
						# Unsharded
						A_		:    int         =				{
						    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wq.weight''']     ),
						    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wk.weight''']     ),
						    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
						    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
						    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
						    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
						    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						A_		:    List[str]         =				{
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.attention_norm.weight'''
						    ].clone(),
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.ffn_norm.weight'''
						    ].clone(),
						}
						A_		:    List[str]         =				permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(__lowercase						,__lowercase						,__lowercase     )
						            for i in range(__lowercase     )
						        ]						,dim=0						,).reshape(__lowercase						,__lowercase     )     )
						A_		:    List[str]         =				permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
						                __lowercase						,__lowercase						,__lowercase     )
						            for i in range(__lowercase     )
						        ]						,dim=0						,).reshape(__lowercase						,__lowercase     )						,__lowercase						,__lowercase						,__lowercase						,)
						A_		:    List[Any]         =				torch.cat(
						    [
						        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
						            __lowercase						,__lowercase						,__lowercase     )
						        for i in range(__lowercase     )
						    ]						,dim=0						,).reshape(__lowercase						,__lowercase     )

						A_		:    str         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(__lowercase     )]						,dim=1     )
						A_		:    str         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(__lowercase     )]						,dim=0     )
						A_		:    Tuple         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(__lowercase     )]						,dim=1     )
						A_		:    Union[str, Any]         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(__lowercase     )]						,dim=0     )

				A_		:    List[Any]         =				inv_freq
				for k, v in state_dict.items():
						A_		:    Dict         =				filename
						param_count += v.numel()
				torch.save(__lowercase						,os.path.join(__lowercase						,__lowercase     )     )

		A_		:    int         =				f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
		if model_size == "7B":
				# Unsharded
				A_		:    Tuple         =				{
				    'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
				    'model.norm.weight': loaded['norm.weight'],
				    'lm_head.weight': loaded['output.weight'],
				}
		else:
				A_		:    List[str]         =				{
				    'model.norm.weight': loaded[0]['norm.weight'],
				    'model.embed_tokens.weight': torch.cat(
				        [loaded[i]['tok_embeddings.weight'] for i in range(__lowercase     )]						,dim=1     ),
				    'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(__lowercase     )]						,dim=0     ),
				}

		for k, v in state_dict.items():
				A_		:    Any         =				filename
				param_count += v.numel()
		torch.save(__lowercase						,os.path.join(__lowercase						,__lowercase     )     )

		# Write configs
		A_		:    List[str]         =				{'total_size': param_count * 2}
		write_json(__lowercase						,os.path.join(__lowercase						,'pytorch_model.bin.index.json'     )     )
		A_		:    Union[str, Any]         =				params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
		A_		:    str         =				params['multiple_of'] if 'multiple_of' in params else 2_56
		A_		:    List[str]         =				LlamaConfig(
		    hidden_size=__lowercase						,intermediate_size=compute_intermediate_size(__lowercase						,__lowercase						,__lowercase     )						,num_attention_heads=params['n_heads']						,num_hidden_layers=params['n_layers']						,rms_norm_eps=params['norm_eps']						,num_key_value_heads=__lowercase						,)
		config.save_pretrained(__lowercase     )

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print('Loading the checkpoint in a Llama model.'     )
		A_		:    List[Any]         =				LlamaForCausalLM.from_pretrained(__lowercase						,torch_dtype=torch.floataa						,low_cpu_mem_usage=__lowercase     )
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print('Saving in the Transformers format.'     )
		model.save_pretrained(__lowercase						,safe_serialization=__lowercase     )
		shutil.rmtree(__lowercase     )




def     UpperCamelCase						(	__lowercase						:			Tuple						,__lowercase						:			Optional[Any]     ):
		'''simple docstring'''




		A_		:    List[str]         =				LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''     )
		A_		:    Dict         =				tokenizer_class(__lowercase     )
		tokenizer.save_pretrained(__lowercase     )




def     UpperCamelCase						(	):
		'''simple docstring'''




		A_		:    Optional[int]         =				argparse.ArgumentParser()
		parser.add_argument(
		    '--input_dir'						,help='Location of LLaMA weights, which contains tokenizer.model and model folders'						,)
		parser.add_argument(
		    '--model_size'						,choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']						,)
		parser.add_argument(
		    '--output_dir'						,help='Location to write HF model and tokenizer'						,)
		parser.add_argument('--safe_serialization'						,type=__lowercase						,help='Whether or not to save using `safetensors`.'     )
		A_		:    List[Any]         =				parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir						,input_base_path=os.path.join(args.input_dir						,args.model_size     )						,model_size=args.model_size						,safe_serialization=args.safe_serialization						,)
		A_		:    Dict         =				os.path.join(args.input_dir						,'tokenizer.model'     )
		write_tokenizer(args.output_dir						,__lowercase     )


if __name__ == "__main__":
	main()







import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

_UpperCAmelCase            =		"""python tqdm regex requests packaging filelock numpy tokenizers""".split()
if sys.version_info < (3, 7):
	pkgs_to_check_at_runtime.append("""dataclasses""")
if sys.version_info < (3, 8):
	pkgs_to_check_at_runtime.append("""importlib_metadata""")

for pkg in pkgs_to_check_at_runtime:
	if pkg in deps:
		if pkg == "tokenizers":
			# must be loaded here, or else tqdm check may fail
			from .utils import is_tokenizers_available

			if not is_tokenizers_available():
				continue  # not required, check version only if installed

		require_version_core(deps[pkg])
	else:
		raise ValueError(F"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")




def     UpperCamelCase						(	__lowercase						:			str						,__lowercase						:			Dict=None     ):
		'''simple docstring'''




		require_version(deps[pkg]						,__lowercase     )






