





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
  from transformers import LlamaTokenizerFast
except ImportError as e:
  warnings.warn(e)
  warnings.warn(
      '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
  )
  lowercase__   :				Tuple      =   None
lowercase__   :				Optional[int]      =   {
    '''7B''': 1_10_08,
    '''13B''': 1_38_24,
    '''30B''': 1_79_20,
    '''65B''': 2_20_16,
    '''70B''': 2_86_72,
}
lowercase__   :				Any      =   {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}






def 	_lowerCAmelCase      (  __snake_case  :			Union[str, Any]			,							__snake_case  :			str=1			,							__snake_case  :			Optional[Any]=2_56      )					->					Any:
     return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3      )      ) + multiple_of - 1) // multiple_of)






def 	_lowerCAmelCase      (  __snake_case  :			Dict      )					->					Tuple:






     with open(_a			,							'r'      ) as f:
          return json.load(_a      )






def 	_lowerCAmelCase      (  __snake_case  :			List[Any]			,							__snake_case  :			str      )					->					Optional[Any]:






     with open(_a			,							'w'      ) as f:
          json.dump(_a			,							_a      )






def 	_lowerCAmelCase      (  __snake_case  :			Union[str, Any]			,							__snake_case  :			List[Any]			,							__snake_case  :			Optional[Any]			,							__snake_case  :			str=True      )					->					Union[str, Any]:
     os.makedirs(_a			,							exist_ok=_a      )
     __A						:			Dict								=	os.path.join(_a			,							'tmp'      )
     os.makedirs(_a			,							exist_ok=_a      )

     __A						:			Tuple								=	read_json(os.path.join(_a			,							'params.json'      )      )
     __A						:			Union[str, Any]								=	NUM_SHARDS[model_size]
     __A						:			Any								=	params['n_layers']
     __A						:			List[Any]								=	params['n_heads']
     __A						:			Tuple								=	n_heads // num_shards
     __A						:			Tuple								=	params['dim']
     __A						:			int								=	dim // n_heads
     __A						:			Tuple								=	1_00_00.0
     __A						:			Tuple								=	1.0 / (base ** (torch.arange(0			,							_a			,							2      ).float() / dims_per_head))

     if "n_kv_heads" in params:
          __A						:			List[Any]								=	params['n_kv_heads']  # for GQA / MQA
          __A						:			Tuple								=	n_heads_per_shard // num_key_value_heads
          __A						:			Optional[int]								=	dim // num_key_value_heads
     else:  # compatibility with other checkpoints
          __A						:			Dict								=	n_heads
          __A						:			Optional[int]								=	n_heads_per_shard
          __A						:			Optional[Any]								=	dim

     # permute for sliced rotary
     def permute(__snake_case  :			Any			,							__snake_case  :			Union[str, Any]=n_heads			,							__snake_case  :			Union[str, Any]=dim			,							__snake_case  :			int=dim      ):
          return w.view(_a			,							dima // n_heads // 2			,							2			,							_a      ).transpose(1			,							2      ).reshape(_a			,							_a      )

     print(f'Fetching all parameters from the checkpoint at {input_base_path}.'      )
     # Load weights
     if model_size == "7B":
          # Not sharded
          # (The sharded implementation would also work, but this is simpler.)
          __A						:			List[str]								=	torch.load(os.path.join(_a			,							'consolidated.00.pth'      )			,							map_location='cpu'      )
     else:
          # Sharded
          __A						:			Any								=	[
              torch.load(os.path.join(_a			,							f'consolidated.{i:02d}.pth'      )			,							map_location='cpu'      )
              for i in range(_a      )
          ]
     __A						:			Optional[int]								=	0
     __A						:			Optional[Any]								=	{'weight_map': {}}
     for layer_i in range(_a      ):
          __A						:			str								=	f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
          if model_size == "7B":
               # Unsharded
               __A						:			Optional[int]								=	{
                   f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
                       loaded[f'layers.{layer_i}.attention.wq.weight']      ),
                   f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
                       loaded[f'layers.{layer_i}.attention.wk.weight']      ),
                   f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
                   f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
                   f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
                   f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
                   f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
                   f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
                   f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
               }
          else:
               # Sharded
               # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
               # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
               # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

               __A						:			Tuple								=	{
                   f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
                       f'layers.{layer_i}.attention_norm.weight'
                   ].clone(),
                   f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
                       f'layers.{layer_i}.ffn_norm.weight'
                   ].clone(),
               }
               __A						:			Any								=	permute(
                   torch.cat(
                       [
                           loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(_a			,							_a			,							_a      )
                           for i in range(_a      )
                       ]			,							dim=0			,							).reshape(_a			,							_a      )      )
               __A						:			List[Any]								=	permute(
                   torch.cat(
                       [
                           loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
                               _a			,							_a			,							_a      )
                           for i in range(_a      )
                       ]			,							dim=0			,							).reshape(_a			,							_a      )			,							_a			,							_a			,							_a			,							)
               __A						:			Union[str, Any]								=	torch.cat(
                   [
                       loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
                           _a			,							_a			,							_a      )
                       for i in range(_a      )
                   ]			,							dim=0			,							).reshape(_a			,							_a      )

               __A						:			str								=	torch.cat(
                   [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(_a      )]			,							dim=1      )
               __A						:			str								=	torch.cat(
                   [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(_a      )]			,							dim=0      )
               __A						:			Optional[int]								=	torch.cat(
                   [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(_a      )]			,							dim=1      )
               __A						:			List[str]								=	torch.cat(
                   [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(_a      )]			,							dim=0      )

          __A						:			int								=	inv_freq
          for k, v in state_dict.items():
               __A						:			List[str]								=	filename
               param_count += v.numel()
          torch.save(_a			,							os.path.join(_a			,							_a      )      )

     __A						:			Optional[int]								=	f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
     if model_size == "7B":
          # Unsharded
          __A						:			Optional[int]								=	{
              'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
              'model.norm.weight': loaded['norm.weight'],
              'lm_head.weight': loaded['output.weight'],
          }
     else:
          __A						:			Tuple								=	{
              'model.norm.weight': loaded[0]['norm.weight'],
              'model.embed_tokens.weight': torch.cat(
                  [loaded[i]['tok_embeddings.weight'] for i in range(_a      )]			,							dim=1      ),
              'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(_a      )]			,							dim=0      ),
          }

     for k, v in state_dict.items():
          __A						:			str								=	filename
          param_count += v.numel()
     torch.save(_a			,							os.path.join(_a			,							_a      )      )

     # Write configs
     __A						:			Dict								=	{'total_size': param_count * 2}
     write_json(_a			,							os.path.join(_a			,							'pytorch_model.bin.index.json'      )      )
     __A						:			int								=	params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
     __A						:			str								=	params['multiple_of'] if 'multiple_of' in params else 2_56
     __A						:			Optional[int]								=	LlamaConfig(
         hidden_size=_a			,							intermediate_size=compute_intermediate_size(_a			,							_a			,							_a      )			,							num_attention_heads=params['n_heads']			,							num_hidden_layers=params['n_layers']			,							rms_norm_eps=params['norm_eps']			,							num_key_value_heads=_a			,							)
     config.save_pretrained(_a      )

     # Make space so we can load the model properly now.
     del state_dict
     del loaded
     gc.collect()

     print('Loading the checkpoint in a Llama model.'      )
     __A						:			Union[str, Any]								=	LlamaForCausalLM.from_pretrained(_a			,							torch_dtype=torch.floataa			,							low_cpu_mem_usage=_a      )
     # Avoid saving this as part of the config.
     del model.config._name_or_path

     print('Saving in the Transformers format.'      )
     model.save_pretrained(_a			,							safe_serialization=_a      )
     shutil.rmtree(_a      )






def 	_lowerCAmelCase      (  __snake_case  :			int			,							__snake_case  :			Optional[int]      )					->					Tuple:
     __A						:			int								=	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
     print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'      )
     __A						:			List[str]								=	tokenizer_class(_a      )
     tokenizer.save_pretrained(_a      )






def 	_lowerCAmelCase      (  )					->					Optional[Any]:
     __A						:			Any								=	argparse.ArgumentParser()
     parser.add_argument(
         '--input_dir'			,							help='Location of LLaMA weights, which contains tokenizer.model and model folders'			,							)
     parser.add_argument(
         '--model_size'			,							choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']			,							)
     parser.add_argument(
         '--output_dir'			,							help='Location to write HF model and tokenizer'			,							)
     parser.add_argument('--safe_serialization'			,							type=_a			,							help='Whether or not to save using `safetensors`.'      )
     __A						:			str								=	parser.parse_args()
     if args.model_size != "tokenizer_only":
          write_model(
              model_path=args.output_dir			,							input_base_path=os.path.join(args.input_dir			,							args.model_size      )			,							model_size=args.model_size			,							safe_serialization=args.safe_serialization			,							)
     __A						:			int								=	os.path.join(args.input_dir			,							'tokenizer.model'      )
     write_tokenizer(args.output_dir			,							_a      )


if __name__ == "__main__":
  main()






'''simple docstring'''




from __future__ import annotations

from math import gcd






def 	_lowerCAmelCase      (  __snake_case  :			int			,							__snake_case  :			int = 2			,							__snake_case  :			int = 1			,							__snake_case  :			int = 3			,							)					->					int | None:
     # A value less than 2 can cause an infinite loop in the algorithm.
     if num < 2:
          raise ValueError('The input value cannot be less than 2'      )

     # Because of the relationship between ``f(f(x))`` and ``f(x)``, this
     # algorithm struggles to find factors that are divisible by two.
     # As a workaround, we specifically check for two and even inputs.
     #   See: https://math.stackexchange.com/a/2856214/165820
     if num > 2 and num % 2 == 0:
          return 2

     # Pollard's Rho algorithm requires a function that returns pseudorandom
     # values between 0 <= X < ``num``.  It doesn't need to be random in the
     # sense that the output value is cryptographically secure or difficult
     # to calculate, it only needs to be random in the sense that all output
     # values should be equally likely to appear.
     # For this reason, Pollard suggested using ``f(x) = (x**2 - 1) % num``
     # However, the success of Pollard's algorithm isn't guaranteed and is
     # determined in part by the initial seed and the chosen random function.
     # To make retries easier, we will instead use ``f(x) = (x**2 + C) % num``
     # where ``C`` is a value that we can modify between each attempt.
     def rand_fn(__snake_case  :			int			,							__snake_case  :			int			,							__snake_case  :			int      ) -> int:
          return (pow(__snake_case			,							2      ) + step) % modulus

     for _ in range(__snake_case      ):
          # These track the position within the cycle detection logic.
          __A						:			int								=	seed
          __A						:			Union[str, Any]								=	seed

          while True:
               # At each iteration, the tortoise moves one step and the hare moves two.
               __A						:			List[Any]								=	rand_fn(__snake_case			,							__snake_case			,							__snake_case      )
               __A						:			Optional[Any]								=	rand_fn(__snake_case			,							__snake_case			,							__snake_case      )
               __A						:			Any								=	rand_fn(__snake_case			,							__snake_case			,							__snake_case      )

               # At some point both the tortoise and the hare will enter a cycle whose
               # length ``p`` is a divisor of ``num``.  Once in that cycle, at some point
               # the tortoise and hare will end up on the same value modulo ``p``.
               # We can detect when this happens because the position difference between
               # the tortoise and the hare will share a common divisor with ``num``.
               __A						:			Optional[int]								=	gcd(hare - tortoise			,							__snake_case      )

               if divisor == 1:
                    # No common divisor yet, just keep searching.
                    continue
               else:
                    # We found a common divisor!
                    if divisor == num:
                         # Unfortunately, the divisor is ``num`` itself and is useless.
                         break
                    else:
                         # The divisor is a nontrivial factor of ``num``!
                         return divisor

        # If we made it here, then this attempt failed.
        # We need to pick a new starting seed for the tortoise and hare
        # in addition to a new step value for the random function.
        # To keep this example implementation deterministic, the
        # new values will be generated based on currently available
        # values instead of using something like ``random.randint``.

        # We can use the hare's position as the new seed.
        # This is actually what Richard Brent's the "optimized" variant does.
          __A						:			Union[str, Any]								=	hare

          # The new step value for the random function can just be incremented.
          # At first the results will be similar to what the old function would
          # have produced, but the value will quickly diverge after a bit.
          step += 1

     # We haven't found a divisor within the requested number of attempts.
     # We were unlucky or ``num`` itself is actually prime.
     return None


if __name__ == "__main__":
  import argparse

  lowercase__   :				str      =   argparse.ArgumentParser()
  parser.add_argument(
      '''num''',
      type=int,
      help='''The value to find a divisor of''',
  )
  parser.add_argument(
      '''--attempts''',
      type=int,
      default=3,
      help='''The number of attempts before giving up''',
  )
  lowercase__   :				Optional[int]      =   parser.parse_args()

  lowercase__   :				int      =   pollard_rho(args.num, attempts=args.attempts)
  if divisor is None:
    print(f"""{args.num} is probably prime""")
  else:
    lowercase__   :				List[str]      =   args.num // divisor
    print(f"""{args.num} = {divisor} * {quotient}""")