






"""simple docstring"""







import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_UpperCamelCase					      =  logging.getLogger(__name__)

_UpperCamelCase					      =  tf.data.AUTOTUNE

def _a			(		):







      """simple docstring"""






      UpperCAmelCase          =	argparse.ArgumentParser(description="""Train a masked language model on TPU."""							)
      parser.add_argument(
          """--pretrained_model_config"""				,	type=_snake_case				,	default="""roberta-base"""				,	help="""The model config to use. Note that we don't copy the model's weights, only the config!"""				,	)
      parser.add_argument(
          """--tokenizer"""				,	type=_snake_case				,	default="""unigram-tokenizer-wikitext"""				,	help="""The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."""				,	)

      parser.add_argument(
          """--per_replica_batch_size"""				,	type=_snake_case				,	default=8				,	help="""Batch size per TPU core."""				,	)

      parser.add_argument(
          """--no_tpu"""				,	action="""store_true"""				,	help="""If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."""				,	)

      parser.add_argument(
          """--tpu_name"""				,	type=_snake_case				,	help="""Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."""				,	default="""local"""				,	)

      parser.add_argument(
          """--tpu_zone"""				,	type=_snake_case				,	help="""Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."""				,	)

      parser.add_argument(
          """--gcp_project"""				,	type=_snake_case				,	help="""Google cloud project name. Only used for non-Colab TPU nodes."""							)

      parser.add_argument(
          """--bfloat16"""				,	action="""store_true"""				,	help="""Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."""				,	)

      parser.add_argument(
          """--train_dataset"""				,	type=_snake_case				,	help="""Path to training dataset to load. If the path begins with `gs://`"""
          """ then the dataset will be loaded from a Google Cloud Storage bucket."""				,	)

      parser.add_argument(
          """--shuffle_buffer_size"""				,	type=_snake_case				,	default=2**18				,	help="""Size of the shuffle buffer (in samples)"""				,	)

      parser.add_argument(
          """--eval_dataset"""				,	type=_snake_case				,	help="""Path to evaluation dataset to load. If the path begins with `gs://`"""
          """ then the dataset will be loaded from a Google Cloud Storage bucket."""				,	)

      parser.add_argument(
          """--num_epochs"""				,	type=_snake_case				,	default=1				,	help="""Number of epochs to train for."""				,	)

      parser.add_argument(
          """--learning_rate"""				,	type=_snake_case				,	default=1E-4				,	help="""Learning rate to use for training."""				,	)

      parser.add_argument(
          """--weight_decay_rate"""				,	type=_snake_case				,	default=1E-3				,	help="""Weight decay rate to use for training."""				,	)

      parser.add_argument(
          """--max_length"""				,	type=_snake_case				,	default=512				,	help="""Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"""				,	)

      parser.add_argument(
          """--mlm_probability"""				,	type=_snake_case				,	default=0.15				,	help="""Fraction of tokens to mask during training."""				,	)

      parser.add_argument("""--output_dir"""				,	type=_snake_case				,	required=_snake_case				,	help="""Path to save model checkpoints to."""							)
      parser.add_argument("""--hub_model_id"""				,	type=_snake_case				,	help="""Model ID to upload to on the Hugging Face Hub."""							)

      UpperCAmelCase          =	parser.parse_args()
      return args

def _a			(		_snake_case							):







      """simple docstring"""






      try:
            if args.tpu_name:
                  UpperCAmelCase          =	tf.distribute.cluster_resolver.TPUClusterResolver(
                      args.tpu_name				,	zone=args.tpu_zone				,	project=args.gcp_project							)
            else:
                  UpperCAmelCase          =	tf.distribute.cluster_resolver.TPUClusterResolver()
      except ValueError:
            raise RuntimeError(
                """Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or """
                """--gcp_project. When running on a TPU VM, use --tpu_name local."""							)

      tf.config.experimental_connect_to_cluster(_snake_case							)
      tf.tpu.experimental.initialize_tpu_system(_snake_case							)

      return tpu

def _a			(		_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	0
      for file in file_list:
            UpperCAmelCase          =	file.split("""/"""							)[-1]
            UpperCAmelCase          =	re.search(R"""-\d+-(\d+)\.tfrecord"""				,	_snake_case							).group(1							)
            UpperCAmelCase          =	int(_snake_case							)
            num_samples += sample_count

      return num_samples

def _a			(		_snake_case				,	_snake_case				,	_snake_case				,	_snake_case				,	_snake_case				,	_snake_case=None							):







      """simple docstring"""






      UpperCAmelCase          =	count_samples(_snake_case							)
      UpperCAmelCase          =	tf.data.Dataset.from_tensor_slices(_snake_case							)
      if shuffle:
            UpperCAmelCase          =	dataset.shuffle(len(_snake_case							)							)
      UpperCAmelCase          =	tf.data.TFRecordDataset(_snake_case				,	num_parallel_reads=_snake_case							)
      # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
      UpperCAmelCase          =	dataset.apply(tf.data.experimental.assert_cardinality(_snake_case							)							)
      UpperCAmelCase          =	dataset.map(_snake_case				,	num_parallel_calls=_snake_case							)
      if shuffle:
            assert shuffle_buffer_size is not None
            UpperCAmelCase          =	dataset.shuffle(args.shuffle_buffer_size							)
      UpperCAmelCase          =	dataset.batch(_snake_case				,	drop_remainder=_snake_case							)
      UpperCAmelCase          =	dataset.map(_snake_case				,	num_parallel_calls=_snake_case							)
      UpperCAmelCase          =	dataset.prefetch(_snake_case							)
      return dataset

def _a			(		_snake_case							):







      """simple docstring"""






      if not args.no_tpu:
            UpperCAmelCase          =	initialize_tpu(_snake_case							)
            UpperCAmelCase          =	tf.distribute.TPUStrategy(_snake_case							)
      else:
            UpperCAmelCase          =	tf.distribute.OneDeviceStrategy(device="""/gpu:0"""							)

      if args.bfloataa:
            tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""							)

      UpperCAmelCase          =	AutoTokenizer.from_pretrained(args.tokenizer							)
      UpperCAmelCase          =	AutoConfig.from_pretrained(args.pretrained_model_config							)
      UpperCAmelCase          =	tokenizer.vocab_size

      UpperCAmelCase          =	tf.io.gfile.glob(os.path.join(args.train_dataset				,	"""*.tfrecord"""							)							)
      if not training_records:
            raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.'''							)
      UpperCAmelCase          =	tf.io.gfile.glob(os.path.join(args.eval_dataset				,	"""*.tfrecord"""							)							)
      if not eval_records:
            raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.'''							)

      UpperCAmelCase          =	count_samples(_snake_case							)

      UpperCAmelCase          =	num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
      UpperCAmelCase          =	steps_per_epoch * args.num_epochs

      with strategy.scope():
            UpperCAmelCase          =	TFAutoModelForMaskedLM.from_config(_snake_case							)
            model(model.dummy_inputs							)  # Pass some dummy inputs through the model to ensure all the weights are built
            UpperCAmelCase      , UpperCAmelCase          =	create_optimizer(
                num_train_steps=_snake_case				,	num_warmup_steps=total_train_steps // 20				,	init_lr=args.learning_rate				,	weight_decay_rate=args.weight_decay_rate				,	)

            # Transformers models compute the right loss for their task by default when labels are passed, and will
            # use this for training unless you specify your own loss function in compile().
            model.compile(optimizer=_snake_case				,	metrics=["""accuracy"""]							)

      def decode_fn(_snake_case							):
            UpperCAmelCase          =	{
                """input_ids""": tf.io.FixedLenFeature(dtype=tf.intaa				,	shape=(args.max_length,)							),
                """attention_mask""": tf.io.FixedLenFeature(dtype=tf.intaa				,	shape=(args.max_length,)							),
            }
            return tf.io.parse_single_example(_snake_case				,	_snake_case							)

      # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
      # use their methods in our data pipeline.
      UpperCAmelCase          =	DataCollatorForLanguageModeling(
          tokenizer=_snake_case				,	mlm_probability=args.mlm_probability				,	mlm=_snake_case				,	return_tensors="""tf"""							)

      def mask_with_collator(_snake_case							):
            # TF really needs an isin() function
            UpperCAmelCase          =	(
                ~tf.cast(batch["""attention_mask"""]				,	tf.bool							)
                | (batch["""input_ids"""] == tokenizer.cls_token_id)
                | (batch["""input_ids"""] == tokenizer.sep_token_id)
            )
            UpperCAmelCase      , UpperCAmelCase          =	data_collator.tf_mask_tokens(
                batch["""input_ids"""]				,	vocab_size=len(_snake_case							)				,	mask_token_id=tokenizer.mask_token_id				,	special_tokens_mask=_snake_case				,	)
            return batch

      UpperCAmelCase          =	args.per_replica_batch_size * strategy.num_replicas_in_sync

      UpperCAmelCase          =	prepare_dataset(
          _snake_case				,	decode_fn=_snake_case				,	mask_fn=_snake_case				,	batch_size=_snake_case				,	shuffle=_snake_case				,	shuffle_buffer_size=args.shuffle_buffer_size				,	)

      UpperCAmelCase          =	prepare_dataset(
          _snake_case				,	decode_fn=_snake_case				,	mask_fn=_snake_case				,	batch_size=_snake_case				,	shuffle=_snake_case				,	)

      UpperCAmelCase          =	[]
      if args.hub_model_id:
            callbacks.append(
                PushToHubCallback(output_dir=args.output_dir				,	hub_model_id=args.hub_model_id				,	tokenizer=_snake_case							)							)

      model.fit(
          _snake_case				,	validation_data=_snake_case				,	epochs=args.num_epochs				,	callbacks=_snake_case				,	)

      model.save_pretrained(args.output_dir							)


if __name__ == "__main__":
  _UpperCamelCase					      =  parse_args()
  main(args)











"""simple docstring"""







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
  from transformers import LlamaTokenizerFast
except ImportError as e:
  warnings.warn(e)
  warnings.warn(
      """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
  )
  _UpperCamelCase					      =  None
_UpperCamelCase					      =  {
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_UpperCamelCase					      =  {
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}

def _a			(		_snake_case				,	_snake_case=1				,	_snake_case=256							):







      """simple docstring"""






      return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3							)							) + multiple_of - 1) // multiple_of)

def _a			(		_snake_case							):







      """simple docstring"""






      with open(_snake_case				,	"""r"""							) as f:
            return json.load(_snake_case							)

def _a			(		_snake_case				,	_snake_case							):







      """simple docstring"""






      with open(_snake_case				,	"""w"""							) as f:
            json.dump(_snake_case				,	_snake_case							)

def _a			(		_snake_case				,	_snake_case				,	_snake_case				,	_snake_case=True							):







      """simple docstring"""






      os.makedirs(_snake_case				,	exist_ok=_snake_case							)
      UpperCAmelCase          =	os.path.join(_snake_case				,	"""tmp"""							)
      os.makedirs(_snake_case				,	exist_ok=_snake_case							)

      UpperCAmelCase          =	read_json(os.path.join(_snake_case				,	"""params.json"""							)							)
      UpperCAmelCase          =	NUM_SHARDS[model_size]
      UpperCAmelCase          =	params["""n_layers"""]
      UpperCAmelCase          =	params["""n_heads"""]
      UpperCAmelCase          =	n_heads // num_shards
      UpperCAmelCase          =	params["""dim"""]
      UpperCAmelCase          =	dim // n_heads
      UpperCAmelCase          =	10000.0
      UpperCAmelCase          =	1.0 / (base ** (torch.arange(0				,	_snake_case				,	2							).float() / dims_per_head))

      if "n_kv_heads" in params:
            UpperCAmelCase          =	params["""n_kv_heads"""]  # for GQA / MQA
            UpperCAmelCase          =	n_heads_per_shard // num_key_value_heads
            UpperCAmelCase          =	dim // num_key_value_heads
      else:  # compatibility with other checkpoints
            UpperCAmelCase          =	n_heads
            UpperCAmelCase          =	n_heads_per_shard
            UpperCAmelCase          =	dim

      # permute for sliced rotary
      def permute(_snake_case				,	_snake_case=n_heads				,	_snake_case=dim				,	_snake_case=dim							):
            return w.view(_snake_case				,	dima // n_heads // 2				,	2				,	_snake_case							).transpose(1				,	2							).reshape(_snake_case				,	_snake_case							)

      print(F'''Fetching all parameters from the checkpoint at {input_base_path}.'''							)
      # Load weights
      if model_size == "7B":
            # Not sharded
            # (The sharded implementation would also work, but this is simpler.)
            UpperCAmelCase          =	torch.load(os.path.join(_snake_case				,	"""consolidated.00.pth"""							)				,	map_location="""cpu"""							)
      else:
            # Sharded
            UpperCAmelCase          =	[
                torch.load(os.path.join(_snake_case				,	F'''consolidated.{i:02d}.pth'''							)				,	map_location="""cpu"""							)
                for i in range(_snake_case							)
            ]
      UpperCAmelCase          =	0
      UpperCAmelCase          =	{"""weight_map""": {}}
      for layer_i in range(_snake_case							):
            UpperCAmelCase          =	F'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
            if model_size == "7B":
                  # Unsharded
                  UpperCAmelCase          =	{
                      F'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
                          loaded[F'''layers.{layer_i}.attention.wq.weight''']							),
                      F'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
                          loaded[F'''layers.{layer_i}.attention.wk.weight''']							),
                      F'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[F'''layers.{layer_i}.attention.wv.weight'''],
                      F'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[F'''layers.{layer_i}.attention.wo.weight'''],
                      F'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w1.weight'''],
                      F'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w2.weight'''],
                      F'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w3.weight'''],
                      F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[F'''layers.{layer_i}.attention_norm.weight'''],
                      F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[F'''layers.{layer_i}.ffn_norm.weight'''],
                  }
            else:
                  # Sharded
                  # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                  # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                  # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                  UpperCAmelCase          =	{
                      F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
                          F'''layers.{layer_i}.attention_norm.weight'''
                      ].clone(),
                      F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
                          F'''layers.{layer_i}.ffn_norm.weight'''
                      ].clone(),
                  }
                  UpperCAmelCase          =	permute(
                      torch.cat(
                          [
                              loaded[i][F'''layers.{layer_i}.attention.wq.weight'''].view(_snake_case				,	_snake_case				,	_snake_case							)
                              for i in range(_snake_case							)
                          ]				,	dim=0				,	).reshape(_snake_case				,	_snake_case							)							)
                  UpperCAmelCase          =	permute(
                      torch.cat(
                          [
                              loaded[i][F'''layers.{layer_i}.attention.wk.weight'''].view(
                                  _snake_case				,	_snake_case				,	_snake_case							)
                              for i in range(_snake_case							)
                          ]				,	dim=0				,	).reshape(_snake_case				,	_snake_case							)				,	_snake_case				,	_snake_case				,	_snake_case				,	)
                  UpperCAmelCase          =	torch.cat(
                      [
                          loaded[i][F'''layers.{layer_i}.attention.wv.weight'''].view(
                              _snake_case				,	_snake_case				,	_snake_case							)
                          for i in range(_snake_case							)
                      ]				,	dim=0				,	).reshape(_snake_case				,	_snake_case							)

                  UpperCAmelCase          =	torch.cat(
                      [loaded[i][F'''layers.{layer_i}.attention.wo.weight'''] for i in range(_snake_case							)]				,	dim=1							)
                  UpperCAmelCase          =	torch.cat(
                      [loaded[i][F'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(_snake_case							)]				,	dim=0							)
                  UpperCAmelCase          =	torch.cat(
                      [loaded[i][F'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(_snake_case							)]				,	dim=1							)
                  UpperCAmelCase          =	torch.cat(
                      [loaded[i][F'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(_snake_case							)]				,	dim=0							)

            UpperCAmelCase          =	inv_freq
            for k, v in state_dict.items():
                  UpperCAmelCase          =	filename
                  param_count += v.numel()
            torch.save(_snake_case				,	os.path.join(_snake_case				,	_snake_case							)							)

      UpperCAmelCase          =	F'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
      if model_size == "7B":
            # Unsharded
            UpperCAmelCase          =	{
                """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
                """model.norm.weight""": loaded["""norm.weight"""],
                """lm_head.weight""": loaded["""output.weight"""],
            }
      else:
            UpperCAmelCase          =	{
                """model.norm.weight""": loaded[0]["""norm.weight"""],
                """model.embed_tokens.weight""": torch.cat(
                    [loaded[i]["""tok_embeddings.weight"""] for i in range(_snake_case							)]				,	dim=1							),
                """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_snake_case							)]				,	dim=0							),
            }

      for k, v in state_dict.items():
            UpperCAmelCase          =	filename
            param_count += v.numel()
      torch.save(_snake_case				,	os.path.join(_snake_case				,	_snake_case							)							)

      # Write configs
      UpperCAmelCase          =	{"""total_size""": param_count * 2}
      write_json(_snake_case				,	os.path.join(_snake_case				,	"""pytorch_model.bin.index.json"""							)							)
      UpperCAmelCase          =	params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
      UpperCAmelCase          =	params["""multiple_of"""] if """multiple_of""" in params else 256
      UpperCAmelCase          =	LlamaConfig(
          hidden_size=_snake_case				,	intermediate_size=compute_intermediate_size(_snake_case				,	_snake_case				,	_snake_case							)				,	num_attention_heads=params["""n_heads"""]				,	num_hidden_layers=params["""n_layers"""]				,	rms_norm_eps=params["""norm_eps"""]				,	num_key_value_heads=_snake_case				,	)
      config.save_pretrained(_snake_case							)

      # Make space so we can load the model properly now.
      del state_dict
      del loaded
      gc.collect()

      print("""Loading the checkpoint in a Llama model."""							)
      UpperCAmelCase          =	LlamaForCausalLM.from_pretrained(_snake_case				,	torch_dtype=torch.floataa				,	low_cpu_mem_usage=_snake_case							)
      # Avoid saving this as part of the config.
      del model.config._name_or_path

      print("""Saving in the Transformers format."""							)
      model.save_pretrained(_snake_case				,	safe_serialization=_snake_case							)
      shutil.rmtree(_snake_case							)

def _a			(		_snake_case				,	_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
      print(F'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''							)
      UpperCAmelCase          =	tokenizer_class(_snake_case							)
      tokenizer.save_pretrained(_snake_case							)

def _a			(		):







      """simple docstring"""






      UpperCAmelCase          =	argparse.ArgumentParser()
      parser.add_argument(
          """--input_dir"""				,	help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""				,	)
      parser.add_argument(
          """--model_size"""				,	choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]				,	)
      parser.add_argument(
          """--output_dir"""				,	help="""Location to write HF model and tokenizer"""				,	)
      parser.add_argument("""--safe_serialization"""				,	type=_snake_case				,	help="""Whether or not to save using `safetensors`."""							)
      UpperCAmelCase          =	parser.parse_args()
      if args.model_size != "tokenizer_only":
            write_model(
                model_path=args.output_dir				,	input_base_path=os.path.join(args.input_dir				,	args.model_size							)				,	model_size=args.model_size				,	safe_serialization=args.safe_serialization				,	)
      UpperCAmelCase          =	os.path.join(args.input_dir				,	"""tokenizer.model"""							)
      write_tokenizer(args.output_dir				,	_snake_case							)


if __name__ == "__main__":
  main()



