



'''simple docstring'''

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Any							,		_SCREAMING_SNAKE_CASE					:						List[Any]							,		_SCREAMING_SNAKE_CASE					:						Dict=0							)      ->						Optional[int]:







	"""simple docstring"""
	# Format the message.
	if name is None:
		lowerCAmelCase   =     None
	else:
		lowerCAmelCase   =     "." * max(0							,		spaces - 2							) + "# {:" + str(50 - spaces							) + "s}"
		lowerCAmelCase   =     fmt.format(_SCREAMING_SNAKE_CASE							)

	# Print and recurse (if needed).
	if isinstance(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							):
		if msg is not None:
			print(_SCREAMING_SNAKE_CASE							)
		for k in val.keys():
			recursive_print(_SCREAMING_SNAKE_CASE							,		val[k]							,		spaces + 2							)
	elif isinstance(_SCREAMING_SNAKE_CASE							,		torch.Tensor							):
		print(_SCREAMING_SNAKE_CASE							,		""":"""							,		val.size()							)
	else:
		print(_SCREAMING_SNAKE_CASE							,		""":"""							,		_SCREAMING_SNAKE_CASE							)






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Optional[Any]							,		_SCREAMING_SNAKE_CASE					:						Tuple							,		_SCREAMING_SNAKE_CASE					:						int							,		_SCREAMING_SNAKE_CASE					:						List[Any]							,		_SCREAMING_SNAKE_CASE					:						Dict							)      ->						Optional[Any]:







	"""simple docstring"""
	# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
	# for compatibility with later versions of NVIDIA Megatron-LM.
	# The inverse operation is performed inside Megatron-LM to read checkpoints:
	# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
	# If param is the weight tensor of the self-attention block, the returned tensor
	# will have to be transposed one more time to be read by HuggingFace GPT2.
	lowerCAmelCase   =     param.size()
	if checkpoint_version == 1.0:
		# version 1.0 stores [num_heads * hidden_size * num_splits, :]
		lowerCAmelCase   =     (num_heads, hidden_size, num_splits) + input_shape[1:]
		lowerCAmelCase   =     param.view(*_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     param.transpose(0							,		2							)
		lowerCAmelCase   =     param.transpose(1							,		2							).contiguous()
	elif checkpoint_version >= 2.0:
		# other versions store [num_heads * num_splits * hidden_size, :]
		lowerCAmelCase   =     (num_heads, num_splits, hidden_size) + input_shape[1:]
		lowerCAmelCase   =     param.view(*_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     param.transpose(0							,		1							).contiguous()
	lowerCAmelCase   =     param.view(*_SCREAMING_SNAKE_CASE							)
	return param






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						int							,		_SCREAMING_SNAKE_CASE					:						str							,		_SCREAMING_SNAKE_CASE					:						Dict							)      ->						Any:







	"""simple docstring"""
	# The converted output model.
	lowerCAmelCase   =     {}

	# old versions did not store training args
	lowerCAmelCase   =     input_state_dict.get("""args"""							,		_SCREAMING_SNAKE_CASE							)
	if ds_args is not None:
		# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
		# from pprint import pprint
		# pprint(vars(ds_args))

		lowerCAmelCase   =     ds_args.padded_vocab_size
		lowerCAmelCase   =     ds_args.max_position_embeddings
		lowerCAmelCase   =     ds_args.hidden_size
		lowerCAmelCase   =     ds_args.num_layers
		lowerCAmelCase   =     ds_args.num_attention_heads
		lowerCAmelCase   =     ds_args.ffn_hidden_size
		# pprint(config)

	# The number of heads.
	lowerCAmelCase   =     config.n_head
	# The hidden_size per head.
	lowerCAmelCase   =     config.n_embd // config.n_head
	# Megatron-LM checkpoint version
	if "checkpoint_version" in input_state_dict.keys():
		lowerCAmelCase   =     input_state_dict["checkpoint_version"]
	else:
		lowerCAmelCase   =     0.0

	# The model.
	lowerCAmelCase   =     input_state_dict["model"]
	# The language model.
	lowerCAmelCase   =     model["language_model"]
	# The embeddings.
	lowerCAmelCase   =     lm["embedding"]

	# The word embeddings.
	lowerCAmelCase   =     embeddings["word_embeddings"]["weight"]
	# Truncate the embedding table to vocab_size rows.
	lowerCAmelCase   =     word_embeddings[: config.vocab_size, :]
	lowerCAmelCase   =     word_embeddings

	# The position embeddings.
	lowerCAmelCase   =     embeddings["position_embeddings"]["weight"]
	# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
	lowerCAmelCase   =     pos_embeddings.size(0							)
	if n_positions != config.n_positions:
		raise ValueError(
		    f'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'							)
	# Store the position embeddings.
	lowerCAmelCase   =     pos_embeddings

	# The transformer.
	lowerCAmelCase   =     lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

	# The regex to extract layer names.
	lowerCAmelCase   =     re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""							)

	# The simple map of names for "automated" rules.
	lowerCAmelCase   =     {
	    "attention.dense": ".attn.c_proj.",
	    "self_attention.dense": ".attn.c_proj.",
	    "mlp.dense_h_to_4h": ".mlp.c_fc.",
	    "mlp.dense_4h_to_h": ".mlp.c_proj.",
	}

	# Extract the layers.
	for key, val in transformer.items():
		# Match the name.
		lowerCAmelCase   =     layer_re.match(_SCREAMING_SNAKE_CASE							)

		# Stop if that's not a layer
		if m is None:
			break

		# The index of the layer.
		lowerCAmelCase   =     int(m.group(1							)							)
		# The name of the operation.
		lowerCAmelCase   =     m.group(2							)
		# Is it a weight or a bias?
		lowerCAmelCase   =     m.group(3							)

		# The name of the layer.
		lowerCAmelCase   =     f'transformer.h.{layer_idx}'

		# For layernorm(s), simply store the layer norm.
		if op_name.endswith("""layernorm"""							):
			lowerCAmelCase   =     "ln_1" if op_name.startswith("""input"""							) else "ln_2"
			lowerCAmelCase   =     val

		# Transpose the QKV matrix.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "weight":
			# Insert a tensor of 1x1xDxD bias.
			lowerCAmelCase   =     torch.tril(torch.ones((n_positions, n_positions)							,		dtype=torch.floataa							)							).view(
			    1							,		1							,		_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
			lowerCAmelCase   =     causal_mask

			# Insert a "dummy" tensor for masked_bias.
			lowerCAmelCase   =     torch.tensor(-1E4							,		dtype=torch.floataa							)
			lowerCAmelCase   =     masked_bias

			lowerCAmelCase   =     fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		3							,		_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
			# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
			lowerCAmelCase   =     out_val.transpose(0							,		1							).contiguous()
			# Store.
			lowerCAmelCase   =     out_val

		# Transpose the bias.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "bias":
			lowerCAmelCase   =     fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		3							,		_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
			# Store. No change of shape.
			lowerCAmelCase   =     out_val

		# Transpose the weights.
		elif weight_or_bias == "weight":
			lowerCAmelCase   =     megatron_to_transformers[op_name]
			lowerCAmelCase   =     val.transpose(0							,		1							)

		# Copy the bias.
		elif weight_or_bias == "bias":
			lowerCAmelCase   =     megatron_to_transformers[op_name]
			lowerCAmelCase   =     val

    # DEBUG.
	assert config.n_layer == layer_idx + 1

	# The final layernorm.
	lowerCAmelCase   =     transformer["final_layernorm.weight"]
	lowerCAmelCase   =     transformer["final_layernorm.bias"]

	# For LM head, transformers' wants the matrix to weight embeddings.
	lowerCAmelCase   =     word_embeddings

	# It should be done!
	return output_state_dict






def 		_snake_case		( )      ->						List[Any]:







	"""simple docstring"""
	# Create the argument parser.
	lowerCAmelCase   =     argparse.ArgumentParser()
	parser.add_argument("""--print-checkpoint-structure"""							,		action="""store_true"""							)
	parser.add_argument(
	    """path_to_checkpoint"""							,		type=_SCREAMING_SNAKE_CASE							,		help="""Path to the checkpoint file (.zip archive or direct .pt file)"""							,		)
	parser.add_argument(
	    """--config_file"""							,		default=""""""							,		type=_SCREAMING_SNAKE_CASE							,		help="""An optional config json file describing the pre-trained model."""							,		)
	lowerCAmelCase   =     parser.parse_args()

	# Extract the basename.
	lowerCAmelCase   =     os.path.dirname(args.path_to_checkpoint							)

	# Load the model.
	# the .zip is very optional, let's keep it for backward compatibility
	print(f'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'							)
	if args.path_to_checkpoint.endswith(""".zip"""							):
		with zipfile.ZipFile(args.path_to_checkpoint							,		"""r"""							) as checkpoint:
			with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""							) as pytorch_dict:
				lowerCAmelCase   =     torch.load(_SCREAMING_SNAKE_CASE							,		map_location="""cpu"""							)
	else:
		lowerCAmelCase   =     torch.load(args.path_to_checkpoint							,		map_location="""cpu"""							)

	lowerCAmelCase   =     input_state_dict.get("""args"""							,		_SCREAMING_SNAKE_CASE							)

	# Read the config, or default to the model released by NVIDIA.
	if args.config_file == "":
		if ds_args is not None:
			if ds_args.bias_gelu_fusion:
				lowerCAmelCase   =     "gelu_fast"
			elif ds_args.openai_gelu:
				lowerCAmelCase   =     "gelu_new"
			else:
				lowerCAmelCase   =     "gelu"
		else:
			# in the very early days this used to be "gelu_new"
			lowerCAmelCase   =     "gelu_new"

		# Spell out all parameters in case the defaults change.
		lowerCAmelCase   =     GPTaConfig(
		    vocab_size=50_257							,		n_positions=1_024							,		n_embd=1_024							,		n_layer=24							,		n_head=16							,		n_inner=4_096							,		activation_function=_SCREAMING_SNAKE_CASE							,		resid_pdrop=0.1							,		embd_pdrop=0.1							,		attn_pdrop=0.1							,		layer_norm_epsilon=1E-5							,		initializer_range=0.02							,		summary_type="""cls_index"""							,		summary_use_proj=_SCREAMING_SNAKE_CASE							,		summary_activation=_SCREAMING_SNAKE_CASE							,		summary_proj_to_labels=_SCREAMING_SNAKE_CASE							,		summary_first_dropout=0.1							,		scale_attn_weights=_SCREAMING_SNAKE_CASE							,		use_cache=_SCREAMING_SNAKE_CASE							,		bos_token_id=50_256							,		eos_token_id=50_256							,		)
	else:
		lowerCAmelCase   =     GPTaConfig.from_json_file(args.config_file							)

	lowerCAmelCase   =     ["GPT2LMHeadModel"]

	# Convert.
	print("""Converting"""							)
	lowerCAmelCase   =     convert_megatron_checkpoint(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)

	# Print the structure of converted state dict.
	if args.print_checkpoint_structure:
		recursive_print(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)

	# Add tokenizer class info to config
	# see https://github.com/huggingface/transformers/issues/13906)
	if ds_args is not None:
		lowerCAmelCase   =     ds_args.tokenizer_type
		if tokenizer_type == "GPT2BPETokenizer":
			lowerCAmelCase   =     "gpt2"
		elif tokenizer_type == "PretrainedFromHF":
			lowerCAmelCase   =     ds_args.tokenizer_name_or_path
		else:
			raise ValueError(f'Unrecognized tokenizer_type {tokenizer_type}'							)
	else:
		lowerCAmelCase   =     "gpt2"

	lowerCAmelCase   =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
	lowerCAmelCase   =     type(_SCREAMING_SNAKE_CASE							).__name__
	lowerCAmelCase   =     tokenizer_class

	# Store the config to file.
	print("""Saving config"""							)
	config.save_pretrained(_SCREAMING_SNAKE_CASE							)

	# Save tokenizer based on args
	print(f'Adding {tokenizer_class} tokenizer files'							)
	tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)

	# Store the state_dict to file.
	lowerCAmelCase   =     os.path.join(_SCREAMING_SNAKE_CASE							,		"""pytorch_model.bin"""							)
	print(f'Saving checkpoint to \"{output_checkpoint_file}\"'							)
	torch.save(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)


####################################################################################################

if __name__ == "__main__":
				main()

####################################################################################################




'''simple docstring'''

import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
				raise RuntimeError('At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training')

# TF training parameters
UpperCAmelCase     		=   False
UpperCAmelCase     		=   False






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Namespace							)      ->						Tuple:







	"""simple docstring"""
	return TrainCommand(_SCREAMING_SNAKE_CASE							)





class __snake_case(				_lowerCAmelCase ):


						'''simple docstring'''



						@staticmethod
						def     __snake_case   ( A_				)     ->     Optional[int]:
							lowerCAmelCase   =     parser.add_parser("""train"""     ,	help="""CLI tool to train a model on a task."""				)

							train_parser.add_argument(
							    """--train_data"""     ,	type=A_     ,	required=A_     ,	help="""path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences."""     ,	)
							train_parser.add_argument(
							    """--column_label"""     ,	type=A_     ,	default=0     ,	help="""Column of the dataset csv file with example labels."""				)
							train_parser.add_argument(
							    """--column_text"""     ,	type=A_     ,	default=1     ,	help="""Column of the dataset csv file with example texts."""				)
							train_parser.add_argument(
							    """--column_id"""     ,	type=A_     ,	default=2     ,	help="""Column of the dataset csv file with example ids."""				)
							train_parser.add_argument(
							    """--skip_first_row"""     ,	action="""store_true"""     ,	help="""Skip the first row of the csv file (headers)."""				)

							train_parser.add_argument("""--validation_data"""     ,	type=A_     ,	default=""""""     ,	help="""path to validation dataset."""				)
							train_parser.add_argument(
							    """--validation_split"""     ,	type=A_     ,	default=0.1     ,	help="""if validation dataset is not provided, fraction of train dataset to use as validation dataset."""     ,	)

							train_parser.add_argument("""--output"""     ,	type=A_     ,	default="""./"""     ,	help="""path to saved the trained model."""				)

							train_parser.add_argument(
							    """--task"""     ,	type=A_     ,	default="""text_classification"""     ,	help="""Task to train the model on."""				)
							train_parser.add_argument(
							    """--model"""     ,	type=A_     ,	default="""bert-base-uncased"""     ,	help="""Model's name or path to stored model."""				)
							train_parser.add_argument("""--train_batch_size"""     ,	type=A_     ,	default=32     ,	help="""Batch size for training."""				)
							train_parser.add_argument("""--valid_batch_size"""     ,	type=A_     ,	default=64     ,	help="""Batch size for validation."""				)
							train_parser.add_argument("""--learning_rate"""     ,	type=A_     ,	default=3e-5     ,	help="""Learning rate."""				)
							train_parser.add_argument("""--adam_epsilon"""     ,	type=A_     ,	default=1e-08     ,	help="""Epsilon for Adam optimizer."""				)
							train_parser.set_defaults(func=A_				)


						def __init__( self     ,	A_				)     ->     Tuple:
							lowerCAmelCase   =     logging.get_logger("""transformers-cli/training"""				)

							lowerCAmelCase   =     """tf""" if is_tf_available() else """torch"""

							os.makedirs(args.output     ,	exist_ok=A_				)
							lowerCAmelCase   =     args.output

							lowerCAmelCase   =     args.column_label
							lowerCAmelCase   =     args.column_text
							lowerCAmelCase   =     args.column_id

							self.logger.info(f'Loading {args.task} pipeline for {args.model}'				)
							if args.task == "text_classification":
								lowerCAmelCase   =     TextClassificationPipeline.from_pretrained(args.model				)
							elif args.task == "token_classification":
								raise NotImplementedError
							elif args.task == "question_answering":
								raise NotImplementedError

							self.logger.info(f'Loading dataset from {args.train_data}'				)
							lowerCAmelCase   =     Processor.create_from_csv(
							    args.train_data     ,	column_label=args.column_label     ,	column_text=args.column_text     ,	column_id=args.column_id     ,	skip_first_row=args.skip_first_row     ,	)
							lowerCAmelCase   =     None
							if args.validation_data:
								self.logger.info(f'Loading validation dataset from {args.validation_data}'				)
								lowerCAmelCase   =     Processor.create_from_csv(
								    args.validation_data     ,	column_label=args.column_label     ,	column_text=args.column_text     ,	column_id=args.column_id     ,	skip_first_row=args.skip_first_row     ,	)

							lowerCAmelCase   =     args.validation_split
							lowerCAmelCase   =     args.train_batch_size
							lowerCAmelCase   =     args.valid_batch_size
							lowerCAmelCase   =     args.learning_rate
							lowerCAmelCase   =     args.adam_epsilon


						def     __snake_case   ( self				)     ->     Optional[int]:
							if self.framework == "tf":
								return self.run_tf()
							return self.run_torch()


						def     __snake_case   ( self				)     ->     Tuple:
							raise NotImplementedError




						def     __snake_case   ( self				)     ->     Tuple:
							self.pipeline.fit(
							    self.train_dataset     ,	validation_data=self.valid_dataset     ,	validation_split=self.validation_split     ,	learning_rate=self.learning_rate     ,	adam_epsilon=self.adam_epsilon     ,	train_batch_size=self.train_batch_size     ,	valid_batch_size=self.valid_batch_size     ,	)

							# Save trained pipeline
							self.pipeline.save_pretrained(self.output				)