



import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    MegatronBertForCausalLM,
						    MegatronBertForMaskedLM,
						    MegatronBertForMultipleChoice,
						    MegatronBertForNextSentencePrediction,
						    MegatronBertForPreTraining,
						    MegatronBertForQuestionAnswering,
						    MegatronBertForSequenceClassification,
						    MegatronBertForTokenClassification,
						    MegatronBertModel,
						)






class      UpperCamelCase    :



	def __init__(       self			,	UpperCAmelCase__			,	UpperCAmelCase__=13			,	UpperCAmelCase__=7			,	UpperCAmelCase__=True			,	UpperCAmelCase__=True			,	UpperCAmelCase__=True			,	UpperCAmelCase__=True			,	UpperCAmelCase__=99			,	UpperCAmelCase__=64			,	UpperCAmelCase__=32			,	UpperCAmelCase__=5			,	UpperCAmelCase__=4			,	UpperCAmelCase__=37			,	UpperCAmelCase__="gelu"			,	UpperCAmelCase__=0.1			,	UpperCAmelCase__=0.1			,	UpperCAmelCase__=512			,	UpperCAmelCase__=16			,	UpperCAmelCase__=2			,	UpperCAmelCase__=0.02			,	UpperCAmelCase__=3			,	UpperCAmelCase__=4			,	UpperCAmelCase__=None			,	):
							A__							=       parent
							A__							=       batch_size
							A__							=       seq_length
							A__							=       is_training
							A__							=       use_input_mask
							A__							=       use_token_type_ids
							A__							=       use_labels
							A__							=       vocab_size
							A__							=       hidden_size
							A__							=       embedding_size
							A__							=       num_hidden_layers
							A__							=       num_attention_heads
							A__							=       intermediate_size
							A__							=       hidden_act
							A__							=       hidden_dropout_prob
							A__							=       attention_probs_dropout_prob
							A__							=       max_position_embeddings
							A__							=       type_vocab_size
							A__							=       type_sequence_label_size
							A__							=       initializer_range
							A__							=       num_labels
							A__							=       num_choices
							A__							=       scope



	def 			__A							(       self		):
							A__							=       ids_tensor([self.batch_size, self.seq_length]			,	self.vocab_size		)

							A__							=       None
							if self.use_input_mask:
													A__							=       random_attention_mask([self.batch_size, self.seq_length]		)

							A__							=       None
							if self.use_token_type_ids:
													A__							=       ids_tensor([self.batch_size, self.seq_length]			,	self.type_vocab_size		)

							A__							=       None
							A__							=       None
							A__							=       None
							if self.use_labels:
													A__							=       ids_tensor([self.batch_size]			,	self.type_sequence_label_size		)
													A__							=       ids_tensor([self.batch_size, self.seq_length]			,	self.num_labels		)
													A__							=       ids_tensor([self.batch_size]			,	self.num_choices		)

							A__							=       self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



	def 			__A							(       self		):
							return MegatronBertConfig(
							    vocab_size=self.vocab_size			,	hidden_size=self.hidden_size			,	num_hidden_layers=self.num_hidden_layers			,	num_attention_heads=self.num_attention_heads			,	intermediate_size=self.intermediate_size			,	embedding_size=self.embedding_size			,	hidden_act=self.hidden_act			,	hidden_dropout_prob=self.hidden_dropout_prob			,	attention_probs_dropout_prob=self.attention_probs_dropout_prob			,	max_position_embeddings=self.max_position_embeddings			,	type_vocab_size=self.type_vocab_size			,	is_decoder=UpperCAmelCase__			,	initializer_range=self.initializer_range			,	)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertModel(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__		)
							A__							=       model(UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__		)
							A__							=       model(UpperCAmelCase__		)

							self.parent.assertEqual(result.last_hidden_state.shape			,	(self.batch_size, self.seq_length, self.hidden_size)		)
							self.parent.assertEqual(result.pooler_output.shape			,	(self.batch_size, self.hidden_size)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertForMaskedLM(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__		)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, self.seq_length, self.vocab_size)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertForCausalLM(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__		)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, self.seq_length, self.vocab_size)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertForNextSentencePrediction(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(
							    UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__			,	)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, 2)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertForPreTraining(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(
							    UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__			,	next_sentence_label=UpperCAmelCase__			,	)
							self.parent.assertEqual(result.prediction_logits.shape			,	(self.batch_size, self.seq_length, self.vocab_size)		)
							self.parent.assertEqual(result.seq_relationship_logits.shape			,	(self.batch_size, 2)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       MegatronBertForQuestionAnswering(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(
							    UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	start_positions=UpperCAmelCase__			,	end_positions=UpperCAmelCase__			,	)
							self.parent.assertEqual(result.start_logits.shape			,	(self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.end_logits.shape			,	(self.batch_size, self.seq_length)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       self.num_labels
							A__							=       MegatronBertForSequenceClassification(UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__		)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, self.num_labels)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       self.num_labels
							A__							=       MegatronBertForTokenClassification(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       model(UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__		)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, self.seq_length, self.num_labels)		)



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		):
							A__							=       self.num_choices
							A__							=       MegatronBertForMultipleChoice(config=UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.eval()
							A__							=       input_ids.unsqueeze(1		).expand(-1			,	self.num_choices			,	-1		).contiguous()
							A__							=       token_type_ids.unsqueeze(1		).expand(-1			,	self.num_choices			,	-1		).contiguous()
							A__							=       input_mask.unsqueeze(1		).expand(-1			,	self.num_choices			,	-1		).contiguous()
							A__							=       model(
							    UpperCAmelCase__			,	attention_mask=UpperCAmelCase__			,	token_type_ids=UpperCAmelCase__			,	labels=UpperCAmelCase__			,	)
							self.parent.assertEqual(result.logits.shape			,	(self.batch_size, self.num_choices)		)



	def 			__A							(       self		):
							A__							=       self.prepare_config_and_inputs()
							(
							    (
							    A__
							)						,       (
							    A__
							)						,       (
							    A__
							)						,       (
							    A__
							)						,       (
							    A__
							)						,       (
							    A__
							)						,       (
							    A__
							)						,       
							)							=       config_and_inputs
							A__							=       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict






@require_torch
class      UpperCamelCase    (						_UpperCAmelCase       ,    _UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   Any				=	(
	    (
	        MegatronBertModel,
	        MegatronBertForMaskedLM,
	        MegatronBertForCausalLM,
	        MegatronBertForMultipleChoice,
	        MegatronBertForNextSentencePrediction,
	        MegatronBertForPreTraining,
	        MegatronBertForQuestionAnswering,
	        MegatronBertForSequenceClassification,
	        MegatronBertForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	lowerCAmelCase	:   Optional[int]				=	(
	    {
	        """feature-extraction""": MegatronBertModel,
	        """fill-mask""": MegatronBertForMaskedLM,
	        """question-answering""": MegatronBertForQuestionAnswering,
	        """text-classification""": MegatronBertForSequenceClassification,
	        """text-generation""": MegatronBertForCausalLM,
	        """token-classification""": MegatronBertForTokenClassification,
	        """zero-shot""": MegatronBertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	lowerCAmelCase	:   Optional[Any]				=	True
	# test_resize_embeddings = False
	lowerCAmelCase	:   Dict				=	False



	def 			__A							(       self			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__=False		):
							A__							=       super()._prepare_for_class(UpperCAmelCase__			,	UpperCAmelCase__			,	return_labels=UpperCAmelCase__		)

							if return_labels:
													if model_class in get_values(UpperCAmelCase__		):
																			A__							=       torch.zeros(
																			    (self.model_tester.batch_size, self.model_tester.seq_length)			,	dtype=torch.long			,	device=UpperCAmelCase__		)
																			A__							=       torch.zeros(
																			    self.model_tester.batch_size			,	dtype=torch.long			,	device=UpperCAmelCase__		)
							return inputs_dict



	def 			__A							(       self		):
							A__							=       MegatronBertModelTester(self		)
							A__							=       ConfigTester(self			,	config_class=UpperCAmelCase__			,	hidden_size=37		)



	def 			__A							(       self		):
							self.config_tester.run_common_tests()



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_model(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_masked_lm(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_pretraining(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_question_answering(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_megatron_bert_for_token_classification(*UpperCAmelCase__		)





def  UpperCamelCase						(       _A			:				str    )->       int:





						"""simple docstring"""





						return torch.tensor(
						    _A			,							dtype=torch.long			,							device=_A			,							)


UpperCAmelCase_							:      int      =  1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class      UpperCamelCase    (						unittest.TestCase     ):



	@slow
	@unittest.skip("Model is not available."		)
	def 			__A							(       self		):
							A__							=       "nvidia/megatron-bert-uncased-345m"
							if "MYDIR" in os.environ:
													A__							=       os.path.join(os.environ["MYDIR"]			,	UpperCAmelCase__		)
							A__							=       MegatronBertModel.from_pretrained(UpperCAmelCase__		)
							model.to(UpperCAmelCase__		)
							model.half()
							A__							=       _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]		)
							with torch.no_grad():
													A__							=       model(UpperCAmelCase__		)[0]
							A__							=       torch.Size((1, 9, 1_024)		)
							self.assertEqual(output.shape			,	UpperCAmelCase__		)

							A__							=       [-0.6_040, -0.2_517, -0.1_025, 0.3_420, -0.6_758, -0.0_017, -0.1_089, -0.1_990, 0.5_728]
							for ii in range(3		):
													for jj in range(3		):
																			A__							=       output[0, ii, jj]
																			A__							=       expected[3 * ii + jj]
																			A__							=       "ii={} jj={} a={} b={}".format(UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__			,	UpperCAmelCase__		)
																			self.assertTrue(math.isclose(UpperCAmelCase__			,	UpperCAmelCase__			,	rel_tol=UpperCAmelCase__			,	abs_tol=UpperCAmelCase__		)			,	msg=UpperCAmelCase__		)










import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
def  UpperCamelCase						(       _A			:				Tuple    )->       Dict:





						"""simple docstring"""





						A__							=       [
						    "encoder.version",
						    "decoder.version",
						    "model.encoder.version",
						    "model.decoder.version",
						    "decoder.output_projection.weight",
						    "_float_tensor",
						    "encoder.embed_positions._float_tensor",
						    "decoder.embed_positions._float_tensor",
						]
						for k in ignore_keys:
												state_dict.pop(_A			,							_A    )
def  UpperCamelCase						(       _A			:				int    )->       Optional[Any]:





						"""simple docstring"""





						A__						,       A__							=       emb.weight.shape
						A__							=       nn.Linear(_A			,							_A			,							bias=_A    )
						A__							=       emb.weight.data
						return lin_layer
def  UpperCamelCase						(       _A			:				str			,							_A			:				Optional[Any]=None    )->       str:





						"""simple docstring"""





						A__							=       {}
						for old_key in state_dict.keys():
												A__							=       old_key
												if "moe_layer.experts." in key:
																		if expert_idx is not None:
																								A__							=       key.replace("moe_layer.experts.0"			,							f"""ffn.experts.expert_{expert_idx}"""    )
																		else:
																								A__							=       key.replace("moe_layer.experts."			,							"ffn.experts.expert_"    )
												if "gate" in key:
																		A__							=       key.replace(".moe_layer.gate.wg"			,							".ffn.router.classifier"    )
												if "fc2" and "experts" not in key:
																		A__							=       key.replace(".fc2."			,							".ffn.fc2."    )
												if "fc1" and "experts" not in key:
																		A__							=       key.replace(".fc1."			,							".ffn.fc1."    )
												if ".encoder_attn." in key:
																		A__							=       key.replace(".encoder_attn."			,							".cross_attention."    )
												if "encoder_attn_layer_norm" in key:
																		A__							=       key.replace("encoder_attn_layer_norm"			,							"cross_attention_layer_norm"    )
												if "final_layer_norm" in key:
																		A__							=       key.replace("final_layer_norm"			,							"ff_layer_norm"    )
												A__							=       state_dict[old_key]
						return new_dict
def  UpperCamelCase						(       _A			:				Tuple			,							_A			:				Tuple			,							_A			:				int			,							_A			:				str			,							_A			:				str = WEIGHTS_NAME    )->       List[str]:





						"""simple docstring"""





						A__							=       []
						A__							=       0
						os.makedirs(_A			,							exist_ok=_A    )

						for expert in range(_A    ):
												A__							=       switch_checkpoint_path + f"""-rank-{expert}.pt"""
												if os.path.isfile(_A    ):
																		A__							=       torch.load(_A    )["model"]
																		remove_ignore_keys_(_A    )
																		A__							=       rename_fairseq_keys(_A			,							_A    )
																		A__							=       os.path.join(
																		    _A			,							weights_name.replace(".bin"			,							f"""-{len(_A    )+1:05d}-of-???.bin"""    )    )
																		torch.save(_A			,							_A    )
																		sharded_state_dicts.append(expert_state.keys()    )
																		total_size += sum([value.numel() for key, value in expert_state.items()]    ) * dtype_byte_size(
																		    expert_state[list(_A    )[0]].dtype    )

    # Add the last block
						A__							=       os.path.join(_A			,							weights_name.replace(".bin"			,							f"""-{len(_A    )+1:05d}-of-???.bin"""    )    )
						A__							=       torch.load(switch_checkpoint_path + "-shared.pt"    )["model"]
						remove_ignore_keys_(_A    )
						A__							=       rename_fairseq_keys(_A			,							_A    )
						A__							=       shared_weights["decoder.embed_tokens.weight"]
						sharded_state_dicts.append(shared_weights.keys()    )

						# If we only have the shared weights (dummy model/experts saved on the same file)
						if len(_A    ) == 1:
												A__							=       os.path.join(_A			,							_A    )
												torch.save(_A			,							_A    )
												return {weights_name: sharded_state_dicts[0]}, None
						else:
												torch.save(_A			,							_A    )
						# Otherwise, let's build the index
						A__							=       {}
						for idx, shard in enumerate(_A    ):
												A__							=       weights_name.replace(".bin"			,							f"""-{idx+1:05d}-of-{len(_A    ):05d}.bin"""    )
												A__							=       os.path.join(_A			,							weights_name.replace(".bin"			,							f"""-{idx+1:05d}-of-???.bin"""    )    )
												os.rename(_A			,							os.path.join(_A			,							_A    )    )
												for key in shard:
																		A__							=       shard_file

    # Add the metadata
						A__							=       {"total_size": total_size}
						A__							=       {"metadata": metadata, "weight_map": weight_map}

						with open(os.path.join(_A			,							_A    )			,							"w"			,							encoding="utf-8"    ) as f:
												A__							=       json.dumps(_A			,							indent=2			,							sort_keys=_A    ) + "\n"
												f.write(_A    )

						return metadata, index


if __name__ == "__main__":
						UpperCAmelCase_							:      str      =  argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    "--nllb_moe_checkpoint_path",
						    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000",
						    type=str,
						    required=False,
						    help="Path to a directory containing a folder per layer. Follows the original Google format.",
						)
						parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model")
						parser.add_argument(
						    "--pytorch_dump_folder_path",
						    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b",
						    type=str,
						    required=False,
						    help="Path to the output pytorch model.",
						)
						UpperCAmelCase_							:      Union[str, Any]      =  parser.parse_args()
						UpperCAmelCase_   ,	UpperCAmelCase_							:      Optional[int]      =  shard_on_the_fly(
						    args.nllb_moe_checkpoint_path,
						    args.pytorch_dump_folder_path,
						    128,
						    args.dtype,
						)

						UpperCAmelCase_							:      Any      =  NllbMoeConfig.from_pretrained(
						    "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
						)
						config.save_pretrained(args.pytorch_dump_folder_path)
						UpperCAmelCase_							:      Tuple      =  NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
						print("Done")
						model.save_pretrained(args.pytorch_dump_folder_path)





