



"""simple docstring"""




import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin






class      UpperCamelCase    (						_UpperCAmelCase     ):
	# to overwrite at feature extractactor specific tests
	lowerCAmelCase	:   List[Any]				=	None
	lowerCAmelCase	:   List[Any]				=	None



	@property
	def 			__A							(       self		):
							return self.feat_extract_tester.prepare_feat_extract_dict()



	def 			__A							(       self		):
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							self.assertTrue(hasattr(UpperCAmelCase__			,	"feature_size"		)		)
							self.assertTrue(hasattr(UpperCAmelCase__			,	"sampling_rate"		)		)
							self.assertTrue(hasattr(UpperCAmelCase__			,	"padding_value"		)		)



	def 			__A							(       self		):
							A__							=       self.feat_extract_tester.prepare_inputs_for_common()
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							self.assertTrue(all(len(UpperCAmelCase__		) == len(UpperCAmelCase__		) for x, y in zip(UpperCAmelCase__			,	processed_features[input_name]		)		)		)

							A__							=       self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCAmelCase__		)
							A__							=       BatchFeature({input_name: speech_inputs}			,	tensor_type="np"		)

							A__							=       processed_features[input_name]

							if len(batch_features_input.shape		) < 3:
													A__							=       batch_features_input[:, :, None]

							self.assertTrue(
							    batch_features_input.shape
							    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)



	@require_torch
	def 			__A							(       self		):
							A__							=       self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCAmelCase__		)
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}			,	tensor_type="pt"		)

							A__							=       processed_features[input_name]

							if len(batch_features_input.shape		) < 3:
													A__							=       batch_features_input[:, :, None]

							self.assertTrue(
							    batch_features_input.shape
							    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)



	@require_tf
	def 			__A							(       self		):
							A__							=       self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCAmelCase__		)
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}			,	tensor_type="tf"		)

							A__							=       processed_features[input_name]

							if len(batch_features_input.shape		) < 3:
													A__							=       batch_features_input[:, :, None]

							self.assertTrue(
							    batch_features_input.shape
							    == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)



	def 			__A							(       self			,	UpperCAmelCase__=False		):
							def _inputs_have_equal_length(UpperCAmelCase__		):
													A__							=       len(input[0]		)
													for input_slice in input[1:]:
																			if len(UpperCAmelCase__		) != length:
																									return False
													return True

							def _inputs_are_equal(UpperCAmelCase__			,	UpperCAmelCase__		):
													if len(UpperCAmelCase__		) != len(UpperCAmelCase__		):
																			return False

													for input_slice_a, input_slice_a in zip(UpperCAmelCase__			,	UpperCAmelCase__		):
																			if not np.allclose(np.asarray(UpperCAmelCase__		)			,	np.asarray(UpperCAmelCase__		)			,	atol=1e-3		):
																									return False
													return True

							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCAmelCase__		)
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							A__							=       self.feat_extract_tester.seq_length_diff
							A__							=       self.feat_extract_tester.max_seq_length + pad_diff
							A__							=       self.feat_extract_tester.min_seq_length
							A__							=       self.feat_extract_tester.batch_size
							A__							=       self.feat_extract_tester.feature_size

							# test padding for List[int] + numpy
							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding=UpperCAmelCase__		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[-1]		)		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="np"		)
							A__							=       input_a[input_name]

							# max_length parameter has to be provided when setting `padding="max_length"`
							with self.assertRaises(UpperCAmelCase__		):
													feat_extract.pad(UpperCAmelCase__			,	padding="max_length"		)[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=UpperCAmelCase__			,	return_tensors="np"		)
							A__							=       input_a[input_name]

							self.assertFalse(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(_inputs_are_equal(UpperCAmelCase__			,	UpperCAmelCase__		)		)
							self.assertTrue(len(input_a[0]		) == pad_min_length		)
							self.assertTrue(len(input_a[1]		) == pad_min_length + pad_diff		)
							self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]		))		)
							self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)		)

							if feature_size > 1:
													self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size		)

							# test padding for `pad_to_multiple_of` for List[int] + numpy
							A__							=       feat_extract.pad(UpperCAmelCase__			,	pad_to_multiple_of=10		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	pad_to_multiple_of=10		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	pad_to_multiple_of=10			,	max_length=UpperCAmelCase__		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	pad_to_multiple_of=10			,	max_length=UpperCAmelCase__			,	return_tensors="np"			,	)
							A__							=       input_a[input_name]

							self.assertTrue(all(len(UpperCAmelCase__		) % 10 == 0 for x in input_a		)		)
							self.assertTrue(_inputs_are_equal(UpperCAmelCase__			,	UpperCAmelCase__		)		)

							A__							=       pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
							self.assertTrue(all(len(UpperCAmelCase__		) == expected_mult_pad_length for x in input_a		)		)
							self.assertEqual(input_a.shape[:2]			,	(batch_size, expected_mult_pad_length)		)

							if feature_size > 1:
													self.assertTrue(input_a.shape[2] == feature_size		)

							# Check padding value is correct
							A__							=       (np.ones(self.feat_extract_tester.feature_size		) * feat_extract.padding_value).sum()
							self.assertTrue(
							    abs(np.asarray(input_a[0]		)[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)		)
							    < 1e-3		)
							self.assertTrue(
							    abs(
							        np.asarray(input_a[1]		)[pad_min_length + pad_diff :].sum()
							        - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)		)
							    < 1e-3		)
							self.assertTrue(
							    abs(
							        np.asarray(input_a[2]		)[pad_min_length + 2 * pad_diff :].sum()
							        - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)		)
							    < 1e-3		)
							self.assertTrue(
							    abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)		) < 1e-3		)
							self.assertTrue(
							    abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)		)
							    < 1e-3		)



	def 			__A							(       self			,	UpperCAmelCase__=False		):
							def _inputs_have_equal_length(UpperCAmelCase__		):
													A__							=       len(input[0]		)
													for input_slice in input[1:]:
																			if len(UpperCAmelCase__		) != length:
																									return False
													return True

							def _inputs_are_equal(UpperCAmelCase__			,	UpperCAmelCase__		):
													if len(UpperCAmelCase__		) != len(UpperCAmelCase__		):
																			return False

													for input_slice_a, input_slice_a in zip(UpperCAmelCase__			,	UpperCAmelCase__		):
																			if not np.allclose(np.asarray(UpperCAmelCase__		)			,	np.asarray(UpperCAmelCase__		)			,	atol=1e-3		):
																									return False
													return True

							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCAmelCase__		)
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							# truncate to smallest
							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)			,	truncation=UpperCAmelCase__		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)		)
							A__							=       input_a[input_name]

							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertFalse(_inputs_have_equal_length(UpperCAmelCase__		)		)

							# truncate to smallest with np
							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)			,	return_tensors="np"			,	truncation=UpperCAmelCase__			,	)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)			,	return_tensors="np"		)
							A__							=       input_a[input_name]

							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(input_a.shape[1] == len(speech_inputs[0]		)		)

							# since truncation forces padding to be smaller than longest input
							# function can't return `np.ndarray`, but has to return list
							self.assertFalse(_inputs_have_equal_length(UpperCAmelCase__		)		)

							# truncate to middle
							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[1]		)			,	truncation=UpperCAmelCase__			,	return_tensors="np"			,	)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[1]		)			,	truncation=UpperCAmelCase__		)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[1]		)			,	return_tensors="np"		)
							A__							=       input_a[input_name]

							self.assertTrue(input_a.shape[1] == len(speech_inputs[1]		)		)
							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(_inputs_are_equal(UpperCAmelCase__			,	UpperCAmelCase__		)		)

							# since truncation forces padding to be smaller than longest input
							# function can't return `np.ndarray`, but has to return list
							self.assertFalse(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertTrue(len(input_a[-1]		) == len(speech_inputs[-1]		)		)

							# padding has to be max_length when setting `truncation=True`
							with self.assertRaises(UpperCAmelCase__		):
													feat_extract.pad(UpperCAmelCase__			,	truncation=UpperCAmelCase__		)[input_name]

							# padding has to be max_length when setting `truncation=True`
							with self.assertRaises(UpperCAmelCase__		):
													feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	truncation=UpperCAmelCase__		)[input_name]

							# padding has to be max_length when setting `truncation=True`
							with self.assertRaises(UpperCAmelCase__		):
													feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	truncation=UpperCAmelCase__		)[input_name]

							# max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
							with self.assertRaises(UpperCAmelCase__		):
													feat_extract.pad(UpperCAmelCase__			,	padding="max_length"			,	truncation=UpperCAmelCase__		)[input_name]

							# test truncation for `pad_to_multiple_of` for List[int] + numpy
							A__							=       12
							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)			,	pad_to_multiple_of=UpperCAmelCase__			,	truncation=UpperCAmelCase__			,	)
							A__							=       input_a[input_name]

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=len(speech_inputs[0]		)			,	pad_to_multiple_of=UpperCAmelCase__			,	)
							A__							=       input_a[input_name]

							# retrieve expected_length as multiple of pad_to_multiple_of
							A__							=       len(speech_inputs[0]		)
							if expected_length % pad_to_multiple_of != 0:
													A__							=       ((len(speech_inputs[0]		) // pad_to_multiple_of) + 1) * pad_to_multiple_of

							self.assertTrue(len(input_a[0]		) == expected_length		)
							self.assertTrue(_inputs_have_equal_length(UpperCAmelCase__		)		)
							self.assertFalse(_inputs_have_equal_length(UpperCAmelCase__		)		)



	def 			__A							(       self		):
							self._check_padding(numpify=UpperCAmelCase__		)



	def 			__A							(       self		):
							self._check_padding(numpify=UpperCAmelCase__		)



	def 			__A							(       self		):
							self._check_truncation(numpify=UpperCAmelCase__		)



	def 			__A							(       self		):
							self._check_truncation(numpify=UpperCAmelCase__		)



	@require_torch
	def 			__A							(       self		):
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common()
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="np"		)[input_name]
							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="pt"		)[input_name]

							self.assertTrue(abs(input_np.astype(np.floataa		).sum() - input_pt.numpy().astype(np.floataa		).sum()		) < 1e-2		)



	@require_tf
	def 			__A							(       self		):
							A__							=       self.feature_extraction_class(**self.feat_extract_dict		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common()
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="np"		)[input_name]
							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="tf"		)[input_name]

							self.assertTrue(abs(input_np.astype(np.floataa		).sum() - input_tf.numpy().astype(np.floataa		).sum()		) < 1e-2		)



	def 			__A							(       self		):
							A__							=       self.feat_extract_dict
							A__							=       True
							A__							=       self.feature_extraction_class(**UpperCAmelCase__		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common()
							A__							=       [len(UpperCAmelCase__		) for x in speech_inputs]
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)

							A__							=       feat_extract.pad(UpperCAmelCase__			,	padding="longest"			,	return_tensors="np"		)
							self.assertIn("attention_mask"			,	UpperCAmelCase__		)
							self.assertListEqual(list(processed.attention_mask.shape		)			,	list(processed[input_name].shape[:2]		)		)
							self.assertListEqual(processed.attention_mask.sum(-1		).tolist()			,	UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       self.feat_extract_dict
							A__							=       True
							A__							=       self.feature_extraction_class(**UpperCAmelCase__		)
							A__							=       self.feat_extract_tester.prepare_inputs_for_common()
							A__							=       [len(UpperCAmelCase__		) for x in speech_inputs]
							A__							=       feat_extract.model_input_names[0]

							A__							=       BatchFeature({input_name: speech_inputs}		)
							A__							=       min(UpperCAmelCase__		)

							A__							=       feat_extract.pad(
							    UpperCAmelCase__			,	padding="max_length"			,	max_length=UpperCAmelCase__			,	truncation=UpperCAmelCase__			,	return_tensors="np"		)
							self.assertIn("attention_mask"			,	UpperCAmelCase__		)
							self.assertListEqual(
							    list(processed_pad.attention_mask.shape		)			,	[processed_pad[input_name].shape[0], max_length]		)
							self.assertListEqual(
							    processed_pad.attention_mask[:, :max_length].sum(-1		).tolist()			,	[max_length for x in speech_inputs]		)










import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
def  UpperCamelCase						(       _A			:				Tuple    )->       Dict:





						"""simple docstring"""





						A__							=       [
						    "encoder.version",
						    "decoder.version",
						    "model.encoder.version",
						    "model.decoder.version",
						    "decoder.output_projection.weight",
						    "_float_tensor",
						    "encoder.embed_positions._float_tensor",
						    "decoder.embed_positions._float_tensor",
						]
						for k in ignore_keys:
												state_dict.pop(_A			,							_A    )
def  UpperCamelCase						(       _A			:				int    )->       Optional[Any]:





						"""simple docstring"""





						A__						,       A__							=       emb.weight.shape
						A__							=       nn.Linear(_A			,							_A			,							bias=_A    )
						A__							=       emb.weight.data
						return lin_layer
def  UpperCamelCase						(       _A			:				str			,							_A			:				Optional[Any]=None    )->       str:





						"""simple docstring"""





						A__							=       {}
						for old_key in state_dict.keys():
												A__							=       old_key
												if "moe_layer.experts." in key:
																		if expert_idx is not None:
																								A__							=       key.replace("moe_layer.experts.0"			,							f"""ffn.experts.expert_{expert_idx}"""    )
																		else:
																								A__							=       key.replace("moe_layer.experts."			,							"ffn.experts.expert_"    )
												if "gate" in key:
																		A__							=       key.replace(".moe_layer.gate.wg"			,							".ffn.router.classifier"    )
												if "fc2" and "experts" not in key:
																		A__							=       key.replace(".fc2."			,							".ffn.fc2."    )
												if "fc1" and "experts" not in key:
																		A__							=       key.replace(".fc1."			,							".ffn.fc1."    )
												if ".encoder_attn." in key:
																		A__							=       key.replace(".encoder_attn."			,							".cross_attention."    )
												if "encoder_attn_layer_norm" in key:
																		A__							=       key.replace("encoder_attn_layer_norm"			,							"cross_attention_layer_norm"    )
												if "final_layer_norm" in key:
																		A__							=       key.replace("final_layer_norm"			,							"ff_layer_norm"    )
												A__							=       state_dict[old_key]
						return new_dict
def  UpperCamelCase						(       _A			:				Tuple			,							_A			:				Tuple			,							_A			:				int			,							_A			:				str			,							_A			:				str = WEIGHTS_NAME    )->       List[str]:





						"""simple docstring"""





						A__							=       []
						A__							=       0
						os.makedirs(_A			,							exist_ok=_A    )

						for expert in range(_A    ):
												A__							=       switch_checkpoint_path + f"""-rank-{expert}.pt"""
												if os.path.isfile(_A    ):
																		A__							=       torch.load(_A    )["model"]
																		remove_ignore_keys_(_A    )
																		A__							=       rename_fairseq_keys(_A			,							_A    )
																		A__							=       os.path.join(
																		    _A			,							weights_name.replace(".bin"			,							f"""-{len(_A    )+1:05d}-of-???.bin"""    )    )
																		torch.save(_A			,							_A    )
																		sharded_state_dicts.append(expert_state.keys()    )
																		total_size += sum([value.numel() for key, value in expert_state.items()]    ) * dtype_byte_size(
																		    expert_state[list(_A    )[0]].dtype    )

    # Add the last block
						A__							=       os.path.join(_A			,							weights_name.replace(".bin"			,							f"""-{len(_A    )+1:05d}-of-???.bin"""    )    )
						A__							=       torch.load(switch_checkpoint_path + "-shared.pt"    )["model"]
						remove_ignore_keys_(_A    )
						A__							=       rename_fairseq_keys(_A			,							_A    )
						A__							=       shared_weights["decoder.embed_tokens.weight"]
						sharded_state_dicts.append(shared_weights.keys()    )

						# If we only have the shared weights (dummy model/experts saved on the same file)
						if len(_A    ) == 1:
												A__							=       os.path.join(_A			,							_A    )
												torch.save(_A			,							_A    )
												return {weights_name: sharded_state_dicts[0]}, None
						else:
												torch.save(_A			,							_A    )
						# Otherwise, let's build the index
						A__							=       {}
						for idx, shard in enumerate(_A    ):
												A__							=       weights_name.replace(".bin"			,							f"""-{idx+1:05d}-of-{len(_A    ):05d}.bin"""    )
												A__							=       os.path.join(_A			,							weights_name.replace(".bin"			,							f"""-{idx+1:05d}-of-???.bin"""    )    )
												os.rename(_A			,							os.path.join(_A			,							_A    )    )
												for key in shard:
																		A__							=       shard_file

    # Add the metadata
						A__							=       {"total_size": total_size}
						A__							=       {"metadata": metadata, "weight_map": weight_map}

						with open(os.path.join(_A			,							_A    )			,							"w"			,							encoding="utf-8"    ) as f:
												A__							=       json.dumps(_A			,							indent=2			,							sort_keys=_A    ) + "\n"
												f.write(_A    )

						return metadata, index


if __name__ == "__main__":
						UpperCAmelCase_							:      str      =  argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    "--nllb_moe_checkpoint_path",
						    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000",
						    type=str,
						    required=False,
						    help="Path to a directory containing a folder per layer. Follows the original Google format.",
						)
						parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model")
						parser.add_argument(
						    "--pytorch_dump_folder_path",
						    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b",
						    type=str,
						    required=False,
						    help="Path to the output pytorch model.",
						)
						UpperCAmelCase_							:      Union[str, Any]      =  parser.parse_args()
						UpperCAmelCase_   ,	UpperCAmelCase_							:      Optional[int]      =  shard_on_the_fly(
						    args.nllb_moe_checkpoint_path,
						    args.pytorch_dump_folder_path,
						    128,
						    args.dtype,
						)

						UpperCAmelCase_							:      Any      =  NllbMoeConfig.from_pretrained(
						    "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
						)
						config.save_pretrained(args.pytorch_dump_folder_path)
						UpperCAmelCase_							:      Tuple      =  NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
						print("Done")
						model.save_pretrained(args.pytorch_dump_folder_path)





