
"""simple docstring"""







from packaging import version

from .import_utils import is_accelerate_available


if is_accelerate_available():
							import accelerate
def    __lowerCAmelCase    (_UpperCamelCase		):
	if not is_accelerate_available():
		return method
	__lowerCAmelCase      :						List[Any]						      =				version.parse(accelerate.__version__		).base_version
	if version.parse(_UpperCamelCase		) < version.parse('0.17.0'		):
		return method

	def wrapper(self	,  *_UpperCamelCase	,  **_UpperCamelCase		):
		if hasattr(self	,  '_hf_hook'		) and hasattr(self._hf_hook	,  'pre_forward'		):
			self._hf_hook.pre_forward(self		)
		return method(self	,  *_UpperCamelCase	,  **_UpperCamelCase		)

	return wrapper

"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__        =		logging.get_logger(__name__)

lowerCamelCase__        =		{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}




class 			A__	(     _lowerCamelCase):
		A_						:	str           =     'nllb-moe'
		A_						:	Optional[Any]           =     ['past_key_values']
		A_						:	List[str]           =     {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}

		def __init__(				self	,		_SCREAMING_SNAKE_CASE=12_81_12	,		_SCREAMING_SNAKE_CASE=10_24	,		_SCREAMING_SNAKE_CASE=12	,		_SCREAMING_SNAKE_CASE=40_96	,		_SCREAMING_SNAKE_CASE=16	,		_SCREAMING_SNAKE_CASE=12	,		_SCREAMING_SNAKE_CASE=40_96	,		_SCREAMING_SNAKE_CASE=16	,		_SCREAMING_SNAKE_CASE=0.05	,		_SCREAMING_SNAKE_CASE=0.05	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE="relu"	,		_SCREAMING_SNAKE_CASE=10_24	,		_SCREAMING_SNAKE_CASE=0.1	,		_SCREAMING_SNAKE_CASE=0.1	,		_SCREAMING_SNAKE_CASE=0.0	,		_SCREAMING_SNAKE_CASE=0.02	,		_SCREAMING_SNAKE_CASE=2	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=False	,		_SCREAMING_SNAKE_CASE="float32"	,		_SCREAMING_SNAKE_CASE=False	,		_SCREAMING_SNAKE_CASE=1_28	,		_SCREAMING_SNAKE_CASE=64	,		_SCREAMING_SNAKE_CASE=4	,		_SCREAMING_SNAKE_CASE=4	,		_SCREAMING_SNAKE_CASE=0.001	,		_SCREAMING_SNAKE_CASE=0.001	,		_SCREAMING_SNAKE_CASE="all"	,		_SCREAMING_SNAKE_CASE=False	,		_SCREAMING_SNAKE_CASE=False	,		_SCREAMING_SNAKE_CASE=1.0	,		_SCREAMING_SNAKE_CASE=0.2	,		_SCREAMING_SNAKE_CASE=1	,		_SCREAMING_SNAKE_CASE=0	,		_SCREAMING_SNAKE_CASE=2	,		_SCREAMING_SNAKE_CASE=False	,		**_SCREAMING_SNAKE_CASE	,		):
			__lowerCAmelCase      :						int						      =				vocab_size
			__lowerCAmelCase      :						str						      =				max_position_embeddings
			__lowerCAmelCase      :						Dict						      =				d_model
			__lowerCAmelCase      :						Tuple						      =				encoder_ffn_dim
			__lowerCAmelCase      :						Optional[Any]						      =				encoder_layers
			__lowerCAmelCase      :						Any						      =				encoder_attention_heads
			__lowerCAmelCase      :						Tuple						      =				decoder_ffn_dim
			__lowerCAmelCase      :						Dict						      =				decoder_layers
			__lowerCAmelCase      :						str						      =				decoder_attention_heads
			__lowerCAmelCase      :						str						      =				dropout
			__lowerCAmelCase      :						List[str]						      =				attention_dropout
			__lowerCAmelCase      :						Optional[int]						      =				activation_dropout
			__lowerCAmelCase      :						List[Any]						      =				activation_function
			__lowerCAmelCase      :						List[str]						      =				init_std
			__lowerCAmelCase      :						Union[str, Any]						      =				encoder_layerdrop
			__lowerCAmelCase      :						List[Any]						      =				decoder_layerdrop
			__lowerCAmelCase      :						Optional[int]						      =				use_cache
			__lowerCAmelCase      :						Optional[Any]						      =				encoder_layers
			__lowerCAmelCase      :						Optional[int]						      =				scale_embedding  # scale factor will be sqrt(d_model) if True
			__lowerCAmelCase      :						Union[str, Any]						      =				router_z_loss_coef
			__lowerCAmelCase      :						Optional[Any]						      =				router_aux_loss_coef
			__lowerCAmelCase      :						int						      =				decoder_sparse_step
			__lowerCAmelCase      :						str						      =				encoder_sparse_step
			__lowerCAmelCase      :						Tuple						      =				num_experts
			__lowerCAmelCase      :						Dict						      =				expert_capacity
			__lowerCAmelCase      :						Union[str, Any]						      =				router_bias
			if router_dtype not in ["float32", "float16", "bfloat16"]:
				raise ValueError(f"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"			)
			__lowerCAmelCase      :						Union[str, Any]						      =				router_dtype

			__lowerCAmelCase      :						Any						      =				router_ignore_padding_tokens
			__lowerCAmelCase      :						str						      =				batch_prioritized_routing
			__lowerCAmelCase      :						Tuple						      =				second_expert_policy
			__lowerCAmelCase      :						List[str]						      =				normalize_router_prob_before_dropping
			__lowerCAmelCase      :						Dict						      =				moe_eval_capacity_token_fraction
			__lowerCAmelCase      :						Union[str, Any]						      =				moe_token_dropout
			__lowerCAmelCase      :						List[Any]						      =				output_router_logits
			super().__init__(
			    pad_token_id=_SCREAMING_SNAKE_CASE	,		bos_token_id=_SCREAMING_SNAKE_CASE	,		eos_token_id=_SCREAMING_SNAKE_CASE	,		is_encoder_decoder=_SCREAMING_SNAKE_CASE	,		decoder_start_token_id=_SCREAMING_SNAKE_CASE	,		**_SCREAMING_SNAKE_CASE	,		)