



"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_							:      List[str]      =  logging.get_logger(__name__)

UpperCAmelCase_							:      Tuple      =  {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}






class      UpperCamelCase    (						SCREAMING_SNAKE_CASE_     ):
	lowerCAmelCase	:   List[str]				=	"""switch_transformers"""
	lowerCAmelCase	:   Optional[int]				=	["""past_key_values"""]
	lowerCAmelCase	:   str				=	{"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}



	def __init__(       self			,	UpperCAmelCase__=32_128			,	UpperCAmelCase__=768			,	UpperCAmelCase__=64			,	UpperCAmelCase__=2_048			,	UpperCAmelCase__=64			,	UpperCAmelCase__=12			,	UpperCAmelCase__=3			,	UpperCAmelCase__=12			,	UpperCAmelCase__=3			,	UpperCAmelCase__=12			,	UpperCAmelCase__=8			,	UpperCAmelCase__=False			,	UpperCAmelCase__=0.01			,	UpperCAmelCase__="float32"			,	UpperCAmelCase__=False			,	UpperCAmelCase__=32			,	UpperCAmelCase__=128			,	UpperCAmelCase__=0.1			,	UpperCAmelCase__=1e-6			,	UpperCAmelCase__=0.001			,	UpperCAmelCase__=0.001			,	UpperCAmelCase__=1.0			,	UpperCAmelCase__="relu"			,	UpperCAmelCase__=True			,	UpperCAmelCase__=False			,	UpperCAmelCase__=True			,	UpperCAmelCase__=0			,	UpperCAmelCase__=1			,	**UpperCAmelCase__			,	):
							A__							=       vocab_size
							A__							=       d_model
							A__							=       d_kv
							A__							=       d_ff

							A__							=       num_sparse_encoder_layers

							A__							=       num_layers
							A__							=       (
							    num_decoder_layers if num_decoder_layers is not None else self.num_layers
							)  # default = symmetry
							A__							=       num_sparse_decoder_layers

							# This tells us, each how many encoder layer we'll have to set a sparse layer.
							if self.num_sparse_encoder_layers > 0:
													A__							=       self.num_layers // self.num_sparse_encoder_layers
							else:
													A__							=       self.num_layers  # HACK: this will create 0 sparse layers

							# This tells us, each how many encoder layer we'll have to set a sparse layer.
							if self.num_sparse_decoder_layers > 0:
													A__							=       self.num_decoder_layers // self.num_sparse_decoder_layers
							else:
													A__							=       self.num_decoder_layers  # HACK: this will create 0 sparse layers

							A__							=       num_heads
							A__							=       num_experts
							A__							=       expert_capacity
							A__							=       router_bias
							A__							=       router_jitter_noise
							if router_dtype not in ["float32", "float16", "bfloat16"]:
													raise ValueError(F"""`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}"""		)
							A__							=       router_dtype

							A__							=       router_ignore_padding_tokens
							A__							=       relative_attention_num_buckets
							A__							=       relative_attention_max_distance

							A__							=       dropout_rate
							A__							=       layer_norm_epsilon
							A__							=       initializer_factor
							A__							=       feed_forward_proj
							A__							=       use_cache
							A__							=       add_router_probs

							A__							=       router_z_loss_coef
							A__							=       router_aux_loss_coef

							A__							=       self.feed_forward_proj.split("-"		)
							A__							=       act_info[-1]
							A__							=       act_info[0] == 'gated'

							if len(__a		) > 1 and act_info[0] != "gated" or len(__a		) > 2:
													raise ValueError(
													    F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
													    "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
													    "\'gated-gelu\' or \'relu\'"		)

							# for backwards compatibility
							if feed_forward_proj == "gated-gelu":
													A__							=       'gelu_new'

							super().__init__(
							    pad_token_id=__a			,	eos_token_id=__a			,	is_encoder_decoder=__a			,	**__a			,	)










from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

UpperCAmelCase_							:      List[Any]      =  [
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
						if pkg in deps:
												if pkg == "tokenizers":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_tokenizers_available

																		if not is_tokenizers_available():
																								continue  # not required, check version only if installed
												elif pkg == "accelerate":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_accelerate_available

																		# Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
																		# Transformers with PyTorch
																		if not is_accelerate_available():
																								continue  # not required, check version only if installed

												require_version_core(deps[pkg])
						else:
												raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')
def  UpperCamelCase						(       _A			:				List[Any]			,							_A			:				int=None    )->       Optional[int]:





						"""simple docstring"""





						require_version(deps[pkg]			,							_A    )





