



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase_							:      Optional[Any]      =  {
    "configuration_distilbert": [
        "DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "DistilBertConfig",
        "DistilBertOnnxConfig",
    ],
    "tokenization_distilbert": ["DistilBertTokenizer"],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_							:      Tuple      =  ["DistilBertTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_							:      Any      =  [
						    "DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "DistilBertForMaskedLM",
						    "DistilBertForMultipleChoice",
						    "DistilBertForQuestionAnswering",
						    "DistilBertForSequenceClassification",
						    "DistilBertForTokenClassification",
						    "DistilBertModel",
						    "DistilBertPreTrainedModel",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_							:      List[str]      =  [
						    "TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "TFDistilBertForMaskedLM",
						    "TFDistilBertForMultipleChoice",
						    "TFDistilBertForQuestionAnswering",
						    "TFDistilBertForSequenceClassification",
						    "TFDistilBertForTokenClassification",
						    "TFDistilBertMainLayer",
						    "TFDistilBertModel",
						    "TFDistilBertPreTrainedModel",
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCAmelCase_							:      str      =  [
						    "FlaxDistilBertForMaskedLM",
						    "FlaxDistilBertForMultipleChoice",
						    "FlaxDistilBertForQuestionAnswering",
						    "FlaxDistilBertForSequenceClassification",
						    "FlaxDistilBertForTokenClassification",
						    "FlaxDistilBertModel",
						    "FlaxDistilBertPreTrainedModel",
						]


if TYPE_CHECKING:
						from .configuration_distilbert import (
						    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    DistilBertConfig,
						    DistilBertOnnxConfig,
						)
						from .tokenization_distilbert import DistilBertTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_distilbert_fast import DistilBertTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_distilbert import (
												    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    DistilBertForMaskedLM,
												    DistilBertForMultipleChoice,
												    DistilBertForQuestionAnswering,
												    DistilBertForSequenceClassification,
												    DistilBertForTokenClassification,
												    DistilBertModel,
												    DistilBertPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_distilbert import (
												    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFDistilBertForMaskedLM,
												    TFDistilBertForMultipleChoice,
												    TFDistilBertForQuestionAnswering,
												    TFDistilBertForSequenceClassification,
												    TFDistilBertForTokenClassification,
												    TFDistilBertMainLayer,
												    TFDistilBertModel,
												    TFDistilBertPreTrainedModel,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_distilbert import (
												    FlaxDistilBertForMaskedLM,
												    FlaxDistilBertForMultipleChoice,
												    FlaxDistilBertForQuestionAnswering,
												    FlaxDistilBertForSequenceClassification,
												    FlaxDistilBertForTokenClassification,
												    FlaxDistilBertModel,
												    FlaxDistilBertPreTrainedModel,
												)

else:
						import sys

						UpperCAmelCase_							:      Any      =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_							:      Dict      =  logging.get_logger(__name__)

UpperCAmelCase_							:      List[Any]      =  {
    "EleutherAI/gpt-neox-20b": "https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class      UpperCamelCase    (						_UpperCAmelCase     ):
	lowerCAmelCase	:   int				=	"""gpt_neox"""



	def __init__(       self			,	UpperCAmelCase__=50_432			,	UpperCAmelCase__=6_144			,	UpperCAmelCase__=44			,	UpperCAmelCase__=64			,	UpperCAmelCase__=24_576			,	UpperCAmelCase__="gelu"			,	UpperCAmelCase__=0.25			,	UpperCAmelCase__=10_000			,	UpperCAmelCase__=0.0			,	UpperCAmelCase__=0.0			,	UpperCAmelCase__=0.1			,	UpperCAmelCase__=2_048			,	UpperCAmelCase__=0.02			,	UpperCAmelCase__=1e-5			,	UpperCAmelCase__=True			,	UpperCAmelCase__=0			,	UpperCAmelCase__=2			,	UpperCAmelCase__=False			,	UpperCAmelCase__=True			,	UpperCAmelCase__=None			,	**UpperCAmelCase__			,	):
							super().__init__(bos_token_id=UpperCAmelCase__			,	eos_token_id=UpperCAmelCase__			,	**UpperCAmelCase__		)
							A__							=       vocab_size
							A__							=       max_position_embeddings
							A__							=       hidden_size
							A__							=       num_hidden_layers
							A__							=       num_attention_heads
							A__							=       intermediate_size
							A__							=       hidden_act
							A__							=       rotary_pct
							A__							=       rotary_emb_base
							A__							=       attention_dropout
							A__							=       hidden_dropout
							A__							=       classifier_dropout
							A__							=       initializer_range
							A__							=       layer_norm_eps
							A__							=       use_cache
							A__							=       tie_word_embeddings
							A__							=       use_parallel_residual
							A__							=       rope_scaling
							self._rope_scaling_validation()

							if self.hidden_size % self.num_attention_heads != 0:
													raise ValueError(
													    "The hidden size is not divisble by the number of attention heads! Make sure to update them!"		)



	def 			__A							(       self		):
							if self.rope_scaling is None:
													return

							if not isinstance(self.rope_scaling			,	UpperCAmelCase__		) or len(self.rope_scaling		) != 2:
													raise ValueError(
													    "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
													    F"""got {self.rope_scaling}"""		)
							A__							=       self.rope_scaling.get("type"			,	UpperCAmelCase__		)
							A__							=       self.rope_scaling.get("factor"			,	UpperCAmelCase__		)
							if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
													raise ValueError(
													    F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""		)
							if rope_scaling_factor is None or not isinstance(UpperCAmelCase__			,	UpperCAmelCase__		) or rope_scaling_factor <= 1.0:
													raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""		)





