


"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowercase    =				{
    """configuration_mobilebert""": [
        """MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """MobileBertConfig""",
        """MobileBertOnnxConfig""",
    ],
    """tokenization_mobilebert""": ["""MobileBertTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    =				["""MobileBertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    =				[
     """MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """MobileBertForMaskedLM""",
     """MobileBertForMultipleChoice""",
     """MobileBertForNextSentencePrediction""",
     """MobileBertForPreTraining""",
     """MobileBertForQuestionAnswering""",
     """MobileBertForSequenceClassification""",
     """MobileBertForTokenClassification""",
     """MobileBertLayer""",
     """MobileBertModel""",
     """MobileBertPreTrainedModel""",
     """load_tf_weights_in_mobilebert""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowercase    =				[
     """TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFMobileBertForMaskedLM""",
     """TFMobileBertForMultipleChoice""",
     """TFMobileBertForNextSentencePrediction""",
     """TFMobileBertForPreTraining""",
     """TFMobileBertForQuestionAnswering""",
     """TFMobileBertForSequenceClassification""",
     """TFMobileBertForTokenClassification""",
     """TFMobileBertMainLayer""",
     """TFMobileBertModel""",
     """TFMobileBertPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_mobilebert import (
     MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
     MobileBertConfig,
     MobileBertOnnxConfig,
 )
 from .tokenization_mobilebert import MobileBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_mobilebert_fast import MobileBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_mobilebert import (
      MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      MobileBertForMaskedLM,
      MobileBertForMultipleChoice,
      MobileBertForNextSentencePrediction,
      MobileBertForPreTraining,
      MobileBertForQuestionAnswering,
      MobileBertForSequenceClassification,
      MobileBertForTokenClassification,
      MobileBertLayer,
      MobileBertModel,
      MobileBertPreTrainedModel,
      load_tf_weights_in_mobilebert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_mobilebert import (
      TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFMobileBertForMaskedLM,
      TFMobileBertForMultipleChoice,
      TFMobileBertForNextSentencePrediction,
      TFMobileBertForPreTraining,
      TFMobileBertForQuestionAnswering,
      TFMobileBertForSequenceClassification,
      TFMobileBertForTokenClassification,
      TFMobileBertMainLayer,
      TFMobileBertModel,
      TFMobileBertPreTrainedModel,
  )

else:
 import sys

 __lowercase    =				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json""",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Union[str, Any]     =	"""gpt_neox"""






    def __init__( self    :   List[str]    ,			__UpperCAmelCase    :   Tuple=50432    ,			__UpperCAmelCase    :   str=6144    ,			__UpperCAmelCase    :   Any=44    ,			__UpperCAmelCase    :   Union[str, Any]=64    ,			__UpperCAmelCase    :   Dict=24576    ,			__UpperCAmelCase    :   Any="gelu"    ,			__UpperCAmelCase    :   List[Any]=0.25    ,			__UpperCAmelCase    :   Optional[Any]=10000    ,			__UpperCAmelCase    :   Tuple=0.0    ,			__UpperCAmelCase    :   Union[str, Any]=0.0    ,			__UpperCAmelCase    :   Optional[int]=0.1    ,			__UpperCAmelCase    :   List[Any]=2048    ,			__UpperCAmelCase    :   Optional[Any]=0.02    ,			__UpperCAmelCase    :   Optional[Any]=1e-5    ,			__UpperCAmelCase    :   Dict=True    ,			__UpperCAmelCase    :   Union[str, Any]=0    ,			__UpperCAmelCase    :   Union[str, Any]=2    ,			__UpperCAmelCase    :   Any=False    ,			__UpperCAmelCase    :   List[Any]=True    ,			__UpperCAmelCase    :   int=None    ,			**__UpperCAmelCase    :   Tuple    ,			):
      super().__init__(bos_token_id=__UpperCAmelCase    ,			eos_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase)
      a						:		List[Any]											=      vocab_size
      a						:		Optional[int]											=      max_position_embeddings
      a						:		List[Any]											=      hidden_size
      a						:		Union[str, Any]											=      num_hidden_layers
      a						:		int											=      num_attention_heads
      a						:		Union[str, Any]											=      intermediate_size
      a						:		Optional[Any]											=      hidden_act
      a						:		Dict											=      rotary_pct
      a						:		Any											=      rotary_emb_base
      a						:		Dict											=      attention_dropout
      a						:		List[str]											=      hidden_dropout
      a						:		List[str]											=      classifier_dropout
      a						:		Any											=      initializer_range
      a						:		Union[str, Any]											=      layer_norm_eps
      a						:		int											=      use_cache
      a						:		int											=      tie_word_embeddings
      a						:		str											=      use_parallel_residual
      a						:		Union[str, Any]											=      rope_scaling
      self._rope_scaling_validation()

      if self.hidden_size % self.num_attention_heads != 0:
        raise ValueError(
            "The hidden size is not divisble by the number of attention heads! Make sure to update them!")






    def      __snake_case							( self    :   Any):
      if self.rope_scaling is None:
        return

      if not isinstance(self.rope_scaling    ,			__UpperCAmelCase) or len(self.rope_scaling) != 2:
        raise ValueError(
            "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
            f'''got {self.rope_scaling}''')
      a						:		str											=      self.rope_scaling.get("type"    ,			__UpperCAmelCase)
      a						:		List[str]											=      self.rope_scaling.get("factor"    ,			__UpperCAmelCase)
      if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''')
      if rope_scaling_factor is None or not isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) or rope_scaling_factor <= 1.0:
        raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''')






