


"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """tanreinama/GPTSAN-2.8B-spout_is_uniform""": (
        """https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"""
    ),
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							List[str]     =	"""gptsan-japanese"""
    UpperCAmelCase	:							int     =	[
        """past_key_values""",
    ]
    UpperCAmelCase	:							Dict     =	{
        """hidden_size""": """d_model""",
        """num_attention_heads""": """num_heads""",
        """num_hidden_layers""": """num_layers""",
    }






    def __init__( self    :   Tuple    ,			__UpperCAmelCase    :   List[Any]=36000    ,			__UpperCAmelCase    :   Optional[int]=1280    ,			__UpperCAmelCase    :   Union[str, Any]=1024    ,			__UpperCAmelCase    :   Tuple=8192    ,			__UpperCAmelCase    :   str=4096    ,			__UpperCAmelCase    :   Tuple=128    ,			__UpperCAmelCase    :   Tuple=10    ,			__UpperCAmelCase    :   Dict=0    ,			__UpperCAmelCase    :   Optional[int]=16    ,			__UpperCAmelCase    :   List[str]=16    ,			__UpperCAmelCase    :   Optional[int]=128    ,			__UpperCAmelCase    :   List[Any]=0.0    ,			__UpperCAmelCase    :   Tuple=1e-5    ,			__UpperCAmelCase    :   List[Any]=False    ,			__UpperCAmelCase    :   List[str]=0.0    ,			__UpperCAmelCase    :   Optional[int]="float32"    ,			__UpperCAmelCase    :   List[str]=False    ,			__UpperCAmelCase    :   Optional[Any]=False    ,			__UpperCAmelCase    :   Optional[int]=False    ,			__UpperCAmelCase    :   List[str]=0.002    ,			__UpperCAmelCase    :   Any=False    ,			__UpperCAmelCase    :   List[Any]=True    ,			__UpperCAmelCase    :   List[str]=35998    ,			__UpperCAmelCase    :   Tuple=35995    ,			__UpperCAmelCase    :   str=35999    ,			**__UpperCAmelCase    :   Union[str, Any]    ,			):
      a						:		Tuple											=      vocab_size
      a						:		str											=      max_position_embeddings
      a						:		List[Any]											=      d_model
      a						:		Dict											=      d_ff
      a						:		List[str]											=      d_ext
      a						:		Any											=      d_spout
      a						:		Optional[Any]											=      num_switch_layers
      a						:		List[Any]											=      num_ext_layers
      a						:		List[str]											=      num_switch_layers + num_ext_layers
      a						:		str											=      num_heads
      a						:		List[Any]											=      num_experts
      a						:		str											=      expert_capacity
      a						:		List[str]											=      dropout_rate
      a						:		Optional[int]											=      layer_norm_epsilon
      a						:		Any											=      router_bias
      a						:		Any											=      router_jitter_noise
      a						:		Dict											=      router_dtype
      a						:		Optional[int]											=      router_ignore_padding_tokens
      a						:		List[str]											=      output_hidden_states
      a						:		List[str]											=      output_attentions
      a						:		Dict											=      initializer_factor
      a						:		Optional[Any]											=      output_router_logits
      a						:		Optional[int]											=      use_cache

      super().__init__(
          separator_token_id=__UpperCAmelCase    ,			pad_token_id=__UpperCAmelCase    ,			eos_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase    ,			)










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json""",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Union[str, Any]     =	"""gpt_neox"""






    def __init__( self    :   List[str]    ,			__UpperCAmelCase    :   Tuple=50432    ,			__UpperCAmelCase    :   str=6144    ,			__UpperCAmelCase    :   Any=44    ,			__UpperCAmelCase    :   Union[str, Any]=64    ,			__UpperCAmelCase    :   Dict=24576    ,			__UpperCAmelCase    :   Any="gelu"    ,			__UpperCAmelCase    :   List[Any]=0.25    ,			__UpperCAmelCase    :   Optional[Any]=10000    ,			__UpperCAmelCase    :   Tuple=0.0    ,			__UpperCAmelCase    :   Union[str, Any]=0.0    ,			__UpperCAmelCase    :   Optional[int]=0.1    ,			__UpperCAmelCase    :   List[Any]=2048    ,			__UpperCAmelCase    :   Optional[Any]=0.02    ,			__UpperCAmelCase    :   Optional[Any]=1e-5    ,			__UpperCAmelCase    :   Dict=True    ,			__UpperCAmelCase    :   Union[str, Any]=0    ,			__UpperCAmelCase    :   Union[str, Any]=2    ,			__UpperCAmelCase    :   Any=False    ,			__UpperCAmelCase    :   List[Any]=True    ,			__UpperCAmelCase    :   int=None    ,			**__UpperCAmelCase    :   Tuple    ,			):
      super().__init__(bos_token_id=__UpperCAmelCase    ,			eos_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase)
      a						:		List[Any]											=      vocab_size
      a						:		Optional[int]											=      max_position_embeddings
      a						:		List[Any]											=      hidden_size
      a						:		Union[str, Any]											=      num_hidden_layers
      a						:		int											=      num_attention_heads
      a						:		Union[str, Any]											=      intermediate_size
      a						:		Optional[Any]											=      hidden_act
      a						:		Dict											=      rotary_pct
      a						:		Any											=      rotary_emb_base
      a						:		Dict											=      attention_dropout
      a						:		List[str]											=      hidden_dropout
      a						:		List[str]											=      classifier_dropout
      a						:		Any											=      initializer_range
      a						:		Union[str, Any]											=      layer_norm_eps
      a						:		int											=      use_cache
      a						:		int											=      tie_word_embeddings
      a						:		str											=      use_parallel_residual
      a						:		Union[str, Any]											=      rope_scaling
      self._rope_scaling_validation()

      if self.hidden_size % self.num_attention_heads != 0:
        raise ValueError(
            "The hidden size is not divisble by the number of attention heads! Make sure to update them!")






    def      __snake_case							( self    :   Any):
      if self.rope_scaling is None:
        return

      if not isinstance(self.rope_scaling    ,			__UpperCAmelCase) or len(self.rope_scaling) != 2:
        raise ValueError(
            "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
            f'''got {self.rope_scaling}''')
      a						:		str											=      self.rope_scaling.get("type"    ,			__UpperCAmelCase)
      a						:		List[str]											=      self.rope_scaling.get("factor"    ,			__UpperCAmelCase)
      if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''')
      if rope_scaling_factor is None or not isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) or rope_scaling_factor <= 1.0:
        raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''')






