




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase    :		Optional[Any]			   =       logging.get_logger(__name__)

lowerCamelCase    :		Tuple			   =       {}







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						'''llama'''
 UpperCamelCase						       =						['''past_key_values''']

 def __init__(    self :  Dict						,							A_ :  List[Any]=32000						,							A_ :  Tuple=4096						,							A_ :  List[Any]=11008						,							A_ :  List[str]=32						,							A_ :  Optional[Any]=32						,							A_ :  int=None						,							A_ :  Any="silu"						,							A_ :  Union[str, Any]=2048						,							A_ :  List[str]=0.02						,							A_ :  Optional[int]=1E-6						,							A_ :  List[str]=True						,							A_ :  Optional[int]=0						,							A_ :  Optional[Any]=1						,							A_ :  Optional[int]=2						,							A_ :  int=1						,							A_ :  Tuple=False						,							A_ :  Tuple=None						,							**A_ :  Optional[int]						,							)       ->     Optional[Any]:

       """simple docstring"""






       lowerCamelCase_							=   vocab_size
       lowerCamelCase_							=   max_position_embeddings
       lowerCamelCase_							=   hidden_size
       lowerCamelCase_							=   intermediate_size
       lowerCamelCase_							=   num_hidden_layers
       lowerCamelCase_							=   num_attention_heads

       # for backward compatibility
       if num_key_value_heads is None:
             lowerCamelCase_							=   num_attention_heads

       lowerCamelCase_							=   num_key_value_heads
       lowerCamelCase_							=   hidden_act
       lowerCamelCase_							=   initializer_range
       lowerCamelCase_							=   rms_norm_eps
       lowerCamelCase_							=   pretraining_tp
       lowerCamelCase_							=   use_cache
       lowerCamelCase_							=   rope_scaling
       self._rope_scaling_validation()

       super().__init__(
           pad_token_id=A_						,							bos_token_id=A_						,							eos_token_id=A_						,							tie_word_embeddings=A_						,							**A_						,							)





 def     a__			(    self :  Optional[Any]  )       ->     Any:

       """simple docstring"""






       if self.rope_scaling is None:
             return

       if not isinstance(self.rope_scaling						,							A_  ) or len(self.rope_scaling  ) != 2:
             raise ValueError(
                 '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                 f"""got {self.rope_scaling}"""  )
       lowerCamelCase_							=   self.rope_scaling.get('type'						,							A_  )
       lowerCamelCase_							=   self.rope_scaling.get('factor'						,							A_  )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
             raise ValueError(
                 f"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""  )
       if rope_scaling_factor is None or not isinstance(A_						,							A_  ) or rope_scaling_factor <= 1.0:
             raise ValueError(f"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase    :		Optional[Any]			   =       logging.get_logger(__name__)

lowerCamelCase    :		Tuple			   =       {
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						'''transfo-xl'''
 UpperCamelCase						       =						['''mems''']
 UpperCamelCase						       =						{
     '''n_token''': '''vocab_size''',
     '''hidden_size''': '''d_model''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }

 def __init__(    self :  Any						,							A_ :  Optional[Any]=267735						,							A_ :  Optional[Any]=[20000, 40000, 200000]						,							A_ :  Union[str, Any]=1024						,							A_ :  Optional[Any]=1024						,							A_ :  Optional[int]=16						,							A_ :  Any=64						,							A_ :  List[Any]=4096						,							A_ :  str=4						,							A_ :  int=False						,							A_ :  List[Any]=18						,							A_ :  Optional[int]=1600						,							A_ :  Union[str, Any]=1000						,							A_ :  Optional[Any]=True						,							A_ :  Optional[int]=True						,							A_ :  List[str]=0						,							A_ :  int=-1						,							A_ :  List[Any]=True						,							A_ :  List[Any]=0.1						,							A_ :  str=0.0						,							A_ :  Dict=True						,							A_ :  Dict="normal"						,							A_ :  Dict=0.01						,							A_ :  Optional[Any]=0.01						,							A_ :  Any=0.02						,							A_ :  int=1E-5						,							A_ :  List[str]=0						,							**A_ :  Optional[Any]						,							)       ->     Optional[int]:

       """simple docstring"""






       lowerCamelCase_							=   vocab_size
       lowerCamelCase_							=   []
       self.cutoffs.extend(A_  )
       if proj_share_all_but_first:
             lowerCamelCase_							=   [False] + [True] * len(self.cutoffs  )
       else:
             lowerCamelCase_							=   [False] + [False] * len(self.cutoffs  )
       lowerCamelCase_							=   d_model
       lowerCamelCase_							=   d_embed
       lowerCamelCase_							=   d_head
       lowerCamelCase_							=   d_inner
       lowerCamelCase_							=   div_val
       lowerCamelCase_							=   pre_lnorm
       lowerCamelCase_							=   n_layer
       lowerCamelCase_							=   n_head
       lowerCamelCase_							=   mem_len
       lowerCamelCase_							=   same_length
       lowerCamelCase_							=   attn_type
       lowerCamelCase_							=   clamp_len
       lowerCamelCase_							=   sample_softmax
       lowerCamelCase_							=   adaptive
       lowerCamelCase_							=   dropout
       lowerCamelCase_							=   dropatt
       lowerCamelCase_							=   untie_r
       lowerCamelCase_							=   init
       lowerCamelCase_							=   init_range
       lowerCamelCase_							=   proj_init_std
       lowerCamelCase_							=   init_std
       lowerCamelCase_							=   layer_norm_epsilon
       super().__init__(eos_token_id=A_						,							**A_  )

 @property
 def     a__			(    self :  Tuple  )       ->     Any:

       """simple docstring"""






       logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )
       return -1





 @max_position_embeddings.setter
 def     a__			(    self :  Dict						,							A_ :  Optional[int]  )       ->     List[Any]:

       """simple docstring"""






       raise NotImplementedError(
           f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )



