




import torch
from torch import nn







class   A(						nn.Module       ):



 '''simple docstring'''


 def __init__(    self :  Dict						,							A_ :  Any						,							A_ :  Tuple						,							A_ :  Dict						,							A_ :  List[str]						,							A_ :  List[Any]=1						,							A_ :  List[Any]=False  )       ->     List[str]:

       """simple docstring"""






       super().__init__()

       lowerCamelCase_							=   n_token
       lowerCamelCase_							=   d_embed
       lowerCamelCase_							=   d_proj

       lowerCamelCase_							=   cutoffs + [n_token]
       lowerCamelCase_							=   [0] + self.cutoffs
       lowerCamelCase_							=   div_val

       lowerCamelCase_							=   self.cutoffs[0]
       lowerCamelCase_							=   len(self.cutoffs  ) - 1
       lowerCamelCase_							=   self.shortlist_size + self.n_clusters

       if self.n_clusters > 0:
             lowerCamelCase_							=   nn.Parameter(torch.zeros(self.n_clusters						,							self.d_embed  )  )
             lowerCamelCase_							=   nn.Parameter(torch.zeros(self.n_clusters  )  )

       lowerCamelCase_							=   nn.ModuleList()
       lowerCamelCase_							=   nn.ParameterList()

       if div_val == 1:
             for i in range(len(self.cutoffs  )  ):
                   if d_proj != d_embed:
                         self.out_projs.append(nn.Parameter(torch.FloatTensor(A_						,							A_  )  )  )
                   else:
                         self.out_projs.append(A_  )

             self.out_layers.append(nn.Linear(A_						,							A_  )  )
       else:
             for i in range(len(self.cutoffs  )  ):
                   lowerCamelCase_					,    lowerCamelCase_							=   self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   lowerCamelCase_							=   d_embed // (div_val**i)

                   self.out_projs.append(nn.Parameter(torch.FloatTensor(A_						,							A_  )  )  )

                   self.out_layers.append(nn.Linear(A_						,							r_idx - l_idx  )  )

       lowerCamelCase_							=   keep_order

 def     a__			(    self :  List[Any]						,							A_ :  List[Any]						,							A_ :  List[str]						,							A_ :  Any						,							A_ :  Tuple  )       ->     Any:

       """simple docstring"""






       if proj is None:
             lowerCamelCase_							=   nn.functional.linear(A_						,							A_						,							bias=A_  )
       else:
             # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
             lowerCamelCase_							=   nn.functional.linear(A_						,							proj.t().contiguous()  )
             lowerCamelCase_							=   nn.functional.linear(A_						,							A_						,							bias=A_  )
             # else:
             #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
             #     if bias is not None:
             #         logit = logit + bias

       return logit

 def     a__			(    self :  int						,							A_ :  Union[str, Any]						,							A_ :  Dict=None						,							A_ :  Union[str, Any]=False  )       ->     int:

       """simple docstring"""






       if labels is not None:
             # Shift so that tokens < n predict n
             lowerCamelCase_							=   hidden[..., :-1, :].contiguous()
             lowerCamelCase_							=   labels[..., 1:].contiguous()
             lowerCamelCase_							=   hidden.view(-1						,							hidden.size(-1  )  )
             lowerCamelCase_							=   labels.view(-1  )
             if hidden.size(0  ) != labels.size(0  ):
                   raise RuntimeError('Input and labels should have the same size in the batch dimension.'  )
       else:
             lowerCamelCase_							=   hidden.view(-1						,							hidden.size(-1  )  )

       if self.n_clusters == 0:
             lowerCamelCase_							=   self._compute_logit(A_						,							self.out_layers[0].weight						,							self.out_layers[0].bias						,							self.out_projs[0]  )
             if labels is not None:
                   lowerCamelCase_							=   labels != -100
                   lowerCamelCase_							=   torch.zeros_like(A_						,							dtype=hidden.dtype						,							device=hidden.device  )
                   lowerCamelCase_							=   (
                       -nn.functional.log_softmax(A_						,							dim=-1  )[mask].gather(1						,							labels[mask].unsqueeze(1  )  ).squeeze(1  )
                   )
             else:
                   lowerCamelCase_							=   nn.functional.log_softmax(A_						,							dim=-1  )
       else:
             # construct weights and biases
             lowerCamelCase_					,    lowerCamelCase_							=   [], []
             for i in range(len(self.cutoffs  )  ):
                   if self.div_val == 1:
                         lowerCamelCase_					,    lowerCamelCase_							=   self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         lowerCamelCase_							=   self.out_layers[0].weight[l_idx:r_idx]
                         lowerCamelCase_							=   self.out_layers[0].bias[l_idx:r_idx]
                   else:
                         lowerCamelCase_							=   self.out_layers[i].weight
                         lowerCamelCase_							=   self.out_layers[i].bias

                   if i == 0:
                         lowerCamelCase_							=   torch.cat([weight_i, self.cluster_weight]						,							dim=0  )
                         lowerCamelCase_							=   torch.cat([bias_i, self.cluster_bias]						,							dim=0  )

                   weights.append(A_  )
                   biases.append(A_  )

             lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_							=   weights[0], biases[0], self.out_projs[0]

             lowerCamelCase_							=   self._compute_logit(A_						,							A_						,							A_						,							A_  )
             lowerCamelCase_							=   nn.functional.log_softmax(A_						,							dim=1  )

             if labels is None:
                   lowerCamelCase_							=   hidden.new_empty((head_logit.size(0  ), self.n_token)  )
             else:
                   lowerCamelCase_							=   torch.zeros_like(A_						,							dtype=hidden.dtype						,							device=hidden.device  )

             lowerCamelCase_							=   0
             lowerCamelCase_							=   [0] + self.cutoffs
             for i in range(len(A_  ) - 1  ):
                   lowerCamelCase_					,    lowerCamelCase_							=   cutoff_values[i], cutoff_values[i + 1]

                   if labels is not None:
                         lowerCamelCase_							=   (labels >= l_idx) & (labels < r_idx)
                         lowerCamelCase_							=   mask_i.nonzero().squeeze()

                         if indices_i.numel() == 0:
                               continue

                         lowerCamelCase_							=   labels.index_select(0						,							A_  ) - l_idx
                         lowerCamelCase_							=   head_logprob.index_select(0						,							A_  )
                         lowerCamelCase_							=   hidden.index_select(0						,							A_  )
                   else:
                         lowerCamelCase_							=   hidden

                   if i == 0:
                         if labels is not None:
                               lowerCamelCase_							=   head_logprob_i.gather(1						,							target_i[:, None]  ).squeeze(1  )
                         else:
                               lowerCamelCase_							=   head_logprob[:, : self.cutoffs[0]]
                   else:
                         lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_							=   weights[i], biases[i], self.out_projs[i]

                         lowerCamelCase_							=   self._compute_logit(A_						,							A_						,							A_						,							A_  )
                         lowerCamelCase_							=   nn.functional.log_softmax(A_						,							dim=1  )
                         lowerCamelCase_							=   self.cutoffs[0] + i - 1  # No probability for the head cluster
                         if labels is not None:
                               lowerCamelCase_							=   head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                   1						,							target_i[:, None]  ).squeeze(1  )
                         else:
                               lowerCamelCase_							=   head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                               lowerCamelCase_							=   logprob_i

                   if labels is not None:
                         if (hasattr(self						,							'keep_order'  ) and self.keep_order) or keep_order:
                               out.index_copy_(0						,							A_						,							-logprob_i  )
                         else:
                               out[offset : offset + logprob_i.size(0  )].copy_(-logprob_i  )
                         offset += logprob_i.size(0  )

       return out





 def     a__			(    self :  str						,							A_ :  Dict  )       ->     str:

       """simple docstring"""






       if self.n_clusters == 0:
             lowerCamelCase_							=   self._compute_logit(A_						,							self.out_layers[0].weight						,							self.out_layers[0].bias						,							self.out_projs[0]  )
             return nn.functional.log_softmax(A_						,							dim=-1  )
       else:
             # construct weights and biases
             lowerCamelCase_					,    lowerCamelCase_							=   [], []
             for i in range(len(self.cutoffs  )  ):
                   if self.div_val == 1:
                         lowerCamelCase_					,    lowerCamelCase_							=   self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         lowerCamelCase_							=   self.out_layers[0].weight[l_idx:r_idx]
                         lowerCamelCase_							=   self.out_layers[0].bias[l_idx:r_idx]
                   else:
                         lowerCamelCase_							=   self.out_layers[i].weight
                         lowerCamelCase_							=   self.out_layers[i].bias

                   if i == 0:
                         lowerCamelCase_							=   torch.cat([weight_i, self.cluster_weight]						,							dim=0  )
                         lowerCamelCase_							=   torch.cat([bias_i, self.cluster_bias]						,							dim=0  )

                   weights.append(A_  )
                   biases.append(A_  )

             lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_							=   weights[0], biases[0], self.out_projs[0]
             lowerCamelCase_							=   self._compute_logit(A_						,							A_						,							A_						,							A_  )

             lowerCamelCase_							=   hidden.new_empty((head_logit.size(0  ), self.n_token)  )
             lowerCamelCase_							=   nn.functional.log_softmax(A_						,							dim=1  )

             lowerCamelCase_							=   [0] + self.cutoffs
             for i in range(len(A_  ) - 1  ):
                   lowerCamelCase_					,    lowerCamelCase_							=   cutoff_values[i], cutoff_values[i + 1]

                   if i == 0:
                         lowerCamelCase_							=   head_logprob[:, : self.cutoffs[0]]
                   else:
                         lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_							=   weights[i], biases[i], self.out_projs[i]

                         lowerCamelCase_							=   self._compute_logit(A_						,							A_						,							A_						,							A_  )
                         lowerCamelCase_							=   nn.functional.log_softmax(A_						,							dim=1  )

                         lowerCamelCase_							=   head_logprob[:, -i] + tail_logprob_i
                         lowerCamelCase_							=   logprob_i

             return out









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase    :		Optional[Any]			   =       logging.get_logger(__name__)

lowerCamelCase    :		Tuple			   =       {
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						'''transfo-xl'''
 UpperCamelCase						       =						['''mems''']
 UpperCamelCase						       =						{
     '''n_token''': '''vocab_size''',
     '''hidden_size''': '''d_model''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }

 def __init__(    self :  Any						,							A_ :  Optional[Any]=267735						,							A_ :  Optional[Any]=[20000, 40000, 200000]						,							A_ :  Union[str, Any]=1024						,							A_ :  Optional[Any]=1024						,							A_ :  Optional[int]=16						,							A_ :  Any=64						,							A_ :  List[Any]=4096						,							A_ :  str=4						,							A_ :  int=False						,							A_ :  List[Any]=18						,							A_ :  Optional[int]=1600						,							A_ :  Union[str, Any]=1000						,							A_ :  Optional[Any]=True						,							A_ :  Optional[int]=True						,							A_ :  List[str]=0						,							A_ :  int=-1						,							A_ :  List[Any]=True						,							A_ :  List[Any]=0.1						,							A_ :  str=0.0						,							A_ :  Dict=True						,							A_ :  Dict="normal"						,							A_ :  Dict=0.01						,							A_ :  Optional[Any]=0.01						,							A_ :  Any=0.02						,							A_ :  int=1E-5						,							A_ :  List[str]=0						,							**A_ :  Optional[Any]						,							)       ->     Optional[int]:

       """simple docstring"""






       lowerCamelCase_							=   vocab_size
       lowerCamelCase_							=   []
       self.cutoffs.extend(A_  )
       if proj_share_all_but_first:
             lowerCamelCase_							=   [False] + [True] * len(self.cutoffs  )
       else:
             lowerCamelCase_							=   [False] + [False] * len(self.cutoffs  )
       lowerCamelCase_							=   d_model
       lowerCamelCase_							=   d_embed
       lowerCamelCase_							=   d_head
       lowerCamelCase_							=   d_inner
       lowerCamelCase_							=   div_val
       lowerCamelCase_							=   pre_lnorm
       lowerCamelCase_							=   n_layer
       lowerCamelCase_							=   n_head
       lowerCamelCase_							=   mem_len
       lowerCamelCase_							=   same_length
       lowerCamelCase_							=   attn_type
       lowerCamelCase_							=   clamp_len
       lowerCamelCase_							=   sample_softmax
       lowerCamelCase_							=   adaptive
       lowerCamelCase_							=   dropout
       lowerCamelCase_							=   dropatt
       lowerCamelCase_							=   untie_r
       lowerCamelCase_							=   init
       lowerCamelCase_							=   init_range
       lowerCamelCase_							=   proj_init_std
       lowerCamelCase_							=   init_std
       lowerCamelCase_							=   layer_norm_epsilon
       super().__init__(eos_token_id=A_						,							**A_  )

 @property
 def     a__			(    self :  Tuple  )       ->     Any:

       """simple docstring"""






       logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )
       return -1





 @max_position_embeddings.setter
 def     a__			(    self :  Dict						,							A_ :  Optional[int]  )       ->     List[Any]:

       """simple docstring"""






       raise NotImplementedError(
           f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )



