

import math
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import SchedulerMixin, SchedulerOutput






class 	UpperCAmelCase__		(	A__						,					A__	):






  """simple docstring"""


  a									= 1






  @register_to_config
  def __init__(							self						:		Union[str, Any]     ,       __lowerCamelCase						:		int = 1000     ,       __lowerCamelCase						:		Optional[Union[np.ndarray, List[float]]] = None )		->							List[str]:
       # set `betas`, `alphas`, `timesteps`
       self.set_timesteps(__lowerCamelCase )

       # standard deviation of the initial noise distribution
       SCREAMING_SNAKE_CASE__   =   1.0

       # For now we only support F-PNDM, i.e. the runge-kutta method
       # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf
       # mainly at formula (9), (12), (13) and the Algorithm 2.
       SCREAMING_SNAKE_CASE__   =   4

       # running values
       SCREAMING_SNAKE_CASE__   =   []






  def 				lowercase_		(							self						:		Any     ,       __lowerCamelCase						:		int     ,       __lowerCamelCase						:		Union[str, torch.device] = None )		->							Dict:
       SCREAMING_SNAKE_CASE__   =   num_inference_steps
       SCREAMING_SNAKE_CASE__   =   torch.linspace(1     ,       0     ,       num_inference_steps + 1 )[:-1]
       SCREAMING_SNAKE_CASE__   =   torch.cat([steps, torch.tensor([0.0] )] )

       if self.config.trained_betas is not None:
            SCREAMING_SNAKE_CASE__   =   torch.tensor(self.config.trained_betas     ,       dtype=torch.floataa )
       else:
            SCREAMING_SNAKE_CASE__   =   torch.sin(steps * math.pi / 2 ) ** 2

       SCREAMING_SNAKE_CASE__   =   (1.0 - self.betas**2) ** 0.5

       SCREAMING_SNAKE_CASE__   =   (torch.atana(self.betas     ,       self.alphas ) / math.pi * 2)[:-1]
       SCREAMING_SNAKE_CASE__   =   timesteps.to(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   []






  def 				lowercase_		(							self						:		Any     ,       __lowerCamelCase						:		torch.FloatTensor     ,       __lowerCamelCase						:		int     ,       __lowerCamelCase						:		torch.FloatTensor     ,       __lowerCamelCase						:		bool = True     ,       )		->							Union[SchedulerOutput, Tuple]:
       if self.num_inference_steps is None:
            raise ValueError(
                '''Number of inference steps is \'None\', you need to run \'set_timesteps\' after creating the scheduler''' )

       SCREAMING_SNAKE_CASE__   =   (self.timesteps == timestep).nonzero().item()
       SCREAMING_SNAKE_CASE__   =   timestep_index + 1

       SCREAMING_SNAKE_CASE__   =   sample * self.betas[timestep_index] + model_output * self.alphas[timestep_index]
       self.ets.append(__lowerCamelCase )

       if len(self.ets ) == 1:
            SCREAMING_SNAKE_CASE__   =   self.ets[-1]
       elif len(self.ets ) == 2:
            SCREAMING_SNAKE_CASE__   =   (3 * self.ets[-1] - self.ets[-2]) / 2
       elif len(self.ets ) == 3:
            SCREAMING_SNAKE_CASE__   =   (23 * self.ets[-1] - 16 * self.ets[-2] + 5 * self.ets[-3]) / 12
       else:
            SCREAMING_SNAKE_CASE__   =   (1 / 24) * (55 * self.ets[-1] - 59 * self.ets[-2] + 37 * self.ets[-3] - 9 * self.ets[-4])

       SCREAMING_SNAKE_CASE__   =   self._get_prev_sample(__lowerCamelCase     ,       __lowerCamelCase     ,       __lowerCamelCase     ,       __lowerCamelCase )

       if not return_dict:
            return (prev_sample,)

       return SchedulerOutput(prev_sample=__lowerCamelCase )






  def 				lowercase_		(							self						:		List[str]     ,       __lowerCamelCase						:		torch.FloatTensor     ,       *__lowerCamelCase						:		Any     ,       **__lowerCamelCase						:		Optional[Any] )		->							torch.FloatTensor:
       return sample






  def 				lowercase_		(							self						:		int     ,       __lowerCamelCase						:		Optional[Any]     ,       __lowerCamelCase						:		Tuple     ,       __lowerCamelCase						:		str     ,       __lowerCamelCase						:		Union[str, Any] )		->							str:
       SCREAMING_SNAKE_CASE__   =   self.alphas[timestep_index]
       SCREAMING_SNAKE_CASE__   =   self.betas[timestep_index]

       SCREAMING_SNAKE_CASE__   =   self.alphas[prev_timestep_index]
       SCREAMING_SNAKE_CASE__   =   self.betas[prev_timestep_index]

       SCREAMING_SNAKE_CASE__   =   (sample - sigma * ets) / max(__lowerCamelCase     ,       1e-8 )
       SCREAMING_SNAKE_CASE__   =   next_alpha * pred + ets * next_sigma

       return prev_sample







  def __len__(							self						:		str )		->							Union[str, Any]:
       return self.config.num_train_timesteps





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE					:  List[str]             =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					:  Tuple             =      {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}






class 	UpperCAmelCase__		(	A__	):






  """simple docstring"""


  a									= "transfo-xl"
  a									= ["mems"]
  a									= {
      "n_token": "vocab_size",
      "hidden_size": "d_model",
      "num_attention_heads": "n_head",
      "num_hidden_layers": "n_layer",
  }






  def __init__(							self						:		Any     ,       __lowerCamelCase						:		int=26_7735     ,       __lowerCamelCase						:		Any=[2_0000, 4_0000, 20_0000]     ,       __lowerCamelCase						:		Dict=1024     ,       __lowerCamelCase						:		Optional[int]=1024     ,       __lowerCamelCase						:		List[Any]=16     ,       __lowerCamelCase						:		Union[str, Any]=64     ,       __lowerCamelCase						:		Dict=4096     ,       __lowerCamelCase						:		int=4     ,       __lowerCamelCase						:		Dict=False     ,       __lowerCamelCase						:		Tuple=18     ,       __lowerCamelCase						:		Optional[int]=1600     ,       __lowerCamelCase						:		str=1000     ,       __lowerCamelCase						:		Any=True     ,       __lowerCamelCase						:		Dict=True     ,       __lowerCamelCase						:		List[Any]=0     ,       __lowerCamelCase						:		int=-1     ,       __lowerCamelCase						:		Tuple=True     ,       __lowerCamelCase						:		List[Any]=0.1     ,       __lowerCamelCase						:		int=0.0     ,       __lowerCamelCase						:		int=True     ,       __lowerCamelCase						:		str="normal"     ,       __lowerCamelCase						:		List[str]=0.01     ,       __lowerCamelCase						:		Any=0.01     ,       __lowerCamelCase						:		List[str]=0.02     ,       __lowerCamelCase						:		List[str]=1e-5     ,       __lowerCamelCase						:		Union[str, Any]=0     ,       **__lowerCamelCase						:		int     ,       )		->							Optional[Any]:
       SCREAMING_SNAKE_CASE__   =   vocab_size
       SCREAMING_SNAKE_CASE__   =   []
       self.cutoffs.extend(__lowerCamelCase )
       if proj_share_all_but_first:
            SCREAMING_SNAKE_CASE__   =   [False] + [True] * len(self.cutoffs )
       else:
            SCREAMING_SNAKE_CASE__   =   [False] + [False] * len(self.cutoffs )
       SCREAMING_SNAKE_CASE__   =   d_model
       SCREAMING_SNAKE_CASE__   =   d_embed
       SCREAMING_SNAKE_CASE__   =   d_head
       SCREAMING_SNAKE_CASE__   =   d_inner
       SCREAMING_SNAKE_CASE__   =   div_val
       SCREAMING_SNAKE_CASE__   =   pre_lnorm
       SCREAMING_SNAKE_CASE__   =   n_layer
       SCREAMING_SNAKE_CASE__   =   n_head
       SCREAMING_SNAKE_CASE__   =   mem_len
       SCREAMING_SNAKE_CASE__   =   same_length
       SCREAMING_SNAKE_CASE__   =   attn_type
       SCREAMING_SNAKE_CASE__   =   clamp_len
       SCREAMING_SNAKE_CASE__   =   sample_softmax
       SCREAMING_SNAKE_CASE__   =   adaptive
       SCREAMING_SNAKE_CASE__   =   dropout
       SCREAMING_SNAKE_CASE__   =   dropatt
       SCREAMING_SNAKE_CASE__   =   untie_r
       SCREAMING_SNAKE_CASE__   =   init
       SCREAMING_SNAKE_CASE__   =   init_range
       SCREAMING_SNAKE_CASE__   =   proj_init_std
       SCREAMING_SNAKE_CASE__   =   init_std
       SCREAMING_SNAKE_CASE__   =   layer_norm_epsilon
       super().__init__(eos_token_id=__lowerCamelCase     ,       **__lowerCamelCase )






  @property
  def 				lowercase_		(							self						:		str )		->							Dict:
       # Message copied from Transformer-XL documentation
       logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )
       return -1







  @max_position_embeddings.setter
  def 				lowercase_		(							self						:		List[str]     ,       __lowerCamelCase						:		Any )		->							List[Any]:
       # Message copied from Transformer-XL documentation
       raise NotImplementedError(
           f'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )


