




import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin



class       a					(       lowercase__     ,       lowercase__      ):

     '''simple docstring'''





     @register_to_config
     def __init__(     self       :      str ,   __snake_case       :      Any = 1_28 ,   __snake_case       :      Optional[int] = 2_56 ,   __snake_case       :      List[Any] = 20_00.0 ,   __snake_case       :      Optional[Any] = 7_68 ,   __snake_case       :      int = 12 ,   __snake_case       :      Any = 12 ,   __snake_case       :      Optional[Any] = 64 ,   __snake_case       :      int = 20_48 ,   __snake_case       :      Union[str, Any] = 0.1 ,   ):
         super().__init__()

         UpperCAmelCase_										=					nn.Sequential(
             nn.Linear(lowercase_ ,   d_model * 4 ,   bias=lowercase_    ) ,   nn.SiLU() ,   nn.Linear(d_model * 4 ,   d_model * 4 ,   bias=lowercase_    ) ,   nn.SiLU() ,   )

         UpperCAmelCase_										=					nn.Embedding(lowercase_ ,   lowercase_    )
         UpperCAmelCase_										=					False

         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   lowercase_ ,   bias=lowercase_    )

         UpperCAmelCase_										=					nn.Dropout(p=lowercase_    )

         UpperCAmelCase_										=					nn.ModuleList()
         for lyr_num in range(lowercase_    ):
             # FiLM conditional T5 decoder
             UpperCAmelCase_										=					DecoderLayer(d_model=lowercase_ ,   d_kv=lowercase_ ,   num_heads=lowercase_ ,   d_ff=lowercase_ ,   dropout_rate=lowercase_    )
             self.decoders.append(lowercase_    )

         UpperCAmelCase_										=					TaLayerNorm(lowercase_    )

         UpperCAmelCase_										=					nn.Dropout(p=lowercase_    )
         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   lowercase_ ,   bias=lowercase_    )
     def   lowerCamelCase_       (     self       :      str ,   __snake_case       :      List[Any] ,   __snake_case       :      Optional[Any]    ):
         UpperCAmelCase_										=					torch.mul(query_input.unsqueeze(-1    ) ,   key_input.unsqueeze(-2    )    )
         return mask.unsqueeze(-3    )




     def   lowerCamelCase_       (     self       :      str ,   __snake_case       :      List[Any] ,   __snake_case       :      List[Any] ,   __snake_case       :      Optional[int]    ):
         UpperCAmelCase_										=					decoder_input_tokens.shape
         assert decoder_noise_time.shape == (batch,)

         # decoder_noise_time is in [0, 1), so rescale to expected timing range.
         UpperCAmelCase_										=					get_timestep_embedding(
             decoder_noise_time * self.config.max_decoder_noise_time ,   embedding_dim=self.config.d_model ,   max_period=self.config.max_decoder_noise_time ,   ).to(dtype=self.dtype    )

         UpperCAmelCase_										=					self.conditioning_emb(lowercase_    ).unsqueeze(1    )

         assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

         UpperCAmelCase_										=					decoder_input_tokens.shape[1]

         # If we want to use relative positions for audio context, we can just offset
         # this sequence by the length of encodings_and_masks.
         UpperCAmelCase_										=					torch.broadcast_to(
             torch.arange(lowercase_ ,   device=decoder_input_tokens.device    ) ,   (batch, seq_length) ,   )

         UpperCAmelCase_										=					self.position_encoding(lowercase_    )

         UpperCAmelCase_										=					self.continuous_inputs_projection(lowercase_    )
         inputs += position_encodings
         UpperCAmelCase_										=					self.dropout(lowercase_    )

         # decoder: No padding present.
         UpperCAmelCase_										=					torch.ones(
             decoder_input_tokens.shape[:2] ,   device=decoder_input_tokens.device ,   dtype=inputs.dtype    )

         # Translate encoding masks to encoder-decoder masks.
         UpperCAmelCase_										=					[(x, self.encoder_decoder_mask(lowercase_ ,   lowercase_    )) for x, y in encodings_and_masks]

         # cross attend style: concat encodings
         UpperCAmelCase_										=					torch.cat([x[0] for x in encodings_and_encdec_masks] ,   dim=1    )
         UpperCAmelCase_										=					torch.cat([x[1] for x in encodings_and_encdec_masks] ,   dim=-1    )

         for lyr in self.decoders:
             UpperCAmelCase_										=					lyr(
                 lowercase_ ,   conditioning_emb=lowercase_ ,   encoder_hidden_states=lowercase_ ,   encoder_attention_mask=lowercase_ ,   )[0]

         UpperCAmelCase_										=					self.decoder_norm(lowercase_    )
         UpperCAmelCase_										=					self.post_dropout(lowercase_    )

         UpperCAmelCase_										=					self.spec_out(lowercase_    )
         return spec_out



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Any ,   __snake_case       :      Any ,   __snake_case       :      Any ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Tuple ,   __snake_case       :      Optional[Any] ,   __snake_case       :      List[Any]=1E-6    ):
         super().__init__()
         UpperCAmelCase_										=					nn.ModuleList()

         # cond self attention: layer 0
         self.layer.append(
             TaLayerSelfAttentionCond(d_model=lowercase_ ,   d_kv=lowercase_ ,   num_heads=lowercase_ ,   dropout_rate=lowercase_    )    )

         # cross attention: layer 1
         self.layer.append(
             TaLayerCrossAttention(
                 d_model=lowercase_ ,   d_kv=lowercase_ ,   num_heads=lowercase_ ,   dropout_rate=lowercase_ ,   layer_norm_epsilon=lowercase_ ,   )    )

         # Film Cond MLP + dropout: last layer
         self.layer.append(
             TaLayerFFCond(d_model=lowercase_ ,   d_ff=lowercase_ ,   dropout_rate=lowercase_ ,   layer_norm_epsilon=lowercase_    )    )




     def   lowerCamelCase_       (     self       :      Dict ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Optional[int]=None ,   __snake_case       :      List[Any]=None ,   __snake_case       :      str=None ,   __snake_case       :      Dict=None ,   __snake_case       :      Optional[int]=None ,   ):
         UpperCAmelCase_										=					self.layer[0](
             lowercase_ ,   conditioning_emb=lowercase_ ,   attention_mask=lowercase_ ,   )

         if encoder_hidden_states is not None:
             UpperCAmelCase_										=					torch.where(encoder_attention_mask > 0 ,   0 ,   -1E10    ).to(
                 encoder_hidden_states.dtype    )

             UpperCAmelCase_										=					self.layer[1](
                 lowercase_ ,   key_value_states=lowercase_ ,   attention_mask=lowercase_ ,   )

         # Apply Film Conditional Feed Forward layer
         UpperCAmelCase_										=					self.layer[-1](lowercase_ ,   lowercase_    )

         return (hidden_states,)



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Optional[int] ,   __snake_case       :      List[str] ,   __snake_case       :      Optional[Any] ,   __snake_case       :      str ,   __snake_case       :      Union[str, Any]    ):
         super().__init__()
         UpperCAmelCase_										=					TaLayerNorm(lowercase_    )
         UpperCAmelCase_										=					TaFiLMLayer(in_features=d_model * 4 ,   out_features=lowercase_    )
         UpperCAmelCase_										=					Attention(query_dim=lowercase_ ,   heads=lowercase_ ,   dim_head=lowercase_ ,   out_bias=lowercase_ ,   scale_qk=lowercase_    )
         UpperCAmelCase_										=					nn.Dropout(lowercase_    )




     def   lowerCamelCase_       (     self       :      str ,   __snake_case       :      str ,   __snake_case       :      Dict=None ,   __snake_case       :      str=None ,   ):
         UpperCAmelCase_										=					self.layer_norm(lowercase_    )

         if conditioning_emb is not None:
             UpperCAmelCase_										=					self.FiLMLayer(lowercase_ ,   lowercase_    )

         # Self-attention block
         UpperCAmelCase_										=					self.attention(lowercase_    )

         UpperCAmelCase_										=					hidden_states + self.dropout(lowercase_    )

         return hidden_states



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Dict ,   __snake_case       :      Union[str, Any] ,   __snake_case       :      List[Any] ,   __snake_case       :      int ,   __snake_case       :      int ,   __snake_case       :      Optional[Any]    ):
         super().__init__()
         UpperCAmelCase_										=					Attention(query_dim=lowercase_ ,   heads=lowercase_ ,   dim_head=lowercase_ ,   out_bias=lowercase_ ,   scale_qk=lowercase_    )
         UpperCAmelCase_										=					TaLayerNorm(lowercase_ ,   eps=lowercase_    )
         UpperCAmelCase_										=					nn.Dropout(lowercase_    )




     def   lowerCamelCase_       (     self       :      Any ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Union[str, Any]=None ,   __snake_case       :      List[str]=None ,   ):
         UpperCAmelCase_										=					self.layer_norm(lowercase_    )
         UpperCAmelCase_										=					self.attention(
             lowercase_ ,   encoder_hidden_states=lowercase_ ,   attention_mask=attention_mask.squeeze(1    ) ,   )
         UpperCAmelCase_										=					hidden_states + self.dropout(lowercase_    )
         return layer_output



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Optional[int] ,   __snake_case       :      str ,   __snake_case       :      Union[str, Any] ,   __snake_case       :      Optional[int] ,   __snake_case       :      List[str]    ):
         super().__init__()
         UpperCAmelCase_										=					TaDenseGatedActDense(d_model=lowercase_ ,   d_ff=lowercase_ ,   dropout_rate=lowercase_    )
         UpperCAmelCase_										=					TaFiLMLayer(in_features=d_model * 4 ,   out_features=lowercase_    )
         UpperCAmelCase_										=					TaLayerNorm(lowercase_ ,   eps=lowercase_    )
         UpperCAmelCase_										=					nn.Dropout(lowercase_    )




     def   lowerCamelCase_       (     self       :      Union[str, Any] ,   __snake_case       :      List[Any] ,   __snake_case       :      Any=None    ):
         UpperCAmelCase_										=					self.layer_norm(lowercase_    )
         if conditioning_emb is not None:
             UpperCAmelCase_										=					self.film(lowercase_ ,   lowercase_    )

         UpperCAmelCase_										=					self.DenseReluDense(lowercase_    )
         UpperCAmelCase_										=					hidden_states + self.dropout(lowercase_    )
         return hidden_states



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      str ,   __snake_case       :      Tuple ,   __snake_case       :      Optional[int] ,   __snake_case       :      List[Any]    ):
         super().__init__()
         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   lowercase_ ,   bias=lowercase_    )
         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   lowercase_ ,   bias=lowercase_    )
         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   lowercase_ ,   bias=lowercase_    )
         UpperCAmelCase_										=					nn.Dropout(lowercase_    )
         UpperCAmelCase_										=					NewGELUActivation()




     def   lowerCamelCase_       (     self       :      Tuple ,   __snake_case       :      List[str]    ):
         UpperCAmelCase_										=					self.act(self.wi_a(lowercase_    )    )
         UpperCAmelCase_										=					self.wi_a(lowercase_    )
         UpperCAmelCase_										=					hidden_gelu * hidden_linear
         UpperCAmelCase_										=					self.dropout(lowercase_    )

         UpperCAmelCase_										=					self.wo(lowercase_    )
         return hidden_states



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Optional[Any] ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Dict=1E-6    ):
         super().__init__()
         UpperCAmelCase_										=					nn.Parameter(torch.ones(lowercase_    )    )
         UpperCAmelCase_										=					eps




     def   lowerCamelCase_       (     self       :      Optional[int] ,   __snake_case       :      Dict    ):
         UpperCAmelCase_										=					hidden_states.to(torch.floataa    ).pow(2    ).mean(-1 ,   keepdim=lowercase_    )
         UpperCAmelCase_										=					hidden_states * torch.rsqrt(variance + self.variance_epsilon    )

         # convert into half-precision if necessary
         if self.weight.dtype in [torch.floataa, torch.bfloataa]:
             UpperCAmelCase_										=					hidden_states.to(self.weight.dtype    )

         return self.weight * hidden_states



class       a					(       nn.Module      ):

     '''simple docstring'''





     def   lowerCamelCase_       (     self       :      Optional[int] ,   __snake_case       :      str    ):
         return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi    ) * (input + 0.044_715 * torch.pow(lowercase_ ,   3.0    ))    ))



class       a					(       nn.Module      ):

     '''simple docstring'''





     def __init__(     self       :      Dict ,   __snake_case       :      Any ,   __snake_case       :      str    ):
         super().__init__()
         UpperCAmelCase_										=					nn.Linear(lowercase_ ,   out_features * 2 ,   bias=lowercase_    )




     def   lowerCamelCase_       (     self       :      Tuple ,   __snake_case       :      Optional[int] ,   __snake_case       :      Dict    ):
         UpperCAmelCase_										=					self.scale_bias(lowercase_    )
         UpperCAmelCase_										=					torch.chunk(lowercase_ ,   2 ,   -1    )
         UpperCAmelCase_										=					x * (1 + scale) + shift
         return x











def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Dict=2_8123				)  ->							str:
    UpperCAmelCase_										=					[1] * (limit + 1)

    for i in range(2	,					int(limit**0.5				) + 1				):
        sum_divs[i * i] += i
        for k in range(i + 1	,					limit // i + 1				):
            sum_divs[k * i] += k + i

    UpperCAmelCase_										=					set()
    UpperCAmelCase_										=					0

    for n in range(1	,					limit + 1				):
        if sum_divs[n] > n:
            abundants.add(__UpperCamelCase				)

        if not any((n - a in abundants) for a in abundants				):
            res += n

    return res


if __name__ == "__main__":
 print(solution())





