




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A    ,				A    ,				A=0.0    ,				A = None    ,				A = "geglu"    ,				A = None    ,				A = False    ,				A = False    ,				A = False    ,				A = False    ,				A = True    ,				A = "layer_norm"    ,				A = False    ,				)		->    str:
         super().__init__()
         snake_case				:    Tuple           =	only_cross_attention

         snake_case				:    Optional[int]           =	(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
         snake_case				:    List[str]           =	(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
            raise ValueError(
                f"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
                f""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""     )

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
            snake_case				:    str           =	AdaLayerNorm(A    ,				A     )
         elif self.use_ada_layer_norm_zero:
            snake_case				:    str           =	AdaLayerNormZero(A    ,				A     )
         else:
            snake_case				:    Tuple           =	nn.LayerNorm(A    ,				elementwise_affine=A     )
         snake_case				:    int           =	Attention(
             query_dim=A    ,				heads=A    ,				dim_head=A    ,				dropout=A    ,				bias=A    ,				cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,				upcast_attention=A    ,				)

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
            # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
            # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
            # the second cross attention block.
            snake_case				:    int           =	(
                AdaLayerNorm(A    ,				A     )
                if self.use_ada_layer_norm
                else nn.LayerNorm(A    ,				elementwise_affine=A     )
            )
            snake_case				:    List[Any]           =	Attention(
                query_dim=A    ,				cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,				heads=A    ,				dim_head=A    ,				dropout=A    ,				bias=A    ,				upcast_attention=A    ,				)  # is self-attn if encoder_hidden_states is none
         else:
            snake_case				:    Any           =	None
            snake_case				:    Any           =	None

         # 3. Feed-forward
         snake_case				:    Optional[Any]           =	nn.LayerNorm(A    ,				elementwise_affine=A     )
         snake_case				:    List[str]           =	FeedForward(A    ,				dropout=A    ,				activation_fn=A    ,				final_dropout=A     )

         # let chunk size default to None
         snake_case				:    List[Any]           =	None
         snake_case				:    List[str]           =	0

      def      UpperCAmelCase				(				self    ,				A    ,				A     )		->    int:
         # Sets chunk feed-forward
         snake_case				:    Any           =	chunk_size
         snake_case				:    Optional[int]           =	dim

      def      UpperCAmelCase				(				self    ,				A    ,				A = None    ,				A = None    ,				A = None    ,				A = None    ,				A = None    ,				A = None    ,				)		->    Optional[int]:
         # Notice that normalization is always applied before the real computation in the following blocks.
         # 1. Self-Attention
         if self.use_ada_layer_norm:
            snake_case				:    str           =	self.norma(A    ,				A     )
         elif self.use_ada_layer_norm_zero:
            snake_case     , snake_case     , snake_case     , snake_case     , snake_case				:    Union[str, Any]           =	self.norma(
                A    ,				A    ,				A    ,				hidden_dtype=hidden_states.dtype     )
         else:
            snake_case				:    Union[str, Any]           =	self.norma(A     )

         snake_case				:    List[str]           =	cross_attention_kwargs if cross_attention_kwargs is not None else {}

         snake_case				:    List[Any]           =	self.attna(
             A    ,				encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,				attention_mask=A    ,				**A    ,				)
         if self.use_ada_layer_norm_zero:
            snake_case				:    Any           =	gate_msa.unsqueeze(1     ) * attn_output
         snake_case				:    Optional[Any]           =	attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
            snake_case				:    str           =	(
                self.norma(A    ,				A     ) if self.use_ada_layer_norm else self.norma(A     )
            )

            snake_case				:    List[str]           =	self.attna(
                A    ,				encoder_hidden_states=A    ,				attention_mask=A    ,				**A    ,				)
            snake_case				:    int           =	attn_output + hidden_states

         # 3. Feed-forward
         snake_case				:    int           =	self.norma(A     )

         if self.use_ada_layer_norm_zero:
            snake_case				:    Dict           =	norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
            # "feed_forward_chunk_size" can be used to save memory
            if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
               raise ValueError(
                   f"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""     )

            snake_case				:    Dict           =	norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
            snake_case				:    Optional[Any]           =	torch.cat(
                [self.ff(A     ) for hid_slice in norm_hidden_states.chunk(A    ,				dim=self._chunk_dim     )]    ,				dim=self._chunk_dim    ,				)
         else:
            snake_case				:    Tuple           =	self.ff(A     )

         if self.use_ada_layer_norm_zero:
            snake_case				:    Union[str, Any]           =	gate_mlp.unsqueeze(1     ) * ff_output

         snake_case				:    int           =	ff_output + hidden_states

         return hidden_states




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A = None    ,				A = 4    ,				A = 0.0    ,				A = "geglu"    ,				A = False    ,				)		->    str:
         super().__init__()
         snake_case				:    Union[str, Any]           =	int(dim * mult     )
         snake_case				:    List[Any]           =	dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
            snake_case				:    Optional[int]           =	GELU(A    ,				A     )
         if activation_fn == "gelu-approximate":
            snake_case				:    str           =	GELU(A    ,				A    ,				approximate="""tanh"""     )
         elif activation_fn == "geglu":
            snake_case				:    Union[str, Any]           =	GEGLU(A    ,				A     )
         elif activation_fn == "geglu-approximate":
            snake_case				:    Any           =	ApproximateGELU(A    ,				A     )

         snake_case				:    List[Any]           =	nn.ModuleList([]     )
         # project in
         self.net.append(A     )
         # project dropout
         self.net.append(nn.Dropout(A     )     )
         # project out
         self.net.append(nn.Linear(A    ,				A     )     )
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
            self.net.append(nn.Dropout(A     )     )

      def      UpperCAmelCase				(				self    ,				A     )		->    Any:
         for module in self.net:
            snake_case				:    Dict           =	module(A     )
         return hidden_states




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A    ,				A = "none"     )		->    str:
         super().__init__()
         snake_case				:    int           =	nn.Linear(A    ,				A     )
         snake_case				:    Dict           =	approximate

      def      UpperCAmelCase				(				self    ,				A     )		->    int:
         if gate.device.type != "mps":
            return F.gelu(A    ,				approximate=self.approximate     )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa     )    ,				approximate=self.approximate     ).to(dtype=gate.dtype     )

      def      UpperCAmelCase				(				self    ,				A     )		->    Union[str, Any]:
         snake_case				:    List[str]           =	self.proj(A     )
         snake_case				:    Tuple           =	self.gelu(A     )
         return hidden_states




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A     )		->    List[str]:
         super().__init__()
         snake_case				:    List[str]           =	nn.Linear(A    ,				dim_out * 2     )

      def      UpperCAmelCase				(				self    ,				A     )		->    Union[str, Any]:
         if gate.device.type != "mps":
            return F.gelu(A     )
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa     )     ).to(dtype=gate.dtype     )

      def      UpperCAmelCase				(				self    ,				A     )		->    Union[str, Any]:
         snake_case     , snake_case				:    Tuple           =	self.proj(A     ).chunk(2    ,				dim=-1     )
         return hidden_states * self.gelu(A     )




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A     )		->    int:
         super().__init__()
         snake_case				:    Any           =	nn.Linear(A    ,				A     )

      def      UpperCAmelCase				(				self    ,				A     )		->    List[Any]:
         snake_case				:    Any           =	self.proj(A     )
         return x * torch.sigmoid(1.7_02 * x     )




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A     )		->    List[Any]:
         super().__init__()
         snake_case				:    List[Any]           =	nn.Embedding(A    ,				A     )
         snake_case				:    List[str]           =	nn.SiLU()
         snake_case				:    Union[str, Any]           =	nn.Linear(A    ,				embedding_dim * 2     )
         snake_case				:    List[str]           =	nn.LayerNorm(A    ,				elementwise_affine=A     )

      def      UpperCAmelCase				(				self    ,				A    ,				A     )		->    Optional[int]:
         snake_case				:    Any           =	self.linear(self.silu(self.emb(A     )     )     )
         snake_case     , snake_case				:    Union[str, Any]           =	torch.chunk(A    ,				2     )
         snake_case				:    Optional[int]           =	self.norm(A     ) * (1 + scale) + shift
         return x




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A     )		->    Any:
         super().__init__()

         snake_case				:    Any           =	CombinedTimestepLabelEmbeddings(A    ,				A     )

         snake_case				:    int           =	nn.SiLU()
         snake_case				:    str           =	nn.Linear(A    ,				6 * embedding_dim    ,				bias=A     )
         snake_case				:    str           =	nn.LayerNorm(A    ,				elementwise_affine=A    ,				eps=1e-6     )

      def      UpperCAmelCase				(				self    ,				A    ,				A    ,				A    ,				A=None     )		->    Tuple:
         snake_case				:    Optional[Any]           =	self.linear(self.silu(self.emb(A    ,				A    ,				hidden_dtype=A     )     )     )
         snake_case     , snake_case     , snake_case     , snake_case     , snake_case     , snake_case				:    str           =	emb.chunk(6    ,				dim=1     )
         snake_case				:    List[str]           =	self.norm(A     ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class 						__lowercase		(nn.Module							):




      """simple docstring"""





      def __init__(				self    ,				A    ,				A    ,				A    ,				A = None    ,				A = 1e-5     )		->    str:
         super().__init__()
         snake_case				:    Optional[Any]           =	num_groups
         snake_case				:    List[str]           =	eps

         if act_fn is None:
            snake_case				:    List[str]           =	None
         else:
            snake_case				:    Optional[Any]           =	get_activation(A     )

         snake_case				:    Tuple           =	nn.Linear(A    ,				out_dim * 2     )

      def      UpperCAmelCase				(				self    ,				A    ,				A     )		->    Optional[Any]:
         if self.act:
            snake_case				:    Any           =	self.act(A     )
         snake_case				:    List[str]           =	self.linear(A     )
         snake_case				:    Optional[Any]           =	emb[:, :, None, None]
         snake_case     , snake_case				:    int           =	emb.chunk(2    ,				dim=1     )

         snake_case				:    int           =	F.group_norm(A    ,				self.num_groups    ,				eps=self.eps     )
         snake_case				:    str           =	x * (1 + scale) + shift
         return x











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase	:	List[str]				=   {
    'configuration_swinv2': ['SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Swinv2Config'],
}


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase	:	Optional[Any]				=   [
           'SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST',
           'Swinv2ForImageClassification',
           'Swinv2ForMaskedImageModeling',
           'Swinv2Model',
           'Swinv2PreTrainedModel',
       ]


if TYPE_CHECKING:
       from .configuration_swinva import SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinvaConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_swinva import (
                  SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST,
                  SwinvaForImageClassification,
                  SwinvaForMaskedImageModeling,
                  SwinvaModel,
                  SwinvaPreTrainedModel,
              )


else:
       import sys

       lowerCamelCase	:	Optional[int]				=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





