


'''simple docstring'''






from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings






@maybe_allow_in_graph
class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case  ,							snake_case  ,							snake_case=0.0  ,							snake_case = None  ,							snake_case = "geglu"  ,							snake_case = None  ,							snake_case = False  ,							snake_case = False  ,							snake_case = False  ,							snake_case = False  ,							snake_case = True  ,							snake_case = "layer_norm"  ,							snake_case = False  ,							)						->  Tuple:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       List[str]							   =only_cross_attention

          _UpperCAmelCase       :       Tuple							   =(num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero'
          _UpperCAmelCase       :       Dict							   =(num_embeds_ada_norm is not None) and norm_type == 'ada_norm'

          if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
             raise ValueError(
                 f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
                 f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.")

          # Define 3 blocks. Each block has its own normalization layer.
          # 1. Self-Attn
          if self.use_ada_layer_norm:
             _UpperCAmelCase       :       Optional[Any]							   =AdaLayerNorm(snake_case  ,							snake_case)
          elif self.use_ada_layer_norm_zero:
             _UpperCAmelCase       :       List[str]							   =AdaLayerNormZero(snake_case  ,							snake_case)
          else:
             _UpperCAmelCase       :       int							   =nn.LayerNorm(snake_case  ,							elementwise_affine=snake_case)
          _UpperCAmelCase       :       Dict							   =Attention(
              query_dim=snake_case  ,							heads=snake_case  ,							dim_head=snake_case  ,							dropout=snake_case  ,							bias=snake_case  ,							cross_attention_dim=cross_attention_dim if only_cross_attention else None  ,							upcast_attention=snake_case  ,							)

          # 2. Cross-Attn
          if cross_attention_dim is not None or double_self_attention:
             # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
             # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
             # the second cross attention block.
             _UpperCAmelCase       :       Tuple							   =(
                 AdaLayerNorm(snake_case  ,							snake_case)
                 if self.use_ada_layer_norm
                 else nn.LayerNorm(snake_case  ,							elementwise_affine=snake_case)
             )
             _UpperCAmelCase       :       str							   =Attention(
                 query_dim=snake_case  ,							cross_attention_dim=cross_attention_dim if not double_self_attention else None  ,							heads=snake_case  ,							dim_head=snake_case  ,							dropout=snake_case  ,							bias=snake_case  ,							upcast_attention=snake_case  ,							)  # is self-attn if encoder_hidden_states is none
          else:
             _UpperCAmelCase       :       str							   =None
             _UpperCAmelCase       :       List[str]							   =None

          # 3. Feed-forward
          _UpperCAmelCase       :       str							   =nn.LayerNorm(snake_case  ,							elementwise_affine=snake_case)
          _UpperCAmelCase       :       List[Any]							   =FeedForward(snake_case  ,							dropout=snake_case  ,							activation_fn=snake_case  ,							final_dropout=snake_case)

          # let chunk size default to None
          _UpperCAmelCase       :       int							   =None
          _UpperCAmelCase       :       Tuple							   =0






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case)						->  Tuple:




          '''simple docstring'''
          # Sets chunk feed-forward
          _UpperCAmelCase       :       Tuple							   =chunk_size
          _UpperCAmelCase       :       Any							   =dim






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case = None  ,							snake_case = None  ,							snake_case = None  ,							snake_case = None  ,							snake_case = None  ,							snake_case = None  ,							)						->  Dict:




          '''simple docstring'''
          # Notice that normalization is always applied before the real computation in the following blocks.
          # 1. Self-Attention
          if self.use_ada_layer_norm:
             _UpperCAmelCase       :       Optional[int]							   =self.norma(snake_case  ,							snake_case)
          elif self.use_ada_layer_norm_zero:
             _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       Any							   =self.norma(
                 snake_case  ,							snake_case  ,							snake_case  ,							hidden_dtype=hidden_states.dtype)
          else:
             _UpperCAmelCase       :       Dict							   =self.norma(snake_case)

          _UpperCAmelCase       :       Union[str, Any]							   =cross_attention_kwargs if cross_attention_kwargs is not None else {}

          _UpperCAmelCase       :       Tuple							   =self.attna(
              snake_case  ,							encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None  ,							attention_mask=snake_case  ,							**snake_case  ,							)
          if self.use_ada_layer_norm_zero:
             _UpperCAmelCase       :       Any							   =gate_msa.unsqueeze(1) * attn_output
          _UpperCAmelCase       :       Tuple							   =attn_output + hidden_states

          # 2. Cross-Attention
          if self.attna is not None:
             _UpperCAmelCase       :       Optional[int]							   =(
                 self.norma(snake_case  ,							snake_case) if self.use_ada_layer_norm else self.norma(snake_case)
             )

             _UpperCAmelCase       :       Dict							   =self.attna(
                 snake_case  ,							encoder_hidden_states=snake_case  ,							attention_mask=snake_case  ,							**snake_case  ,							)
             _UpperCAmelCase       :       Union[str, Any]							   =attn_output + hidden_states

          # 3. Feed-forward
          _UpperCAmelCase       :       Optional[Any]							   =self.norma(snake_case)

          if self.use_ada_layer_norm_zero:
             _UpperCAmelCase       :       Union[str, Any]							   =norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

          if self._chunk_size is not None:
             # "feed_forward_chunk_size" can be used to save memory
             if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                raise ValueError(
                    f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.")

             _UpperCAmelCase       :       Any							   =norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
             _UpperCAmelCase       :       List[str]							   =torch.cat(
                 [self.ff(snake_case) for hid_slice in norm_hidden_states.chunk(snake_case  ,							dim=self._chunk_dim)]  ,							dim=self._chunk_dim  ,							)
          else:
             _UpperCAmelCase       :       List[Any]							   =self.ff(snake_case)

          if self.use_ada_layer_norm_zero:
             _UpperCAmelCase       :       List[str]							   =gate_mlp.unsqueeze(1) * ff_output

          _UpperCAmelCase       :       Any							   =ff_output + hidden_states

          return hidden_states






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case = None  ,							snake_case = 4  ,							snake_case = 0.0  ,							snake_case = "geglu"  ,							snake_case = False  ,							)						->  str:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       List[Any]							   =int(dim * mult)
          _UpperCAmelCase       :       Tuple							   =dim_out if dim_out is not None else dim

          if activation_fn == "gelu":
             _UpperCAmelCase       :       int							   =GELU(snake_case  ,							snake_case)
          if activation_fn == "gelu-approximate":
             _UpperCAmelCase       :       List[str]							   =GELU(snake_case  ,							snake_case  ,							approximate='tanh')
          elif activation_fn == "geglu":
             _UpperCAmelCase       :       Optional[Any]							   =GEGLU(snake_case  ,							snake_case)
          elif activation_fn == "geglu-approximate":
             _UpperCAmelCase       :       Union[str, Any]							   =ApproximateGELU(snake_case  ,							snake_case)

          _UpperCAmelCase       :       Dict							   =nn.ModuleList([])
          # project in
          self.net.append(snake_case)
          # project dropout
          self.net.append(nn.Dropout(snake_case))
          # project out
          self.net.append(nn.Linear(snake_case  ,							snake_case))
          # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
          if final_dropout:
             self.net.append(nn.Dropout(snake_case))






       def       lowerCAmelCase		(      self  ,							snake_case)						->  Optional[int]:




          '''simple docstring'''
          for module in self.net:
             _UpperCAmelCase       :       Optional[Any]							   =module(snake_case)
          return hidden_states






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case  ,							snake_case = "none")						->  Union[str, Any]:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       Union[str, Any]							   =nn.Linear(snake_case  ,							snake_case)
          _UpperCAmelCase       :       Dict							   =approximate






       def       lowerCAmelCase		(      self  ,							snake_case)						->  Optional[Any]:




          '''simple docstring'''
          if gate.device.type != "mps":
             return F.gelu(snake_case  ,							approximate=self.approximate)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa)  ,							approximate=self.approximate).to(dtype=gate.dtype)






       def       lowerCAmelCase		(      self  ,							snake_case)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase       :       List[Any]							   =self.proj(snake_case)
          _UpperCAmelCase       :       Union[str, Any]							   =self.gelu(snake_case)
          return hidden_states






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case)						->  Optional[Any]:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       List[str]							   =nn.Linear(snake_case  ,							dim_out * 2)






       def       lowerCAmelCase		(      self  ,							snake_case)						->  int:




          '''simple docstring'''
          if gate.device.type != "mps":
             return F.gelu(snake_case)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa)).to(dtype=gate.dtype)






       def       lowerCAmelCase		(      self  ,							snake_case)						->  int:




          '''simple docstring'''
          _UpperCAmelCase   ,	_UpperCAmelCase       :       Dict							   =self.proj(snake_case).chunk(2  ,							dim=-1)
          return hidden_states * self.gelu(snake_case)






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case)						->  Any:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       List[Any]							   =nn.Linear(snake_case  ,							snake_case)






       def       lowerCAmelCase		(      self  ,							snake_case)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =self.proj(snake_case)
          return x * torch.sigmoid(1.7_02 * x)






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case)						->  Any:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       Optional[int]							   =nn.Embedding(snake_case  ,							snake_case)
          _UpperCAmelCase       :       Optional[Any]							   =nn.SiLU()
          _UpperCAmelCase       :       Dict							   =nn.Linear(snake_case  ,							embedding_dim * 2)
          _UpperCAmelCase       :       Any							   =nn.LayerNorm(snake_case  ,							elementwise_affine=snake_case)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[int]							   =self.linear(self.silu(self.emb(snake_case)))
          _UpperCAmelCase   ,	_UpperCAmelCase       :       List[Any]							   =torch.chunk(snake_case  ,							2)
          _UpperCAmelCase       :       str							   =self.norm(snake_case) * (1 + scale) + shift
          return x






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case)						->  Any:




          '''simple docstring'''
          super().__init__()

          _UpperCAmelCase       :       List[Any]							   =CombinedTimestepLabelEmbeddings(snake_case  ,							snake_case)

          _UpperCAmelCase       :       List[str]							   =nn.SiLU()
          _UpperCAmelCase       :       Optional[Any]							   =nn.Linear(snake_case  ,							6 * embedding_dim  ,							bias=snake_case)
          _UpperCAmelCase       :       Optional[Any]							   =nn.LayerNorm(snake_case  ,							elementwise_affine=snake_case  ,							eps=1E-6)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case  ,							snake_case  ,							snake_case=None)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =self.linear(self.silu(self.emb(snake_case  ,							snake_case  ,							hidden_dtype=snake_case)))
          _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       Tuple							   =emb.chunk(6  ,							dim=1)
          _UpperCAmelCase       :       Optional[int]							   =self.norm(snake_case) * (1 + scale_msa[:, None]) + shift_msa[:, None]
          return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class  __magic_name__							(		nn.Module    ):






       def __init__(      self  ,							snake_case  ,							snake_case  ,							snake_case  ,							snake_case = None  ,							snake_case = 1E-5)						->  str:




          '''simple docstring'''
          super().__init__()
          _UpperCAmelCase       :       str							   =num_groups
          _UpperCAmelCase       :       int							   =eps

          if act_fn is None:
             _UpperCAmelCase       :       Tuple							   =None
          else:
             _UpperCAmelCase       :       Optional[int]							   =get_activation(snake_case)

          _UpperCAmelCase       :       List[str]							   =nn.Linear(snake_case  ,							out_dim * 2)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case)						->  str:




          '''simple docstring'''
          if self.act:
             _UpperCAmelCase       :       Any							   =self.act(snake_case)
          _UpperCAmelCase       :       Optional[int]							   =self.linear(snake_case)
          _UpperCAmelCase       :       str							   =emb[:, :, None, None]
          _UpperCAmelCase   ,	_UpperCAmelCase       :       str							   =emb.chunk(2  ,							dim=1)

          _UpperCAmelCase       :       Dict							   =F.group_norm(snake_case  ,							self.num_groups  ,							eps=self.eps)
          _UpperCAmelCase       :       Tuple							   =x * (1 + scale) + shift
          return x





'''simple docstring'''






from __future__ import annotations

import bisect





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int = 0	,   __lowerCamelCase	:							int = -1							):







   '''simple docstring'''




   if hi < 0:
      _UpperCAmelCase       :       int							   =len(__lowerCamelCase							)

   while lo < hi:
      _UpperCAmelCase       :       Dict							   =lo + (hi - lo) // 2
      if sorted_collection[mid] < item:
         _UpperCAmelCase       :       int							   =mid + 1
      else:
         _UpperCAmelCase       :       Union[str, Any]							   =mid

   return lo





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int = 0	,   __lowerCamelCase	:							int = -1							):







   '''simple docstring'''




   if hi < 0:
      _UpperCAmelCase       :       str							   =len(__lowerCamelCase							)

   while lo < hi:
      _UpperCAmelCase       :       List[Any]							   =lo + (hi - lo) // 2
      if sorted_collection[mid] <= item:
         _UpperCAmelCase       :       Dict							   =mid + 1
      else:
         _UpperCAmelCase       :       Tuple							   =mid

   return lo





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int = 0	,   __lowerCamelCase	:							int = -1							):







   '''simple docstring'''




   sorted_collection.insert(bisect_left(__lowerCamelCase	,   __lowerCamelCase	,   __lowerCamelCase	,   __lowerCamelCase							)	,   __lowerCamelCase							)





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int = 0	,   __lowerCamelCase	:							int = -1							):







   '''simple docstring'''




   sorted_collection.insert(bisect_right(__lowerCamelCase	,   __lowerCamelCase	,   __lowerCamelCase	,   __lowerCamelCase							)	,   __lowerCamelCase							)





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int							):







   '''simple docstring'''




   _UpperCAmelCase       :       Optional[int]							   =0
   _UpperCAmelCase       :       Any							   =len(__lowerCamelCase							) - 1

   while left <= right:
      _UpperCAmelCase       :       int							   =left + (right - left) // 2
      _UpperCAmelCase       :       Union[str, Any]							   =sorted_collection[midpoint]
      if current_item == item:
         return midpoint
      elif item < current_item:
         _UpperCAmelCase       :       Optional[Any]							   =midpoint - 1
      else:
         _UpperCAmelCase       :       int							   =midpoint + 1
   return None





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int							):







   '''simple docstring'''




   _UpperCAmelCase       :       Any							   =bisect.bisect_left(__lowerCamelCase	,   __lowerCamelCase							)
   if index != len(__lowerCamelCase							) and sorted_collection[index] == item:
      return index
   return None





def lowerCamelCase__      (						__lowerCamelCase	:							list[int]	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int							):







   '''simple docstring'''




   if right < left:
      return None

   _UpperCAmelCase       :       str							   =left + (right - left) // 2

   if sorted_collection[midpoint] == item:
      return midpoint
   elif sorted_collection[midpoint] > item:
      return binary_search_by_recursion(__lowerCamelCase	,   __lowerCamelCase	,   __lowerCamelCase	,   midpoint - 1							)
   else:
      return binary_search_by_recursion(__lowerCamelCase	,   __lowerCamelCase	,   midpoint + 1	,   __lowerCamelCase							)


if __name__ == "__main__":
 lowercase							=input('Enter numbers separated by comma:\n').strip()
 lowercase							=sorted(int(item) for item in user_input.split(','))
 lowercase							=int(input('Enter a single number to be found in the list:\n'))
 lowercase							=binary_search(collection, target)
 if result is None:
  print(F"""{target} was not found in {collection}.""")
 else:
  print(F"""{target} was found at position {result} in {collection}.""")

