




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__			:       Tuple         =							logging.get_logger(__name__)

lowerCamelCase__			:       Optional[Any]         =							{
    """tiiuae/falcon-40b""": """https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json""",
    """tiiuae/falcon-7b""": """https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json""",
}





class 	lowerCamelCase_							(				__A ):


 '''simple docstring'''







 lowercase_       =   'falcon'
 lowercase_       =   ['past_key_values']



 def __init__(							self  :		Tuple    ,    _lowerCAmelCase  :		Optional[Any]=65_024    ,    _lowerCAmelCase  :		Tuple=4_544    ,    _lowerCAmelCase  :		Union[str, Any]=32    ,    _lowerCAmelCase  :		int=71    ,    _lowerCAmelCase  :		str=1E-5    ,    _lowerCAmelCase  :		Union[str, Any]=0.02    ,    _lowerCAmelCase  :		int=True    ,    _lowerCAmelCase  :		List[Any]=0.0    ,    _lowerCAmelCase  :		List[Any]=0.0    ,    _lowerCAmelCase  :		Optional[Any]=None    ,    _lowerCAmelCase  :		Optional[Any]=False    ,    _lowerCAmelCase  :		Union[str, Any]=False    ,    _lowerCAmelCase  :		Union[str, Any]=True    ,    _lowerCAmelCase  :		Optional[Any]=True    ,    _lowerCAmelCase  :		Union[str, Any]=False    ,    _lowerCAmelCase  :		Optional[Any]=11    ,    _lowerCAmelCase  :		Union[str, Any]=11    ,    **_lowerCAmelCase  :		List[Any]    ,    ):
       SCREAMING_SNAKE_CASE_     						=      vocab_size
       # Backward compatibility with n_embed kwarg
       SCREAMING_SNAKE_CASE_     						=      kwargs.pop('n_embed'    ,    __lowercase						)
       SCREAMING_SNAKE_CASE_     						=      hidden_size if n_embed is None else n_embed
       SCREAMING_SNAKE_CASE_     						=      num_hidden_layers
       SCREAMING_SNAKE_CASE_     						=      num_attention_heads
       SCREAMING_SNAKE_CASE_     						=      layer_norm_epsilon
       SCREAMING_SNAKE_CASE_     						=      initializer_range
       SCREAMING_SNAKE_CASE_     						=      use_cache
       SCREAMING_SNAKE_CASE_     						=      hidden_dropout
       SCREAMING_SNAKE_CASE_     						=      attention_dropout

       SCREAMING_SNAKE_CASE_     						=      bos_token_id
       SCREAMING_SNAKE_CASE_     						=      eos_token_id
       SCREAMING_SNAKE_CASE_     						=      num_attention_heads if num_kv_heads is None else num_kv_heads
       SCREAMING_SNAKE_CASE_     						=      alibi
       SCREAMING_SNAKE_CASE_     						=      new_decoder_architecture
       SCREAMING_SNAKE_CASE_     						=      multi_query  # Ignored when new_decoder_architecture is True
       SCREAMING_SNAKE_CASE_     						=      parallel_attn
       SCREAMING_SNAKE_CASE_     						=      bias

       super().__init__(bos_token_id=__lowercase    ,    eos_token_id=__lowercase    ,    **__lowercase						)



 @property
 def 				lowerCAmelCase_  (							self  :		int						):
       return self.hidden_size // self.num_attention_heads




 @property
 def 				lowerCAmelCase_  (							self  :		Tuple						):
       return not self.alibi





import torch
from torch import nn
from transformers import CLIPPreTrainedModel, CLIPVisionModel

from ...models.attention import BasicTransformerBlock
from ...utils import logging


lowerCamelCase__			:       List[Any]         =							logging.get_logger(__name__)  # pylint: disable=invalid-name





class 	lowerCamelCase_							(				_SCREAMING_SNAKE_CASE ):


 '''simple docstring'''







 def __init__(							self  :		List[Any]    ,    _lowerCAmelCase  :		Dict    ,    _lowerCAmelCase  :		Tuple=768						):
       super().__init__(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      proj_size

       SCREAMING_SNAKE_CASE_     						=      CLIPVisionModel(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      PaintByExampleMapper(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      nn.LayerNorm(config.hidden_size						)
       SCREAMING_SNAKE_CASE_     						=      nn.Linear(config.hidden_size    ,    self.proj_size						)

       # uncondition for scaling
       SCREAMING_SNAKE_CASE_     						=      nn.Parameter(torch.randn((1, 1, self.proj_size)						)						)




 def 				lowerCAmelCase_  (							self  :		Any    ,    _lowerCAmelCase  :		int    ,    _lowerCAmelCase  :		int=False						):
       SCREAMING_SNAKE_CASE_     						=      self.model(pixel_values=_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      clip_output.pooler_output
       SCREAMING_SNAKE_CASE_     						=      self.mapper(latent_states[:, None]						)
       SCREAMING_SNAKE_CASE_     						=      self.final_layer_norm(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      self.proj_out(_lowerCAmelCase						)
       if return_uncond_vector:
             return latent_states, self.uncond_vector

       return latent_states





class 	lowerCamelCase_							(				nn.Module ):


 '''simple docstring'''







 def __init__(							self  :		int    ,    _lowerCAmelCase  :		Optional[Any]						):
       super().__init__()
       SCREAMING_SNAKE_CASE_     						=      (config.num_hidden_layers + 1) // 5
       SCREAMING_SNAKE_CASE_     						=      config.hidden_size
       SCREAMING_SNAKE_CASE_     						=      1
       SCREAMING_SNAKE_CASE_     						=      nn.ModuleList(
           [
               BasicTransformerBlock(_lowerCAmelCase    ,    _lowerCAmelCase    ,    _lowerCAmelCase    ,    activation_fn='gelu'    ,    attention_bias=_lowerCAmelCase						)
               for _ in range(_lowerCAmelCase						)
           ]						)




 def 				lowerCAmelCase_  (							self  :		Union[str, Any]    ,    _lowerCAmelCase  :		Optional[Any]						):
       for block in self.blocks:
             SCREAMING_SNAKE_CASE_     						=      block(_lowerCAmelCase						)

       return hidden_states