




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[Any]							) ->			Union[str, Any]:
      return (torch.arange(state.num_processes							) + 1.0 + (state.num_processes * state.process_index)).to(state.device							)

def  UpperCAmelCase_							(				__UpperCAmelCase						:     str							) ->			List[str]:
      SCREAMING_SNAKE_CASE_     						=      create_tensor(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_     						=      gather(__UpperCAmelCase							)
      assert gathered_tensor.tolist() == list(range(1      ,      state.num_processes**2 + 1							)							)

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Tuple							) ->			int:
      SCREAMING_SNAKE_CASE_     						=      [state.process_index]
      SCREAMING_SNAKE_CASE_     						=      gather_object(__UpperCAmelCase							)
      assert len(__UpperCAmelCase							) == state.num_processes, f"{gathered_obj}, {len(__UpperCAmelCase							)} != {state.num_processes}"
      assert gathered_obj == list(range(state.num_processes							)							), f"{gathered_obj} != {list(range(state.num_processes							)							)}"

def  UpperCAmelCase_							(				__UpperCAmelCase						:     List[str]							) ->			int:
      SCREAMING_SNAKE_CASE_     						=      create_tensor(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_     						=      broadcast(__UpperCAmelCase							)
      assert broadcasted_tensor.shape == torch.Size([state.num_processes]							)
      assert broadcasted_tensor.tolist() == list(range(1      ,      state.num_processes + 1							)							)

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[Any]							) ->			Dict:
      # We need to pad the tensor with one more element if we are the main process
      # to ensure that we can pad
      if state.is_main_process:
            SCREAMING_SNAKE_CASE_     						=      torch.arange(state.num_processes + 1							).to(state.device							)
      else:
            SCREAMING_SNAKE_CASE_     						=      torch.arange(state.num_processes							).to(state.device							)
      SCREAMING_SNAKE_CASE_     						=      pad_across_processes(__UpperCAmelCase							)
      assert padded_tensor.shape == torch.Size([state.num_processes + 1]							)
      if not state.is_main_process:
            assert padded_tensor.tolist() == list(range(0      ,      state.num_processes							)							) + [0]

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[Any]							) ->			Union[str, Any]:
      # For now runs on only two processes
      if state.num_processes != 2:
            return
      SCREAMING_SNAKE_CASE_     						=      create_tensor(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_     						=      reduce(__UpperCAmelCase      ,      'sum'							)
      SCREAMING_SNAKE_CASE_     						=      torch.tensor([4.0, 6]							).to(state.device							)
      assert torch.allclose(__UpperCAmelCase      ,      __UpperCAmelCase							), f"{reduced_tensor} != {truth_tensor}"

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Tuple							) ->			List[str]:
      # For now runs on only two processes
      if state.num_processes != 2:
            return
      SCREAMING_SNAKE_CASE_     						=      create_tensor(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_     						=      reduce(__UpperCAmelCase      ,      'mean'							)
      SCREAMING_SNAKE_CASE_     						=      torch.tensor([2.0, 3]							).to(state.device							)
      assert torch.allclose(__UpperCAmelCase      ,      __UpperCAmelCase							), f"{reduced_tensor} != {truth_tensor}"

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Dict							) ->			str:
      # For xla_spawn (TPUs)
      main()

def  UpperCAmelCase_							(				) ->			Any:
      SCREAMING_SNAKE_CASE_     						=      PartialState()
      state.print(f"State: {state}"							)
      state.print('testing gather'							)
      test_gather(__UpperCAmelCase							)
      state.print('testing gather_object'							)
      test_gather_object(__UpperCAmelCase							)
      state.print('testing broadcast'							)
      test_broadcast(__UpperCAmelCase							)
      state.print('testing pad_across_processes'							)
      test_pad_across_processes(__UpperCAmelCase							)
      state.print('testing reduce_sum'							)
      test_reduce_sum(__UpperCAmelCase							)
      state.print('testing reduce_mean'							)
      test_reduce_mean(__UpperCAmelCase							)


if __name__ == "__main__":
    main()





import torch
from torch import nn
from transformers import CLIPPreTrainedModel, CLIPVisionModel

from ...models.attention import BasicTransformerBlock
from ...utils import logging


lowerCamelCase__			:       List[Any]         =							logging.get_logger(__name__)  # pylint: disable=invalid-name





class 	lowerCamelCase_							(				_SCREAMING_SNAKE_CASE ):


 '''simple docstring'''







 def __init__(							self  :		List[Any]    ,    _lowerCAmelCase  :		Dict    ,    _lowerCAmelCase  :		Tuple=768						):
       super().__init__(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      proj_size

       SCREAMING_SNAKE_CASE_     						=      CLIPVisionModel(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      PaintByExampleMapper(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      nn.LayerNorm(config.hidden_size						)
       SCREAMING_SNAKE_CASE_     						=      nn.Linear(config.hidden_size    ,    self.proj_size						)

       # uncondition for scaling
       SCREAMING_SNAKE_CASE_     						=      nn.Parameter(torch.randn((1, 1, self.proj_size)						)						)




 def 				lowerCAmelCase_  (							self  :		Any    ,    _lowerCAmelCase  :		int    ,    _lowerCAmelCase  :		int=False						):
       SCREAMING_SNAKE_CASE_     						=      self.model(pixel_values=_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      clip_output.pooler_output
       SCREAMING_SNAKE_CASE_     						=      self.mapper(latent_states[:, None]						)
       SCREAMING_SNAKE_CASE_     						=      self.final_layer_norm(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      self.proj_out(_lowerCAmelCase						)
       if return_uncond_vector:
             return latent_states, self.uncond_vector

       return latent_states





class 	lowerCamelCase_							(				nn.Module ):


 '''simple docstring'''







 def __init__(							self  :		int    ,    _lowerCAmelCase  :		Optional[Any]						):
       super().__init__()
       SCREAMING_SNAKE_CASE_     						=      (config.num_hidden_layers + 1) // 5
       SCREAMING_SNAKE_CASE_     						=      config.hidden_size
       SCREAMING_SNAKE_CASE_     						=      1
       SCREAMING_SNAKE_CASE_     						=      nn.ModuleList(
           [
               BasicTransformerBlock(_lowerCAmelCase    ,    _lowerCAmelCase    ,    _lowerCAmelCase    ,    activation_fn='gelu'    ,    attention_bias=_lowerCAmelCase						)
               for _ in range(_lowerCAmelCase						)
           ]						)




 def 				lowerCAmelCase_  (							self  :		Union[str, Any]    ,    _lowerCAmelCase  :		Optional[Any]						):
       for block in self.blocks:
             SCREAMING_SNAKE_CASE_     						=      block(_lowerCAmelCase						)

       return hidden_states