





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_snake_case           =			logging.getLogger(__name__)





class  lowerCAmelCase							(       lowercase_ ):







  def   UpperCAmelCase					(      self  :Optional[int]   ,							_lowercase  :int   ,							_lowercase  :Union[str, Any]   ,							_lowercase  :int=None   ,							_lowercase  :Dict=None ):






     '''simple docstring'''





     lowercase__			       =  self.layer[current_layer](_lowercase   ,							_lowercase   ,							head_mask[current_layer] )

     lowercase__			       =  layer_outputs[0]

     return hidden_states





@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'							,							lowercase_							,							)
class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :Tuple   ,							_lowercase  :Dict ):






     '''simple docstring'''





     super().__init__(_lowercase )

     lowercase__			       =  BertEncoderWithPabee(_lowercase )

     self.init_weights()
     lowercase__			       =  0
     lowercase__			       =  0
     lowercase__			       =  0

     lowercase__			       =  0







  def   UpperCAmelCase					(      self  :Union[str, Any]   ,							_lowercase  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  threshold







  def   UpperCAmelCase					(      self  :List[str]   ,							_lowercase  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  patience







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  0
     lowercase__			       =  0







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  self.inference_layers_num / self.inference_instances_num
     lowercase__			       =  (
         f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
         f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
     )
     print(_lowercase )







  @add_start_docstrings_to_model_forward(_lowercase )
  def   UpperCAmelCase					(      self  :Union[str, Any]   ,							_lowercase  :Any=None   ,							_lowercase  :Any=None   ,							_lowercase  :Optional[int]=None   ,							_lowercase  :Tuple=None   ,							_lowercase  :Any=None   ,							_lowercase  :Tuple=None   ,							_lowercase  :Optional[Any]=None   ,							_lowercase  :List[str]=None   ,							_lowercase  :str=None   ,							_lowercase  :Tuple=None   ,							_lowercase  :Union[str, Any]=False   ,							):






     '''simple docstring'''





     if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time" )
     elif input_ids is not None:
        lowercase__			       =  input_ids.size()
     elif inputs_embeds is not None:
        lowercase__			       =  inputs_embeds.size()[:-1]
     else:
        raise ValueError("You have to specify either input_ids or inputs_embeds" )

     lowercase__			       =  input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
        lowercase__			       =  torch.ones(_lowercase   ,							device=_lowercase )
     if token_type_ids is None:
        lowercase__			       =  torch.zeros(_lowercase   ,							dtype=torch.long   ,							device=_lowercase )

     # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
     # ourselves in which case we just need to make it broadcastable to all heads.
     lowercase__			       =  self.get_extended_attention_mask(_lowercase   ,							_lowercase   ,							_lowercase )

     # If a 2D ou 3D attention mask is provided for the cross-attention
     # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
     if self.config.is_decoder and encoder_hidden_states is not None:
        lowercase__       ,							lowercase__       ,							lowercase__			       =  encoder_hidden_states.size()
        lowercase__			       =  (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
           lowercase__			       =  torch.ones(_lowercase   ,							device=_lowercase )
        lowercase__			       =  self.invert_attention_mask(_lowercase )
     else:
        lowercase__			       =  None

     # Prepare head mask if needed
     # 1.0 in head_mask indicate we keep the head
     # attention_probs has shape bsz x n_heads x N x N
     # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
     # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
     lowercase__			       =  self.get_head_mask(_lowercase   ,							self.config.num_hidden_layers )

     lowercase__			       =  self.embeddings(
         input_ids=_lowercase   ,							position_ids=_lowercase   ,							token_type_ids=_lowercase   ,							inputs_embeds=_lowercase )
     lowercase__			       =  embedding_output

     if self.training:
        lowercase__			       =  []
        for i in range(self.config.num_hidden_layers ):
           lowercase__			       =  self.encoder.adaptive_forward(
               _lowercase   ,							current_layer=_lowercase   ,							attention_mask=_lowercase   ,							head_mask=_lowercase )

           lowercase__			       =  self.pooler(_lowercase )
           lowercase__			       =  output_layers[i](output_dropout(_lowercase ) )
           res.append(_lowercase )
     elif self.patience == 0:  # Use all layers for inference
        lowercase__			       =  self.encoder(
            _lowercase   ,							attention_mask=_lowercase   ,							head_mask=_lowercase   ,							encoder_hidden_states=_lowercase   ,							encoder_attention_mask=_lowercase   ,							)
        lowercase__			       =  self.pooler(encoder_outputs[0] )
        lowercase__			       =  [output_layers[self.config.num_hidden_layers - 1](_lowercase )]
     else:
        lowercase__			       =  0
        lowercase__			       =  None
        lowercase__			       =  0
        for i in range(self.config.num_hidden_layers ):
           calculated_layer_num += 1
           lowercase__			       =  self.encoder.adaptive_forward(
               _lowercase   ,							current_layer=_lowercase   ,							attention_mask=_lowercase   ,							head_mask=_lowercase )

           lowercase__			       =  self.pooler(_lowercase )
           lowercase__			       =  output_layers[i](_lowercase )
           if regression:
              lowercase__			       =  logits.detach()
              if patient_result is not None:
                 lowercase__			       =  patient_result.detach()
              if (patient_result is not None) and torch.abs(patient_result - labels ) < self.regression_threshold:
                 patient_counter += 1
              else:
                 lowercase__			       =  0
           else:
              lowercase__			       =  logits.detach().argmax(dim=1 )
              if patient_result is not None:
                 lowercase__			       =  patient_result.detach().argmax(dim=1 )
              if (patient_result is not None) and torch.all(labels.eq(_lowercase ) ):
                 patient_counter += 1
              else:
                 lowercase__			       =  0

           lowercase__			       =  logits
           if patient_counter == self.patience:
              break
        lowercase__			       =  [patient_result]
        self.inference_layers_num += calculated_layer_num
        self.inference_instances_num += 1

     return res





@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '							,							lowercase_							,							)
class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :Optional[int]   ,							_lowercase  :List[Any] ):






     '''simple docstring'''





     super().__init__(_lowercase )
     lowercase__			       =  config.num_labels

     lowercase__			       =  BertModelWithPabee(_lowercase )
     lowercase__			       =  nn.Dropout(config.hidden_dropout_prob )
     lowercase__			       =  nn.ModuleList(
         [nn.Linear(config.hidden_size   ,							self.config.num_labels ) for _ in range(config.num_hidden_layers )] )

     self.init_weights()







  @add_start_docstrings_to_model_forward(_lowercase )
  def   UpperCAmelCase					(      self  :Any   ,							_lowercase  :Tuple=None   ,							_lowercase  :Optional[int]=None   ,							_lowercase  :Dict=None   ,							_lowercase  :Optional[Any]=None   ,							_lowercase  :str=None   ,							_lowercase  :List[str]=None   ,							_lowercase  :Any=None   ,							):






     '''simple docstring'''





     lowercase__			       =  self.bert(
         input_ids=_lowercase   ,							attention_mask=_lowercase   ,							token_type_ids=_lowercase   ,							position_ids=_lowercase   ,							head_mask=_lowercase   ,							inputs_embeds=_lowercase   ,							output_dropout=self.dropout   ,							output_layers=self.classifiers   ,							regression=self.num_labels == 1   ,							)

     lowercase__			       =  (logits[-1],)

     if labels is not None:
        lowercase__			       =  None
        lowercase__			       =  0
        for ix, logits_item in enumerate(_lowercase ):
           if self.num_labels == 1:
              #  We are doing regression
              lowercase__			       =  MSELoss()
              lowercase__			       =  loss_fct(logits_item.view(-1 )   ,							labels.view(-1 ) )
           else:
              lowercase__			       =  CrossEntropyLoss()
              lowercase__			       =  loss_fct(logits_item.view(-1   ,							self.num_labels )   ,							labels.view(-1 ) )
           if total_loss is None:
              lowercase__			       =  loss
           else:
              total_loss += loss * (ix + 1)
           total_weights += ix + 1
        lowercase__			       =  (total_loss / total_weights,) + outputs

     return outputs













import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

import diffusers
from diffusers import (
    AutoencoderKL,
    EulerDiscreteScheduler,
    StableDiffusionLatentUpscalePipeline,
    StableDiffusionPipeline,
    UNetaDConditionModel,
)
from diffusers.schedulers import KarrasDiffusionSchedulers
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


def _A       (   __magic_name__	):
   lowercase__			       =  [tensor.shape for tensor in tensor_list]
   return all(shape == shapes[0] for shape in shapes[1:]	)





class  lowerCAmelCase							(       lowercase_							,							lowercase_							,							lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      StableDiffusionLatentUpscalePipeline
  __lowerCamelCase		=      TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {
      'height',
      'width',
      'cross_attention_kwargs',
      'negative_prompt_embeds',
      'prompt_embeds',
  }
  __lowerCamelCase		=      PipelineTesterMixin.required_optional_params - {'num_images_per_prompt'}
  __lowerCamelCase		=      TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
  __lowerCamelCase		=      frozenset(
      [] )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
  __lowerCamelCase		=      frozenset([] )

  __lowerCamelCase		=      True







  @property
  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     lowercase__			       =  1
     lowercase__			       =  4
     lowercase__			       =  (16, 16)

     lowercase__			       =  floats_tensor((batch_size, num_channels) + sizes   ,							rng=random.Random(0 ) ).to(_lowercase )
     return image







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     torch.manual_seed(0 )
     lowercase__			       =  UNetaDConditionModel(
         act_fn="gelu"   ,							attention_head_dim=8   ,							norm_num_groups=_lowercase   ,							block_out_channels=[32, 32, 64, 64]   ,							time_cond_proj_dim=1_60   ,							conv_in_kernel=1   ,							conv_out_kernel=1   ,							cross_attention_dim=32   ,							down_block_types=(
             "KDownBlock2D",
             "KCrossAttnDownBlock2D",
             "KCrossAttnDownBlock2D",
             "KCrossAttnDownBlock2D",
         )   ,							in_channels=8   ,							mid_block_type=_lowercase   ,							only_cross_attention=_lowercase   ,							out_channels=5   ,							resnet_time_scale_shift="scale_shift"   ,							time_embedding_type="fourier"   ,							timestep_post_act="gelu"   ,							up_block_types=("KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KUpBlock2D")   ,							)
     lowercase__			       =  AutoencoderKL(
         block_out_channels=[32, 32, 64, 64]   ,							in_channels=3   ,							out_channels=3   ,							down_block_types=[
             "DownEncoderBlock2D",
             "DownEncoderBlock2D",
             "DownEncoderBlock2D",
             "DownEncoderBlock2D",
         ]   ,							up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"]   ,							latent_channels=4   ,							)
     lowercase__			       =  EulerDiscreteScheduler(prediction_type="sample" )
     lowercase__			       =  CLIPTextConfig(
         bos_token_id=0   ,							eos_token_id=2   ,							hidden_size=32   ,							intermediate_size=37   ,							layer_norm_eps=1e-05   ,							num_attention_heads=4   ,							num_hidden_layers=5   ,							pad_token_id=1   ,							vocab_size=10_00   ,							hidden_act="quick_gelu"   ,							projection_dim=5_12   ,							)
     lowercase__			       =  CLIPTextModel(_lowercase )
     lowercase__			       =  CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip" )

     lowercase__			       =  {
         "unet": model.eval(),
         "vae": vae.eval(),
         "scheduler": scheduler,
         "text_encoder": text_encoder,
         "tokenizer": tokenizer,
     }

     return components







  def   UpperCAmelCase					(      self  :Dict   ,							_lowercase  :Union[str, Any]   ,							_lowercase  :int=0 ):






     '''simple docstring'''





     if str(_lowercase ).startswith("mps" ):
        lowercase__			       =  torch.manual_seed(_lowercase )
     else:
        lowercase__			       =  torch.Generator(device=_lowercase ).manual_seed(_lowercase )
     lowercase__			       =  {
         "prompt": "A painting of a squirrel eating a burger",
         "image": self.dummy_image.cpu(),
         "generator": generator,
         "num_inference_steps": 2,
         "output_type": "numpy",
     }
     return inputs







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  "cpu"

     lowercase__			       =  self.get_dummy_components()
     lowercase__			       =  self.pipeline_class(**_lowercase )
     pipe.to(_lowercase )
     pipe.set_progress_bar_config(disable=_lowercase )

     lowercase__			       =  self.get_dummy_inputs(_lowercase )
     lowercase__			       =  pipe(**_lowercase ).images
     lowercase__			       =  image[0, -3:, -3:, -1]

     self.assertEqual(image.shape   ,							(1, 2_56, 2_56, 3) )
     lowercase__			       =  np.array(
         [0.47222412, 0.41921633, 0.44717434, 0.46874192, 0.42588258, 0.46150726, 0.4677534, 0.45583832, 0.48579055] )
     lowercase__			       =  np.abs(image_slice.flatten() - expected_slice ).max()
     self.assertLessEqual(_lowercase   ,							1e-3 )







  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     super().test_attention_slicing_forward_pass(expected_max_diff=7e-3 )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     super().test_cpu_offload_forward_pass(expected_max_diff=3e-3 )







  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3 )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     super().test_inference_batch_single_identical(expected_max_diff=7e-3 )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     super().test_pt_np_pil_outputs_equivalent(expected_max_diff=3e-3 )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     super().test_save_load_local(expected_max_difference=3e-3 )







  def   UpperCAmelCase					(      self  :Tuple ):






     '''simple docstring'''





     super().test_save_load_optional_components(expected_max_difference=3e-3 )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  [
         "DDIMScheduler",
         "DDPMScheduler",
         "PNDMScheduler",
         "HeunDiscreteScheduler",
         "EulerAncestralDiscreteScheduler",
         "KDPM2DiscreteScheduler",
         "KDPM2AncestralDiscreteScheduler",
         "DPMSolverSDEScheduler",
     ]
     lowercase__			       =  self.get_dummy_components()
     lowercase__			       =  self.pipeline_class(**_lowercase )

     # make sure that PNDM does not need warm-up
     pipe.scheduler.register_to_config(skip_prk_steps=_lowercase )

     pipe.to(_lowercase )
     pipe.set_progress_bar_config(disable=_lowercase )
     lowercase__			       =  self.get_dummy_inputs(_lowercase )
     lowercase__			       =  2

     lowercase__			       =  []
     for scheduler_enum in KarrasDiffusionSchedulers:
        if scheduler_enum.name in skip_schedulers:
           # no sigma schedulers are not supported
           # no schedulers
           continue

        lowercase__			       =  getattr(_lowercase   ,							scheduler_enum.name )
        lowercase__			       =  scheduler_cls.from_config(pipe.scheduler.config )
        lowercase__			       =  pipe(**_lowercase )[0]
        outputs.append(_lowercase )

     assert check_same_shape(_lowercase )





@require_torch_gpu
@slow
class  lowerCAmelCase							(       unittest.TestCase ):







  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     super().tearDown()
     gc.collect()
     torch.cuda.empty_cache()







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  torch.manual_seed(33 )

     lowercase__			       =  StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4"   ,							torch_dtype=torch.floataa )
     pipe.to("cuda" )

     lowercase__			       =  StableDiffusionLatentUpscalePipeline.from_pretrained(
         "stabilityai/sd-x2-latent-upscaler"   ,							torch_dtype=torch.floataa )
     upscaler.to("cuda" )

     lowercase__			       =  "a photo of an astronaut high resolution, unreal engine, ultra realistic"

     lowercase__			       =  pipe(_lowercase   ,							generator=_lowercase   ,							output_type="latent" ).images

     lowercase__			       =  upscaler(
         prompt=_lowercase   ,							image=_lowercase   ,							num_inference_steps=20   ,							guidance_scale=0   ,							generator=_lowercase   ,							output_type="np"   ,							).images[0]

     lowercase__			       =  load_numpy(
         "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/astronaut_1024.npy" )
     assert np.abs((expected_image - image).mean() ) < 5e-2







  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     lowercase__			       =  torch.manual_seed(33 )

     lowercase__			       =  StableDiffusionLatentUpscalePipeline.from_pretrained(
         "stabilityai/sd-x2-latent-upscaler"   ,							torch_dtype=torch.floataa )
     upscaler.to("cuda" )

     lowercase__			       =  "the temple of fire by Ross Tran and Gerardo Dottori, oil on canvas"

     lowercase__			       =  load_image(
         "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_512.png" )

     lowercase__			       =  upscaler(
         prompt=_lowercase   ,							image=_lowercase   ,							num_inference_steps=20   ,							guidance_scale=0   ,							generator=_lowercase   ,							output_type="np"   ,							).images[0]

     lowercase__			       =  load_numpy(
         "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_1024.npy" )
     assert np.abs((expected_image - image).max() ) < 5e-2






