





import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
         CTRLForSequenceClassification,
         CTRLLMHeadModel,
         CTRLModel,
     )



class     _UpperCAmelCase   :

  """simple docstring"""


  def __init__(    self			:		Any ,  lowerCAmelCase_			:		List[Any] ,  lowerCAmelCase_			:		Any=1_4 ,  lowerCAmelCase_			:		str=7 ,  lowerCAmelCase_			:		Tuple=True ,  lowerCAmelCase_			:		str=True ,  lowerCAmelCase_			:		Optional[Any]=True ,  lowerCAmelCase_			:		List[Any]=True ,  lowerCAmelCase_			:		Union[str, Any]=True ,  lowerCAmelCase_			:		Union[str, Any]=9_9 ,  lowerCAmelCase_			:		List[Any]=3_2 ,  lowerCAmelCase_			:		Optional[Any]=5 ,  lowerCAmelCase_			:		List[str]=4 ,  lowerCAmelCase_			:		Any=3_7 ,  lowerCAmelCase_			:		Optional[Any]="gelu" ,  lowerCAmelCase_			:		int=0.1 ,  lowerCAmelCase_			:		List[Any]=0.1 ,  lowerCAmelCase_			:		int=5_1_2 ,  lowerCAmelCase_			:		Dict=1_6 ,  lowerCAmelCase_			:		Any=2 ,  lowerCAmelCase_			:		Optional[Any]=0.02 ,  lowerCAmelCase_			:		str=3 ,  lowerCAmelCase_			:		Dict=4 ,  lowerCAmelCase_			:		Dict=None ,  )							->						Any:
     __lowerCAmelCase							     =     parent
     __lowerCAmelCase							     =     batch_size
     __lowerCAmelCase							     =     seq_length
     __lowerCAmelCase							     =     is_training
     __lowerCAmelCase							     =     use_token_type_ids
     __lowerCAmelCase							     =     use_input_mask
     __lowerCAmelCase							     =     use_labels
     __lowerCAmelCase							     =     use_mc_token_ids
     __lowerCAmelCase							     =     vocab_size
     __lowerCAmelCase							     =     hidden_size
     __lowerCAmelCase							     =     num_hidden_layers
     __lowerCAmelCase							     =     num_attention_heads
     __lowerCAmelCase							     =     intermediate_size
     __lowerCAmelCase							     =     hidden_act
     __lowerCAmelCase							     =     hidden_dropout_prob
     __lowerCAmelCase							     =     attention_probs_dropout_prob
     __lowerCAmelCase							     =     max_position_embeddings
     __lowerCAmelCase							     =     type_vocab_size
     __lowerCAmelCase							     =     type_sequence_label_size
     __lowerCAmelCase							     =     initializer_range
     __lowerCAmelCase							     =     num_labels
     __lowerCAmelCase							     =     num_choices
     __lowerCAmelCase							     =     scope
     __lowerCAmelCase							     =     self.vocab_size - 1






  def     lowercase		(    self			:		Dict					)							->						str:
     __lowerCAmelCase							     =     ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size					)

     __lowerCAmelCase							     =     None
     if self.use_input_mask:
        __lowerCAmelCase							     =     random_attention_mask([self.batch_size, self.seq_length]					)

     __lowerCAmelCase							     =     None
     if self.use_token_type_ids:
        __lowerCAmelCase							     =     ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size					)

     __lowerCAmelCase							     =     None
     if self.use_mc_token_ids:
        __lowerCAmelCase							     =     ids_tensor([self.batch_size, self.num_choices] ,  self.seq_length					)

     __lowerCAmelCase							     =     None
     __lowerCAmelCase							     =     None
     __lowerCAmelCase							     =     None
     if self.use_labels:
        __lowerCAmelCase							     =     ids_tensor([self.batch_size] ,  self.type_sequence_label_size					)
        __lowerCAmelCase							     =     ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels					)
        __lowerCAmelCase							     =     ids_tensor([self.batch_size] ,  self.num_choices					)

     __lowerCAmelCase							     =     self.get_config()

     __lowerCAmelCase							     =     ids_tensor([self.num_hidden_layers, self.num_attention_heads] ,  2					)

     return (
         config,
         input_ids,
         input_mask,
         head_mask,
         token_type_ids,
         mc_token_ids,
         sequence_labels,
         token_labels,
         choice_labels,
     )






  def     lowercase		(    self			:		List[Any]					)							->						List[str]:
     return CTRLConfig(
         vocab_size=self.vocab_size ,  n_embd=self.hidden_size ,  n_layer=self.num_hidden_layers ,  n_head=self.num_attention_heads ,  n_positions=self.max_position_embeddings ,  pad_token_id=self.pad_token_id ,  )






  def     lowercase		(    self			:		List[Any] ,  lowerCAmelCase_			:		List[str] ,  lowerCAmelCase_			:		int ,  lowerCAmelCase_			:		Any ,  lowerCAmelCase_			:		Any ,  lowerCAmelCase_			:		int ,  *lowerCAmelCase_			:		List[Any]					)							->						int:
     __lowerCAmelCase							     =     CTRLModel(config=lowerCAmelCase_					)
     model.to(lowerCAmelCase_					)
     model.eval()

     model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  head_mask=lowerCAmelCase_					)
     model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_					)
     __lowerCAmelCase							     =     model(lowerCAmelCase_					)
     self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)					)
     self.parent.assertEqual(len(result.past_key_values					) ,  config.n_layer					)






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		Union[str, Any] ,  lowerCAmelCase_			:		List[str] ,  lowerCAmelCase_			:		Dict ,  lowerCAmelCase_			:		Optional[Any] ,  lowerCAmelCase_			:		Optional[Any] ,  *lowerCAmelCase_			:		Tuple					)							->						int:
     __lowerCAmelCase							     =     CTRLLMHeadModel(lowerCAmelCase_					)
     model.to(lowerCAmelCase_					)
     model.eval()

     __lowerCAmelCase							     =     model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  labels=lowerCAmelCase_					)
     self.parent.assertEqual(result.loss.shape ,  ()					)
     self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)					)






  def     lowercase		(    self			:		List[str]					)							->						List[Any]:
     __lowerCAmelCase							     =     self.prepare_config_and_inputs()

     (
         (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     (
         __lowerCAmelCase
     )     ,     
     )							     =     config_and_inputs

     __lowerCAmelCase							     =     {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'head_mask': head_mask}

     return config, inputs_dict






  def     lowercase		(    self			:		str ,  lowerCAmelCase_			:		List[str] ,  lowerCAmelCase_			:		Any ,  lowerCAmelCase_			:		Tuple ,  lowerCAmelCase_			:		Optional[int] ,  *lowerCAmelCase_			:		List[str]					)							->						List[Any]:
     __lowerCAmelCase							     =     self.num_labels
     __lowerCAmelCase							     =     CTRLForSequenceClassification(lowerCAmelCase_					)
     model.to(lowerCAmelCase_					)
     model.eval()
     __lowerCAmelCase							     =     ids_tensor([self.batch_size] ,  self.type_sequence_label_size					)
     __lowerCAmelCase							     =     model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  labels=lowerCAmelCase_					)
     self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)					)





@require_torch
class     _UpperCAmelCase   (   _UpperCamelCase		,   _UpperCamelCase		,   _UpperCamelCase		,   unittest.TestCase   ):

  """simple docstring"""


  a_      =			(CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
  a_      =			(CTRLLMHeadModel,) if is_torch_available() else ()
  a_      =			(
      {
          """feature-extraction""": CTRLModel,
          """text-classification""": CTRLForSequenceClassification,
          """text-generation""": CTRLLMHeadModel,
          """zero-shot""": CTRLForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  a_      =			True
  a_      =			False
  a_      =			False






  def     lowercase		(    self			:		int ,  lowerCAmelCase_			:		Any ,  lowerCAmelCase_			:		List[Any] ,  lowerCAmelCase_			:		Dict ,  lowerCAmelCase_			:		Dict ,  lowerCAmelCase_			:		Tuple					)							->						Any:
     if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
        # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
        # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
        # config could not be created.
        return True

     return False






  def     lowercase		(    self			:		Optional[int]					)							->						List[str]:
     __lowerCAmelCase							     =     CTRLModelTester(self					)
     __lowerCAmelCase							     =     ConfigTester(self ,  config_class=lowerCAmelCase_ ,  n_embd=3_7					)






  def     lowercase		(    self			:		str					)							->						str:
     super().tearDown()
     # clean-up as much as possible GPU memory occupied by PyTorch
     gc.collect()
     torch.cuda.empty_cache()






  def     lowercase		(    self			:		Any					)							->						Any:
     self.config_tester.run_common_tests()






  def     lowercase		(    self			:		Optional[Any]					)							->						List[str]:
     __lowerCAmelCase							     =     self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_ctrl_model(*lowerCAmelCase_					)






  def     lowercase		(    self			:		str					)							->						List[Any]:
     __lowerCAmelCase							     =     self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_lm_head_model(*lowerCAmelCase_					)






  @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'					)
  def     lowercase		(    self			:		Optional[int]					)							->						Optional[int]:
     pass






  @slow
  def     lowercase		(    self			:		int					)							->						Any:
     for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        __lowerCAmelCase							     =     CTRLModel.from_pretrained(lowerCAmelCase_					)
        self.assertIsNotNone(lowerCAmelCase_					)






  @unittest.skip('The model doesn\'t support left padding'					)  # and it's not used enough to be worth fixing :)
  def     lowercase		(    self			:		str					)							->						List[str]:
     pass





@require_torch
class     _UpperCAmelCase   (   unittest.TestCase   ):

  """simple docstring"""


  def     lowercase		(    self			:		Tuple					)							->						Tuple:
     super().tearDown()
     # clean-up as much as possible GPU memory occupied by PyTorch
     gc.collect()
     torch.cuda.empty_cache()






  @slow
  def     lowercase		(    self			:		Optional[Any]					)							->						Union[str, Any]:
     __lowerCAmelCase							     =     CTRLLMHeadModel.from_pretrained('ctrl'					)
     model.to(lowerCAmelCase_					)
     __lowerCAmelCase							     =     torch.tensor(
         [[1_1_8_5_9, 0, 1_6_1_1, 8]] ,  dtype=torch.long ,  device=lowerCAmelCase_					)  # Legal the president is
     __lowerCAmelCase							     =     [
         1_1_8_5_9,
         0,
         1_6_1_1,
         8,
         5,
         1_5_0,
         2_6_4_4_9,
         2,
         1_9,
         3_4_8,
         4_6_9,
         3,
         2_5_9_5,
         4_8,
         2_0_7_4_0,
         2_4_6_5_3_3,
         2_4_6_5_3_3,
         1_9,
         3_0,
         5,
     ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

     __lowerCAmelCase							     =     model.generate(lowerCAmelCase_ ,  do_sample=lowerCAmelCase_					)
     self.assertListEqual(output_ids[0].tolist() ,  lowerCAmelCase_					)








from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .modeling_utils import ModelMixin



@dataclass
class     _UpperCAmelCase   (   _UpperCamelCase   ):

  """simple docstring"""


  a_      =			42





class     _UpperCAmelCase   (   _UpperCamelCase		,   _UpperCamelCase   ):

  """simple docstring"""


  @register_to_config
  def __init__(    self			:		Optional[int] ,  lowerCAmelCase_			:		int = 1_6 ,  lowerCAmelCase_			:		int = 8_8 ,  lowerCAmelCase_			:		Optional[int] = None ,  lowerCAmelCase_			:		Optional[int] = None ,  lowerCAmelCase_			:		int = 1 ,  lowerCAmelCase_			:		float = 0.0 ,  lowerCAmelCase_			:		int = 3_2 ,  lowerCAmelCase_			:		Optional[int] = None ,  lowerCAmelCase_			:		bool = False ,  lowerCAmelCase_			:		Optional[int] = None ,  lowerCAmelCase_			:		str = "geglu" ,  lowerCAmelCase_			:		bool = True ,  lowerCAmelCase_			:		bool = True ,  )							->						Dict:
     super().__init__()
     __lowerCAmelCase							     =     num_attention_heads
     __lowerCAmelCase							     =     attention_head_dim
     __lowerCAmelCase							     =     num_attention_heads * attention_head_dim

     __lowerCAmelCase							     =     in_channels

     __lowerCAmelCase							     =     torch.nn.GroupNorm(num_groups=lowerCAmelCase_ ,  num_channels=lowerCAmelCase_ ,  eps=1e-6 ,  affine=lowerCAmelCase_					)
     __lowerCAmelCase							     =     nn.Linear(lowerCAmelCase_ ,  lowerCAmelCase_					)

     # 3. Define transformers blocks
     __lowerCAmelCase							     =     nn.ModuleList(
         [
             BasicTransformerBlock(
                 lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  dropout=lowerCAmelCase_ ,  cross_attention_dim=lowerCAmelCase_ ,  activation_fn=lowerCAmelCase_ ,  attention_bias=lowerCAmelCase_ ,  double_self_attention=lowerCAmelCase_ ,  norm_elementwise_affine=lowerCAmelCase_ ,  )
             for d in range(lowerCAmelCase_					)
         ]					)

     __lowerCAmelCase							     =     nn.Linear(lowerCAmelCase_ ,  lowerCAmelCase_					)






  def     lowercase		(    self			:		Dict ,  lowerCAmelCase_			:		Tuple ,  lowerCAmelCase_			:		List[Any]=None ,  lowerCAmelCase_			:		Optional[int]=None ,  lowerCAmelCase_			:		Union[str, Any]=None ,  lowerCAmelCase_			:		Optional[int]=1 ,  lowerCAmelCase_			:		Optional[int]=None ,  lowerCAmelCase_			:		bool = True ,  )							->						str:
     __lowerCAmelCase     ,     __lowerCAmelCase     ,     __lowerCAmelCase     ,     __lowerCAmelCase							     =     hidden_states.shape
     __lowerCAmelCase							     =     batch_frames // num_frames

     __lowerCAmelCase							     =     hidden_states

     __lowerCAmelCase							     =     hidden_states[None, :].reshape(lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_					)
     __lowerCAmelCase							     =     hidden_states.permute(0 ,  2 ,  1 ,  3 ,  4					)

     __lowerCAmelCase							     =     self.norm(lowerCAmelCase_					)
     __lowerCAmelCase							     =     hidden_states.permute(0 ,  3 ,  4 ,  2 ,  1					).reshape(batch_size * height * width ,  lowerCAmelCase_ ,  lowerCAmelCase_					)

     __lowerCAmelCase							     =     self.proj_in(lowerCAmelCase_					)

     # 2. Blocks
     for block in self.transformer_blocks:
        __lowerCAmelCase							     =     block(
            lowerCAmelCase_ ,  encoder_hidden_states=lowerCAmelCase_ ,  timestep=lowerCAmelCase_ ,  cross_attention_kwargs=lowerCAmelCase_ ,  class_labels=lowerCAmelCase_ ,  )

     # 3. Output
     __lowerCAmelCase							     =     self.proj_out(lowerCAmelCase_					)
     __lowerCAmelCase							     =     (
         hidden_states[None, None, :]
         .reshape(lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_					)
         .permute(0 ,  3 ,  4 ,  1 ,  2					)
         .contiguous()
     )
     __lowerCAmelCase							     =     hidden_states.reshape(lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_					)

     __lowerCAmelCase							     =     hidden_states + residual

     if not return_dict:
        return (output,)

     return TransformerTemporalModelOutput(sample=lowerCAmelCase_					)

