'''simple docstring'''




import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      FlaubertForMultipleChoice,
      FlaubertForQuestionAnswering,
      FlaubertForQuestionAnsweringSimple,
      FlaubertForSequenceClassification,
      FlaubertForTokenClassification,
      FlaubertModel,
      FlaubertWithLMHeadModel,
  )
  from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class    a__				(       UpperCAmelCase__			):






     def __init__(			self		:		Union[str, Any]  ,    a		:		List[Any]  ,    a		:		Optional[int]=13  ,    a		:		str=7  ,    a		:		List[Any]=True  ,    a		:		int=True  ,    a		:		Any=True  ,    a		:		Any=True  ,    a		:		List[Any]=True  ,    a		:		int=False  ,    a		:		Optional[Any]=False  ,    a		:		Dict=False  ,    a		:		Dict=2  ,    a		:		Optional[int]=99  ,    a		:		List[str]=0  ,    a		:		List[Any]=32  ,    a		:		Union[str, Any]=5  ,    a		:		List[str]=4  ,    a		:		int=0.1  ,    a		:		List[Any]=0.1  ,    a		:		List[Any]=5_12  ,    a		:		Tuple=12  ,    a		:		int=2  ,    a		:		List[str]=0.02  ,    a		:		int=3  ,    a		:		Optional[int]=4  ,    a		:		int="last"  ,    a		:		Any=None  ,    a		:		str=None  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  parent
         __lowerCamelCase          =  batch_size
         __lowerCamelCase          =  seq_length
         __lowerCamelCase          =  is_training
         __lowerCamelCase          =  use_input_lengths
         __lowerCamelCase          =  use_token_type_ids
         __lowerCamelCase          =  use_labels
         __lowerCamelCase          =  gelu_activation
         __lowerCamelCase          =  sinusoidal_embeddings
         __lowerCamelCase          =  causal
         __lowerCamelCase          =  asm
         __lowerCamelCase          =  n_langs
         __lowerCamelCase          =  vocab_size
         __lowerCamelCase          =  n_special
         __lowerCamelCase          =  hidden_size
         __lowerCamelCase          =  num_hidden_layers
         __lowerCamelCase          =  num_attention_heads
         __lowerCamelCase          =  hidden_dropout_prob
         __lowerCamelCase          =  attention_probs_dropout_prob
         __lowerCamelCase          =  max_position_embeddings
         __lowerCamelCase          =  type_vocab_size
         __lowerCamelCase          =  type_sequence_label_size
         __lowerCamelCase          =  initializer_range
         __lowerCamelCase          =  num_labels
         __lowerCamelCase          =  num_choices
         __lowerCamelCase          =  summary_type
         __lowerCamelCase          =  use_proj
         __lowerCamelCase          =  scope






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size   )
         __lowerCamelCase          =  random_attention_mask([self.batch_size, self.seq_length]   )

         __lowerCamelCase          =  None
         if self.use_input_lengths:
             __lowerCamelCase          =  (
                 ids_tensor([self.batch_size]  ,    vocab_size=2   ) + self.seq_length - 2
             )  # small variation of seq_length

         __lowerCamelCase          =  None
         if self.use_token_type_ids:
             __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.n_langs   )

         __lowerCamelCase          =  None
         __lowerCamelCase          =  None
         __lowerCamelCase          =  None
         if self.use_labels:
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    self.type_sequence_label_size   )
             __lowerCamelCase          =  ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels   )
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    2   ).float()
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    self.num_choices   )

         __lowerCamelCase          =  self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         return FlaubertConfig(
             vocab_size=self.vocab_size  ,    n_special=self.n_special  ,    emb_dim=self.hidden_size  ,    n_layers=self.num_hidden_layers  ,    n_heads=self.num_attention_heads  ,    dropout=self.hidden_dropout_prob  ,    attention_dropout=self.attention_probs_dropout_prob  ,    gelu_activation=self.gelu_activation  ,    sinusoidal_embeddings=self.sinusoidal_embeddings  ,    asm=self.asm  ,    causal=self.causal  ,    n_langs=self.n_langs  ,    max_position_embeddings=self.max_position_embeddings  ,    initializer_range=self.initializer_range  ,    summary_type=self.summary_type  ,    use_proj=self.use_proj  ,    )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    a		:		int  ,    a		:		Any  ,    a		:		Tuple  ,    a		:		str  ,    a		:		int  ,    a		:		Optional[int]  ,    a		:		List[str]  ,    a		:		Optional[int]  ,    a		:		List[str]  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertModel(config=a   )
         model.to(a   )
         model.eval()
         __lowerCamelCase          =  model(a  ,    lengths=a  ,    langs=a   )
         __lowerCamelCase          =  model(a  ,    langs=a   )
         __lowerCamelCase          =  model(a   )
         self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.hidden_size)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict  ,    a		:		Optional[Any]  ,    a		:		Any  ,    a		:		Any  ,    a		:		Union[str, Any]  ,    a		:		Dict  ,    a		:		List[str]  ,    a		:		Optional[Any]  ,    a		:		Optional[Any]  ,    a		:		str  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertWithLMHeadModel(a   )
         model.to(a   )
         model.eval()

         __lowerCamelCase          =  model(a  ,    token_type_ids=a  ,    labels=a   )
         self.parent.assertEqual(result.loss.shape  ,    ()   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]  ,    a		:		List[Any]  ,    a		:		Any  ,    a		:		Tuple  ,    a		:		List[str]  ,    a		:		Tuple  ,    a		:		str  ,    a		:		Dict  ,    a		:		int  ,    a		:		List[str]  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertForQuestionAnsweringSimple(a   )
         model.to(a   )
         model.eval()

         __lowerCamelCase          =  model(a   )

         __lowerCamelCase          =  model(a  ,    start_positions=a  ,    end_positions=a   )
         self.parent.assertEqual(result.start_logits.shape  ,    (self.batch_size, self.seq_length)   )
         self.parent.assertEqual(result.end_logits.shape  ,    (self.batch_size, self.seq_length)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]  ,    a		:		Union[str, Any]  ,    a		:		List[str]  ,    a		:		List[str]  ,    a		:		List[Any]  ,    a		:		Dict  ,    a		:		List[Any]  ,    a		:		List[str]  ,    a		:		List[Any]  ,    a		:		Tuple  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertForQuestionAnswering(a   )
         model.to(a   )
         model.eval()

         __lowerCamelCase          =  model(a   )

         __lowerCamelCase          =  model(
             a  ,    start_positions=a  ,    end_positions=a  ,    cls_index=a  ,    is_impossible=a  ,    p_mask=a  ,    )

         __lowerCamelCase          =  model(
             a  ,    start_positions=a  ,    end_positions=a  ,    cls_index=a  ,    is_impossible=a  ,    )

         ((__lowerCamelCase)     ,  )          =  result_with_labels.to_tuple()

         __lowerCamelCase          =  model(a  ,    start_positions=a  ,    end_positions=a   )

         ((__lowerCamelCase)     ,  )          =  result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape  ,    ()   )
         self.parent.assertEqual(result.start_top_log_probs.shape  ,    (self.batch_size, model.config.start_n_top)   )
         self.parent.assertEqual(result.start_top_index.shape  ,    (self.batch_size, model.config.start_n_top)   )
         self.parent.assertEqual(
             result.end_top_log_probs.shape  ,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
         self.parent.assertEqual(
             result.end_top_index.shape  ,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
         self.parent.assertEqual(result.cls_logits.shape  ,    (self.batch_size,)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]  ,    a		:		str  ,    a		:		List[Any]  ,    a		:		Optional[int]  ,    a		:		Union[str, Any]  ,    a		:		Any  ,    a		:		Optional[Any]  ,    a		:		Optional[int]  ,    a		:		Any  ,    a		:		Optional[int]  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertForSequenceClassification(a   )
         model.to(a   )
         model.eval()

         __lowerCamelCase          =  model(a   )
         __lowerCamelCase          =  model(a  ,    labels=a   )

         self.parent.assertEqual(result.loss.shape  ,    ()   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.type_sequence_label_size)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		int  ,    a		:		List[str]  ,    a		:		List[Any]  ,    a		:		List[Any]  ,    a		:		Any  ,    a		:		Dict  ,    a		:		Union[str, Any]  ,    a		:		Dict  ,    a		:		int  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_labels
         __lowerCamelCase          =  FlaubertForTokenClassification(a   )
         model.to(a   )
         model.eval()

         __lowerCamelCase          =  model(a  ,    attention_mask=a  ,    labels=a   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.num_labels)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		Optional[int]  ,    a		:		Union[str, Any]  ,    a		:		str  ,    a		:		int  ,    a		:		Any  ,    a		:		Dict  ,    a		:		Tuple  ,    a		:		Tuple  ,    a		:		Optional[int]  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_choices
         __lowerCamelCase          =  FlaubertForMultipleChoice(config=a   )
         model.to(a   )
         model.eval()
         __lowerCamelCase          =  input_ids.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  token_type_ids.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  input_mask.unsqueeze(1   ).expand(-1  ,    self.num_choices  ,    -1   ).contiguous()
         __lowerCamelCase          =  model(
             a  ,    attention_mask=a  ,    token_type_ids=a  ,    labels=a  ,    )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_choices)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.prepare_config_and_inputs()
         (
             (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  (
             __lowerCamelCase
         )     ,  
         )          =  config_and_inputs
         __lowerCamelCase          =  {
             '''input_ids''': input_ids,
             '''token_type_ids''': token_type_ids,
             '''lengths''': input_lengths,
             '''attention_mask''': input_mask,
         }
         return config, inputs_dict




@require_torch
class    a__				(       UpperCAmelCase__							,       UpperCAmelCase__							,       unittest.TestCase			):
     lowerCamelCase  :					Tuple							=(
         (
             FlaubertModel,
             FlaubertWithLMHeadModel,
             FlaubertForQuestionAnswering,
             FlaubertForQuestionAnsweringSimple,
             FlaubertForSequenceClassification,
             FlaubertForTokenClassification,
             FlaubertForMultipleChoice,
         )
         if is_torch_available()
         else ()
     )
     lowerCamelCase  :					int							=(
         {
             "feature-extraction": FlaubertModel,
             "fill-mask": FlaubertWithLMHeadModel,
             "question-answering": FlaubertForQuestionAnsweringSimple,
             "text-classification": FlaubertForSequenceClassification,
             "token-classification": FlaubertForTokenClassification,
             "zero-shot": FlaubertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]  ,    a		:		List[str]  ,    a		:		Tuple  ,    a		:		Union[str, Any]  ,    a		:		List[str]  ,    a		:		List[str]   ):





         """simple docstring"""



         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('''Fast'''   )
         ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

         return False






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]  ,    a		:		Tuple  ,    a		:		int  ,    a		:		Dict=False   ):





         """simple docstring"""



         __lowerCamelCase          =  super()._prepare_for_class(a  ,    a  ,    return_labels=a   )

         if return_labels:
             if model_class.__name__ == "FlaubertForQuestionAnswering":
                 __lowerCamelCase          =  torch.zeros(
                     self.model_tester.batch_size  ,    dtype=torch.long  ,    device=a   )
                 __lowerCamelCase          =  torch.zeros(
                     self.model_tester.batch_size  ,    dtype=torch.long  ,    device=a   )

         return inputs_dict






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertModelTester(self   )
         __lowerCamelCase          =  ConfigTester(self  ,    config_class=a  ,    emb_dim=37   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         self.config_tester.run_common_tests()






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_model(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_lm_head(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_simple_qa(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_qa(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_sequence_classif(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_token_classif(*a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_multiple_choice(*a   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple   ):





         """simple docstring"""



         for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __lowerCamelCase          =  FlaubertModel.from_pretrained(a   )
             self.assertIsNotNone(a   )






     @slow
     @require_torch_gpu
     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase     ,  __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
             # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
             if model_class == FlaubertForMultipleChoice:
                 return

             __lowerCamelCase          =  True
             __lowerCamelCase          =  model_class(config=a   )

             __lowerCamelCase          =  self._prepare_for_class(a  ,    a   )
             __lowerCamelCase          =  torch.jit.trace(
                 a  ,    (inputs_dict['''input_ids'''].to('''cpu'''   ), inputs_dict['''attention_mask'''].to('''cpu'''   ))   )

             with tempfile.TemporaryDirectory() as tmp:
                 torch.jit.save(a  ,    os.path.join(a  ,    '''traced_model.pt'''   )   )
                 __lowerCamelCase          =  torch.jit.load(os.path.join(a  ,    '''traced_model.pt'''   )  ,    map_location=a   )
                 loaded(inputs_dict['''input_ids'''].to(a   )  ,    inputs_dict['''attention_mask'''].to(a   )   )




@require_torch
class    a__				(       unittest.TestCase			):






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  FlaubertModel.from_pretrained('''flaubert/flaubert_base_cased'''   )
         __lowerCamelCase          =  torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]   )
         with torch.no_grad():
             __lowerCamelCase          =  model(a   )[0]
         __lowerCamelCase          =  torch.Size((1, 11, 7_68)   )
         self.assertEqual(output.shape  ,    a   )
         __lowerCamelCase          =  torch.tensor(
             [[[-2.62_51, -1.42_98, -0.02_27], [-2.85_10, -1.63_87, 0.22_58], [-2.81_14, -1.18_32, -0.30_66]]]   )

         self.assertTrue(torch.allclose(output[:, :3, :3]  ,    a  ,    atol=1e-4   )   )

'''simple docstring'''




import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class    a__				(       UpperCAmelCase__							,       UpperCAmelCase__			):






     @register_to_config
     def __init__(			self		:		Any  ,    *,
         a		:		int = 4  ,    a		:		int = 7_68  ,    a		:		int  ,    a		:		Optional[int]  ,    ):





         """simple docstring"""



         super().__init__()

         __lowerCamelCase          =  nn.Parameter(torch.zeros(a   )   )

         # parameters for additional clip time embeddings
         __lowerCamelCase          =  nn.Linear(a  ,    a   )
         __lowerCamelCase          =  nn.Linear(a  ,    a   )

         # parameters for encoder hidden states
         __lowerCamelCase          =  clip_extra_context_tokens
         __lowerCamelCase          =  nn.Linear(
             a  ,    self.clip_extra_context_tokens * cross_attention_dim   )
         __lowerCamelCase          =  nn.Linear(a  ,    a   )
         __lowerCamelCase          =  nn.LayerNorm(a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    *, a		:		Union[str, Any]  ,    a		:		Any  ,    a		:		str  ,    a		:		Any   ):





         """simple docstring"""



         if do_classifier_free_guidance:
             # Add the classifier free guidance embeddings to the image embeddings
             __lowerCamelCase          =  image_embeddings.shape[0]
             __lowerCamelCase          =  self.learned_classifier_free_guidance_embeddings.unsqueeze(0   )
             __lowerCamelCase          =  classifier_free_guidance_embeddings.expand(
                 a  ,    -1   )
             __lowerCamelCase          =  torch.cat([classifier_free_guidance_embeddings, image_embeddings]  ,    dim=0   )

         # The image embeddings batch size and the text embeddings batch size are equal
         assert image_embeddings.shape[0] == prompt_embeds.shape[0]

         __lowerCamelCase          =  prompt_embeds.shape[0]

         # "Specifically, we modify the architecture described in Nichol et al. (2021) by projecting and
         # adding CLIP embeddings to the existing timestep embedding, ...
         __lowerCamelCase          =  self.embedding_proj(a   )
         __lowerCamelCase          =  self.clip_image_embeddings_project_to_time_embeddings(a   )
         __lowerCamelCase          =  time_projected_image_embeddings + time_projected_prompt_embeds

         # ... and by projecting CLIP embeddings into four
         # extra tokens of context that are concatenated to the sequence of outputs from the GLIDE text encoder"
         __lowerCamelCase          =  self.clip_extra_context_tokens_proj(a   )
         __lowerCamelCase          =  clip_extra_context_tokens.reshape(a  ,    -1  ,    self.clip_extra_context_tokens   )
         __lowerCamelCase          =  clip_extra_context_tokens.permute(0  ,    2  ,    1   )

         __lowerCamelCase          =  self.encoder_hidden_states_proj(a   )
         __lowerCamelCase          =  self.text_encoder_hidden_states_norm(a   )
         __lowerCamelCase          =  torch.cat([clip_extra_context_tokens, text_encoder_hidden_states]  ,    dim=1   )

         return text_encoder_hidden_states, additive_clip_time_embeddings
