




"""simple docstring"""



import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


SCREAMING_SNAKE_CASE    =	1E-4

if is_torch_available():
      import torch

      from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
      from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder




@require_torch
class      UpperCAmelCase_  :




    def __init__(   self						:	Optional[int]     ,			snake_case_						:	List[Any]     ,			snake_case_						:	Tuple=16     ,			snake_case_						:	Dict=13     ,			snake_case_						:	Optional[Any]=7     ,			snake_case_						:	Optional[Any]=14     ,			snake_case_						:	List[str]=10     ,			snake_case_						:	Optional[Any]=19     ,			snake_case_						:	List[str]=5     ,			snake_case_						:	Optional[Any]=4     ,			snake_case_						:	str=True     ,			snake_case_						:	Tuple=16     ,			snake_case_						:	Union[str, Any]=2     ,			snake_case_						:	Tuple=4     ,			snake_case_						:	Optional[int]=4     ,			snake_case_						:	List[Any]="gelu"     ,			snake_case_						:	int=0.1     ,			snake_case_						:	Dict=0.1     ,			snake_case_						:	Dict=[1, 2, 3, 4, 5]     ,			snake_case_						:	str=25     ,			snake_case_						:	List[str]=5     ,			)				->							str:




     '''simple docstring'''

     A__	=    d_model
     A__	=    parent
     A__	=    batch_size
     A__	=    prediction_length
     A__	=    context_length
     A__	=    cardinality
     A__	=    num_time_features
     A__	=    lags_sequence
     A__	=    embedding_dimension
     A__	=    is_training
     A__	=    hidden_size
     A__	=    num_hidden_layers
     A__	=    num_attention_heads
     A__	=    intermediate_size
     A__	=    hidden_act
     A__	=    hidden_dropout_prob
     A__	=    attention_probs_dropout_prob

     A__	=    context_length
     A__	=    prediction_length + label_length
     A__	=    label_length

     A__	=    moving_average
     A__	=    autocorrelation_factor




    def 							__magic_name__							(   self						:	List[str]				)				->							str:




     '''simple docstring'''

     return AutoformerConfig(
         d_model=self.d_model     ,			encoder_layers=self.num_hidden_layers     ,			decoder_layers=self.num_hidden_layers     ,			encoder_attention_heads=self.num_attention_heads     ,			decoder_attention_heads=self.num_attention_heads     ,			encoder_ffn_dim=self.intermediate_size     ,			decoder_ffn_dim=self.intermediate_size     ,			dropout=self.hidden_dropout_prob     ,			attention_dropout=self.attention_probs_dropout_prob     ,			prediction_length=self.prediction_length     ,			context_length=self.context_length     ,			label_length=self.label_length     ,			lags_sequence=self.lags_sequence     ,			num_time_features=self.num_time_features     ,			num_static_categorical_features=1     ,			cardinality=[self.cardinality]     ,			embedding_dimension=[self.embedding_dimension]     ,			moving_average=self.moving_average     ,			)




    def 							__magic_name__							(   self						:	Optional[int]     ,			snake_case_						:	List[Any]				)				->							Tuple:




     '''simple docstring'''

     A__	=    config.context_length + max(config.lags_sequence				)

     A__	=    ids_tensor([self.batch_size, 1]     ,			config.cardinality[0]				)
     A__	=    floats_tensor([self.batch_size, _past_length, config.num_time_features]				)
     A__	=    floats_tensor([self.batch_size, _past_length]				)
     A__	=    floats_tensor([self.batch_size, _past_length]				) > 0.5

     # decoder inputs
     A__	=    floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]				)
     A__	=    floats_tensor([self.batch_size, config.prediction_length]				)

     A__	=    {
         "past_values": past_values,
         "static_categorical_features": static_categorical_features,
         "past_time_features": past_time_features,
         "past_observed_mask": past_observed_mask,
         "future_time_features": future_time_features,
         "future_values": future_values,
     }
     return inputs_dict




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    self.get_config()
     A__	=    self.prepare_autoformer_inputs_dict(snake_case_				)
     return config, inputs_dict




    def 							__magic_name__							(   self						:	int				)				->							Union[str, Any]:




     '''simple docstring'''

     A__,				A__	=    self.prepare_config_and_inputs()
     return config, inputs_dict




    def 							__magic_name__							(   self						:	Optional[int]     ,			snake_case_						:	Optional[Any]     ,			snake_case_						:	Any				)				->							List[str]:




     '''simple docstring'''

     A__	=    AutoformerModel(config=snake_case_				).to(snake_case_				).eval()
     A__	=    model(**snake_case_				)

     A__	=    outputs.encoder_last_hidden_state
     A__	=    outputs.last_hidden_state

     with tempfile.TemporaryDirectory() as tmpdirname:
      A__	=    model.get_encoder()
      encoder.save_pretrained(snake_case_				)
      A__	=    AutoformerEncoder.from_pretrained(snake_case_				).to(snake_case_				)

     A__,				A__,				A__,				A__,				A__	=    model.create_network_inputs(**snake_case_				)
     A__,				A__	=    model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]				)

     A__	=    torch.cat(
         (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])     ,			dim=-1     ,			)
     A__	=    encoder(inputs_embeds=snake_case_				)[0]
     self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3				)

     A__	=    (
         torch.mean(transformer_inputs[:, : config.context_length, ...]     ,			dim=1				)
         .unsqueeze(1				)
         .repeat(1     ,			config.prediction_length     ,			1				)
     )
     A__	=    torch.zeros(
         [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]     ,			device=enc_input.device     ,			)

     A__	=    torch.cat(
         (
             torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)     ,			dim=1				),
             feature[:, config.context_length - config.label_length :, ...],
         )     ,			dim=-1     ,			)
     A__	=    torch.cat(
         (
             torch.cat((trend_input[:, -config.label_length :, ...], mean)     ,			dim=1				),
             feature[:, config.context_length - config.label_length :, ...],
         )     ,			dim=-1     ,			)

     with tempfile.TemporaryDirectory() as tmpdirname:
      A__	=    model.get_decoder()
      decoder.save_pretrained(snake_case_				)
      A__	=    AutoformerDecoder.from_pretrained(snake_case_				).to(snake_case_				)

     A__	=    decoder(
         trend=snake_case_     ,			inputs_embeds=snake_case_     ,			encoder_hidden_states=snake_case_     ,			)[0]

     self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3				)






@require_torch
class      UpperCAmelCase_  (						A_,       A_,       unittest.TestCase							):
    lowercase__						 =				(AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
    lowercase__						 =				(AutoformerForPrediction,) if is_torch_available() else ()
    lowercase__						 =				{'''feature-extraction''': AutoformerModel} if is_torch_available() else {}
    lowercase__						 =				False
    lowercase__						 =				False
    lowercase__						 =				False
    lowercase__						 =				False
    lowercase__						 =				False
    lowercase__						 =				False




    def 							__magic_name__							(   self						:	List[Any]				)				->							Tuple:




     '''simple docstring'''

     A__	=    AutoformerModelTester(self				)
     A__	=    ConfigTester(self     ,			config_class=snake_case_     ,			has_text_modality=snake_case_				)




    def 							__magic_name__							(   self						:	Optional[int]				)				->							Optional[Any]:




     '''simple docstring'''

     self.config_tester.run_common_tests()




    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							Any:




     '''simple docstring'''

     A__,				A__	=    self.model_tester.prepare_config_and_inputs()
     for model_class in self.all_model_classes:
      A__	=    model_class(snake_case_				)

      with tempfile.TemporaryDirectory() as tmpdirname:
       model.save_pretrained(snake_case_				)
       A__,				A__	=    model_class.from_pretrained(snake_case_     ,			output_loading_info=snake_case_				)
      self.assertEqual(info["missing_keys"]     ,			[]				)




    def 							__magic_name__							(   self						:	Any				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.check_encoder_decoder_model_standalone(*snake_case_				)




    @unittest.skip(reason="Model has no tokens embeddings"				)
    def 							__magic_name__							(   self						:	str				)				->							int:




     '''simple docstring'''

     pass




    def 							__magic_name__							(   self						:	Dict				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    inspect.signature(getattr(snake_case_     ,			"forward"				)				)
     # The main input is the name of the argument after `self`
     A__	=    list(model_signature.parameters.keys()				)[1]
     self.assertEqual(AutoformerModel.main_input_name     ,			snake_case_				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Any:




     '''simple docstring'''

     A__,				A__	=    self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      A__	=    model_class(snake_case_				)
      A__	=    inspect.signature(model.forward				)
      # signature.parameters is an OrderedDict => so arg_names order is deterministic
      A__	=    [*signature.parameters.keys()]

      A__	=    [
          "past_values",
          "past_time_features",
          "past_observed_mask",
          "static_categorical_features",
          "static_real_features",
          "future_values",
          "future_time_features",
      ]

      if model.__class__.__name__ in ["AutoformerForPrediction"]:
       expected_arg_names.append("future_observed_mask"				)

      expected_arg_names.extend(
          [
              "decoder_attention_mask",
              "head_mask",
              "decoder_head_mask",
              "cross_attn_head_mask",
              "encoder_outputs",
              "past_key_values",
              "output_hidden_states",
              "output_attentions",
              "use_cache",
              "return_dict",
          ]				)

      self.assertListEqual(arg_names[: len(snake_case_				)]     ,			snake_case_				)




    def 							__magic_name__							(   self						:	int				)				->							Optional[Any]:




     '''simple docstring'''

     A__,				A__	=    self.model_tester.prepare_config_and_inputs_for_common()
     A__	=    True

     A__	=    getattr(self.model_tester     ,			"seq_length"     ,			snake_case_				)
     A__	=    getattr(self.model_tester     ,			"decoder_seq_length"     ,			snake_case_				)
     A__	=    getattr(self.model_tester     ,			"encoder_seq_length"     ,			snake_case_				)
     A__	=    getattr(self.model_tester     ,			"d_model"     ,			snake_case_				)
     A__	=    getattr(self.model_tester     ,			"num_attention_heads"     ,			snake_case_				)
     A__	=    d_model // num_attention_heads

     for model_class in self.all_model_classes:
      A__	=    True
      A__	=    False
      A__	=    True
      A__	=    model_class(snake_case_				)
      model.to(snake_case_				)
      model.eval()
      with torch.no_grad():
       A__	=    model(**self._prepare_for_class(snake_case_     ,			snake_case_				)				)
      A__	=    outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
      self.assertEqual(len(snake_case_				)     ,			self.model_tester.num_hidden_layers				)

      # check that output_attentions also work using config
      del inputs_dict["output_attentions"]
      A__	=    True
      A__	=    model_class(snake_case_				)
      model.to(snake_case_				)
      model.eval()
      with torch.no_grad():
       A__	=    model(**self._prepare_for_class(snake_case_     ,			snake_case_				)				)
      A__	=    outputs.encoder_attentions
      self.assertEqual(len(snake_case_				)     ,			self.model_tester.num_hidden_layers				)

      self.assertListEqual(
          list(attentions[0].shape[-3:]				)     ,			[self.model_tester.num_attention_heads, encoder_seq_length, dim]     ,			)
      A__	=    len(snake_case_				)

      A__	=    7

      if "last_hidden_state" in outputs:
       correct_outlen += 1

      if "trend" in outputs:
       correct_outlen += 1

      if "past_key_values" in outputs:
       correct_outlen += 1  # past_key_values have been returned

      if "loss" in outputs:
       correct_outlen += 1

      if "params" in outputs:
       correct_outlen += 1

      self.assertEqual(snake_case_     ,			snake_case_				)

      # decoder attentions
      A__	=    outputs.decoder_attentions
      self.assertIsInstance(snake_case_     ,			(list, tuple)				)
      self.assertEqual(len(snake_case_				)     ,			self.model_tester.num_hidden_layers				)
      self.assertListEqual(
          list(decoder_attentions[0].shape[-3:]				)     ,			[self.model_tester.num_attention_heads, decoder_seq_length, dim]     ,			)

      # cross attentions
      A__	=    outputs.cross_attentions
      self.assertIsInstance(snake_case_     ,			(list, tuple)				)
      self.assertEqual(len(snake_case_				)     ,			self.model_tester.num_hidden_layers				)
      self.assertListEqual(
          list(cross_attentions[0].shape[-3:]				)     ,			[self.model_tester.num_attention_heads, decoder_seq_length, dim]     ,			)

     # Check attention is always last and order is fine
     A__	=    True
     A__	=    True
     A__	=    model_class(snake_case_				)
     model.to(snake_case_				)
     model.eval()
     with torch.no_grad():
      A__	=    model(**self._prepare_for_class(snake_case_     ,			snake_case_				)				)

     self.assertEqual(out_len + 2     ,			len(snake_case_				)				)

     A__	=    outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

     self.assertEqual(len(snake_case_				)     ,			self.model_tester.num_hidden_layers				)
     self.assertListEqual(
         list(self_attentions[0].shape[-3:]				)     ,			[self.model_tester.num_attention_heads, encoder_seq_length, dim]     ,			)




    @is_flaky()
    def 							__magic_name__							(   self						:	List[str]				)				->							Dict:




     '''simple docstring'''

     super().test_retain_grad_hidden_states_attentions()







def   _SCREAMING_SNAKE_CASE       (							lowercase_="train-batch.pt"						)						->		Dict:
 A__	=    hf_hub_download(repo_id="hf-internal-testing/tourism-monthly-batch"      ,			filename=lowercase_      ,			repo_type="dataset"						)
 A__	=    torch.load(lowercase_      ,			map_location=lowercase_						)
 return batch






@require_torch
@slow
class      UpperCAmelCase_  (						unittest.TestCase							):




    def 							__magic_name__							(   self						:	List[str]				)				->							int:




     '''simple docstring'''

     A__	=    AutoformerModel.from_pretrained("huggingface/autoformer-tourism-monthly"				).to(snake_case_				)
     A__	=    prepare_batch()

     with torch.no_grad():
      A__	=    model(
          past_values=batch["past_values"]     ,			past_time_features=batch["past_time_features"]     ,			past_observed_mask=batch["past_observed_mask"]     ,			static_categorical_features=batch["static_categorical_features"]     ,			future_values=batch["future_values"]     ,			future_time_features=batch["future_time_features"]     ,			)[0]

     A__	=    torch.Size(
         (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)				)
     self.assertEqual(output.shape     ,			snake_case_				)

     A__	=    torch.tensor(
         [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]     ,			device=snake_case_				)
     self.assertTrue(torch.allclose(output[0, :3, :3]     ,			snake_case_     ,			atol=snake_case_				)				)




    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							Any:




     '''simple docstring'''

     A__	=    AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly"				).to(snake_case_				)
     A__	=    prepare_batch("val-batch.pt"				)
     with torch.no_grad():
      A__	=    model(
          past_values=batch["past_values"]     ,			past_time_features=batch["past_time_features"]     ,			past_observed_mask=batch["past_observed_mask"]     ,			static_categorical_features=batch["static_categorical_features"]     ,			).encoder_last_hidden_state
     A__	=    torch.Size((64, model.config.context_length, model.config.d_model)				)
     self.assertEqual(output.shape     ,			snake_case_				)

     A__	=    torch.tensor(
         [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]     ,			device=snake_case_				)
     self.assertTrue(torch.allclose(output[0, :3, :3]     ,			snake_case_     ,			atol=snake_case_				)				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							Dict:




     '''simple docstring'''

     A__	=    AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly"				).to(snake_case_				)
     A__	=    prepare_batch("val-batch.pt"				)
     with torch.no_grad():
      A__	=    model.generate(
          static_categorical_features=batch["static_categorical_features"]     ,			past_time_features=batch["past_time_features"]     ,			past_values=batch["past_values"]     ,			future_time_features=batch["future_time_features"]     ,			past_observed_mask=batch["past_observed_mask"]     ,			)
     A__	=    torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)				)
     self.assertEqual(outputs.sequences.shape     ,			snake_case_				)

     A__	=    torch.tensor([3130.6763, 4056.5293, 7053.0786]     ,			device=snake_case_				)
     A__	=    outputs.sequences.mean(dim=1				)
     self.assertTrue(torch.allclose(mean_prediction[0, -3:]     ,			snake_case_     ,			rtol=1e-1				)				)











"""simple docstring"""



from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE    =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE    =	{
    "SCUT-DLVCLab/lilt-roberta-en-base": (
        "https://huggingface.co/SCUT-DLVCLab/lilt-roberta-en-base/resolve/main/config.json"
    ),
}




class      UpperCAmelCase_  (						A_							):
    lowercase__						 =				'''lilt'''




    def __init__(   self						:	List[str]     ,			snake_case_						:	Any=30_522     ,			snake_case_						:	Optional[Any]=768     ,			snake_case_						:	Union[str, Any]=12     ,			snake_case_						:	Union[str, Any]=12     ,			snake_case_						:	Any=3_072     ,			snake_case_						:	List[str]="gelu"     ,			snake_case_						:	Union[str, Any]=0.1     ,			snake_case_						:	List[Any]=0.1     ,			snake_case_						:	Any=512     ,			snake_case_						:	Optional[Any]=2     ,			snake_case_						:	List[str]=0.02     ,			snake_case_						:	Optional[Any]=1e-12     ,			snake_case_						:	List[Any]=0     ,			snake_case_						:	Any="absolute"     ,			snake_case_						:	str=None     ,			snake_case_						:	int=4     ,			snake_case_						:	int=1_024     ,			**snake_case_						:	Tuple     ,			)				->							List[Any]:




     '''simple docstring'''

     super().__init__(pad_token_id=snake_case_     ,			**snake_case_				)

     A__	=    vocab_size
     A__	=    hidden_size
     A__	=    num_hidden_layers
     A__	=    num_attention_heads
     A__	=    hidden_act
     A__	=    intermediate_size
     A__	=    hidden_dropout_prob
     A__	=    attention_probs_dropout_prob
     A__	=    max_position_embeddings
     A__	=    type_vocab_size
     A__	=    initializer_range
     A__	=    layer_norm_eps
     A__	=    position_embedding_type
     A__	=    classifier_dropout
     A__	=    channel_shrink_ratio
     A__	=    max_ad_position_embeddings





