


'''simple docstring'''






import inspect
import unittest

from transformers import SegformerConfig, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_MAPPING,
     SegformerForImageClassification,
     SegformerForSemanticSegmentation,
     SegformerModel,
 )
 from transformers.models.segformer.modeling_segformer import SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import SegformerImageProcessor






class  __magic_name__							(		lowerCAmelCase    ):






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =self.config_class(**self.inputs_dict)
          self.parent.assertTrue(hasattr(snake_case  ,							'hidden_sizes'))
          self.parent.assertTrue(hasattr(snake_case  ,							'num_attention_heads'))
          self.parent.assertTrue(hasattr(snake_case  ,							'num_encoder_blocks'))






class  __magic_name__							:






       def __init__(      self  ,							snake_case  ,							snake_case=1_3  ,							snake_case=6_4  ,							snake_case=3  ,							snake_case=4  ,							snake_case=[2, 2, 2, 2]  ,							snake_case=[8, 4, 2, 1]  ,							snake_case=[1_6, 3_2, 6_4, 1_2_8]  ,							snake_case=[1, 4, 8, 1_6]  ,							snake_case=[1, 2, 4, 8]  ,							snake_case=True  ,							snake_case=True  ,							snake_case="gelu"  ,							snake_case=0.1  ,							snake_case=0.1  ,							snake_case=0.02  ,							snake_case=3  ,							snake_case=None  ,							)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =parent
          _UpperCAmelCase       :       List[str]							   =batch_size
          _UpperCAmelCase       :       List[Any]							   =image_size
          _UpperCAmelCase       :       List[Any]							   =num_channels
          _UpperCAmelCase       :       Any							   =num_encoder_blocks
          _UpperCAmelCase       :       Optional[int]							   =sr_ratios
          _UpperCAmelCase       :       Optional[Any]							   =depths
          _UpperCAmelCase       :       List[str]							   =hidden_sizes
          _UpperCAmelCase       :       int							   =downsampling_rates
          _UpperCAmelCase       :       Dict							   =num_attention_heads
          _UpperCAmelCase       :       int							   =is_training
          _UpperCAmelCase       :       Dict							   =use_labels
          _UpperCAmelCase       :       List[str]							   =hidden_act
          _UpperCAmelCase       :       List[Any]							   =hidden_dropout_prob
          _UpperCAmelCase       :       str							   =attention_probs_dropout_prob
          _UpperCAmelCase       :       str							   =initializer_range
          _UpperCAmelCase       :       Optional[Any]							   =num_labels
          _UpperCAmelCase       :       Any							   =scope






       def       lowerCAmelCase		(      self)						->  List[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

          _UpperCAmelCase       :       Any							   =None
          if self.use_labels:
             _UpperCAmelCase       :       Union[str, Any]							   =ids_tensor([self.batch_size, self.image_size, self.image_size]  ,							self.num_labels)

          _UpperCAmelCase       :       int							   =self.get_config()
          return config, pixel_values, labels






       def       lowerCAmelCase		(      self)						->  List[Any]:




          '''simple docstring'''
          return SegformerConfig(
              image_size=self.image_size  ,							num_channels=self.num_channels  ,							num_encoder_blocks=self.num_encoder_blocks  ,							depths=self.depths  ,							hidden_sizes=self.hidden_sizes  ,							num_attention_heads=self.num_attention_heads  ,							hidden_act=self.hidden_act  ,							hidden_dropout_prob=self.hidden_dropout_prob  ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,							initializer_range=self.initializer_range  ,							)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case  ,							snake_case)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =SegformerModel(config=snake_case)
          model.to(snake_case)
          model.eval()
          _UpperCAmelCase       :       List[Any]							   =model(snake_case)
          _UpperCAmelCase       :       Dict							   =self.image_size // (self.downsampling_rates[-1] * 2)
          self.parent.assertEqual(
              result.last_hidden_state.shape  ,							(self.batch_size, self.hidden_sizes[-1], expected_height, expected_width))






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case  ,							snake_case)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =self.num_labels
          _UpperCAmelCase       :       str							   =SegformerForSemanticSegmentation(snake_case)
          model.to(snake_case)
          model.eval()
          _UpperCAmelCase       :       Optional[int]							   =model(snake_case)
          self.parent.assertEqual(
              result.logits.shape  ,							(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
          _UpperCAmelCase       :       int							   =model(snake_case  ,							labels=snake_case)
          self.parent.assertEqual(
              result.logits.shape  ,							(self.batch_size, self.num_labels, self.image_size // 4, self.image_size // 4))
          self.parent.assertGreater(result.loss  ,							0.0)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case  ,							snake_case)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[Any]							   =1
          _UpperCAmelCase       :       List[str]							   =SegformerForSemanticSegmentation(config=snake_case)
          model.to(snake_case)
          model.eval()
          _UpperCAmelCase       :       Union[str, Any]							   =torch.randint(0  ,							1  ,							(self.batch_size, self.image_size, self.image_size)).to(snake_case)
          _UpperCAmelCase       :       Optional[int]							   =model(snake_case  ,							labels=snake_case)
          self.parent.assertGreater(result.loss  ,							0.0)






       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =self.prepare_config_and_inputs()
          _UpperCAmelCase       :       List[Any]							   =config_and_inputs
          _UpperCAmelCase       :       Optional[int]							   ={'pixel_values': pixel_values}
          return config, inputs_dict






@require_torch
class  __magic_name__							(		lowerCAmelCase ,lowerCAmelCase ,unittest.TestCase    ):
       UpperCAmelCase										=(
           (
               SegformerModel,
               SegformerForSemanticSegmentation,
               SegformerForImageClassification,
           )
           if is_torch_available()
           else ()
       )
       UpperCAmelCase										=(
           {
               "feature-extraction": SegformerModel,
               "image-classification": SegformerForImageClassification,
               "image-segmentation": SegformerForSemanticSegmentation,
           }
           if is_torch_available()
           else {}
       )

       UpperCAmelCase										=True
       UpperCAmelCase										=False
       UpperCAmelCase										=False
       UpperCAmelCase										=False






       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase       :       Tuple							   =SegformerModelTester(self)
          _UpperCAmelCase       :       Tuple							   =SegformerConfigTester(self  ,							config_class=snake_case)






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          self.config_tester.run_common_tests()






       def       lowerCAmelCase		(      self)						->  Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*snake_case)






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[int]							   =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_binary_image_segmentation(*snake_case)






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_image_segmentation(*snake_case)






       @unittest.skip('SegFormer does not use inputs_embeds')
       def       lowerCAmelCase		(      self)						->  List[Any]:




          '''simple docstring'''
          pass






       @unittest.skip('SegFormer does not have get_input_embeddings method and get_output_embeddings methods')
       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          pass






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       Tuple							   =self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
             _UpperCAmelCase       :       List[str]							   =model_class(snake_case)
             _UpperCAmelCase       :       Dict							   =inspect.signature(model.forward)
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             _UpperCAmelCase       :       Optional[int]							   =[*signature.parameters.keys()]

             _UpperCAmelCase       :       Optional[int]							   =['pixel_values']
             self.assertListEqual(arg_names[:1]  ,							snake_case)






       def       lowerCAmelCase		(      self)						->  Dict:




          '''simple docstring'''
          _UpperCAmelCase       :       Union[str, Any]							   =self.model_tester.prepare_config_and_inputs_for_common()
          _UpperCAmelCase       :       Tuple							   =True

          for model_class in self.all_model_classes:
             _UpperCAmelCase       :       List[Any]							   =True
             _UpperCAmelCase       :       Dict							   =False
             _UpperCAmelCase       :       Optional[Any]							   =True
             _UpperCAmelCase       :       Any							   =model_class(snake_case)
             model.to(snake_case)
             model.eval()
             with torch.no_grad():
                _UpperCAmelCase       :       List[Any]							   =model(**self._prepare_for_class(snake_case  ,							snake_case))
             _UpperCAmelCase       :       str							   =outputs.attentions

             _UpperCAmelCase       :       Dict							   =sum(self.model_tester.depths)
             self.assertEqual(len(snake_case)  ,							snake_case)

             # check that output_attentions also work using config
             del inputs_dict["output_attentions"]
             _UpperCAmelCase       :       str							   =True
             _UpperCAmelCase       :       str							   =model_class(snake_case)
             model.to(snake_case)
             model.eval()
             with torch.no_grad():
                _UpperCAmelCase       :       Tuple							   =model(**self._prepare_for_class(snake_case  ,							snake_case))
             _UpperCAmelCase       :       List[str]							   =outputs.attentions

             self.assertEqual(len(snake_case)  ,							snake_case)

             # verify the first attentions (first block, first layer)
             _UpperCAmelCase       :       str							   =(self.model_tester.image_size // 4) ** 2
             _UpperCAmelCase       :       Optional[int]							   =(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
             self.assertListEqual(
                 list(attentions[0].shape[-3:])  ,							[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]  ,							)

             # verify the last attentions (last block, last layer)
             _UpperCAmelCase       :       Tuple							   =(self.model_tester.image_size // 3_2) ** 2
             _UpperCAmelCase       :       Union[str, Any]							   =(self.model_tester.image_size // (3_2 * self.model_tester.sr_ratios[-1])) ** 2
             self.assertListEqual(
                 list(attentions[-1].shape[-3:])  ,							[self.model_tester.num_attention_heads[-1], expected_seq_len, expected_reduced_seq_len]  ,							)
             _UpperCAmelCase       :       Any							   =len(snake_case)

             # Check attention is always last and order is fine
             _UpperCAmelCase       :       Union[str, Any]							   =True
             _UpperCAmelCase       :       int							   =True
             _UpperCAmelCase       :       List[Any]							   =model_class(snake_case)
             model.to(snake_case)
             model.eval()
             with torch.no_grad():
                _UpperCAmelCase       :       Any							   =model(**self._prepare_for_class(snake_case  ,							snake_case))

             self.assertEqual(out_len + 1  ,							len(snake_case))

             _UpperCAmelCase       :       Dict							   =outputs.attentions

             self.assertEqual(len(snake_case)  ,							snake_case)
             # verify the first attentions (first block, first layer)
             _UpperCAmelCase       :       Optional[int]							   =(self.model_tester.image_size // 4) ** 2
             _UpperCAmelCase       :       Any							   =(self.model_tester.image_size // (4 * self.model_tester.sr_ratios[0])) ** 2
             self.assertListEqual(
                 list(self_attentions[0].shape[-3:])  ,							[self.model_tester.num_attention_heads[0], expected_seq_len, expected_reduced_seq_len]  ,							)






       def       lowerCAmelCase		(      self)						->  int:




          '''simple docstring'''
          def check_hidden_states_output(snake_case  ,							snake_case  ,							snake_case):
             _UpperCAmelCase       :       int							   =model_class(snake_case)
             model.to(snake_case)
             model.eval()

             with torch.no_grad():
                _UpperCAmelCase       :       Union[str, Any]							   =model(**self._prepare_for_class(snake_case  ,							snake_case))

             _UpperCAmelCase       :       Optional[int]							   =outputs.hidden_states

             _UpperCAmelCase       :       Dict							   =self.model_tester.num_encoder_blocks
             self.assertEqual(len(snake_case)  ,							snake_case)

             # verify the first hidden states (first block)
             self.assertListEqual(
                 list(hidden_states[0].shape[-3:])  ,							[
                     self.model_tester.hidden_sizes[0],
                     self.model_tester.image_size // 4,
                     self.model_tester.image_size // 4,
                 ]  ,							)

          _UpperCAmelCase       :       List[Any]							   =self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
             _UpperCAmelCase       :       Optional[int]							   =True
             check_hidden_states_output(snake_case  ,							snake_case  ,							snake_case)

             # check that output_hidden_states also work using config
             del inputs_dict["output_hidden_states"]
             _UpperCAmelCase       :       Union[str, Any]							   =True

             check_hidden_states_output(snake_case  ,							snake_case  ,							snake_case)






       def       lowerCAmelCase		(      self)						->  List[Any]:




          '''simple docstring'''
          if not self.model_tester.is_training:
             return

          _UpperCAmelCase       :       Optional[Any]							   =self.model_tester.prepare_config_and_inputs_for_common()
          _UpperCAmelCase       :       Dict							   =True

          for model_class in self.all_model_classes:
             if model_class in get_values(snake_case):
                continue

             _UpperCAmelCase       :       List[str]							   =model_class(snake_case)
             model.to(snake_case)
             model.train()
             _UpperCAmelCase       :       Any							   =self._prepare_for_class(snake_case  ,							snake_case  ,							return_labels=snake_case)
             _UpperCAmelCase       :       List[str]							   =model(**snake_case).loss
             loss.backward()






       @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.')
       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          pass






       @slow
       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          for model_name in SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             _UpperCAmelCase       :       List[Any]							   =SegformerModel.from_pretrained(snake_case)
             self.assertIsNotNone(snake_case)






def lowerCamelCase__      (						):







   '''simple docstring'''




   _UpperCAmelCase       :       Optional[Any]							   =Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							)
   return image






@require_torch
class  __magic_name__							(		unittest.TestCase    ):






       @slow
       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =SegformerImageProcessor(
              image_scale=(5_1_2, 5_1_2)  ,							keep_ratio=snake_case  ,							align=snake_case  ,							do_random_crop=snake_case)
          _UpperCAmelCase       :       Optional[Any]							   =SegformerForSemanticSegmentation.from_pretrained('nvidia/segformer-b0-finetuned-ade-512-512').to(
              snake_case)

          _UpperCAmelCase       :       Union[str, Any]							   =prepare_img()
          _UpperCAmelCase       :       Optional[Any]							   =image_processor(images=snake_case  ,							return_tensors='pt')
          _UpperCAmelCase       :       str							   =encoded_inputs.pixel_values.to(snake_case)

          with torch.no_grad():
             _UpperCAmelCase       :       Any							   =model(snake_case)

          _UpperCAmelCase       :       Optional[int]							   =torch.Size((1, model.config.num_labels, 1_2_8, 1_2_8))
          self.assertEqual(outputs.logits.shape  ,							snake_case)

          _UpperCAmelCase       :       Union[str, Any]							   =torch.tensor(
              [
                  [[-4.63_10, -5.52_32, -6.23_56], [-5.19_21, -6.14_44, -6.59_96], [-5.44_24, -6.27_90, -6.75_74]],
                  [[-12.13_91, -13.31_22, -13.95_54], [-12.87_32, -13.93_52, -14.35_63], [-12.94_38, -13.82_26, -14.25_13]],
                  [[-12.51_34, -13.46_86, -14.49_15], [-12.86_69, -14.43_43, -14.77_58], [-13.25_23, -14.58_19, -15.06_94]],
              ]).to(snake_case)
          self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]  ,							snake_case  ,							atol=1E-4))






       @slow
       def       lowerCAmelCase		(      self)						->  int:




          '''simple docstring'''
          _UpperCAmelCase       :       List[Any]							   =SegformerImageProcessor(
              image_scale=(5_1_2, 5_1_2)  ,							keep_ratio=snake_case  ,							align=snake_case  ,							do_random_crop=snake_case)
          _UpperCAmelCase       :       str							   =SegformerForSemanticSegmentation.from_pretrained(
              'nvidia/segformer-b1-finetuned-cityscapes-1024-1024').to(snake_case)

          _UpperCAmelCase       :       Tuple							   =prepare_img()
          _UpperCAmelCase       :       Optional[Any]							   =image_processor(images=snake_case  ,							return_tensors='pt')
          _UpperCAmelCase       :       str							   =encoded_inputs.pixel_values.to(snake_case)

          with torch.no_grad():
             _UpperCAmelCase       :       Optional[int]							   =model(snake_case)

          _UpperCAmelCase       :       Optional[Any]							   =torch.Size((1, model.config.num_labels, 1_2_8, 1_2_8))
          self.assertEqual(outputs.logits.shape  ,							snake_case)

          _UpperCAmelCase       :       Optional[Any]							   =torch.tensor(
              [
                  [[-13.57_48, -13.91_11, -12.65_00], [-14.35_00, -15.36_83, -14.23_28], [-14.75_32, -16.04_24, -15.60_87]],
                  [[-17.16_51, -15.87_25, -12.96_53], [-17.25_80, -17.37_18, -14.82_23], [-16.60_58, -16.87_83, -16.74_52]],
                  [[-3.64_56, -3.02_09, -1.42_03], [-3.07_97, -3.19_59, -2.00_00], [-1.87_57, -1.92_17, -1.69_97]],
              ]).to(snake_case)
          self.assertTrue(torch.allclose(outputs.logits[0, :3, :3, :3]  ,							snake_case  ,							atol=1E-1))






       @slow
       def       lowerCAmelCase		(      self)						->  List[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[int]							   =SegformerImageProcessor(
              image_scale=(5_1_2, 5_1_2)  ,							keep_ratio=snake_case  ,							align=snake_case  ,							do_random_crop=snake_case)
          _UpperCAmelCase       :       Union[str, Any]							   =SegformerForSemanticSegmentation.from_pretrained('nvidia/segformer-b0-finetuned-ade-512-512').to(
              snake_case)

          _UpperCAmelCase       :       List[str]							   =prepare_img()
          _UpperCAmelCase       :       str							   =image_processor(images=snake_case  ,							return_tensors='pt')
          _UpperCAmelCase       :       Union[str, Any]							   =encoded_inputs.pixel_values.to(snake_case)

          with torch.no_grad():
             _UpperCAmelCase       :       Optional[Any]							   =model(snake_case)

          _UpperCAmelCase       :       Any							   =outputs.logits.detach().cpu()

          _UpperCAmelCase       :       Union[str, Any]							   =image_processor.post_process_semantic_segmentation(outputs=snake_case  ,							target_sizes=[(5_0_0, 3_0_0)])
          _UpperCAmelCase       :       List[str]							   =torch.Size((5_0_0, 3_0_0))
          self.assertEqual(segmentation[0].shape  ,							snake_case)

          _UpperCAmelCase       :       Union[str, Any]							   =image_processor.post_process_semantic_segmentation(outputs=snake_case)
          _UpperCAmelCase       :       int							   =torch.Size((1_2_8, 1_2_8))
          self.assertEqual(segmentation[0].shape  ,							snake_case)





'''simple docstring'''






import unittest

import numpy as np

from transformers import RobertaPreLayerNormConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
 import jax.numpy as jnp

 from transformers.models.roberta_prelayernorm.modeling_flax_roberta_prelayernorm import (
     FlaxRobertaPreLayerNormForCausalLM,
     FlaxRobertaPreLayerNormForMaskedLM,
     FlaxRobertaPreLayerNormForMultipleChoice,
     FlaxRobertaPreLayerNormForQuestionAnswering,
     FlaxRobertaPreLayerNormForSequenceClassification,
     FlaxRobertaPreLayerNormForTokenClassification,
     FlaxRobertaPreLayerNormModel,
 )






class  __magic_name__							(		unittest.TestCase    ):






       def __init__(      self  ,							snake_case  ,							snake_case=1_3  ,							snake_case=7  ,							snake_case=True  ,							snake_case=True  ,							snake_case=True  ,							snake_case=True  ,							snake_case=9_9  ,							snake_case=3_2  ,							snake_case=5  ,							snake_case=4  ,							snake_case=3_7  ,							snake_case="gelu"  ,							snake_case=0.1  ,							snake_case=0.1  ,							snake_case=5_1_2  ,							snake_case=1_6  ,							snake_case=2  ,							snake_case=0.02  ,							snake_case=4  ,							)						->  Dict:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =parent
          _UpperCAmelCase       :       Dict							   =batch_size
          _UpperCAmelCase       :       List[Any]							   =seq_length
          _UpperCAmelCase       :       List[str]							   =is_training
          _UpperCAmelCase       :       Optional[int]							   =use_attention_mask
          _UpperCAmelCase       :       Dict							   =use_token_type_ids
          _UpperCAmelCase       :       Dict							   =use_labels
          _UpperCAmelCase       :       Optional[Any]							   =vocab_size
          _UpperCAmelCase       :       str							   =hidden_size
          _UpperCAmelCase       :       Dict							   =num_hidden_layers
          _UpperCAmelCase       :       Tuple							   =num_attention_heads
          _UpperCAmelCase       :       List[str]							   =intermediate_size
          _UpperCAmelCase       :       List[str]							   =hidden_act
          _UpperCAmelCase       :       int							   =hidden_dropout_prob
          _UpperCAmelCase       :       Optional[int]							   =attention_probs_dropout_prob
          _UpperCAmelCase       :       Optional[Any]							   =max_position_embeddings
          _UpperCAmelCase       :       Union[str, Any]							   =type_vocab_size
          _UpperCAmelCase       :       Dict							   =type_sequence_label_size
          _UpperCAmelCase       :       Union[str, Any]							   =initializer_range
          _UpperCAmelCase       :       Any							   =num_choices






       def       lowerCAmelCase		(      self)						->  Dict:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =ids_tensor([self.batch_size, self.seq_length]  ,							self.vocab_size)

          _UpperCAmelCase       :       str							   =None
          if self.use_attention_mask:
             _UpperCAmelCase       :       Dict							   =random_attention_mask([self.batch_size, self.seq_length])

          _UpperCAmelCase       :       Optional[Any]							   =None
          if self.use_token_type_ids:
             _UpperCAmelCase       :       Tuple							   =ids_tensor([self.batch_size, self.seq_length]  ,							self.type_vocab_size)

          _UpperCAmelCase       :       Union[str, Any]							   =RobertaPreLayerNormConfig(
              vocab_size=self.vocab_size  ,							hidden_size=self.hidden_size  ,							num_hidden_layers=self.num_hidden_layers  ,							num_attention_heads=self.num_attention_heads  ,							intermediate_size=self.intermediate_size  ,							hidden_act=self.hidden_act  ,							hidden_dropout_prob=self.hidden_dropout_prob  ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,							max_position_embeddings=self.max_position_embeddings  ,							type_vocab_size=self.type_vocab_size  ,							is_decoder=snake_case  ,							initializer_range=self.initializer_range  ,							)

          return config, input_ids, token_type_ids, attention_mask






       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =self.prepare_config_and_inputs()
          _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       str							   =config_and_inputs
          _UpperCAmelCase       :       List[Any]							   ={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
          return config, inputs_dict






       def       lowerCAmelCase		(      self)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Tuple							   =self.prepare_config_and_inputs()
          _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       Optional[int]							   =config_and_inputs

          _UpperCAmelCase       :       Tuple							   =True
          _UpperCAmelCase       :       Any							   =floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          _UpperCAmelCase       :       Optional[Any]							   =ids_tensor([self.batch_size, self.seq_length]  ,							vocab_size=2)

          return (
              config,
              input_ids,
              token_type_ids,
              encoder_hidden_states,
              encoder_attention_mask,
          )






@require_flax
# Copied from tests.models.roberta.test_modelling_flax_roberta.FlaxRobertaPreLayerNormModelTest with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta-base->andreasmadsen/efficient_mlm_m0.40
class  __magic_name__							(		lowerCAmelCase ,unittest.TestCase    ):
       UpperCAmelCase										=True

       UpperCAmelCase										=(
           (
               FlaxRobertaPreLayerNormModel,
               FlaxRobertaPreLayerNormForCausalLM,
               FlaxRobertaPreLayerNormForMaskedLM,
               FlaxRobertaPreLayerNormForSequenceClassification,
               FlaxRobertaPreLayerNormForTokenClassification,
               FlaxRobertaPreLayerNormForMultipleChoice,
               FlaxRobertaPreLayerNormForQuestionAnswering,
           )
           if is_flax_available()
           else ()
       )






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       int							   =FlaxRobertaPreLayerNormModelTester(self)






       @slow
       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          for model_class_name in self.all_model_classes:
             _UpperCAmelCase       :       List[str]							   =model_class_name.from_pretrained('andreasmadsen/efficient_mlm_m0.40'  ,							from_pt=snake_case)
             _UpperCAmelCase       :       Dict							   =model(np.ones((1, 1)))
             self.assertIsNotNone(snake_case)






@require_flax
class  __magic_name__							(		unittest.TestCase    ):






       @slow
       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase       :       Tuple							   =FlaxRobertaPreLayerNormForMaskedLM.from_pretrained('andreasmadsen/efficient_mlm_m0.40'  ,							from_pt=snake_case)

          _UpperCAmelCase       :       Optional[int]							   =np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]  ,							dtype=jnp.intaa)
          _UpperCAmelCase       :       str							   =model(snake_case)[0]
          _UpperCAmelCase       :       int							   =[1, 1_1, 5_0_2_6_5]
          self.assertEqual(list(output.shape)  ,							snake_case)
          # compare the actual values for a slice.
          _UpperCAmelCase       :       List[str]							   =np.array(
              [[[40.48_80, 18.01_99, -5.23_67], [-1.88_77, -4.08_85, 10.70_85], [-2.26_13, -5.61_10, 7.26_65]]]  ,							dtype=np.floataa)
          self.assertTrue(np.allclose(output[:, :3, :3]  ,							snake_case  ,							atol=1E-4))






       @slow
       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =FlaxRobertaPreLayerNormModel.from_pretrained('andreasmadsen/efficient_mlm_m0.40'  ,							from_pt=snake_case)

          _UpperCAmelCase       :       List[str]							   =np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]  ,							dtype=jnp.intaa)
          _UpperCAmelCase       :       Tuple							   =model(snake_case)[0]
          # compare the actual values for a slice.
          _UpperCAmelCase       :       List[str]							   =np.array(
              [[[0.02_08, -0.03_56, 0.02_37], [-0.15_69, -0.04_11, -0.26_26], [0.18_79, 0.01_25, -0.00_89]]]  ,							dtype=np.floataa)
          self.assertTrue(np.allclose(output[:, :3, :3]  ,							snake_case  ,							atol=1E-4))

