'''simple docstring'''

import copy
import inspect
import unittest

from transformers import PretrainedConfig, SwiftFormerConfig
from transformers.testing_utils import (
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch
				from torch import nn

				from transformers import SwiftFormerForImageClassification, SwiftFormerModel
				from transformers.models.swiftformer.modeling_swiftformer import SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
				from PIL import Image

				from transformers import ViTImageProcessor

class __snake_case:


						'''simple docstring'''



						def __init__( self     ,	A_     ,	A_=13     ,	A_=3     ,	A_=True     ,	A_=True     ,	A_=0.1     ,	A_=0.1     ,	A_=224     ,	A_=1000     ,	A_=[3, 3, 6, 4]     ,	A_=[48, 56, 112, 220]     ,	)     ->     int:
							lowerCAmelCase   =     parent
							lowerCAmelCase   =     batch_size
							lowerCAmelCase   =     num_channels
							lowerCAmelCase   =     is_training
							lowerCAmelCase   =     use_labels
							lowerCAmelCase   =     hidden_dropout_prob
							lowerCAmelCase   =     attention_probs_dropout_prob
							lowerCAmelCase   =     num_labels
							lowerCAmelCase   =     image_size
							lowerCAmelCase   =     layer_depths
							lowerCAmelCase   =     embed_dims


						def     __snake_case   ( self				)     ->     int:
							lowerCAmelCase   =     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

							lowerCAmelCase   =     None
							if self.use_labels:
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.num_labels				)

							lowerCAmelCase   =     self.get_config()

							return config, pixel_values, labels


						def     __snake_case   ( self				)     ->     Tuple:
							return SwiftFormerConfig(
							    depths=self.layer_depths     ,	embed_dims=self.embed_dims     ,	mlp_ratio=4     ,	downsamples=[True, True, True, True]     ,	hidden_act="""gelu"""     ,	num_labels=self.num_labels     ,	down_patch_size=3     ,	down_stride=2     ,	down_pad=1     ,	drop_rate=0.0     ,	drop_path_rate=0.0     ,	use_layer_scale=UpperCAmelCase_     ,	layer_scale_init_value=1e-5     ,	)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_				)     ->     List[Any]:
							lowerCAmelCase   =     SwiftFormerModel(config=UpperCAmelCase_				)
							model.to(UpperCAmelCase_				)
							model.eval()
							lowerCAmelCase   =     model(UpperCAmelCase_				)
							self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.embed_dims[-1], 7, 7)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_				)     ->     Any:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     SwiftFormerForImageClassification(UpperCAmelCase_				)
							model.to(UpperCAmelCase_				)
							model.eval()
							lowerCAmelCase   =     model(UpperCAmelCase_     ,	labels=UpperCAmelCase_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels)				)

							lowerCAmelCase   =     SwiftFormerForImageClassification(UpperCAmelCase_				)
							model.to(UpperCAmelCase_				)
							model.eval()

							lowerCAmelCase   =     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)
							lowerCAmelCase   =     model(UpperCAmelCase_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels)				)




						def     __snake_case   ( self				)     ->     Tuple:
							(lowerCAmelCase)   =     self.prepare_config_and_inputs()
							lowerCAmelCase   =     {"pixel_values": pixel_values}
							return config, inputs_dict







@require_torch
class __snake_case(				snake_case__		,							snake_case__		,							unittest.TestCase ):


						'''simple docstring'''



						UpperCAmelCase							:   int									=		(SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else ()
						UpperCAmelCase							:   Optional[Any]									=		(
						    {"feature-extraction": SwiftFormerModel, "image-classification": SwiftFormerForImageClassification}
						    if is_torch_available()
						    else {}
						)

						UpperCAmelCase							:   int									=		False
						UpperCAmelCase							:   str									=		False
						UpperCAmelCase							:   int									=		False
						UpperCAmelCase							:   Union[str, Any]									=		False
						UpperCAmelCase							:   Optional[Any]									=		False


						def     __snake_case   ( self				)     ->     List[str]:
							lowerCAmelCase   =     SwiftFormerModelTester(self				)
							lowerCAmelCase   =     ConfigTester(
							    self     ,	config_class=UpperCAmelCase_     ,	has_text_modality=UpperCAmelCase_     ,	hidden_size=37     ,	num_attention_heads=12     ,	num_hidden_layers=12     ,	)


						def     __snake_case   ( self				)     ->     List[str]:
							self.config_tester.run_common_tests()


						@unittest.skip(reason="""SwiftFormer does not use inputs_embeds"""				)
						def     __snake_case   ( self				)     ->     str:
							pass


						def     __snake_case   ( self				)     ->     List[Any]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     model_class(UpperCAmelCase_				)
								lowerCAmelCase   =     model.get_output_embeddings()
								self.assertTrue(x is None or isinstance(UpperCAmelCase_     ,	nn.Linear				)				)


						def     __snake_case   ( self				)     ->     Dict:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     model_class(UpperCAmelCase_				)
								lowerCAmelCase   =     inspect.signature(model.forward				)
								# signature.parameters is an OrderedDict => so arg_names order is deterministic
								lowerCAmelCase   =     [*signature.parameters.keys()]

								lowerCAmelCase   =     ["pixel_values"]
								self.assertListEqual(arg_names[:1]     ,	UpperCAmelCase_				)


						def     __snake_case   ( self				)     ->     List[str]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*UpperCAmelCase_				)


						def     __snake_case   ( self				)     ->     Optional[Any]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*UpperCAmelCase_				)


						@slow
						def     __snake_case   ( self				)     ->     Optional[Any]:
							for model_name in SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								lowerCAmelCase   =     SwiftFormerModel.from_pretrained(UpperCAmelCase_				)
								self.assertIsNotNone(UpperCAmelCase_				)


						@unittest.skip(reason="""SwiftFormer does not output attentions"""				)
						def     __snake_case   ( self				)     ->     Union[str, Any]:
							pass


						def     __snake_case   ( self				)     ->     Any:
							def check_hidden_states_output(A_     ,	A_     ,	A_				):
								lowerCAmelCase   =     model_class(UpperCAmelCase_				)
								model.to(UpperCAmelCase_				)
								model.eval()

								with torch.no_grad():
									lowerCAmelCase   =     model(**self._prepare_for_class(UpperCAmelCase_     ,	UpperCAmelCase_				)				)

								lowerCAmelCase   =     outputs.hidden_states

								lowerCAmelCase   =     8
								self.assertEqual(len(UpperCAmelCase_				)     ,	UpperCAmelCase_				)  # TODO

								# SwiftFormer's feature maps are of shape (batch_size, embed_dims, height, width)
								# with the width and height being successively divided by 2, after every 2 blocks
								for i in range(len(UpperCAmelCase_				)				):
									self.assertEqual(
									    hidden_states[i].shape     ,	torch.Size(
									        [
									            self.model_tester.batch_size,
									            self.model_tester.embed_dims[i // 2],
									            (self.model_tester.image_size // 4) // 2 ** (i // 2),
									            (self.model_tester.image_size // 4) // 2 ** (i // 2),
									        ]				)     ,	)

							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     True
								check_hidden_states_output(UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_				)

								# check that output_hidden_states also work using config
								del inputs_dict["output_hidden_states"]
								lowerCAmelCase   =     True

								check_hidden_states_output(UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_				)


						def     __snake_case   ( self				)     ->     List[Any]:
							def _config_zero_init(A_				):
								lowerCAmelCase   =     copy.deepcopy(UpperCAmelCase_				)
								for key in configs_no_init.__dict__.keys():
									if "_range" in key or "_std" in key or "initializer_factor" in key or "layer_scale" in key:
										setattr(UpperCAmelCase_     ,	UpperCAmelCase_     ,	1e-10				)
									if isinstance(getattr(UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_				)     ,	UpperCAmelCase_				):
										lowerCAmelCase   =     _config_zero_init(getattr(UpperCAmelCase_     ,	UpperCAmelCase_				)				)
										setattr(UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_				)
								return configs_no_init

							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()

							lowerCAmelCase   =     _config_zero_init(UpperCAmelCase_				)
							for model_class in self.all_model_classes:
								lowerCAmelCase   =     model_class(config=UpperCAmelCase_				)
								for name, param in model.named_parameters():
									if param.requires_grad:
										self.assertIn(
										    ((param.data.mean() * 1e9) / 1e9).round().item()     ,	[0.0, 1.0]     ,	msg=f'Parameter {name} of model {model_class} seems not properly initialized'     ,	)




						@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""				)
						def     __snake_case   ( self				)     ->     int:
							pass







def 		_snake_case		( )      ->						Any:







	"""simple docstring"""
	lowerCAmelCase   =     Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""							)
	return image





@require_torch
@require_vision
class __snake_case(				unittest.TestCase ):


						'''simple docstring'''



						@cached_property
						def     __snake_case   ( self				)     ->     List[str]:
							return ViTImageProcessor.from_pretrained("""MBZUAI/swiftformer-xs"""				) if is_vision_available() else None




						@slow
						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     SwiftFormerForImageClassification.from_pretrained("""MBZUAI/swiftformer-xs"""				).to(UpperCAmelCase_				)

							lowerCAmelCase   =     self.default_image_processor
							lowerCAmelCase   =     prepare_img()
							lowerCAmelCase   =     image_processor(images=UpperCAmelCase_     ,	return_tensors="""pt"""				).to(UpperCAmelCase_				)

							# forward pass
							with torch.no_grad():
								lowerCAmelCase   =     model(**UpperCAmelCase_				)

							# verify the logits
							lowerCAmelCase   =     torch.Size((1, 1000)				)
							self.assertEqual(outputs.logits.shape     ,	UpperCAmelCase_				)

							lowerCAmelCase   =     torch.tensor([[-2.17_03e00, 2.11_07e00, -2.08_11e00]]				).to(UpperCAmelCase_				)
							self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,	UpperCAmelCase_     ,	atol=1e-4				)				)




'''simple docstring'''

from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFConvBertForMaskedLM,
				    TFConvBertForMultipleChoice,
				    TFConvBertForQuestionAnswering,
				    TFConvBertForSequenceClassification,
				    TFConvBertForTokenClassification,
				    TFConvBertModel,
				)

class __snake_case:


						'''simple docstring'''



						def __init__( self     ,	A_     ,	A_=13     ,	A_=7     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=99     ,	A_=32     ,	A_=2     ,	A_=4     ,	A_=37     ,	A_="gelu"     ,	A_=0.1     ,	A_=0.1     ,	A_=512     ,	A_=16     ,	A_=2     ,	A_=0.0_2     ,	A_=3     ,	A_=4     ,	A_=None     ,	)     ->     Dict:
							lowerCAmelCase   =     parent
							lowerCAmelCase   =     13
							lowerCAmelCase   =     7
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     99
							lowerCAmelCase   =     384
							lowerCAmelCase   =     2
							lowerCAmelCase   =     4
							lowerCAmelCase   =     37
							lowerCAmelCase   =     """gelu"""
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     512
							lowerCAmelCase   =     16
							lowerCAmelCase   =     2
							lowerCAmelCase   =     0.0_2
							lowerCAmelCase   =     3
							lowerCAmelCase   =     4
							lowerCAmelCase   =     128
							lowerCAmelCase   =     2
							lowerCAmelCase   =     9
							lowerCAmelCase   =     1
							lowerCAmelCase   =     None


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size				)

							lowerCAmelCase   =     None
							if self.use_input_mask:
								lowerCAmelCase   =     random_attention_mask([self.batch_size, self.seq_length]				)

							lowerCAmelCase   =     None
							if self.use_token_type_ids:
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size				)

							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							if self.use_labels:
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.type_sequence_label_size				)
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels				)
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.num_choices				)

							lowerCAmelCase   =     ConvBertConfig(
							    vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	initializer_range=self.initializer_range     ,	return_dict=A_     ,	)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     int:
							lowerCAmelCase   =     TFConvBertModel(config=A_				)
							lowerCAmelCase   =     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

							lowerCAmelCase   =     [input_ids, input_mask]
							lowerCAmelCase   =     model(A_				)

							lowerCAmelCase   =     model(A_				)

							self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     List[Any]:
							lowerCAmelCase   =     TFConvBertForMaskedLM(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForSequenceClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Any:
							lowerCAmelCase   =     self.num_choices
							lowerCAmelCase   =     TFConvBertForMultipleChoice(config=A_				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     {
							    """input_ids""": multiple_choice_inputs_ids,
							    """attention_mask""": multiple_choice_input_mask,
							    """token_type_ids""": multiple_choice_token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForTokenClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertForQuestionAnswering(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)				)




						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     self.prepare_config_and_inputs()
							(
							    (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       
							)   =     config_and_inputs
							lowerCAmelCase   =     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict







@require_tf
class __snake_case(				_lowerCAmelCase		,							_lowerCAmelCase		,							unittest.TestCase ):


						'''simple docstring'''



						UpperCAmelCase							:   Optional[int]									=		(
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase							:   Union[str, Any]									=		(
						    {
						        "feature-extraction": TFConvBertModel,
						        "fill-mask": TFConvBertForMaskedLM,
						        "question-answering": TFConvBertForQuestionAnswering,
						        "text-classification": TFConvBertForSequenceClassification,
						        "token-classification": TFConvBertForTokenClassification,
						        "zero-shot": TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCAmelCase							:   Union[str, Any]									=		False
						UpperCAmelCase							:   Optional[int]									=		False
						UpperCAmelCase							:   Dict									=		False


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertModelTester(self				)
							lowerCAmelCase   =     ConfigTester(self     ,	config_class=A_     ,	hidden_size=37				)


						def     __snake_case   ( self				)     ->     Tuple:
							self.config_tester.run_common_tests()


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*A_				)


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*A_				)


						def     __snake_case   ( self				)     ->     List[str]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*A_				)


						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*A_				)


						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True

							if hasattr(A_     ,	"""use_cache"""				):
								lowerCAmelCase   =     True

							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     self._prepare_for_class(A_     ,	A_				)
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     len(model(A_				)				)

								with tempfile.TemporaryDirectory() as tmpdirname:
									model.save_pretrained(A_     ,	saved_model=A_				)
									lowerCAmelCase   =     os.path.join(A_     ,	"""saved_model"""     ,	"""1"""				)
									lowerCAmelCase   =     tf.keras.models.load_model(A_				)
									lowerCAmelCase   =     model(A_				)

									if self.is_encoder_decoder:
										lowerCAmelCase   =     outputs["""encoder_hidden_states"""]
										lowerCAmelCase   =     outputs["""encoder_attentions"""]
									else:
										lowerCAmelCase   =     outputs["""hidden_states"""]
										lowerCAmelCase   =     outputs["""attentions"""]

									self.assertEqual(len(A_				)     ,	A_				)

									lowerCAmelCase   =     getattr(
									    self.model_tester     ,	"""expected_num_hidden_layers"""     ,	self.model_tester.num_hidden_layers + 1				)

									self.assertEqual(len(A_				)     ,	A_				)
									self.assertListEqual(
									    list(output_hidden_states[0].shape[-2:]				)     ,	[self.model_tester.seq_length, self.model_tester.hidden_size]     ,	)

									self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
									self.assertListEqual(
									    list(output_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)


						@slow
						def     __snake_case   ( self				)     ->     Optional[Any]:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							self.assertIsNotNone(A_				)




						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""decoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							def check_decoder_attentions_output(A_				):
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(out_len % 2     ,	0				)
								lowerCAmelCase   =     outputs.decoder_attentions
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]     ,	)

							def check_encoder_attentions_output(A_				):
								lowerCAmelCase   =     [
								    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
								]
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     True
								lowerCAmelCase   =     False
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								if self.is_encoder_decoder:
									lowerCAmelCase   =     model_class(A_				)
									lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
									self.assertEqual(config.output_hidden_states     ,	A_				)
									check_decoder_attentions_output(A_				)

								# Check that output attentions can also be changed via the config
								del inputs_dict["output_attentions"]
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								# Check attention is always last and order is fine
								lowerCAmelCase   =     True
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)

								self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)     ,	len(A_				)				)
								self.assertEqual(model.config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)







@require_tf
class __snake_case(				unittest.TestCase ):


						'''simple docstring'''



						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							lowerCAmelCase   =     tf.constant([[0, 1, 2, 3, 4, 5]]				)
							lowerCAmelCase   =     model(A_				)[0]

							lowerCAmelCase   =     [1, 6, 768]
							self.assertEqual(output.shape     ,	A_				)

							lowerCAmelCase   =     tf.constant(
							    [
							        [
							            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
							            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
							            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
							        ]
							    ]				)
							tf.debugging.assert_near(output[:, :3, :3]     ,	A_     ,	atol=1e-4				)