
"""simple docstring"""







from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowerCamelCase__        =		logging.get_logger(__name__)

lowerCamelCase__        =		{
    """microsoft/swin-tiny-patch4-window7-224""": (
        """https://huggingface.co/microsoft/swin-tiny-patch4-window7-224/resolve/main/config.json"""
    ),
    # See all Swin models at https://huggingface.co/models?filter=swin
}




class 			A__	(     _lowerCamelCase	,					_lowerCamelCase):
		A_						:	str           =     'swin'

		A_						:	Union[str, Any]           =     {
		    'num_attention_heads': 'num_heads',
		    'num_hidden_layers': 'num_layers',
		}

		def __init__(				self	,		_SCREAMING_SNAKE_CASE=2_24	,		_SCREAMING_SNAKE_CASE=4	,		_SCREAMING_SNAKE_CASE=3	,		_SCREAMING_SNAKE_CASE=96	,		_SCREAMING_SNAKE_CASE=[2, 2, 6, 2]	,		_SCREAMING_SNAKE_CASE=[3, 6, 12, 24]	,		_SCREAMING_SNAKE_CASE=7	,		_SCREAMING_SNAKE_CASE=4.0	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=0.0	,		_SCREAMING_SNAKE_CASE=0.0	,		_SCREAMING_SNAKE_CASE=0.1	,		_SCREAMING_SNAKE_CASE="gelu"	,		_SCREAMING_SNAKE_CASE=False	,		_SCREAMING_SNAKE_CASE=0.02	,		_SCREAMING_SNAKE_CASE=1E-5	,		_SCREAMING_SNAKE_CASE=32	,		_SCREAMING_SNAKE_CASE=None	,		_SCREAMING_SNAKE_CASE=None	,		**_SCREAMING_SNAKE_CASE	,		):
			super().__init__(**_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						Union[str, Any]						      =				image_size
			__lowerCAmelCase      :						Union[str, Any]						      =				patch_size
			__lowerCAmelCase      :						Any						      =				num_channels
			__lowerCAmelCase      :						List[Any]						      =				embed_dim
			__lowerCAmelCase      :						List[str]						      =				depths
			__lowerCAmelCase      :						Union[str, Any]						      =				len(_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						List[str]						      =				num_heads
			__lowerCAmelCase      :						List[str]						      =				window_size
			__lowerCAmelCase      :						Any						      =				mlp_ratio
			__lowerCAmelCase      :						int						      =				qkv_bias
			__lowerCAmelCase      :						Union[str, Any]						      =				hidden_dropout_prob
			__lowerCAmelCase      :						Any						      =				attention_probs_dropout_prob
			__lowerCAmelCase      :						Tuple						      =				drop_path_rate
			__lowerCAmelCase      :						List[str]						      =				hidden_act
			__lowerCAmelCase      :						Optional[int]						      =				use_absolute_embeddings
			__lowerCAmelCase      :						Optional[int]						      =				layer_norm_eps
			__lowerCAmelCase      :						Optional[int]						      =				initializer_range
			__lowerCAmelCase      :						Optional[Any]						      =				encoder_stride
			# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
			# this indicates the channel dimension after the last stage of the model
			__lowerCAmelCase      :						List[Any]						      =				int(embed_dim * 2 ** (len(_SCREAMING_SNAKE_CASE			) - 1)			)
			__lowerCAmelCase      :						Tuple						      =				['stem'] + [f"stage{idx}" for idx in range(1	,		len(_SCREAMING_SNAKE_CASE			) + 1			)]
			__lowerCAmelCase      ,       __lowerCAmelCase      :						Dict						      =				get_aligned_output_features_output_indices(
			    out_features=_SCREAMING_SNAKE_CASE	,		out_indices=_SCREAMING_SNAKE_CASE	,		stage_names=self.stage_names			)




class 			A__	(     _lowerCamelCase):
		A_						:	str           =     version.parse('1.11')

		@property
		def 							__lowerCamelCase							(				self			):
			return OrderedDict(
			    [
			        ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
			    ]			)







		@property
		def 							__lowerCamelCase							(				self			):
			return 1E-4

"""simple docstring"""







import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class 			A__	:

		def __init__(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE=13	,		_SCREAMING_SNAKE_CASE=7	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=True	,		_SCREAMING_SNAKE_CASE=99	,		_SCREAMING_SNAKE_CASE=16	,		_SCREAMING_SNAKE_CASE=36	,		_SCREAMING_SNAKE_CASE=6	,		_SCREAMING_SNAKE_CASE=6	,		_SCREAMING_SNAKE_CASE=6	,		_SCREAMING_SNAKE_CASE=37	,		_SCREAMING_SNAKE_CASE="gelu"	,		_SCREAMING_SNAKE_CASE=0.1	,		_SCREAMING_SNAKE_CASE=0.1	,		_SCREAMING_SNAKE_CASE=5_12	,		_SCREAMING_SNAKE_CASE=16	,		_SCREAMING_SNAKE_CASE=2	,		_SCREAMING_SNAKE_CASE=0.02	,		_SCREAMING_SNAKE_CASE=3	,		_SCREAMING_SNAKE_CASE=4	,		_SCREAMING_SNAKE_CASE=None	,		):
			__lowerCAmelCase      :						List[str]						      =				parent
			__lowerCAmelCase      :						Optional[int]						      =				batch_size
			__lowerCAmelCase      :						List[Any]						      =				seq_length
			__lowerCAmelCase      :						List[Any]						      =				is_training
			__lowerCAmelCase      :						Tuple						      =				use_input_mask
			__lowerCAmelCase      :						Tuple						      =				use_token_type_ids
			__lowerCAmelCase      :						Optional[int]						      =				use_labels
			__lowerCAmelCase      :						Tuple						      =				vocab_size
			__lowerCAmelCase      :						List[Any]						      =				embedding_size
			__lowerCAmelCase      :						List[Any]						      =				hidden_size
			__lowerCAmelCase      :						Union[str, Any]						      =				num_hidden_layers
			__lowerCAmelCase      :						int						      =				num_hidden_groups
			__lowerCAmelCase      :						List[Any]						      =				num_attention_heads
			__lowerCAmelCase      :						Dict						      =				intermediate_size
			__lowerCAmelCase      :						List[str]						      =				hidden_act
			__lowerCAmelCase      :						Optional[Any]						      =				hidden_dropout_prob
			__lowerCAmelCase      :						List[Any]						      =				attention_probs_dropout_prob
			__lowerCAmelCase      :						Any						      =				max_position_embeddings
			__lowerCAmelCase      :						str						      =				type_vocab_size
			__lowerCAmelCase      :						List[Any]						      =				type_sequence_label_size
			__lowerCAmelCase      :						List[Any]						      =				initializer_range
			__lowerCAmelCase      :						Tuple						      =				num_labels
			__lowerCAmelCase      :						int						      =				num_choices
			__lowerCAmelCase      :						Dict						      =				scope

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Dict						      =				ids_tensor([self.batch_size, self.seq_length]	,		self.vocab_size			)

			__lowerCAmelCase      :						Union[str, Any]						      =				None
			if self.use_input_mask:
				__lowerCAmelCase      :						List[Any]						      =				random_attention_mask([self.batch_size, self.seq_length]			)

			__lowerCAmelCase      :						Any						      =				None
			if self.use_token_type_ids:
				__lowerCAmelCase      :						List[str]						      =				ids_tensor([self.batch_size, self.seq_length]	,		self.type_vocab_size			)

			__lowerCAmelCase      :						Optional[int]						      =				None
			__lowerCAmelCase      :						Optional[Any]						      =				None
			__lowerCAmelCase      :						Any						      =				None
			if self.use_labels:
				__lowerCAmelCase      :						List[str]						      =				ids_tensor([self.batch_size]	,		self.type_sequence_label_size			)
				__lowerCAmelCase      :						Optional[int]						      =				ids_tensor([self.batch_size, self.seq_length]	,		self.num_labels			)
				__lowerCAmelCase      :						int						      =				ids_tensor([self.batch_size]	,		self.num_choices			)

			__lowerCAmelCase      :						Any						      =				self.get_config()

			return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

		def 							__lowerCamelCase							(				self			):
			return AlbertConfig(
			    vocab_size=self.vocab_size	,		hidden_size=self.hidden_size	,		num_hidden_layers=self.num_hidden_layers	,		num_attention_heads=self.num_attention_heads	,		intermediate_size=self.intermediate_size	,		hidden_act=self.hidden_act	,		hidden_dropout_prob=self.hidden_dropout_prob	,		attention_probs_dropout_prob=self.attention_probs_dropout_prob	,		max_position_embeddings=self.max_position_embeddings	,		type_vocab_size=self.type_vocab_size	,		initializer_range=self.initializer_range	,		num_hidden_groups=self.num_hidden_groups	,		)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						List[str]						      =				AlbertModel(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						Dict						      =				model(_SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Optional[Any]						      =				model(_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						List[Any]						      =				model(_SCREAMING_SNAKE_CASE			)
			self.parent.assertEqual(result.last_hidden_state.shape	,		(self.batch_size, self.seq_length, self.hidden_size)			)
			self.parent.assertEqual(result.pooler_output.shape	,		(self.batch_size, self.hidden_size)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						str						      =				AlbertForPreTraining(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						Any						      =				model(
			    _SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		labels=_SCREAMING_SNAKE_CASE	,		sentence_order_label=_SCREAMING_SNAKE_CASE	,		)
			self.parent.assertEqual(result.prediction_logits.shape	,		(self.batch_size, self.seq_length, self.vocab_size)			)
			self.parent.assertEqual(result.sop_logits.shape	,		(self.batch_size, config.num_labels)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Optional[int]						      =				AlbertForMaskedLM(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						Any						      =				model(_SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		labels=_SCREAMING_SNAKE_CASE			)
			self.parent.assertEqual(result.logits.shape	,		(self.batch_size, self.seq_length, self.vocab_size)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Optional[int]						      =				AlbertForQuestionAnswering(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						List[str]						      =				model(
			    _SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		start_positions=_SCREAMING_SNAKE_CASE	,		end_positions=_SCREAMING_SNAKE_CASE	,		)
			self.parent.assertEqual(result.start_logits.shape	,		(self.batch_size, self.seq_length)			)
			self.parent.assertEqual(result.end_logits.shape	,		(self.batch_size, self.seq_length)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Optional[int]						      =				self.num_labels
			__lowerCAmelCase      :						str						      =				AlbertForSequenceClassification(_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						str						      =				model(_SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		labels=_SCREAMING_SNAKE_CASE			)
			self.parent.assertEqual(result.logits.shape	,		(self.batch_size, self.num_labels)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Optional[Any]						      =				self.num_labels
			__lowerCAmelCase      :						List[str]						      =				AlbertForTokenClassification(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						Dict						      =				model(_SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		labels=_SCREAMING_SNAKE_CASE			)
			self.parent.assertEqual(result.logits.shape	,		(self.batch_size, self.seq_length, self.num_labels)			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						str						      =				self.num_choices
			__lowerCAmelCase      :						Dict						      =				AlbertForMultipleChoice(config=_SCREAMING_SNAKE_CASE			)
			model.to(_SCREAMING_SNAKE_CASE			)
			model.eval()
			__lowerCAmelCase      :						Union[str, Any]						      =				input_ids.unsqueeze(1			).expand(-1	,		self.num_choices	,		-1			).contiguous()
			__lowerCAmelCase      :						Dict						      =				token_type_ids.unsqueeze(1			).expand(-1	,		self.num_choices	,		-1			).contiguous()
			__lowerCAmelCase      :						List[str]						      =				input_mask.unsqueeze(1			).expand(-1	,		self.num_choices	,		-1			).contiguous()
			__lowerCAmelCase      :						Dict						      =				model(
			    _SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE	,		token_type_ids=_SCREAMING_SNAKE_CASE	,		labels=_SCREAMING_SNAKE_CASE	,		)
			self.parent.assertEqual(result.logits.shape	,		(self.batch_size, self.num_choices)			)







		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						List[Any]						      =				self.prepare_config_and_inputs()
			(
			    (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       (
			    __lowerCAmelCase
			)      ,       
			)      :						int						      =				config_and_inputs
			__lowerCAmelCase      :						Tuple						      =				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
			return config, inputs_dict




@require_torch
class 			A__	(     _lowerCamelCase	,					_lowerCamelCase	,					unittest.TestCase):
		A_						:	List[str]           =     (
		    (
		        AlbertModel,
		        AlbertForPreTraining,
		        AlbertForMaskedLM,
		        AlbertForMultipleChoice,
		        AlbertForSequenceClassification,
		        AlbertForTokenClassification,
		        AlbertForQuestionAnswering,
		    )
		    if is_torch_available()
		    else ()
		)
		A_						:	str           =     (
		    {
		        'feature-extraction': AlbertModel,
		        'fill-mask': AlbertForMaskedLM,
		        'question-answering': AlbertForQuestionAnswering,
		        'text-classification': AlbertForSequenceClassification,
		        'token-classification': AlbertForTokenClassification,
		        'zero-shot': AlbertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		A_						:	Union[str, Any]           =     True

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE=False			):
			__lowerCAmelCase      :						Optional[Any]						      =				super()._prepare_for_class(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		return_labels=_SCREAMING_SNAKE_CASE			)

			if return_labels:
				if model_class in get_values(_SCREAMING_SNAKE_CASE			):
					__lowerCAmelCase      :						List[Any]						      =				torch.zeros(
					    (self.model_tester.batch_size, self.model_tester.seq_length)	,		dtype=torch.long	,		device=_SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase      :						List[Any]						      =				torch.zeros(
					    self.model_tester.batch_size	,		dtype=torch.long	,		device=_SCREAMING_SNAKE_CASE			)
			return inputs_dict

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						List[str]						      =				AlbertModelTester(self			)
			__lowerCAmelCase      :						List[str]						      =				ConfigTester(self	,		config_class=_SCREAMING_SNAKE_CASE	,		hidden_size=37			)

		def 							__lowerCamelCase							(				self			):
			self.config_tester.run_common_tests()

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Any						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Tuple						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_pretraining(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Tuple						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_lm(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Optional[Any]						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						List[Any]						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Dict						      =				self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Optional[Any]						      =				self.model_tester.prepare_config_and_inputs()
			for type in ["absolute", "relative_key", "relative_key_query"]:
				__lowerCAmelCase      :						Optional[Any]						      =				type
				self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE			)







		@slow
		def 							__lowerCamelCase							(				self			):
			for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
				__lowerCAmelCase      :						Optional[int]						      =				AlbertModel.from_pretrained(_SCREAMING_SNAKE_CASE			)
				self.assertIsNotNone(_SCREAMING_SNAKE_CASE			)




@require_torch
class 			A__	(     unittest.TestCase):

		@slow
		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						str						      =				AlbertModel.from_pretrained('albert-base-v2'			)
			__lowerCAmelCase      :						List[str]						      =				torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]			)
			__lowerCAmelCase      :						Optional[Any]						      =				torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]			)
			with torch.no_grad():
				__lowerCAmelCase      :						Dict						      =				model(_SCREAMING_SNAKE_CASE	,		attention_mask=_SCREAMING_SNAKE_CASE			)[0]
			__lowerCAmelCase      :						Dict						      =				torch.Size((1, 11, 7_68)			)
			self.assertEqual(output.shape	,		_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						List[Any]						      =				torch.tensor(
			    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]			)

			self.assertTrue(torch.allclose(output[:, 1:4, 1:4]	,		_SCREAMING_SNAKE_CASE	,		atol=1E-4			)			)