
'''simple docstring'''



from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


_lowercase    =		logging.get_logger(__name__)

# General docstring
_lowercase    =		"""RegNetConfig"""

# Base docstring
_lowercase    =		"""facebook/regnet-y-040"""
_lowercase    =		[1, 1088, 7, 7]

# Image classification docstring
_lowercase    =		"""facebook/regnet-y-040"""
_lowercase    =		"""tabby, tabby cat"""

_lowercase    =		[
    """facebook/regnet-y-040""",
    # See all regnet models at https://huggingface.co/models?filter=regnet
]




class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase = 3   ,       _lowercase = 1   ,       _lowercase = 1   ,       _lowercase = "relu"   ,       ):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			nn.Convad(
												    _lowercase   ,       _lowercase   ,       kernel_size=_lowercase   ,       stride=_lowercase   ,       padding=kernel_size // 2   ,       groups=_lowercase   ,       bias=_lowercase   ,       )
												_lowerCAmelCase  						=			nn.BatchNormad(_lowercase				)
												_lowerCAmelCase  						=			ACTaFN[activation] if activation is not None else nn.Identity()




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.convolution(_lowercase				)
												_lowerCAmelCase  						=			self.normalization(_lowercase				)
												_lowerCAmelCase  						=			self.activation(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase				):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			RegNetConvLayer(
												    config.num_channels   ,       config.embedding_size   ,       kernel_size=3   ,       stride=2   ,       activation=config.hidden_act				)
												_lowerCAmelCase  						=			config.num_channels




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			pixel_values.shape[1]
												if num_channels != self.num_channels:
																		raise ValueError(
																		    """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""				)
												_lowerCAmelCase  						=			self.embedder(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase = 2				):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			nn.Convad(_lowercase   ,       _lowercase   ,       kernel_size=1   ,       stride=_lowercase   ,       bias=_lowercase				)
												_lowerCAmelCase  						=			nn.BatchNormad(_lowercase				)




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.convolution(_lowercase				)
												_lowerCAmelCase  						=			self.normalization(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												super().__init__()

												_lowerCAmelCase  						=			nn.AdaptiveAvgPoolad((1, 1)				)
												_lowerCAmelCase  						=			nn.Sequential(
												    nn.Convad(_lowercase   ,       _lowercase   ,       kernel_size=1				)   ,       nn.ReLU()   ,       nn.Convad(_lowercase   ,       _lowercase   ,       kernel_size=1				)   ,       nn.Sigmoid()   ,       )




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.pooler(_lowercase				)
												_lowerCAmelCase  						=			self.attention(_lowercase				)
												_lowerCAmelCase  						=			hidden_state * attention
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase = 1				):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			in_channels != out_channels or stride != 1
												_lowerCAmelCase  						=			max(1   ,       out_channels // config.groups_width				)
												_lowerCAmelCase  						=			(
												    RegNetShortCut(_lowercase   ,       _lowercase   ,       stride=_lowercase				) if should_apply_shortcut else nn.Identity()
												)
												_lowerCAmelCase  						=			nn.Sequential(
												    RegNetConvLayer(_lowercase   ,       _lowercase   ,       kernel_size=1   ,       activation=config.hidden_act				)   ,       RegNetConvLayer(_lowercase   ,       _lowercase   ,       stride=_lowercase   ,       groups=_lowercase   ,       activation=config.hidden_act				)   ,       RegNetConvLayer(_lowercase   ,       _lowercase   ,       kernel_size=1   ,       activation=_lowercase				)   ,       )
												_lowerCAmelCase  						=			ACTaFN[config.hidden_act]




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			hidden_state
												_lowerCAmelCase  						=			self.layer(_lowercase				)
												_lowerCAmelCase  						=			self.shortcut(_lowercase				)
												hidden_state += residual
												_lowerCAmelCase  						=			self.activation(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase = 1				):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			in_channels != out_channels or stride != 1
												_lowerCAmelCase  						=			max(1   ,       out_channels // config.groups_width				)
												_lowerCAmelCase  						=			(
												    RegNetShortCut(_lowercase   ,       _lowercase   ,       stride=_lowercase				) if should_apply_shortcut else nn.Identity()
												)
												_lowerCAmelCase  						=			nn.Sequential(
												    RegNetConvLayer(_lowercase   ,       _lowercase   ,       kernel_size=1   ,       activation=config.hidden_act				)   ,       RegNetConvLayer(_lowercase   ,       _lowercase   ,       stride=_lowercase   ,       groups=_lowercase   ,       activation=config.hidden_act				)   ,       RegNetSELayer(_lowercase   ,       reduced_channels=int(round(in_channels / 4				)				)				)   ,       RegNetConvLayer(_lowercase   ,       _lowercase   ,       kernel_size=1   ,       activation=_lowercase				)   ,       )
												_lowerCAmelCase  						=			ACTaFN[config.hidden_act]




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			hidden_state
												_lowerCAmelCase  						=			self.layer(_lowercase				)
												_lowerCAmelCase  						=			self.shortcut(_lowercase				)
												hidden_state += residual
												_lowerCAmelCase  						=			self.activation(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase = 2   ,       _lowercase = 2   ,       ):

												"""simple docstring"""



												super().__init__()

												_lowerCAmelCase  						=			RegNetXLayer if config.layer_type == """x""" else RegNetYLayer

												_lowerCAmelCase  						=			nn.Sequential(
												    # downsampling is done in the first layer with stride of 2
												    layer(
												        _lowercase   ,       _lowercase   ,       _lowercase   ,       stride=_lowercase   ,       )   ,       *[layer(_lowercase   ,       _lowercase   ,       _lowercase				) for _ in range(depth - 1				)]   ,       )




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.layers(_lowercase				)
												return hidden_state






class   UpperCAmelCase_					(		nn.Module			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase				):

												"""simple docstring"""



												super().__init__()
												_lowerCAmelCase  						=			nn.ModuleList([]				)
												# based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
												self.stages.append(
												    RegNetStage(
												        _lowercase   ,       config.embedding_size   ,       config.hidden_sizes[0]   ,       stride=2 if config.downsample_in_first_stage else 1   ,       depth=config.depths[0]   ,       )				)
												_lowerCAmelCase  						=			zip(config.hidden_sizes   ,       config.hidden_sizes[1:]				)
												for (in_channels, out_channels), depth in zip(_lowercase   ,       config.depths[1:]				):
																		self.stages.append(RegNetStage(_lowercase   ,       _lowercase   ,       _lowercase   ,       depth=_lowercase				)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase = False   ,       _lowercase = True				):

												"""simple docstring"""



												_lowerCAmelCase  						=			() if output_hidden_states else None

												for stage_module in self.stages:
																		if output_hidden_states:
																								_lowerCAmelCase  						=			hidden_states + (hidden_state,)

																		_lowerCAmelCase  						=			stage_module(_lowercase				)

												if output_hidden_states:
																		_lowerCAmelCase  						=			hidden_states + (hidden_state,)

												if not return_dict:
																		return tuple(v for v in [hidden_state, hidden_states] if v is not None				)

												return BaseModelOutputWithNoAttention(last_hidden_state=_lowercase   ,       hidden_states=_lowercase				)






class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






						'''simple docstring'''






						_lowercase		:				List[Any]									=       RegNetConfig
						_lowercase		:				Union[str, Any]									=       '''regnet'''
						_lowercase		:				Tuple									=       '''pixel_values'''
						_lowercase		:				Any									=       True




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												if isinstance(_lowercase   ,       nn.Convad				):
																		nn.init.kaiming_normal_(module.weight   ,       mode="""fan_out"""   ,       nonlinearity="""relu"""				)
												elif isinstance(_lowercase   ,       (nn.BatchNormad, nn.GroupNorm)				):
																		nn.init.constant_(module.weight   ,       1				)
																		nn.init.constant_(module.bias   ,       0				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase=False				):

												"""simple docstring"""



												if isinstance(_lowercase   ,       _lowercase				):
																		_lowerCAmelCase  						=			value






_lowercase    =		R"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

_lowercase    =		R"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
"""




@add_start_docstrings(
    '''The bare RegNet model outputting raw features without any specific head on top.'''	,							_SCREAMING_SNAKE_CASE	,							)
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase				):

												"""simple docstring"""



												super().__init__(_lowercase				)
												_lowerCAmelCase  						=			config
												_lowerCAmelCase  						=			RegNetEmbeddings(_lowercase				)
												_lowerCAmelCase  						=			RegNetEncoder(_lowercase				)
												_lowerCAmelCase  						=			nn.AdaptiveAvgPoolad((1, 1)				)
												# Initialize weights and apply final processing
												self.post_init()




						@add_start_docstrings_to_model_forward(_lowercase				)
						@add_code_sample_docstrings(
						    checkpoint=_CHECKPOINT_FOR_DOC   ,       output_type=_lowercase   ,       config_class=_CONFIG_FOR_DOC   ,       modality="""vision"""   ,       expected_output=_EXPECTED_OUTPUT_SHAPE   ,       )
						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase = None   ,       _lowercase = None				):

												"""simple docstring"""



												_lowerCAmelCase  						=			(
												    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
												)
												_lowerCAmelCase  						=			return_dict if return_dict is not None else self.config.use_return_dict

												_lowerCAmelCase  						=			self.embedder(_lowercase				)

												_lowerCAmelCase  						=			self.encoder(
												    _lowercase   ,       output_hidden_states=_lowercase   ,       return_dict=_lowercase				)

												_lowerCAmelCase  						=			encoder_outputs[0]

												_lowerCAmelCase  						=			self.pooler(_lowercase				)

												if not return_dict:
																		return (last_hidden_state, pooled_output) + encoder_outputs[1:]

												return BaseModelOutputWithPoolingAndNoAttention(
												    last_hidden_state=_lowercase   ,       pooler_output=_lowercase   ,       hidden_states=encoder_outputs.hidden_states   ,       )






@add_start_docstrings(
    '''
    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    '''	,							_SCREAMING_SNAKE_CASE	,							)
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase				):

												"""simple docstring"""



												super().__init__(_lowercase				)
												_lowerCAmelCase  						=			config.num_labels
												_lowerCAmelCase  						=			RegNetModel(_lowercase				)
												# classification head
												_lowerCAmelCase  						=			nn.Sequential(
												    nn.Flatten()   ,       nn.Linear(config.hidden_sizes[-1]   ,       config.num_labels				) if config.num_labels > 0 else nn.Identity()   ,       )
												# initialize weights and apply final processing
												self.post_init()




						@add_start_docstrings_to_model_forward(_lowercase				)
						@add_code_sample_docstrings(
						    checkpoint=_IMAGE_CLASS_CHECKPOINT   ,       output_type=_lowercase   ,       config_class=_CONFIG_FOR_DOC   ,       expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT   ,       )
						def  _lowercase     ( self   ,       _lowercase = None   ,       _lowercase = None   ,       _lowercase = None   ,       _lowercase = None   ,       ):

												"""simple docstring"""



												_lowerCAmelCase  						=			return_dict if return_dict is not None else self.config.use_return_dict

												_lowerCAmelCase  						=			self.regnet(_lowercase   ,       output_hidden_states=_lowercase   ,       return_dict=_lowercase				)

												_lowerCAmelCase  						=			outputs.pooler_output if return_dict else outputs[1]

												_lowerCAmelCase  						=			self.classifier(_lowercase				)

												_lowerCAmelCase  						=			None

												if labels is not None:
																		if self.config.problem_type is None:
																								if self.num_labels == 1:
																														_lowerCAmelCase  						=			"""regression"""
																								elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																														_lowerCAmelCase  						=			"""single_label_classification"""
																								else:
																														_lowerCAmelCase  						=			"""multi_label_classification"""
																		if self.config.problem_type == "regression":
																								_lowerCAmelCase  						=			MSELoss()
																								if self.num_labels == 1:
																														_lowerCAmelCase  						=			loss_fct(logits.squeeze()   ,       labels.squeeze()				)
																								else:
																														_lowerCAmelCase  						=			loss_fct(_lowercase   ,       _lowercase				)
																		elif self.config.problem_type == "single_label_classification":
																								_lowerCAmelCase  						=			CrossEntropyLoss()
																								_lowerCAmelCase  						=			loss_fct(logits.view(-1   ,       self.num_labels				)   ,       labels.view(-1				)				)
																		elif self.config.problem_type == "multi_label_classification":
																								_lowerCAmelCase  						=			BCEWithLogitsLoss()
																								_lowerCAmelCase  						=			loss_fct(_lowercase   ,       _lowercase				)

												if not return_dict:
																		_lowerCAmelCase  						=			(logits,) + outputs[2:]
																		return (loss,) + output if loss is not None else output

												return ImageClassifierOutputWithNoAttention(loss=_lowercase   ,       logits=_lowercase   ,       hidden_states=outputs.hidden_states				)


'''simple docstring'''



from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TFConvBertForMaskedLM,
						    TFConvBertForMultipleChoice,
						    TFConvBertForQuestionAnswering,
						    TFConvBertForSequenceClassification,
						    TFConvBertForTokenClassification,
						    TFConvBertModel,
						)




class   UpperCAmelCase_					:






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase=13   ,       _lowercase=7   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=True   ,       _lowercase=99   ,       _lowercase=32   ,       _lowercase=2   ,       _lowercase=4   ,       _lowercase=37   ,       _lowercase="gelu"   ,       _lowercase=0.1   ,       _lowercase=0.1   ,       _lowercase=512   ,       _lowercase=16   ,       _lowercase=2   ,       _lowercase=0.02   ,       _lowercase=3   ,       _lowercase=4   ,       _lowercase=None   ,       ):

												"""simple docstring"""



												_lowerCAmelCase  						=			parent
												_lowerCAmelCase  						=			13
												_lowerCAmelCase  						=			7
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			99
												_lowerCAmelCase  						=			384
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			4
												_lowerCAmelCase  						=			37
												_lowerCAmelCase  						=			"""gelu"""
												_lowerCAmelCase  						=			0.1
												_lowerCAmelCase  						=			0.1
												_lowerCAmelCase  						=			512
												_lowerCAmelCase  						=			16
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			0.02
												_lowerCAmelCase  						=			3
												_lowerCAmelCase  						=			4
												_lowerCAmelCase  						=			128
												_lowerCAmelCase  						=			2
												_lowerCAmelCase  						=			9
												_lowerCAmelCase  						=			1
												_lowerCAmelCase  						=			None




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.vocab_size				)

												_lowerCAmelCase  						=			None
												if self.use_input_mask:
																		_lowerCAmelCase  						=			random_attention_mask([self.batch_size, self.seq_length]				)

												_lowerCAmelCase  						=			None
												if self.use_token_type_ids:
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.type_vocab_size				)

												_lowerCAmelCase  						=			None
												_lowerCAmelCase  						=			None
												_lowerCAmelCase  						=			None
												if self.use_labels:
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size]   ,       self.type_sequence_label_size				)
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size, self.seq_length]   ,       self.num_labels				)
																		_lowerCAmelCase  						=			ids_tensor([self.batch_size]   ,       self.num_choices				)

												_lowerCAmelCase  						=			ConvBertConfig(
												    vocab_size=self.vocab_size   ,       hidden_size=self.hidden_size   ,       num_hidden_layers=self.num_hidden_layers   ,       num_attention_heads=self.num_attention_heads   ,       intermediate_size=self.intermediate_size   ,       hidden_act=self.hidden_act   ,       hidden_dropout_prob=self.hidden_dropout_prob   ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,       max_position_embeddings=self.max_position_embeddings   ,       type_vocab_size=self.type_vocab_size   ,       initializer_range=self.initializer_range   ,       return_dict=_lowercase   ,       )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel(config=_lowercase				)
												_lowerCAmelCase  						=			{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

												_lowerCAmelCase  						=			[input_ids, input_mask]
												_lowerCAmelCase  						=			model(_lowercase				)

												_lowerCAmelCase  						=			model(_lowercase				)

												self.parent.assertEqual(result.last_hidden_state.shape   ,       (self.batch_size, self.seq_length, self.hidden_size)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertForMaskedLM(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.seq_length, self.vocab_size)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_labels
												_lowerCAmelCase  						=			TFConvBertForSequenceClassification(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.num_labels)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_choices
												_lowerCAmelCase  						=			TFConvBertForMultipleChoice(config=_lowercase				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			tf.tile(tf.expand_dims(_lowercase   ,       1				)   ,       (1, self.num_choices, 1)				)
												_lowerCAmelCase  						=			{
												    """input_ids""": multiple_choice_inputs_ids,
												    """attention_mask""": multiple_choice_input_mask,
												    """token_type_ids""": multiple_choice_token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.num_choices)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.num_labels
												_lowerCAmelCase  						=			TFConvBertForTokenClassification(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.logits.shape   ,       (self.batch_size, self.seq_length, self.num_labels)				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertForQuestionAnswering(config=_lowercase				)
												_lowerCAmelCase  						=			{
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												_lowerCAmelCase  						=			model(_lowercase				)
												self.parent.assertEqual(result.start_logits.shape   ,       (self.batch_size, self.seq_length)				)
												self.parent.assertEqual(result.end_logits.shape   ,       (self.batch_size, self.seq_length)				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.prepare_config_and_inputs()
												(
												    (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   (
												    _lowerCAmelCase
												)						,   
												)  						=			config_and_inputs
												_lowerCAmelCase  						=			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict






@require_tf
class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE	,							_SCREAMING_SNAKE_CASE	,							unittest.TestCase			):






						'''simple docstring'''






						_lowercase		:				Union[str, Any]									=       (
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						_lowercase		:				str									=       (
						    {
						        '''feature-extraction''': TFConvBertModel,
						        '''fill-mask''': TFConvBertForMaskedLM,
						        '''question-answering''': TFConvBertForQuestionAnswering,
						        '''text-classification''': TFConvBertForSequenceClassification,
						        '''token-classification''': TFConvBertForTokenClassification,
						        '''zero-shot''': TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						_lowercase		:				Optional[Any]									=       False
						_lowercase		:				Dict									=       False
						_lowercase		:				Any									=       False




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModelTester(self				)
												_lowerCAmelCase  						=			ConfigTester(self   ,       config_class=_lowercase   ,       hidden_size=37				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												self.config_tester.run_common_tests()




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*_lowercase				)




						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase						,   _lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			True

												if hasattr(_lowercase   ,       """use_cache"""				):
																		_lowerCAmelCase  						=			True

												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """encoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)

												for model_class in self.all_model_classes:
																		_lowerCAmelCase  						=			self._prepare_for_class(_lowercase   ,       _lowercase				)
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			len(model(_lowercase				)				)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																								model.save_pretrained(_lowercase   ,       saved_model=_lowercase				)
																								_lowerCAmelCase  						=			os.path.join(_lowercase   ,       """saved_model"""   ,       """1"""				)
																								_lowerCAmelCase  						=			tf.keras.models.load_model(_lowercase				)
																								_lowerCAmelCase  						=			model(_lowercase				)

																								if self.is_encoder_decoder:
																														_lowerCAmelCase  						=			outputs["""encoder_hidden_states"""]
																														_lowerCAmelCase  						=			outputs["""encoder_attentions"""]
																								else:
																														_lowerCAmelCase  						=			outputs["""hidden_states"""]
																														_lowerCAmelCase  						=			outputs["""attentions"""]

																								self.assertEqual(len(_lowercase				)   ,       _lowercase				)

																								_lowerCAmelCase  						=			getattr(
																								    self.model_tester   ,       """expected_num_hidden_layers"""   ,       self.model_tester.num_hidden_layers + 1				)

																								self.assertEqual(len(_lowercase				)   ,       _lowercase				)
																								self.assertListEqual(
																								    list(output_hidden_states[0].shape[-2:]				)   ,       [self.model_tester.seq_length, self.model_tester.hidden_size]   ,       )

																								self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																								self.assertListEqual(
																								    list(output_attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,       )




						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
												self.assertIsNotNone(_lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase						,   _lowerCAmelCase  						=			self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase  						=			True
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """decoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """encoder_seq_length"""   ,       self.model_tester.seq_length				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)
												_lowerCAmelCase  						=			getattr(self.model_tester   ,       """key_length"""   ,       _lowercase				)

												def check_decoder_attentions_output(_lowercase				):
																		_lowerCAmelCase  						=			len(_lowercase				)
																		self.assertEqual(out_len % 2   ,       0				)
																		_lowerCAmelCase  						=			outputs.decoder_attentions
																		self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																		self.assertListEqual(
																		    list(decoder_attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]   ,       )

												def check_encoder_attentions_output(_lowercase				):
																		_lowerCAmelCase  						=			[
																		    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																		]
																		self.assertEqual(len(_lowercase				)   ,       self.model_tester.num_hidden_layers				)
																		self.assertListEqual(
																		    list(attentions[0].shape[-3:]				)   ,       [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,       )

												for model_class in self.all_model_classes:
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			False
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																		_lowerCAmelCase  						=			len(_lowercase				)
																		self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)

																		if self.is_encoder_decoder:
																								_lowerCAmelCase  						=			model_class(_lowercase				)
																								_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																								self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																								check_decoder_attentions_output(_lowercase				)

																		# Check that output attentions can also be changed via the config
																		del inputs_dict["output_attentions"]
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)
																		self.assertEqual(config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)

																		# Check attention is always last and order is fine
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			True
																		_lowerCAmelCase  						=			model_class(_lowercase				)
																		_lowerCAmelCase  						=			model(self._prepare_for_class(_lowercase   ,       _lowercase				)				)

																		self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)   ,       len(_lowercase				)				)
																		self.assertEqual(model.config.output_hidden_states   ,       _lowercase				)
																		check_encoder_attentions_output(_lowercase				)






@require_tf
class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						@slow
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
												_lowerCAmelCase  						=			tf.constant([[0, 1, 2, 3, 4, 5]]				)
												_lowerCAmelCase  						=			model(_lowercase				)[0]

												_lowerCAmelCase  						=			[1, 6, 768]
												self.assertEqual(output.shape   ,       _lowercase				)

												_lowerCAmelCase  						=			tf.constant(
												    [
												        [
												            [-0.0347_5493, -0.468_6034, -0.3063_8832],
												            [0.2263_7248, -0.2698_8646, -0.742_3424],
												            [0.1032_4868, -0.4501_3508, -0.5828_0784],
												        ]
												    ]				)
												tf.debugging.assert_near(output[:, :3, :3]   ,       _lowercase   ,       atol=1e-4				)
