'''simple docstring'''

import argparse
import os.path as osp
import re

import torch
from safetensors.torch import load_file, save_file


# =================#
# UNet Conversion #
# =================#

UpperCAmelCase     		=   [
    # (stable-diffusion, HF Diffusers)
    ('time_embed.0.weight', 'time_embedding.linear_1.weight'),
    ('time_embed.0.bias', 'time_embedding.linear_1.bias'),
    ('time_embed.2.weight', 'time_embedding.linear_2.weight'),
    ('time_embed.2.bias', 'time_embedding.linear_2.bias'),
    ('input_blocks.0.0.weight', 'conv_in.weight'),
    ('input_blocks.0.0.bias', 'conv_in.bias'),
    ('out.0.weight', 'conv_norm_out.weight'),
    ('out.0.bias', 'conv_norm_out.bias'),
    ('out.2.weight', 'conv_out.weight'),
    ('out.2.bias', 'conv_out.bias'),
]

UpperCAmelCase     		=   [
    # (stable-diffusion, HF Diffusers)
    ('in_layers.0', 'norm1'),
    ('in_layers.2', 'conv1'),
    ('out_layers.0', 'norm2'),
    ('out_layers.3', 'conv2'),
    ('emb_layers.1', 'time_emb_proj'),
    ('skip_connection', 'conv_shortcut'),
]

UpperCAmelCase     		=   []
# hardcoded number of downblocks and resnets/attentions...
# would need smarter logic for other networks.
for i in range(4):
				# loop over downblocks/upblocks

				for j in range(2):
								# loop over resnets/attentions for downblocks
								UpperCAmelCase     		=   F'''down_blocks.{i}.resnets.{j}.'''
								UpperCAmelCase     		=   F'''input_blocks.{3*i + j + 1}.0.'''
								unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))

								if i < 3:
												# no attention layers in down_blocks.3
												UpperCAmelCase     		=   F'''down_blocks.{i}.attentions.{j}.'''
												UpperCAmelCase     		=   F'''input_blocks.{3*i + j + 1}.1.'''
												unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))

				for j in range(3):
								# loop over resnets/attentions for upblocks
								UpperCAmelCase     		=   F'''up_blocks.{i}.resnets.{j}.'''
								UpperCAmelCase     		=   F'''output_blocks.{3*i + j}.0.'''
								unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))

								if i > 0:
												# no attention layers in up_blocks.0
												UpperCAmelCase     		=   F'''up_blocks.{i}.attentions.{j}.'''
												UpperCAmelCase     		=   F'''output_blocks.{3*i + j}.1.'''
												unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))

				if i < 3:
								# no downsample in down_blocks.3
								UpperCAmelCase     		=   F'''down_blocks.{i}.downsamplers.0.conv.'''
								UpperCAmelCase     		=   F'''input_blocks.{3*(i+1)}.0.op.'''
								unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))

								# no upsample in up_blocks.3
								UpperCAmelCase     		=   F'''up_blocks.{i}.upsamplers.0.'''
								UpperCAmelCase     		=   F'''output_blocks.{3*i + 2}.{1 if i == 0 else 2}.'''
								unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))

UpperCAmelCase     		=   'mid_block.attentions.0.'
UpperCAmelCase     		=   'middle_block.1.'
unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))

for j in range(2):
				UpperCAmelCase     		=   F'''mid_block.resnets.{j}.'''
				UpperCAmelCase     		=   F'''middle_block.{2*j}.'''
				unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Optional[int]							)      ->						Tuple:







	"""simple docstring"""
	lowerCAmelCase   =     {k: k for k in unet_state_dict.keys()}
	for sd_name, hf_name in unet_conversion_map:
		lowerCAmelCase   =     sd_name
	for k, v in mapping.items():
		if "resnets" in k:
			for sd_part, hf_part in unet_conversion_map_resnet:
				lowerCAmelCase   =     v.replace(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
			lowerCAmelCase   =     v
	for k, v in mapping.items():
		for sd_part, hf_part in unet_conversion_map_layer:
			lowerCAmelCase   =     v.replace(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     v
	lowerCAmelCase   =     {v: unet_state_dict[k] for k, v in mapping.items()}
	return new_state_dict


# ================#
# VAE Conversion #
# ================#

UpperCAmelCase     		=   [
    # (stable-diffusion, HF Diffusers)
    ('nin_shortcut', 'conv_shortcut'),
    ('norm_out', 'conv_norm_out'),
    ('mid.attn_1.', 'mid_block.attentions.0.'),
]

for i in range(4):
				# down_blocks have two resnets
				for j in range(2):
								UpperCAmelCase     		=   F'''encoder.down_blocks.{i}.resnets.{j}.'''
								UpperCAmelCase     		=   F'''encoder.down.{i}.block.{j}.'''
								vae_conversion_map.append((sd_down_prefix, hf_down_prefix))

				if i < 3:
								UpperCAmelCase     		=   F'''down_blocks.{i}.downsamplers.0.'''
								UpperCAmelCase     		=   F'''down.{i}.downsample.'''
								vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix))

								UpperCAmelCase     		=   F'''up_blocks.{i}.upsamplers.0.'''
								UpperCAmelCase     		=   F'''up.{3-i}.upsample.'''
								vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix))

				# up_blocks have three resnets
				# also, up blocks in hf are numbered in reverse from sd
				for j in range(3):
								UpperCAmelCase     		=   F'''decoder.up_blocks.{i}.resnets.{j}.'''
								UpperCAmelCase     		=   F'''decoder.up.{3-i}.block.{j}.'''
								vae_conversion_map.append((sd_up_prefix, hf_up_prefix))

# this part accounts for mid blocks in both the encoder and the decoder
for i in range(2):
				UpperCAmelCase     		=   F'''mid_block.resnets.{i}.'''
				UpperCAmelCase     		=   F'''mid.block_{i+1}.'''
				vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix))


UpperCAmelCase     		=   [
    # (stable-diffusion, HF Diffusers)
    ('norm.', 'group_norm.'),
    ('q.', 'query.'),
    ('k.', 'key.'),
    ('v.', 'value.'),
    ('proj_out.', 'proj_attn.'),
]






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						List[Any]							)      ->						Optional[Any]:







	"""simple docstring"""
	return w.reshape(*w.shape							,		1							,		1							)






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Optional[Any]							)      ->						Tuple:







	"""simple docstring"""
	lowerCAmelCase   =     {k: k for k in vae_state_dict.keys()}
	for k, v in mapping.items():
		for sd_part, hf_part in vae_conversion_map:
			lowerCAmelCase   =     v.replace(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     v
	for k, v in mapping.items():
		if "attentions" in k:
			for sd_part, hf_part in vae_conversion_map_attn:
				lowerCAmelCase   =     v.replace(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)
			lowerCAmelCase   =     v
	lowerCAmelCase   =     {v: vae_state_dict[k] for k, v in mapping.items()}
	lowerCAmelCase   =     ["""q""", """k""", """v""", """proj_out"""]
	for k, v in new_state_dict.items():
		for weight_name in weights_to_convert:
			if f'mid.attn_1.{weight_name}.weight' in k:
				print(f'Reshaping {k} for SD format'							)
				lowerCAmelCase   =     reshape_weight_for_sd(_SCREAMING_SNAKE_CASE							)
	return new_state_dict


# =========================#
# Text Encoder Conversion #
# =========================#


UpperCAmelCase     		=   [
    # (stable-diffusion, HF Diffusers)
    ('resblocks.', 'text_model.encoder.layers.'),
    ('ln_1', 'layer_norm1'),
    ('ln_2', 'layer_norm2'),
    ('.c_fc.', '.fc1.'),
    ('.c_proj.', '.fc2.'),
    ('.attn', '.self_attn'),
    ('ln_final.', 'transformer.text_model.final_layer_norm.'),
    ('token_embedding.weight', 'transformer.text_model.embeddings.token_embedding.weight'),
    ('positional_embedding', 'transformer.text_model.embeddings.position_embedding.weight'),
]
UpperCAmelCase     		=   {re.escape(x[1]): x[0] for x in textenc_conversion_lst}
UpperCAmelCase     		=   re.compile('|'.join(protected.keys()))

# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp
UpperCAmelCase     		=   {'q': 0, 'k': 1, 'v': 2}






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						int							)      ->						Optional[int]:







	"""simple docstring"""
	lowerCAmelCase   =     {}
	lowerCAmelCase   =     {}
	lowerCAmelCase   =     {}
	for k, v in text_enc_dict.items():
		if (
		    k.endswith(""".self_attn.q_proj.weight"""							)
		    or k.endswith(""".self_attn.k_proj.weight"""							)
		    or k.endswith(""".self_attn.v_proj.weight"""							)
		):
			lowerCAmelCase   =     k[: -len(""".q_proj.weight"""							)]
			lowerCAmelCase   =     k[-len("""q_proj.weight"""							)]
			if k_pre not in capture_qkv_weight:
				lowerCAmelCase   =     [None, None, None]
			lowerCAmelCase   =     v
			continue

		if (
		    k.endswith(""".self_attn.q_proj.bias"""							)
		    or k.endswith(""".self_attn.k_proj.bias"""							)
		    or k.endswith(""".self_attn.v_proj.bias"""							)
		):
			lowerCAmelCase   =     k[: -len(""".q_proj.bias"""							)]
			lowerCAmelCase   =     k[-len("""q_proj.bias"""							)]
			if k_pre not in capture_qkv_bias:
				lowerCAmelCase   =     [None, None, None]
			lowerCAmelCase   =     v
			continue

		lowerCAmelCase   =     textenc_pattern.sub(lambda _SCREAMING_SNAKE_CASE							: protected[re.escape(m.group(0							)							)]							,		_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     v

	for k_pre, tensors in capture_qkv_weight.items():
		if None in tensors:
			raise Exception("""CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing"""							)
		lowerCAmelCase   =     textenc_pattern.sub(lambda _SCREAMING_SNAKE_CASE							: protected[re.escape(m.group(0							)							)]							,		_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     torch.cat(_SCREAMING_SNAKE_CASE							)

	for k_pre, tensors in capture_qkv_bias.items():
		if None in tensors:
			raise Exception("""CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing"""							)
		lowerCAmelCase   =     textenc_pattern.sub(lambda _SCREAMING_SNAKE_CASE							: protected[re.escape(m.group(0							)							)]							,		_SCREAMING_SNAKE_CASE							)
		lowerCAmelCase   =     torch.cat(_SCREAMING_SNAKE_CASE							)

	return new_state_dict






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						List[Any]							)      ->						Tuple:







	"""simple docstring"""
	return text_enc_dict


if __name__ == "__main__":
				UpperCAmelCase     		=   argparse.ArgumentParser()

				parser.add_argument('--model_path', default=None, type=str, required=True, help='Path to the model to convert.')
				parser.add_argument('--checkpoint_path', default=None, type=str, required=True, help='Path to the output model.')
				parser.add_argument('--half', action='store_true', help='Save weights in half precision.')
				parser.add_argument(
				    '--use_safetensors', action='store_true', help='Save weights use safetensors, default is ckpt.'
				)

				UpperCAmelCase     		=   parser.parse_args()

				assert args.model_path is not None, "Must provide a model path!"

				assert args.checkpoint_path is not None, "Must provide a checkpoint path!"

				# Path for safetensors
				UpperCAmelCase     		=   osp.join(args.model_path, 'unet', 'diffusion_pytorch_model.safetensors')
				UpperCAmelCase     		=   osp.join(args.model_path, 'vae', 'diffusion_pytorch_model.safetensors')
				UpperCAmelCase     		=   osp.join(args.model_path, 'text_encoder', 'model.safetensors')

				# Load models from safetensors if it exists, if it doesn't pytorch
				if osp.exists(unet_path):
								UpperCAmelCase     		=   load_file(unet_path, device='cpu')
				else:
								UpperCAmelCase     		=   osp.join(args.model_path, 'unet', 'diffusion_pytorch_model.bin')
								UpperCAmelCase     		=   torch.load(unet_path, map_location='cpu')

				if osp.exists(vae_path):
								UpperCAmelCase     		=   load_file(vae_path, device='cpu')
				else:
								UpperCAmelCase     		=   osp.join(args.model_path, 'vae', 'diffusion_pytorch_model.bin')
								UpperCAmelCase     		=   torch.load(vae_path, map_location='cpu')

				if osp.exists(text_enc_path):
								UpperCAmelCase     		=   load_file(text_enc_path, device='cpu')
				else:
								UpperCAmelCase     		=   osp.join(args.model_path, 'text_encoder', 'pytorch_model.bin')
								UpperCAmelCase     		=   torch.load(text_enc_path, map_location='cpu')

				# Convert the UNet model
				UpperCAmelCase     		=   convert_unet_state_dict(unet_state_dict)
				UpperCAmelCase     		=   {'model.diffusion_model.' + k: v for k, v in unet_state_dict.items()}

				# Convert the VAE model
				UpperCAmelCase     		=   convert_vae_state_dict(vae_state_dict)
				UpperCAmelCase     		=   {'first_stage_model.' + k: v for k, v in vae_state_dict.items()}

				# Easiest way to identify v2.0 model seems to be that the text encoder (OpenCLIP) is deeper
				UpperCAmelCase     		=   'text_model.encoder.layers.22.layer_norm2.bias' in text_enc_dict

				if is_vaa_model:
								# Need to add the tag 'transformer' in advance so we can knock it out from the final layer-norm
								UpperCAmelCase     		=   {'transformer.' + k: v for k, v in text_enc_dict.items()}
								UpperCAmelCase     		=   convert_text_enc_state_dict_vaa(text_enc_dict)
								UpperCAmelCase     		=   {'cond_stage_model.model.' + k: v for k, v in text_enc_dict.items()}
				else:
								UpperCAmelCase     		=   convert_text_enc_state_dict(text_enc_dict)
								UpperCAmelCase     		=   {'cond_stage_model.transformer.' + k: v for k, v in text_enc_dict.items()}

				# Put together new checkpoint
				UpperCAmelCase     		=   {**unet_state_dict, **vae_state_dict, **text_enc_dict}
				if args.half:
								UpperCAmelCase     		=   {k: v.half() for k, v in state_dict.items()}

				if args.use_safetensors:
								save_file(state_dict, args.checkpoint_path)
				else:
								UpperCAmelCase     		=   {'state_dict': state_dict}
								torch.save(state_dict, args.checkpoint_path)




'''simple docstring'''

from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFConvBertForMaskedLM,
				    TFConvBertForMultipleChoice,
				    TFConvBertForQuestionAnswering,
				    TFConvBertForSequenceClassification,
				    TFConvBertForTokenClassification,
				    TFConvBertModel,
				)

class __snake_case:


						'''simple docstring'''



						def __init__( self     ,	A_     ,	A_=13     ,	A_=7     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=99     ,	A_=32     ,	A_=2     ,	A_=4     ,	A_=37     ,	A_="gelu"     ,	A_=0.1     ,	A_=0.1     ,	A_=512     ,	A_=16     ,	A_=2     ,	A_=0.0_2     ,	A_=3     ,	A_=4     ,	A_=None     ,	)     ->     Dict:
							lowerCAmelCase   =     parent
							lowerCAmelCase   =     13
							lowerCAmelCase   =     7
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     99
							lowerCAmelCase   =     384
							lowerCAmelCase   =     2
							lowerCAmelCase   =     4
							lowerCAmelCase   =     37
							lowerCAmelCase   =     """gelu"""
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     512
							lowerCAmelCase   =     16
							lowerCAmelCase   =     2
							lowerCAmelCase   =     0.0_2
							lowerCAmelCase   =     3
							lowerCAmelCase   =     4
							lowerCAmelCase   =     128
							lowerCAmelCase   =     2
							lowerCAmelCase   =     9
							lowerCAmelCase   =     1
							lowerCAmelCase   =     None


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size				)

							lowerCAmelCase   =     None
							if self.use_input_mask:
								lowerCAmelCase   =     random_attention_mask([self.batch_size, self.seq_length]				)

							lowerCAmelCase   =     None
							if self.use_token_type_ids:
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size				)

							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							if self.use_labels:
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.type_sequence_label_size				)
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels				)
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.num_choices				)

							lowerCAmelCase   =     ConvBertConfig(
							    vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	initializer_range=self.initializer_range     ,	return_dict=A_     ,	)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     int:
							lowerCAmelCase   =     TFConvBertModel(config=A_				)
							lowerCAmelCase   =     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

							lowerCAmelCase   =     [input_ids, input_mask]
							lowerCAmelCase   =     model(A_				)

							lowerCAmelCase   =     model(A_				)

							self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     List[Any]:
							lowerCAmelCase   =     TFConvBertForMaskedLM(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForSequenceClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Any:
							lowerCAmelCase   =     self.num_choices
							lowerCAmelCase   =     TFConvBertForMultipleChoice(config=A_				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     {
							    """input_ids""": multiple_choice_inputs_ids,
							    """attention_mask""": multiple_choice_input_mask,
							    """token_type_ids""": multiple_choice_token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForTokenClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertForQuestionAnswering(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)				)




						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     self.prepare_config_and_inputs()
							(
							    (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       
							)   =     config_and_inputs
							lowerCAmelCase   =     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict







@require_tf
class __snake_case(				_lowerCAmelCase		,							_lowerCAmelCase		,							unittest.TestCase ):


						'''simple docstring'''



						UpperCAmelCase							:   Optional[int]									=		(
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase							:   Union[str, Any]									=		(
						    {
						        "feature-extraction": TFConvBertModel,
						        "fill-mask": TFConvBertForMaskedLM,
						        "question-answering": TFConvBertForQuestionAnswering,
						        "text-classification": TFConvBertForSequenceClassification,
						        "token-classification": TFConvBertForTokenClassification,
						        "zero-shot": TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCAmelCase							:   Union[str, Any]									=		False
						UpperCAmelCase							:   Optional[int]									=		False
						UpperCAmelCase							:   Dict									=		False


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertModelTester(self				)
							lowerCAmelCase   =     ConfigTester(self     ,	config_class=A_     ,	hidden_size=37				)


						def     __snake_case   ( self				)     ->     Tuple:
							self.config_tester.run_common_tests()


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*A_				)


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*A_				)


						def     __snake_case   ( self				)     ->     List[str]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*A_				)


						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*A_				)


						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True

							if hasattr(A_     ,	"""use_cache"""				):
								lowerCAmelCase   =     True

							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     self._prepare_for_class(A_     ,	A_				)
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     len(model(A_				)				)

								with tempfile.TemporaryDirectory() as tmpdirname:
									model.save_pretrained(A_     ,	saved_model=A_				)
									lowerCAmelCase   =     os.path.join(A_     ,	"""saved_model"""     ,	"""1"""				)
									lowerCAmelCase   =     tf.keras.models.load_model(A_				)
									lowerCAmelCase   =     model(A_				)

									if self.is_encoder_decoder:
										lowerCAmelCase   =     outputs["""encoder_hidden_states"""]
										lowerCAmelCase   =     outputs["""encoder_attentions"""]
									else:
										lowerCAmelCase   =     outputs["""hidden_states"""]
										lowerCAmelCase   =     outputs["""attentions"""]

									self.assertEqual(len(A_				)     ,	A_				)

									lowerCAmelCase   =     getattr(
									    self.model_tester     ,	"""expected_num_hidden_layers"""     ,	self.model_tester.num_hidden_layers + 1				)

									self.assertEqual(len(A_				)     ,	A_				)
									self.assertListEqual(
									    list(output_hidden_states[0].shape[-2:]				)     ,	[self.model_tester.seq_length, self.model_tester.hidden_size]     ,	)

									self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
									self.assertListEqual(
									    list(output_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)


						@slow
						def     __snake_case   ( self				)     ->     Optional[Any]:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							self.assertIsNotNone(A_				)




						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""decoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							def check_decoder_attentions_output(A_				):
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(out_len % 2     ,	0				)
								lowerCAmelCase   =     outputs.decoder_attentions
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]     ,	)

							def check_encoder_attentions_output(A_				):
								lowerCAmelCase   =     [
								    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
								]
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     True
								lowerCAmelCase   =     False
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								if self.is_encoder_decoder:
									lowerCAmelCase   =     model_class(A_				)
									lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
									self.assertEqual(config.output_hidden_states     ,	A_				)
									check_decoder_attentions_output(A_				)

								# Check that output attentions can also be changed via the config
								del inputs_dict["output_attentions"]
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								# Check attention is always last and order is fine
								lowerCAmelCase   =     True
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)

								self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)     ,	len(A_				)				)
								self.assertEqual(model.config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)







@require_tf
class __snake_case(				unittest.TestCase ):


						'''simple docstring'''



						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							lowerCAmelCase   =     tf.constant([[0, 1, 2, 3, 4, 5]]				)
							lowerCAmelCase   =     model(A_				)[0]

							lowerCAmelCase   =     [1, 6, 768]
							self.assertEqual(output.shape     ,	A_				)

							lowerCAmelCase   =     tf.constant(
							    [
							        [
							            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
							            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
							            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
							        ]
							    ]				)
							tf.debugging.assert_near(output[:, :3, :3]     ,	A_     ,	atol=1e-4				)