



'''simple docstring'''

import json
import logging
import math
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from datasets import Dataset, load_dataset

import transformers
from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_MASKED_LM_MAPPING,
    AutoConfig,
    AutoModelForMaskedLM,
    AutoTokenizer,
    DataCollatorForWholeWordMask,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


UpperCAmelCase     		=   logging.getLogger(__name__)
UpperCAmelCase     		=   list(MODEL_FOR_MASKED_LM_MAPPING.keys())
UpperCAmelCase     		=   tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

@dataclass
class __snake_case:


						'''simple docstring'''



						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={
						        "help": (
						            "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
						        )
						    }		,							)
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(_lowerCAmelCase )}		,							)
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={
						        "help": (
						            "Override some existing default config settings when a model is trained from scratch. Example: "
						            "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
						        )
						    }		,							)
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "Pretrained config name or path if not the same as model_name"} )
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} )
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}		,							)
						UpperCAmelCase							:   bool									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}		,							)
						UpperCAmelCase							:   str									=		field(
						    default="main"		,							metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}		,							)
						UpperCAmelCase							:   bool									=		field(
						    default=_lowerCAmelCase		,							metadata={
						        "help": (
						            "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
						            "with private models)."
						        )
						    }		,							)


						def     __snake_case   ( self				)     ->     int:
							if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
								raise ValueError(
								    """--config_overrides can't be used in combination with --config_name or --model_name_or_path"""				)







@dataclass
class __snake_case:


						'''simple docstring'''



						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "The name of the dataset to use (via the datasets library)."} )
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} )
						UpperCAmelCase							:   Optional[str]									=		field(default=_lowerCAmelCase		,							metadata={"help": "The input training data file (a text file)."} )
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}		,							)
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "An optional input train ref data file for whole word masking in Chinese."}		,							)
						UpperCAmelCase							:   Optional[str]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "An optional input validation ref data file for whole word masking in Chinese."}		,							)
						UpperCAmelCase							:   bool									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "Overwrite the cached training and evaluation sets"} )
						UpperCAmelCase							:   Optional[int]									=		field(
						    default=5		,							metadata={
						        "help": "The percentage of the train set used as validation set in case there's no validation split"
						    }		,							)
						UpperCAmelCase							:   Optional[int]									=		field(
						    default=_lowerCAmelCase		,							metadata={
						        "help": (
						            "The maximum total input sequence length after tokenization. Sequences longer "
						            "than this will be truncated. Default to the max input length of the model."
						        )
						    }		,							)
						UpperCAmelCase							:   Optional[int]									=		field(
						    default=_lowerCAmelCase		,							metadata={"help": "The number of processes to use for the preprocessing."}		,							)
						UpperCAmelCase							:   float									=		field(
						    default=0.15		,							metadata={"help": "Ratio of tokens to mask for masked language modeling loss"} )
						UpperCAmelCase							:   bool									=		field(
						    default=_lowerCAmelCase		,							metadata={
						        "help": (
						            "Whether to pad all samples to `max_seq_length`. "
						            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
						        )
						    }		,							)


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							if self.train_file is not None:
								lowerCAmelCase   =     self.train_file.split("""."""				)[-1]
								assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
							if self.validation_file is not None:
								lowerCAmelCase   =     self.validation_file.split("""."""				)[-1]
								assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."







def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						List[Any]							,		_SCREAMING_SNAKE_CASE					:						List[str]							)      ->						Any:







	"""simple docstring"""
	with open(_SCREAMING_SNAKE_CASE							,		"""r"""							,		encoding="""utf-8"""							) as f:
		lowerCAmelCase   =     [json.loads(_SCREAMING_SNAKE_CASE							) for line in f.read().splitlines() if (len(_SCREAMING_SNAKE_CASE							) > 0 and not line.isspace())]
	assert len(_SCREAMING_SNAKE_CASE							) == len(_SCREAMING_SNAKE_CASE							)

	lowerCAmelCase   =     {c: dataset[c] for c in dataset.column_names}
	lowerCAmelCase   =     refs
	return Dataset.from_dict(_SCREAMING_SNAKE_CASE							)






def 		_snake_case		( )      ->						List[str]:







	"""simple docstring"""
	# See all possible arguments in src/transformers/training_args.py
	# or by passing the --help flag to this script.
	# We now keep distinct sets of args, for a cleaner separation of concerns.

	lowerCAmelCase   =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)
	if len(sys.argv							) == 2 and sys.argv[1].endswith(""".json"""							):
		# If we pass only one argument to the script and it's the path to a json file,
		# let's parse it to get our arguments.
		lowerCAmelCase,       lowerCAmelCase,       lowerCAmelCase   =     parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]							)							)
	else:
		lowerCAmelCase,       lowerCAmelCase,       lowerCAmelCase   =     parser.parse_args_into_dataclasses()

	# Detecting last checkpoint.
	lowerCAmelCase   =     None
	if os.path.isdir(training_args.output_dir							) and training_args.do_train and not training_args.overwrite_output_dir:
		lowerCAmelCase   =     get_last_checkpoint(training_args.output_dir							)
		if last_checkpoint is None and len(os.listdir(training_args.output_dir							)							) > 0:
			raise ValueError(
			    f'Output directory ({training_args.output_dir}) already exists and is not empty. '
			    """Use --overwrite_output_dir to overcome."""							)
		elif last_checkpoint is not None:
			logger.info(
			    f'Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change '
			    """the `--output_dir` or add `--overwrite_output_dir` to train from scratch."""							)

    # Setup logging
	logging.basicConfig(
	    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""							,		datefmt="""%m/%d/%Y %H:%M:%S"""							,		handlers=[logging.StreamHandler(sys.stdout							)]							,		)
	logger.setLevel(logging.INFO if is_main_process(training_args.local_rank							) else logging.WARN							)

	# Log on each process the small summary:
	logger.warning(
	    f'Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}'
	    + f'distributed training: {bool(training_args.local_rank != -1							)}, 16-bits training: {training_args.fpaa}'							)
	# Set the verbosity to info of the Transformers logger (on main process only):
	if is_main_process(training_args.local_rank							):
		transformers.utils.logging.set_verbosity_info()
		transformers.utils.logging.enable_default_handler()
		transformers.utils.logging.enable_explicit_format()
	logger.info("""Training/evaluation parameters %s"""							,		_SCREAMING_SNAKE_CASE							)

	# Set seed before initializing model.
	set_seed(training_args.seed							)

	# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
	# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
	# (the dataset will be downloaded automatically from the datasets Hub).
	#
	# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
	# 'text' is found. You can easily tweak this behavior (see below).
	#
	# In distributed training, the load_dataset function guarantee that only one local process can concurrently
	# download the dataset.
	if data_args.dataset_name is not None:
		# Downloading and loading a dataset from the hub.
		lowerCAmelCase   =     load_dataset(data_args.dataset_name							,		data_args.dataset_config_name							)
		if "validation" not in datasets.keys():
			lowerCAmelCase   =     load_dataset(
			    data_args.dataset_name							,		data_args.dataset_config_name							,		split=f'train[:{data_args.validation_split_percentage}%]'							,		)
			lowerCAmelCase   =     load_dataset(
			    data_args.dataset_name							,		data_args.dataset_config_name							,		split=f'train[{data_args.validation_split_percentage}%:]'							,		)
	else:
		lowerCAmelCase   =     {}
		if data_args.train_file is not None:
			lowerCAmelCase   =     data_args.train_file
		if data_args.validation_file is not None:
			lowerCAmelCase   =     data_args.validation_file
		lowerCAmelCase   =     data_args.train_file.split("""."""							)[-1]
		if extension == "txt":
			lowerCAmelCase   =     """text"""
		lowerCAmelCase   =     load_dataset(_SCREAMING_SNAKE_CASE							,		data_files=_SCREAMING_SNAKE_CASE							)
	# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
	# https://huggingface.co/docs/datasets/loading_datasets.html.

	# Load pretrained model and tokenizer
	#
	# Distributed training:
	# The .from_pretrained methods guarantee that only one local process can concurrently
	# download model & vocab.
	lowerCAmelCase   =     {
	    """cache_dir""": model_args.cache_dir,
	    """revision""": model_args.model_revision,
	    """use_auth_token""": True if model_args.use_auth_token else None,
	}
	if model_args.config_name:
		lowerCAmelCase   =     AutoConfig.from_pretrained(model_args.config_name							,		**_SCREAMING_SNAKE_CASE							)
	elif model_args.model_name_or_path:
		lowerCAmelCase   =     AutoConfig.from_pretrained(model_args.model_name_or_path							,		**_SCREAMING_SNAKE_CASE							)
	else:
		lowerCAmelCase   =     CONFIG_MAPPING[model_args.model_type]()
		logger.warning("""You are instantiating a new config instance from scratch."""							)
		if model_args.config_overrides is not None:
			logger.info(f'Overriding config: {model_args.config_overrides}'							)
			config.update_from_string(model_args.config_overrides							)
			logger.info(f'New config: {config}'							)

	lowerCAmelCase   =     {
	    """cache_dir""": model_args.cache_dir,
	    """use_fast""": model_args.use_fast_tokenizer,
	    """revision""": model_args.model_revision,
	    """use_auth_token""": True if model_args.use_auth_token else None,
	}
	if model_args.tokenizer_name:
		lowerCAmelCase   =     AutoTokenizer.from_pretrained(model_args.tokenizer_name							,		**_SCREAMING_SNAKE_CASE							)
	elif model_args.model_name_or_path:
		lowerCAmelCase   =     AutoTokenizer.from_pretrained(model_args.model_name_or_path							,		**_SCREAMING_SNAKE_CASE							)
	else:
		raise ValueError(
		    """You are instantiating a new tokenizer from scratch. This is not supported by this script."""
		    """You can do it from another script, save it, and load it from here, using --tokenizer_name."""							)

	if model_args.model_name_or_path:
		lowerCAmelCase   =     AutoModelForMaskedLM.from_pretrained(
		    model_args.model_name_or_path							,		from_tf=bool(""".ckpt""" in model_args.model_name_or_path							)							,		config=_SCREAMING_SNAKE_CASE							,		cache_dir=model_args.cache_dir							,		revision=model_args.model_revision							,		use_auth_token=True if model_args.use_auth_token else None							,		)
	else:
		logger.info("""Training new model from scratch"""							)
		lowerCAmelCase   =     AutoModelForMaskedLM.from_config(_SCREAMING_SNAKE_CASE							)

	model.resize_token_embeddings(len(_SCREAMING_SNAKE_CASE							)							)

	# Preprocessing the datasets.
	# First we tokenize all the texts.
	if training_args.do_train:
		lowerCAmelCase   =     datasets["""train"""].column_names
	else:
		lowerCAmelCase   =     datasets["""validation"""].column_names
	lowerCAmelCase   =     """text""" if """text""" in column_names else column_names[0]

	lowerCAmelCase   =     """max_length""" if data_args.pad_to_max_length else False

	def tokenize_function(_SCREAMING_SNAKE_CASE					:						List[Any]							):
		# Remove empty lines
		lowerCAmelCase   =     [line for line in examples["""text"""] if len(_SCREAMING_SNAKE_CASE							) > 0 and not line.isspace()]
		return tokenizer(examples["""text"""]							,		padding=_SCREAMING_SNAKE_CASE							,		truncation=_SCREAMING_SNAKE_CASE							,		max_length=data_args.max_seq_length							)

	lowerCAmelCase   =     datasets.map(
	    _SCREAMING_SNAKE_CASE							,		batched=_SCREAMING_SNAKE_CASE							,		num_proc=data_args.preprocessing_num_workers							,		remove_columns=[text_column_name]							,		load_from_cache_file=not data_args.overwrite_cache							,		)

	# Add the chinese references if provided
	if data_args.train_ref_file is not None:
		lowerCAmelCase   =     add_chinese_references(tokenized_datasets["""train"""]							,		data_args.train_ref_file							)
	if data_args.validation_ref_file is not None:
		lowerCAmelCase   =     add_chinese_references(
		    tokenized_datasets["""validation"""]							,		data_args.validation_ref_file							)
	# If we have ref files, need to avoid it removed by trainer
	lowerCAmelCase   =     data_args.train_ref_file or data_args.validation_ref_file
	if has_ref:
		lowerCAmelCase   =     False

	# Data collator
	# This one will take care of randomly masking the tokens.
	lowerCAmelCase   =     DataCollatorForWholeWordMask(tokenizer=_SCREAMING_SNAKE_CASE							,		mlm_probability=data_args.mlm_probability							)

	# Initialize our Trainer
	lowerCAmelCase   =     Trainer(
	    model=_SCREAMING_SNAKE_CASE							,		args=_SCREAMING_SNAKE_CASE							,		train_dataset=tokenized_datasets["""train"""] if training_args.do_train else None							,		eval_dataset=tokenized_datasets["""validation"""] if training_args.do_eval else None							,		tokenizer=_SCREAMING_SNAKE_CASE							,		data_collator=_SCREAMING_SNAKE_CASE							,		)

	# Training
	if training_args.do_train:
		if last_checkpoint is not None:
			lowerCAmelCase   =     last_checkpoint
		elif model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path							):
			lowerCAmelCase   =     model_args.model_name_or_path
		else:
			lowerCAmelCase   =     None
		lowerCAmelCase   =     trainer.train(resume_from_checkpoint=_SCREAMING_SNAKE_CASE							)
		trainer.save_model()  # Saves the tokenizer too for easy upload

		lowerCAmelCase   =     os.path.join(training_args.output_dir							,		"""train_results.txt"""							)
		if trainer.is_world_process_zero():
			with open(_SCREAMING_SNAKE_CASE							,		"""w"""							) as writer:
				logger.info("""***** Train results *****"""							)
				for key, value in sorted(train_result.metrics.items()							):
					logger.info(f'  {key} = {value}'							)
					writer.write(f'{key} = {value}\n'							)

            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
			trainer.state.save_to_json(os.path.join(training_args.output_dir							,		"""trainer_state.json"""							)							)

    # Evaluation
	lowerCAmelCase   =     {}
	if training_args.do_eval:
		logger.info("""*** Evaluate ***"""							)

		lowerCAmelCase   =     trainer.evaluate()

		lowerCAmelCase   =     math.exp(eval_output["""eval_loss"""]							)
		lowerCAmelCase   =     perplexity

		lowerCAmelCase   =     os.path.join(training_args.output_dir							,		"""eval_results_mlm_wwm.txt"""							)
		if trainer.is_world_process_zero():
			with open(_SCREAMING_SNAKE_CASE							,		"""w"""							) as writer:
				logger.info("""***** Eval results *****"""							)
				for key, value in sorted(results.items()							):
					logger.info(f'  {key} = {value}'							)
					writer.write(f'{key} = {value}\n'							)

	return results






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						List[str]							)      ->						Dict:







	"""simple docstring"""
	# For xla_spawn (TPUs)
	main()


if __name__ == "__main__":
				main()




'''simple docstring'''

from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFConvBertForMaskedLM,
				    TFConvBertForMultipleChoice,
				    TFConvBertForQuestionAnswering,
				    TFConvBertForSequenceClassification,
				    TFConvBertForTokenClassification,
				    TFConvBertModel,
				)

class __snake_case:


						'''simple docstring'''



						def __init__( self     ,	A_     ,	A_=13     ,	A_=7     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=True     ,	A_=99     ,	A_=32     ,	A_=2     ,	A_=4     ,	A_=37     ,	A_="gelu"     ,	A_=0.1     ,	A_=0.1     ,	A_=512     ,	A_=16     ,	A_=2     ,	A_=0.0_2     ,	A_=3     ,	A_=4     ,	A_=None     ,	)     ->     Dict:
							lowerCAmelCase   =     parent
							lowerCAmelCase   =     13
							lowerCAmelCase   =     7
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True
							lowerCAmelCase   =     99
							lowerCAmelCase   =     384
							lowerCAmelCase   =     2
							lowerCAmelCase   =     4
							lowerCAmelCase   =     37
							lowerCAmelCase   =     """gelu"""
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     0.1
							lowerCAmelCase   =     512
							lowerCAmelCase   =     16
							lowerCAmelCase   =     2
							lowerCAmelCase   =     0.0_2
							lowerCAmelCase   =     3
							lowerCAmelCase   =     4
							lowerCAmelCase   =     128
							lowerCAmelCase   =     2
							lowerCAmelCase   =     9
							lowerCAmelCase   =     1
							lowerCAmelCase   =     None


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size				)

							lowerCAmelCase   =     None
							if self.use_input_mask:
								lowerCAmelCase   =     random_attention_mask([self.batch_size, self.seq_length]				)

							lowerCAmelCase   =     None
							if self.use_token_type_ids:
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size				)

							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							lowerCAmelCase   =     None
							if self.use_labels:
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.type_sequence_label_size				)
								lowerCAmelCase   =     ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels				)
								lowerCAmelCase   =     ids_tensor([self.batch_size]     ,	self.num_choices				)

							lowerCAmelCase   =     ConvBertConfig(
							    vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	initializer_range=self.initializer_range     ,	return_dict=A_     ,	)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     int:
							lowerCAmelCase   =     TFConvBertModel(config=A_				)
							lowerCAmelCase   =     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

							lowerCAmelCase   =     [input_ids, input_mask]
							lowerCAmelCase   =     model(A_				)

							lowerCAmelCase   =     model(A_				)

							self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     List[Any]:
							lowerCAmelCase   =     TFConvBertForMaskedLM(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForSequenceClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Any:
							lowerCAmelCase   =     self.num_choices
							lowerCAmelCase   =     TFConvBertForMultipleChoice(config=A_				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     tf.tile(tf.expand_dims(A_     ,	1				)     ,	(1, self.num_choices, 1)				)
							lowerCAmelCase   =     {
							    """input_ids""": multiple_choice_inputs_ids,
							    """attention_mask""": multiple_choice_input_mask,
							    """token_type_ids""": multiple_choice_token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.num_labels
							lowerCAmelCase   =     TFConvBertForTokenClassification(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}
							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertForQuestionAnswering(config=A_				)
							lowerCAmelCase   =     {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							    """token_type_ids""": token_type_ids,
							}

							lowerCAmelCase   =     model(A_				)
							self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)				)
							self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)				)




						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     self.prepare_config_and_inputs()
							(
							    (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       (
							    lowerCAmelCase
							),       
							)   =     config_and_inputs
							lowerCAmelCase   =     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict







@require_tf
class __snake_case(				_lowerCAmelCase		,							_lowerCAmelCase		,							unittest.TestCase ):


						'''simple docstring'''



						UpperCAmelCase							:   Optional[int]									=		(
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase							:   Union[str, Any]									=		(
						    {
						        "feature-extraction": TFConvBertModel,
						        "fill-mask": TFConvBertForMaskedLM,
						        "question-answering": TFConvBertForQuestionAnswering,
						        "text-classification": TFConvBertForSequenceClassification,
						        "token-classification": TFConvBertForTokenClassification,
						        "zero-shot": TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCAmelCase							:   Union[str, Any]									=		False
						UpperCAmelCase							:   Optional[int]									=		False
						UpperCAmelCase							:   Dict									=		False


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     TFConvBertModelTester(self				)
							lowerCAmelCase   =     ConfigTester(self     ,	config_class=A_     ,	hidden_size=37				)


						def     __snake_case   ( self				)     ->     Tuple:
							self.config_tester.run_common_tests()


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*A_				)


						def     __snake_case   ( self				)     ->     Optional[int]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*A_				)


						def     __snake_case   ( self				)     ->     List[str]:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*A_				)


						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*A_				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*A_				)


						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     True

							if hasattr(A_     ,	"""use_cache"""				):
								lowerCAmelCase   =     True

							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     self._prepare_for_class(A_     ,	A_				)
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     len(model(A_				)				)

								with tempfile.TemporaryDirectory() as tmpdirname:
									model.save_pretrained(A_     ,	saved_model=A_				)
									lowerCAmelCase   =     os.path.join(A_     ,	"""saved_model"""     ,	"""1"""				)
									lowerCAmelCase   =     tf.keras.models.load_model(A_				)
									lowerCAmelCase   =     model(A_				)

									if self.is_encoder_decoder:
										lowerCAmelCase   =     outputs["""encoder_hidden_states"""]
										lowerCAmelCase   =     outputs["""encoder_attentions"""]
									else:
										lowerCAmelCase   =     outputs["""hidden_states"""]
										lowerCAmelCase   =     outputs["""attentions"""]

									self.assertEqual(len(A_				)     ,	A_				)

									lowerCAmelCase   =     getattr(
									    self.model_tester     ,	"""expected_num_hidden_layers"""     ,	self.model_tester.num_hidden_layers + 1				)

									self.assertEqual(len(A_				)     ,	A_				)
									self.assertListEqual(
									    list(output_hidden_states[0].shape[-2:]				)     ,	[self.model_tester.seq_length, self.model_tester.hidden_size]     ,	)

									self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
									self.assertListEqual(
									    list(output_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)


						@slow
						def     __snake_case   ( self				)     ->     Optional[Any]:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							self.assertIsNotNone(A_				)




						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase,       lowerCAmelCase   =     self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase   =     True
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""decoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""encoder_seq_length"""     ,	self.model_tester.seq_length				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)
							lowerCAmelCase   =     getattr(self.model_tester     ,	"""key_length"""     ,	A_				)

							def check_decoder_attentions_output(A_				):
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(out_len % 2     ,	0				)
								lowerCAmelCase   =     outputs.decoder_attentions
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]     ,	)

							def check_encoder_attentions_output(A_				):
								lowerCAmelCase   =     [
								    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
								]
								self.assertEqual(len(A_				)     ,	self.model_tester.num_hidden_layers				)
								self.assertListEqual(
								    list(attentions[0].shape[-3:]				)     ,	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]     ,	)

							for model_class in self.all_model_classes:
								lowerCAmelCase   =     True
								lowerCAmelCase   =     False
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								lowerCAmelCase   =     len(A_				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								if self.is_encoder_decoder:
									lowerCAmelCase   =     model_class(A_				)
									lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
									self.assertEqual(config.output_hidden_states     ,	A_				)
									check_decoder_attentions_output(A_				)

								# Check that output attentions can also be changed via the config
								del inputs_dict["output_attentions"]
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)
								self.assertEqual(config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)

								# Check attention is always last and order is fine
								lowerCAmelCase   =     True
								lowerCAmelCase   =     True
								lowerCAmelCase   =     model_class(A_				)
								lowerCAmelCase   =     model(self._prepare_for_class(A_     ,	A_				)				)

								self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)     ,	len(A_				)				)
								self.assertEqual(model.config.output_hidden_states     ,	A_				)
								check_encoder_attentions_output(A_				)







@require_tf
class __snake_case(				unittest.TestCase ):


						'''simple docstring'''



						@slow
						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""				)
							lowerCAmelCase   =     tf.constant([[0, 1, 2, 3, 4, 5]]				)
							lowerCAmelCase   =     model(A_				)[0]

							lowerCAmelCase   =     [1, 6, 768]
							self.assertEqual(output.shape     ,	A_				)

							lowerCAmelCase   =     tf.constant(
							    [
							        [
							            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
							            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
							            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
							        ]
							    ]				)
							tf.debugging.assert_near(output[:, :3, :3]     ,	A_     ,	atol=1e-4				)