


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
		lowerCamelCase						=		True
		from torch.cuda.amp import autocast


lowerCamelCase						=		logging.getLogger(__name__)





@dataclass
class  _a							:
	_a  :  str				   =							field(
	    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''})
	_a  :  Optional[str]				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}					,						)
	_a  :  Optional[bool]				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''})
	_a  :  Optional[bool]				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''Whether to log verbose messages or not.'''}					,						)
	_a  :  Optional[float]				   =							field(
	    default=2.0					,						metadata={'''help''': '''Maximum temperature for gumbel softmax.'''})
	_a  :  Optional[float]				   =							field(
	    default=0.5					,						metadata={'''help''': '''Minimum temperature for gumbel softmax.'''})
	_a  :  Optional[float]				   =							field(
	    default=0.99_9995					,						metadata={'''help''': '''Decay of gumbel temperature during training.'''})






def 					lowerCamelCase_   (	_a     ,       _a	):




		"""simple docstring"""






		logging.basicConfig(
		    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''     ,       datefmt='''%m/%d/%Y %H:%M:%S'''     ,       handlers=[logging.StreamHandler(sys.stdout	)]     ,       )
		lowerCAmelCase__       :      Optional[Any]   					= logging.WARNING
		if model_args.verbose_logging:
				lowerCAmelCase__       :      List[Any]   					= logging.DEBUG
		elif trainer_utils.is_main_process(training_args.local_rank	):
				lowerCAmelCase__       :      Tuple   					= logging.INFO
		logger.setLevel(_a	)





@dataclass
class  _a							:
	_a  :  str				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''})
	_a  :  Optional[str]				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''})
	_a  :  Optional[str]				   =							field(
	    default='''train'''					,						metadata={
	        '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
	    }					,						)
	_a  :  Optional[str]				   =							field(
	    default='''validation'''					,						metadata={
	        '''help''': (
	            '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
	        )
	    }					,						)
	_a  :  Optional[str]				   =							field(
	    default='''file'''					,						metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}					,						)
	_a  :  bool				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''})
	_a  :  Optional[int]				   =							field(
	    default=1					,						metadata={
	        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
	    }					,						)
	_a  :  Optional[int]				   =							field(
	    default=_lowercase					,						metadata={'''help''': '''The number of processes to use for the preprocessing.'''}					,						)
	_a  :  Optional[float]				   =							field(
	    default=20.0					,						metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''})





@dataclass
class  _a							:
	_a  :  WavaVecaForPreTraining
	_a  :  WavaVecaFeatureExtractor
	_a  :  Union[bool, str]				   =							"longest"
	_a  :  Optional[int]				   =							None
	_a  :  Optional[int]				   =							None




	def __call__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		List[Dict[str, Union[List[int], torch.Tensor]]]		)->	Dict[str, torch.Tensor]:
			# reformat list to dict and set to pytorch format
			lowerCAmelCase__       :      Union[str, Any]   					= self.feature_extractor.pad(
			    _SCREAMING_SNAKE_CASE		, max_length=self.max_length		, padding=self.padding		, pad_to_multiple_of=self.pad_to_multiple_of		, return_tensors='''pt'''		, )
			lowerCAmelCase__       :      List[str]   					= self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]		)

			lowerCAmelCase__       :      Union[str, Any]   					= batch['''input_values'''].shape[0]

			# make sure that no loss is computed on padded inputs
			if batch["attention_mask"] is not None:
					# compute real output lengths according to convolution formula
					lowerCAmelCase__       :      Optional[int]   					= self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1		)		).to(
					    torch.long		)

					lowerCAmelCase__       :      Any   					= torch.zeros(
					    (batch_size, mask_indices_seq_length)		, dtype=torch.long		, device=batch['''input_values'''].device		)

					# these two operations makes sure that all values
					# before the output lengths indices are attended to
					lowerCAmelCase__       :      str   					= 1
					lowerCAmelCase__       :      Tuple   					= attention_mask.flip([-1]		).cumsum(-1		).flip([-1]		).bool()

			# sample randomly masked indices
			lowerCAmelCase__       :      str   					= _compute_mask_indices(
			    (batch_size, mask_indices_seq_length)		, self.model.config.mask_time_prob		, self.model.config.mask_time_length		, attention_mask=_SCREAMING_SNAKE_CASE		, min_masks=2		, )

			return batch





class  _a							(    _lowercase):




	def __init__( self						:		Any		, *_SCREAMING_SNAKE_CASE						:		int		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=1		, _SCREAMING_SNAKE_CASE						:		Optional[int]=0		, _SCREAMING_SNAKE_CASE						:		Optional[int]=1.0		, **_SCREAMING_SNAKE_CASE						:		Optional[int]		)->	Any:
			super().__init__(*_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		)
			lowerCAmelCase__       :      Dict   					= 0
			lowerCAmelCase__       :      List[Any]   					= max_gumbel_temp
			lowerCAmelCase__       :      str   					= min_gumbel_temp
			lowerCAmelCase__       :      Optional[int]   					= gumbel_temp_decay




	def 							UpperCAmelCase__( self						:		Any		, _SCREAMING_SNAKE_CASE						:		nn.Module		, _SCREAMING_SNAKE_CASE						:		Dict[str, Union[torch.Tensor, Any]]		)->	torch.Tensor:
			model.train()
			lowerCAmelCase__       :      Tuple   					= self._prepare_inputs(_SCREAMING_SNAKE_CASE		)

			if self.use_amp:
					with autocast():
							lowerCAmelCase__       :      Union[str, Any]   					= self.compute_loss(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)
			else:
					lowerCAmelCase__       :      Tuple   					= self.compute_loss(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		)

			if self.args.n_gpu > 1 or self.deepspeed:
					if model.module.config.ctc_loss_reduction == "mean":
							lowerCAmelCase__       :      Tuple   					= loss.mean()
					elif model.module.config.ctc_loss_reduction == "sum":
							lowerCAmelCase__       :      Tuple   					= loss.sum() / (inputs['''mask_time_indices''']).sum()
					else:
							raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'		)

			if self.args.gradient_accumulation_steps > 1:
					lowerCAmelCase__       :      Optional[Any]   					= loss / self.args.gradient_accumulation_steps

			if self.use_amp:
					self.scaler.scale(_SCREAMING_SNAKE_CASE		).backward()
			elif self.use_apex:
					with amp.scale_loss(_SCREAMING_SNAKE_CASE		, self.optimizer		) as scaled_loss:
							scaled_loss.backward()
			elif self.deepspeed:
					self.deepspeed.backward(_SCREAMING_SNAKE_CASE		)
			else:
					loss.backward()

			self.num_update_step += 1
			# make sure gumbel softmax temperature is decayed
			if self.args.n_gpu > 1 or self.deepspeed:
					model.module.set_gumbel_temperature(
					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		, self.min_gumbel_temp		)		)
			else:
					model.set_gumbel_temperature(
					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		, self.min_gumbel_temp		)		)

			return loss.detach()






def 					lowerCamelCase_   (	):




		"""simple docstring"""






		lowerCAmelCase__       :      Union[str, Any]   					= HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)	)

		lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__       :      Any   					= parser.parse_args_into_dataclasses()
		configure_logger(_a     ,       _a	)

		# Downloading and loading a dataset from the hub.
		lowerCAmelCase__       :      Dict   					= load_dataset(data_args.dataset_name     ,       data_args.dataset_config_name     ,       cache_dir=model_args.cache_dir	)

		if "validation" not in datasets.keys():
				# make sure only "validation" and "train" keys remain"
				lowerCAmelCase__       :      int   					= DatasetDict()
				lowerCAmelCase__       :      str   					= load_dataset(
				    data_args.dataset_name     ,       data_args.dataset_config_name     ,       split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'     ,       cache_dir=model_args.cache_dir     ,       )
				lowerCAmelCase__       :      Optional[Any]   					= load_dataset(
				    data_args.dataset_name     ,       data_args.dataset_config_name     ,       split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'     ,       cache_dir=model_args.cache_dir     ,       )
		else:
				# make sure only "validation" and "train" keys remain"
				lowerCAmelCase__       :      Tuple   					= DatasetDict()
				lowerCAmelCase__       :      str   					= load_dataset(
				    data_args.dataset_name     ,       data_args.dataset_config_name     ,       split='''validation'''     ,       cache_dir=model_args.cache_dir     ,       )
				lowerCAmelCase__       :      List[str]   					= load_dataset(
				    data_args.dataset_name     ,       data_args.dataset_config_name     ,       split=f'{data_args.train_split_name}'     ,       cache_dir=model_args.cache_dir     ,       )

		# only normalized-inputs-training is supported
		lowerCAmelCase__       :      Tuple   					= WavaVecaFeatureExtractor.from_pretrained(
		    model_args.model_name_or_path     ,       cache_dir=model_args.cache_dir     ,       do_normalize=_a	)

		def prepare_dataset(_a	):
				# check that all files have the correct sampling rate
				lowerCAmelCase__					,     lowerCAmelCase__       :      Dict   					= librosa.load(batch[data_args.speech_file_column]     ,       sr=feature_extractor.sampling_rate	)
				return batch

		# load audio files into numpy arrays
		lowerCAmelCase__       :      Any   					= datasets.map(
		    _a     ,       num_proc=data_args.preprocessing_num_workers     ,       remove_columns=datasets['''train'''].column_names	)

		# filter audio files that are too long
		lowerCAmelCase__       :      str   					= vectorized_datasets.filter(
		    lambda _a	: len(data['''speech''']	) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate	)	)

		def normalize(_a	):
				return feature_extractor(batch['''speech''']     ,       sampling_rate=feature_extractor.sampling_rate	)

		# normalize and transform to `BatchFeatures`
		lowerCAmelCase__       :      List[Any]   					= vectorized_datasets.map(
		    _a     ,       batched=_a     ,       num_proc=data_args.preprocessing_num_workers     ,       load_from_cache_file=not data_args.overwrite_cache     ,       remove_columns=vectorized_datasets['''train'''].column_names     ,       )

		# pretraining is only supported for "newer" stable layer norm architecture
		# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
		lowerCAmelCase__       :      Dict   					= WavaVecaConfig.from_pretrained(
		    model_args.model_name_or_path     ,       cache_dir=model_args.cache_dir     ,       gradient_checkpointing=training_args.gradient_checkpointing     ,       )

		if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
				raise ValueError(
				    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
				    ''' ``config.feat_extract_norm=\'layer\''''	)

		lowerCAmelCase__       :      Dict   					= WavaVecaForPreTraining(_a	)

		lowerCAmelCase__       :      Optional[int]   					= DataCollatorForWavaVecaPretraining(model=_a     ,       feature_extractor=_a	)

		lowerCAmelCase__       :      Tuple   					= WavaVecaPreTrainer(
		    model=_a     ,       data_collator=_a     ,       args=_a     ,       train_dataset=vectorized_datasets['''train''']     ,       eval_dataset=vectorized_datasets['''validation''']     ,       tokenizer=_a     ,       max_gumbel_temp=model_args.max_gumbel_temperature     ,       min_gumbel_temp=model_args.min_gumbel_temperature     ,       gumbel_temp_decay=model_args.gumbel_temperature_decay     ,       )
		trainer.train()


if __name__ == "__main__":
		main()








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{
    '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json''',
    '''YituTech/conv-bert-medium-small''': (
        '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json'''
    ),
    '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json''',
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}





class  _a							(    _lowercase):
	_a  :  Dict				   =							'''convbert'''




	def __init__( self						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		str=3_0522		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=768		, _SCREAMING_SNAKE_CASE						:		Tuple=12		, _SCREAMING_SNAKE_CASE						:		str=12		, _SCREAMING_SNAKE_CASE						:		Any=3072		, _SCREAMING_SNAKE_CASE						:		List[Any]="gelu"		, _SCREAMING_SNAKE_CASE						:		Dict=0.1		, _SCREAMING_SNAKE_CASE						:		List[str]=0.1		, _SCREAMING_SNAKE_CASE						:		int=512		, _SCREAMING_SNAKE_CASE						:		Any=2		, _SCREAMING_SNAKE_CASE						:		Optional[int]=0.02		, _SCREAMING_SNAKE_CASE						:		Optional[int]=1E-12		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=1		, _SCREAMING_SNAKE_CASE						:		Optional[int]=0		, _SCREAMING_SNAKE_CASE						:		Dict=2		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=768		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=2		, _SCREAMING_SNAKE_CASE						:		Tuple=9		, _SCREAMING_SNAKE_CASE						:		int=1		, _SCREAMING_SNAKE_CASE						:		Tuple=None		, **_SCREAMING_SNAKE_CASE						:		List[str]		, )->	List[str]:
			super().__init__(
			    pad_token_id=_SCREAMING_SNAKE_CASE		, bos_token_id=_SCREAMING_SNAKE_CASE		, eos_token_id=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		, )

			lowerCAmelCase__       :      Union[str, Any]   					= vocab_size
			lowerCAmelCase__       :      List[Any]   					= hidden_size
			lowerCAmelCase__       :      Tuple   					= num_hidden_layers
			lowerCAmelCase__       :      Union[str, Any]   					= num_attention_heads
			lowerCAmelCase__       :      Optional[Any]   					= intermediate_size
			lowerCAmelCase__       :      Tuple   					= hidden_act
			lowerCAmelCase__       :      Dict   					= hidden_dropout_prob
			lowerCAmelCase__       :      str   					= attention_probs_dropout_prob
			lowerCAmelCase__       :      str   					= max_position_embeddings
			lowerCAmelCase__       :      Tuple   					= type_vocab_size
			lowerCAmelCase__       :      Optional[Any]   					= initializer_range
			lowerCAmelCase__       :      List[Any]   					= layer_norm_eps
			lowerCAmelCase__       :      int   					= embedding_size
			lowerCAmelCase__       :      Union[str, Any]   					= head_ratio
			lowerCAmelCase__       :      Optional[int]   					= conv_kernel_size
			lowerCAmelCase__       :      List[str]   					= num_groups
			lowerCAmelCase__       :      Dict   					= classifier_dropout





class  _a							(    _lowercase):




	@property
	def 							UpperCAmelCase__( self						:		Tuple		)->	Mapping[str, Mapping[int, str]]:
			if self.task == "multiple-choice":
					lowerCAmelCase__       :      Optional[Any]   					= {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
			else:
					lowerCAmelCase__       :      List[str]   					= {0: '''batch''', 1: '''sequence'''}
			return OrderedDict(
			    [
			        ('''input_ids''', dynamic_axis),
			        ('''attention_mask''', dynamic_axis),
			        ('''token_type_ids''', dynamic_axis),
			    ]		)




