


"""simple docstring"""







import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
		a__    :							Tuple            =					True
		from torch.cuda.amp import autocast


a__    :							str            =					logging.getLogger(__name__)




@dataclass
class 			UpperCamelCase_     :

	"""simple docstring"""





	snake_case__    :  str    =      field(
	    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"})
	snake_case__    :  Optional[str]    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}      ,  )
	snake_case__    :  Optional[bool]    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "Whether to freeze the feature extractor layers of the model."})
	snake_case__    :  Optional[bool]    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "Whether to log verbose messages or not."}      ,  )
	snake_case__    :  Optional[float]    =      field(
	    default=2.0      ,  metadata={"help": "Maximum temperature for gumbel softmax."})
	snake_case__    :  Optional[float]    =      field(
	    default=0.5      ,  metadata={"help": "Minimum temperature for gumbel softmax."})
	snake_case__    :  Optional[float]    =      field(
	    default=0.99_99_95      ,  metadata={"help": "Decay of gumbel temperature during training."})




def        UpperCAmelCase__	(lowerCAmelCase_  ,						lowerCAmelCase_       ):




					'''simple docstring'''


					logging.basicConfig(
					    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"  ,						datefmt="%m/%d/%Y %H:%M:%S"  ,						handlers=[logging.StreamHandler(sys.stdout       )]  ,						)
					__SCREAMING_SNAKE_CASE						       =    logging.WARNING
					if model_args.verbose_logging:
										__SCREAMING_SNAKE_CASE						       =    logging.DEBUG
					elif trainer_utils.is_main_process(training_args.local_rank       ):
										__SCREAMING_SNAKE_CASE						       =    logging.INFO
					logger.setLevel(lowerCAmelCase_       )




@dataclass
class 			UpperCamelCase_     :

	"""simple docstring"""





	snake_case__    :  str    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "The name of the dataset to use (via the datasets library)."})
	snake_case__    :  Optional[str]    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "The configuration name of the dataset to use (via the datasets library)."})
	snake_case__    :  Optional[str]    =      field(
	    default="train"      ,  metadata={
	        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
	    }      ,  )
	snake_case__    :  Optional[str]    =      field(
	    default="validation"      ,  metadata={
	        "help": (
	            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
	        )
	    }      ,  )
	snake_case__    :  Optional[str]    =      field(
	    default="file"      ,  metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}      ,  )
	snake_case__    :  bool    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "Overwrite the cached preprocessed datasets or not."})
	snake_case__    :  Optional[int]    =      field(
	    default=1      ,  metadata={
	        "help": "The percentage of the train set used as validation set in case there's no validation split"
	    }      ,  )
	snake_case__    :  Optional[int]    =      field(
	    default=UpperCamelCase      ,  metadata={"help": "The number of processes to use for the preprocessing."}      ,  )
	snake_case__    :  Optional[float]    =      field(
	    default=20.0      ,  metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"})




@dataclass
class 			UpperCamelCase_     :

	"""simple docstring"""





	snake_case__    :  WavaVecaForPreTraining
	snake_case__    :  WavaVecaFeatureExtractor
	snake_case__    :  Union[bool, str]    =      "longest"
	snake_case__    :  Optional[int]    =      None
	snake_case__    :  Optional[int]    =      None




	def __call__(			self      :    Dict      ,      UpperCAmelCase__      :    List[Dict[str, Union[List[int], torch.Tensor]]]       )	->     Dict[str, torch.Tensor]:
						# reformat list to dict and set to pytorch format
						__SCREAMING_SNAKE_CASE						       =    self.feature_extractor.pad(
						    UpperCAmelCase__      ,      max_length=self.max_length      ,      padding=self.padding      ,      pad_to_multiple_of=self.pad_to_multiple_of      ,      return_tensors="pt"      ,      )
						__SCREAMING_SNAKE_CASE						       =    self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]       )

						__SCREAMING_SNAKE_CASE						       =    batch["input_values"].shape[0]

						# make sure that no loss is computed on padded inputs
						if batch["attention_mask"] is not None:
											# compute real output lengths according to convolution formula
											__SCREAMING_SNAKE_CASE						       =    self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1       )       ).to(
											    torch.long       )

											__SCREAMING_SNAKE_CASE						       =    torch.zeros(
											    (batch_size, mask_indices_seq_length)      ,      dtype=torch.long      ,      device=batch["input_values"].device       )

											# these two operations makes sure that all values
											# before the output lengths indices are attended to
											__SCREAMING_SNAKE_CASE						       =    1
											__SCREAMING_SNAKE_CASE						       =    attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

						# sample randomly masked indices
						__SCREAMING_SNAKE_CASE						       =    _compute_mask_indices(
						    (batch_size, mask_indices_seq_length)      ,      self.model.config.mask_time_prob      ,      self.model.config.mask_time_length      ,      attention_mask=UpperCAmelCase__      ,      min_masks=2      ,      )

						return batch




class 			UpperCamelCase_     (			UpperCamelCase):

	"""simple docstring"""





	def __init__(			self      :    int      ,      *UpperCAmelCase__      :    List[Any]      ,      UpperCAmelCase__      :    Dict=1      ,      UpperCAmelCase__      :    Any=0      ,      UpperCAmelCase__      :    Tuple=1.0      ,      **UpperCAmelCase__      :    int       )	->     Optional[int]:
						super().__init__(*UpperCAmelCase__      ,      **UpperCAmelCase__       )
						__SCREAMING_SNAKE_CASE						       =    0
						__SCREAMING_SNAKE_CASE						       =    max_gumbel_temp
						__SCREAMING_SNAKE_CASE						       =    min_gumbel_temp
						__SCREAMING_SNAKE_CASE						       =    gumbel_temp_decay




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    nn.Module      ,      UpperCAmelCase__      :    Dict[str, Union[torch.Tensor, Any]]       )	->     torch.Tensor:
						model.train()
						__SCREAMING_SNAKE_CASE						       =    self._prepare_inputs(UpperCAmelCase__       )

						if self.use_amp:
											with autocast():
																__SCREAMING_SNAKE_CASE						       =    self.compute_loss(UpperCAmelCase__      ,      UpperCAmelCase__       )
						else:
											__SCREAMING_SNAKE_CASE						       =    self.compute_loss(UpperCAmelCase__      ,      UpperCAmelCase__       )

						if self.args.n_gpu > 1 or self.deepspeed:
											if model.module.config.ctc_loss_reduction == "mean":
																__SCREAMING_SNAKE_CASE						       =    loss.mean()
											elif model.module.config.ctc_loss_reduction == "sum":
																__SCREAMING_SNAKE_CASE						       =    loss.sum() / (inputs["mask_time_indices"]).sum()
											else:
																raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""       )

						if self.args.gradient_accumulation_steps > 1:
											__SCREAMING_SNAKE_CASE						       =    loss / self.args.gradient_accumulation_steps

						if self.use_amp:
											self.scaler.scale(UpperCAmelCase__       ).backward()
						elif self.use_apex:
											with amp.scale_loss(UpperCAmelCase__      ,      self.optimizer       ) as scaled_loss:
																scaled_loss.backward()
						elif self.deepspeed:
											self.deepspeed.backward(UpperCAmelCase__       )
						else:
											loss.backward()

						self.num_update_step += 1
						# make sure gumbel softmax temperature is decayed
						if self.args.n_gpu > 1 or self.deepspeed:
											model.module.set_gumbel_temperature(
											    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,      self.min_gumbel_temp       )       )
						else:
											model.set_gumbel_temperature(
											    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,      self.min_gumbel_temp       )       )

						return loss.detach()




def        UpperCAmelCase__	():




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						       =    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

					__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    parser.parse_args_into_dataclasses()
					configure_logger(lowerCAmelCase_  ,						lowerCAmelCase_       )

					# Downloading and loading a dataset from the hub.
					__SCREAMING_SNAKE_CASE						       =    load_dataset(data_args.dataset_name  ,						data_args.dataset_config_name  ,						cache_dir=model_args.cache_dir       )

					if "validation" not in datasets.keys():
										# make sure only "validation" and "train" keys remain"
										__SCREAMING_SNAKE_CASE						       =    DatasetDict()
										__SCREAMING_SNAKE_CASE						       =    load_dataset(
										    data_args.dataset_name  ,						data_args.dataset_config_name  ,						split=f"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""  ,						cache_dir=model_args.cache_dir  ,						)
										__SCREAMING_SNAKE_CASE						       =    load_dataset(
										    data_args.dataset_name  ,						data_args.dataset_config_name  ,						split=f"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""  ,						cache_dir=model_args.cache_dir  ,						)
					else:
										# make sure only "validation" and "train" keys remain"
										__SCREAMING_SNAKE_CASE						       =    DatasetDict()
										__SCREAMING_SNAKE_CASE						       =    load_dataset(
										    data_args.dataset_name  ,						data_args.dataset_config_name  ,						split="validation"  ,						cache_dir=model_args.cache_dir  ,						)
										__SCREAMING_SNAKE_CASE						       =    load_dataset(
										    data_args.dataset_name  ,						data_args.dataset_config_name  ,						split=f"""{data_args.train_split_name}"""  ,						cache_dir=model_args.cache_dir  ,						)

					# only normalized-inputs-training is supported
					__SCREAMING_SNAKE_CASE						       =    WavaVecaFeatureExtractor.from_pretrained(
					    model_args.model_name_or_path  ,						cache_dir=model_args.cache_dir  ,						do_normalize=lowerCAmelCase_       )

					def prepare_dataset(lowerCAmelCase_       ):
										# check that all files have the correct sampling rate
										__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE						       =    librosa.load(batch[data_args.speech_file_column]  ,						sr=feature_extractor.sampling_rate       )
										return batch

					# load audio files into numpy arrays
					__SCREAMING_SNAKE_CASE						       =    datasets.map(
					    lowerCAmelCase_  ,						num_proc=data_args.preprocessing_num_workers  ,						remove_columns=datasets["train"].column_names       )

					# filter audio files that are too long
					__SCREAMING_SNAKE_CASE						       =    vectorized_datasets.filter(
					    lambda lowerCAmelCase_       : len(data["speech"]       ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate       )       )

					def normalize(lowerCAmelCase_       ):
										return feature_extractor(batch["speech"]  ,						sampling_rate=feature_extractor.sampling_rate       )

					# normalize and transform to `BatchFeatures`
					__SCREAMING_SNAKE_CASE						       =    vectorized_datasets.map(
					    lowerCAmelCase_  ,						batched=lowerCAmelCase_  ,						num_proc=data_args.preprocessing_num_workers  ,						load_from_cache_file=not data_args.overwrite_cache  ,						remove_columns=vectorized_datasets["train"].column_names  ,						)

					# pretraining is only supported for "newer" stable layer norm architecture
					# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
					__SCREAMING_SNAKE_CASE						       =    WavaVecaConfig.from_pretrained(
					    model_args.model_name_or_path  ,						cache_dir=model_args.cache_dir  ,						gradient_checkpointing=training_args.gradient_checkpointing  ,						)

					if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
										raise ValueError(
										    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
										    " ``config.feat_extract_norm='layer'"       )

					__SCREAMING_SNAKE_CASE						       =    WavaVecaForPreTraining(lowerCAmelCase_       )

					__SCREAMING_SNAKE_CASE						       =    DataCollatorForWavaVecaPretraining(model=lowerCAmelCase_  ,						feature_extractor=lowerCAmelCase_       )

					__SCREAMING_SNAKE_CASE						       =    WavaVecaPreTrainer(
					    model=lowerCAmelCase_  ,						data_collator=lowerCAmelCase_  ,						args=lowerCAmelCase_  ,						train_dataset=vectorized_datasets["train"]  ,						eval_dataset=vectorized_datasets["validation"]  ,						tokenizer=lowerCAmelCase_  ,						max_gumbel_temp=model_args.max_gumbel_temperature  ,						min_gumbel_temp=model_args.min_gumbel_temperature  ,						gumbel_temp_decay=model_args.gumbel_temperature_decay  ,						)
					trainer.train()


if __name__ == "__main__":
		main()




"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import _LazyModule


a__    :							Optional[int]            =					{'''tokenization_byt5''': ['''ByT5Tokenizer''']}


if TYPE_CHECKING:
		from .tokenization_byta import ByTaTokenizer
else:
		import sys

		a__    :							Any            =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
