






"""simple docstring"""



from math import ceil
from typing import List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import BatchFeature, SequenceFeatureExtractor
from ...utils import TensorType, logging


A__	:    Union[str, Any]			 = logging.get_logger(__name__)







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:str             =			["audio_values", "audio_mask"]


			def __init__(				self					:  Tuple       ,	snake_case__					:  Optional[int]=2048       ,	snake_case__					:  List[Any]=1       ,	snake_case__					:  Dict=[16, 16]       ,	snake_case__					:  Union[str, Any]=128       ,	snake_case__					:  Any=4_4100       ,	snake_case__					:  Dict=86       ,	snake_case__					:  Tuple=2048       ,	snake_case__					:  List[Any]=0.0       ,	**snake_case__					:  List[str]       ,	):
										super().__init__(
										    feature_size=snake_case__       ,	sampling_rate=snake_case__       ,	padding_value=snake_case__       ,	**snake_case__       ,	)

										lowerCamelCase_	:							Union[str, Any]									=spectrogram_length
										lowerCamelCase_	:							List[str]									=num_channels
										lowerCamelCase_	:							int									=patch_size
										lowerCamelCase_	:							Union[str, Any]									=feature_size // self.patch_size[1]
										lowerCamelCase_	:							int									=n_fft
										lowerCamelCase_	:							Optional[int]									=sampling_rate // hop_length_to_sampling_rate
										lowerCamelCase_	:							str									=sampling_rate
										lowerCamelCase_	:							List[str]									=padding_value
										lowerCamelCase_	:							List[str]									=mel_filter_bank(
										    num_frequency_bins=1 + n_fft // 2       ,	num_mel_filters=snake_case__       ,	min_frequency=0.0       ,	max_frequency=22_050.0       ,	sampling_rate=snake_case__       ,	norm="slaney"       ,	mel_scale="slaney"       ,	).T


			def      UpperCAmelCase__						(				self					:  Any       ,	snake_case__					:  np.array ):
										lowerCamelCase_	:							Optional[int]									=spectrogram(
										    snake_case__       ,	window_function(self.n_fft       ,	"hann" )       ,	frame_length=self.n_fft       ,	hop_length=self.hop_length       ,	power=2.0       ,	mel_filters=self.mel_filters.T       ,	log_mel="dB"       ,	db_range=80.0       ,	)
										lowerCamelCase_	:							Tuple									=log_spec[:, :-1]
										lowerCamelCase_	:							Any									=log_spec - 20.0
										lowerCamelCase_	:							List[str]									=np.clip(log_spec / 40.0       ,	-2.0       ,	0.0 ) + 1.0
										return log_spec


			def __call__(				self					:  int       ,	snake_case__					:  Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]       ,	snake_case__					:  Optional[Union[str, TensorType]] = None       ,	snake_case__					:  Optional[bool] = True       ,	snake_case__					:  Optional[int] = None       ,	snake_case__					:  bool = False       ,	snake_case__					:  bool = False       ,	**snake_case__					:  int       ,	):

										if sampling_rate is not None:
																	if sampling_rate != self.sampling_rate:
																								raise ValueError(
																								    "This feature extractor is set to support sampling rate"
																								    F""" of {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled"""
																								    F""" with {self.sampling_rate} and not {sampling_rate}.""" )
										else:
																	logger.warning(
																	    "It is strongly recommended to pass the `sampling_rate` argument to this function. "
																	    "Failing to do so can result in silent errors that might be hard to debug." )

										lowerCamelCase_	:							Dict									=isinstance(snake_case__       ,	np.ndarray ) and len(raw_speech.shape ) > 1
										if is_batched_numpy and len(raw_speech.shape ) > 2:
																	raise ValueError(F"""Only mono-channel audio is supported for input to {self}""" )
										lowerCamelCase_	:							List[str]									=is_batched_numpy or (
										    isinstance(snake_case__       ,	(list, tuple) ) and (isinstance(raw_speech[0]       ,	(np.ndarray, tuple, list) ))
										)
										if is_batched:
																	lowerCamelCase_	:							Dict									=[np.asarray([speech]       ,	dtype=np.floataa ).T for speech in raw_speech]
										elif not is_batched and not isinstance(snake_case__       ,	np.ndarray ):
																	lowerCamelCase_	:							Optional[Any]									=np.asarray(snake_case__       ,	dtype=np.floataa )
										elif isinstance(snake_case__       ,	np.ndarray ) and raw_speech.dtype is np.dtype(np.floataa ):
																	lowerCamelCase_	:							Any									=raw_speech.astype(np.floataa )
										# always return batch
										if not is_batched:
																	lowerCamelCase_	:							Optional[Any]									=[np.asarray([raw_speech] ).T]

										# Convert audio signals to log mel spectrograms, truncate by time axis
										lowerCamelCase_	:							Optional[Any]									=[
										    self._np_extract_fbank_features(waveform.squeeze() ).T[: self.spectrogram_length] for waveform in raw_speech
										]
										if isinstance(audio_features[0]       ,	snake_case__ ):
																	lowerCamelCase_	:							Union[str, Any]									=[np.asarray(snake_case__       ,	dtype=np.floataa ) for feature in audio_features]

										# Create audio attention mask
										lowerCamelCase_	:							int									=max(
										    [ceil(feature.shape[0] / self.patch_size[0] ) * self.freq_len for feature in audio_features] )  # The maximum number of audio patches in a batch
										if return_attention_mask:
																	lowerCamelCase_	:							Optional[int]									=[
																	    (ceil(feature.shape[0] / self.patch_size[0] ) * self.freq_len) * [1]
																	    + (max_patch_len - ceil(feature.shape[0] / self.patch_size[0] ) * self.freq_len) * [0]
																	    for feature in audio_features
																	]
																	lowerCamelCase_	:							Union[str, Any]									=np.array(snake_case__ ).astype(np.floataa )

										# convert into correct format for padding
										lowerCamelCase_	:							int									=max_patch_len // self.freq_len * self.patch_size[0]  # The maximum audio size in a batch
										lowerCamelCase_	:							Dict									=np.ones([len(snake_case__ ), 1, max_time_len, self.feature_size] ).astype(np.floataa )
										lowerCamelCase_	:							Any									=padded_audio_features * self.padding_value
										for i in range(len(snake_case__ ) ):
																	lowerCamelCase_	:							Optional[Any]									=audio_features[i]
																	lowerCamelCase_	:							Union[str, Any]									=feature

										# return as BatchFeature
										if return_attention_mask:
																	lowerCamelCase_	:							Dict									={"audio_values": padded_audio_features, "audio_mask": audio_mask}
										else:
																	lowerCamelCase_	:							Any									={"audio_values": padded_audio_features}

										lowerCamelCase_	:							List[Any]									=BatchFeature(data=snake_case__       ,	tensor_type=snake_case__ )
										return encoded_inputs











"""simple docstring"""



import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


A__	:    List[str]			 = datasets.utils.logging.get_logger(__name__)







@dataclass
class        lowercase__      (							datasets.BuilderConfig       ):
			_UpperCAmelCase		:int             =			10000
			_UpperCAmelCase		:Optional[List[str]]             =			None
			_UpperCAmelCase		:Optional[datasets.Features]             =			None







class        lowercase__      (							datasets.ArrowBasedBuilder       ):
			_UpperCAmelCase		:Optional[int]             =			ParquetConfig


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										return datasets.DatasetInfo(features=self.config.features )


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  List[Any] ):
										if not self.config.data_files:
																	raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}""" )
										lowerCamelCase_	:							str									=dl_manager.download_and_extract(self.config.data_files )
										if isinstance(snake_case__       ,	(str, list, tuple) ):
																	lowerCamelCase_	:							Dict									=data_files
																	if isinstance(snake_case__       ,	snake_case__ ):
																								lowerCamelCase_	:							List[Any]									=[files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	lowerCamelCase_	:							Any									=[dl_manager.iter_files(snake_case__ ) for file in files]
																	return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={"files": files} )]
										lowerCamelCase_	:							Optional[int]									=[]
										for split_name, files in data_files.items():
																	if isinstance(snake_case__       ,	snake_case__ ):
																								lowerCamelCase_	:							Optional[int]									=[files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	lowerCamelCase_	:							int									=[dl_manager.iter_files(snake_case__ ) for file in files]
																	# Infer features is they are stoed in the arrow schema
																	if self.info.features is None:
																								for file in itertools.chain.from_iterable(snake_case__ ):
																															with open(snake_case__       ,	"rb" ) as f:
																																						lowerCamelCase_	:							List[Any]									=datasets.Features.from_arrow_schema(pq.read_schema(snake_case__ ) )
																															break
																	splits.append(datasets.SplitGenerator(name=snake_case__       ,	gen_kwargs={"files": files} ) )
										return splits


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  pa.Table ):
										if self.info.features is not None:
																	# more expensive cast to support nested features with keys in a different order
																	# allows str <-> int/float or str to Audio for example
																	lowerCamelCase_	:							List[str]									=table_cast(snake_case__       ,	self.info.features.arrow_schema )
										return pa_table


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  Optional[Any] ):
										lowerCamelCase_	:							Tuple									=self.info.features.arrow_schema if self.info.features is not None else None
										if self.info.features is not None and self.config.columns is not None:
																	if sorted(field.name for field in schema ) != sorted(self.config.columns ):
																								raise ValueError(
																								    F"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'""" )
										for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__ ) ):
																	with open(snake_case__       ,	"rb" ) as f:
																								lowerCamelCase_	:							List[str]									=pq.ParquetFile(snake_case__ )
																								try:
																															for batch_idx, record_batch in enumerate(
																															    parquet_file.iter_batches(batch_size=self.config.batch_size       ,	columns=self.config.columns ) ):
																																						lowerCamelCase_	:							Union[str, Any]									=pa.Table.from_batches([record_batch] )
																																						# Uncomment for debugging (will print the Arrow table size and elements)
																																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																						yield F"""{file_idx}_{batch_idx}""", self._cast_table(snake_case__ )
																								except ValueError as e:
																															logger.error(F"""Failed to read file '{file}' with error {type(snake_case__ )}: {e}""" )
																															raise



