


"""simple docstring"""
import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


__SCREAMING_SNAKE_CASE							:      List[Any]						=      logging.get_logger(__name__)





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						Dict							 =      ["""input_values""", """attention_mask"""]



							def __init__(			self							: Tuple						,    UpperCAmelCase_							: int = 1						,    UpperCAmelCase_							: int = 16_000						,    UpperCAmelCase_							: float = 0.0						,    UpperCAmelCase_							: bool = False						,    UpperCAmelCase_							: int = 80						,    UpperCAmelCase_							: int = 16						,    UpperCAmelCase_							: int = 64						,    UpperCAmelCase_							: str = "hann_window"						,    UpperCAmelCase_							: float = 1.0						,    UpperCAmelCase_							: float = 80						,    UpperCAmelCase_							: float = 7_600						,    UpperCAmelCase_							: float = 1E-10						,    UpperCAmelCase_							: int = 2						,    UpperCAmelCase_							: bool = True						,    **UpperCAmelCase_							: int						,    )   ->Dict:



														"""simple docstring"""



														super().__init__(feature_size=UpperCAmelCase_						,    sampling_rate=UpperCAmelCase_						,    padding_value=UpperCAmelCase_						,    **UpperCAmelCase_    )
														snake_case_          =						do_normalize
														snake_case_          =						return_attention_mask

														snake_case_          =						num_mel_bins
														snake_case_          =						hop_length
														snake_case_          =						win_length
														snake_case_          =						win_function
														snake_case_          =						frame_signal_scale
														snake_case_          =						fmin
														snake_case_          =						fmax
														snake_case_          =						mel_floor
														snake_case_          =						reduction_factor

														snake_case_          =						win_length * sampling_rate // 1_000
														snake_case_          =						hop_length * sampling_rate // 1_000
														snake_case_          =						optimal_fft_length(self.sample_size    )
														snake_case_          =						(self.n_fft // 2) + 1

														snake_case_          =						window_function(window_length=self.sample_size						,    name=self.win_function						,    periodic=UpperCAmelCase_    )

														snake_case_          =						mel_filter_bank(
														    num_frequency_bins=self.n_freqs						,    num_mel_filters=self.num_mel_bins						,    min_frequency=self.fmin						,    max_frequency=self.fmax						,    sampling_rate=self.sampling_rate						,    norm="""slaney"""						,    mel_scale="""slaney"""						,    )

														if frame_signal_scale != 1.0:
																					warnings.warn(
																					    """The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers"""						,    UpperCAmelCase_						,    )
														if reduction_factor != 2.0:
																					warnings.warn(
																					    """The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers"""						,    UpperCAmelCase_						,    )



							@staticmethod
							# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
							def 						lowerCAmelCase    (			UpperCAmelCase_							: List[np.ndarray]						,    UpperCAmelCase_							: List[np.ndarray]						,    UpperCAmelCase_							: float = 0.0    )   ->List[np.ndarray]:



														"""simple docstring"""



														if attention_mask is not None:
																					snake_case_          =						np.array(UpperCAmelCase_						,    np.intaa    )
																					snake_case_          =						[]

																					for vector, length in zip(UpperCAmelCase_						,    attention_mask.sum(-1    )    ):
																												snake_case_          =						(vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7    )
																												if length < normed_slice.shape[0]:
																																			snake_case_          =						padding_value

																												normed_input_values.append(UpperCAmelCase_    )
														else:
																					snake_case_          =						[(x - x.mean()) / np.sqrt(x.var() + 1E-7    ) for x in input_values]

														return normed_input_values



							def 						lowerCAmelCase    (			self							: int						,    UpperCAmelCase_							: np.ndarray						,    )   ->np.ndarray:



														"""simple docstring"""



														snake_case_          =						spectrogram(
														    UpperCAmelCase_						,    window=self.window						,    frame_length=self.sample_size						,    hop_length=self.sample_stride						,    fft_length=self.n_fft						,    mel_filters=self.mel_filters						,    mel_floor=self.mel_floor						,    log_mel="""log10"""						,    )
														return log_mel_spec.T



							def __call__(			self							: str						,    UpperCAmelCase_							: Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None						,    UpperCAmelCase_							: Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None						,    UpperCAmelCase_							: Union[bool, str, PaddingStrategy] = False						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: bool = False						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: Optional[bool] = None						,    UpperCAmelCase_							: Optional[Union[str, TensorType]] = None						,    UpperCAmelCase_							: Optional[int] = None						,    **UpperCAmelCase_							: List[str]						,    )   ->BatchFeature:



														"""simple docstring"""



														if audio is None and audio_target is None:
																					raise ValueError("""You must provide either `audio` or `audio_target` values."""    )

														if sampling_rate is not None:
																					if sampling_rate != self.sampling_rate:
																												raise ValueError(
																												    F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
																												    F""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
																												    F""" {self.sampling_rate} and not {sampling_rate}."""    )
														else:
																					logger.warning(
																					    """It is strongly recommended to pass the ``sampling_rate`` argument to this function. """
																					    """Failing to do so can result in silent errors that might be hard to debug."""    )

														if audio is not None:
																					snake_case_          =						self._process_audio(
																					    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    **UpperCAmelCase_						,    )
														else:
																					snake_case_          =						None

														if audio_target is not None:
																					snake_case_          =						self._process_audio(
																					    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    **UpperCAmelCase_						,    )

																					if inputs is None:
																												return inputs_target
																					else:
																												snake_case_          =						inputs_target["""input_values"""]
																												snake_case_          =						inputs_target.get("""attention_mask"""    )
																												if decoder_attention_mask is not None:
																																			snake_case_          =						decoder_attention_mask

														return inputs



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]						,    UpperCAmelCase_							: bool = False						,    UpperCAmelCase_							: Union[bool, str, PaddingStrategy] = False						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: bool = False						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: Optional[bool] = None						,    UpperCAmelCase_							: Optional[Union[str, TensorType]] = None						,    **UpperCAmelCase_							: Dict						,    )   ->BatchFeature:



														"""simple docstring"""



														snake_case_          =						isinstance(UpperCAmelCase_						,    np.ndarray    ) and len(speech.shape    ) > 1
														if is_batched_numpy and len(speech.shape    ) > 2:
																					raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""    )
														snake_case_          =						is_batched_numpy or (
														    isinstance(UpperCAmelCase_						,    (list, tuple)    ) and (isinstance(speech[0]						,    (np.ndarray, tuple, list)    ))
														)

														if is_batched:
																					snake_case_          =						[np.asarray(UpperCAmelCase_						,    dtype=np.floataa    ) for speech in speech]
														elif not is_batched and not isinstance(UpperCAmelCase_						,    np.ndarray    ):
																					snake_case_          =						np.asarray(UpperCAmelCase_						,    dtype=np.floataa    )
														elif isinstance(UpperCAmelCase_						,    np.ndarray    ) and speech.dtype is np.dtype(np.floataa    ):
																					snake_case_          =						speech.astype(np.floataa    )

														# always return batch
														if not is_batched:
																					snake_case_          =						[speech]

														# needed to make pad() work on spectrogram inputs
														snake_case_          =						self.feature_size

														# convert into correct format for padding
														if is_target:
																					snake_case_          =						[self._extract_mel_features(UpperCAmelCase_    ) for waveform in speech]
																					snake_case_          =						BatchFeature({"""input_values""": features}    )
																					snake_case_          =						self.num_mel_bins
														else:
																					snake_case_          =						BatchFeature({"""input_values""": speech}    )

														snake_case_          =						self.pad(
														    UpperCAmelCase_						,    padding=UpperCAmelCase_						,    max_length=UpperCAmelCase_						,    truncation=UpperCAmelCase_						,    pad_to_multiple_of=UpperCAmelCase_						,    return_attention_mask=UpperCAmelCase_						,    **UpperCAmelCase_						,    )

														snake_case_          =						feature_size_hack

														# convert input values to correct format
														snake_case_          =						padded_inputs["""input_values"""]
														if not isinstance(input_values[0]						,    np.ndarray    ):
																					snake_case_          =						[np.asarray(UpperCAmelCase_						,    dtype=np.floataa    ) for array in input_values]
														elif (
														    not isinstance(UpperCAmelCase_						,    np.ndarray    )
														    and isinstance(input_values[0]						,    np.ndarray    )
														    and input_values[0].dtype is np.dtype(np.floataa    )
														):
																					snake_case_          =						[array.astype(np.floataa    ) for array in input_values]
														elif isinstance(UpperCAmelCase_						,    np.ndarray    ) and input_values.dtype is np.dtype(np.floataa    ):
																					snake_case_          =						input_values.astype(np.floataa    )

														# convert attention_mask to correct format
														snake_case_          =						padded_inputs.get("""attention_mask"""    )
														if attention_mask is not None:
																					snake_case_          =						[np.asarray(UpperCAmelCase_						,    dtype=np.intaa    ) for array in attention_mask]

														# zero-mean and unit-variance normalization
														if not is_target and self.do_normalize:
																					snake_case_          =						(
																					    attention_mask
																					    if self._get_padding_strategies(UpperCAmelCase_						,    max_length=UpperCAmelCase_    ) is not PaddingStrategy.DO_NOT_PAD
																					    else None
																					)
																					snake_case_          =						self.zero_mean_unit_var_norm(
																					    padded_inputs["""input_values"""]						,    attention_mask=UpperCAmelCase_						,    padding_value=self.padding_value    )

														if return_tensors is not None:
																					snake_case_          =						padded_inputs.convert_to_tensors(UpperCAmelCase_    )

														return padded_inputs



							def 						lowerCAmelCase    (			self							: Any    )   ->Dict[str, Any]:



														"""simple docstring"""



														snake_case_          =						super().to_dict()

														# Don't serialize these as they are derived from the other properties.
														snake_case_          =						["""window""", """mel_filters""", """sample_size""", """sample_stride""", """n_fft""", """n_freqs"""]
														for name in names:
																					if name in output:
																												del output[name]

														return output







"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be trained."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""./"""  ,						metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-train"""  ,						metadata={"""help""": """Name or path of training dataset."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-valid"""  ,						metadata={"""help""": """Name or path of validation dataset."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size for training."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size for evaluation."""})
							__lowercase:						Optional[float]							 =      field(default=0.1  ,						metadata={"""help""": """Value of weight decay."""})
							__lowercase:						Optional[int]							 =      field(
							    default=1_00_00  ,						metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""})
							__lowercase:						Optional[float]							 =      field(default=2E-4  ,						metadata={"""help""": """Learning rate fo training."""})
							__lowercase:						Optional[str]							 =      field(default="""cosine"""  ,						metadata={"""help""": """Learning rate."""})
							__lowercase:						Optional[int]							 =      field(
							    default=7_50  ,						metadata={"""help""": """Number of warmup steps in the learning rate schedule."""})
							__lowercase:						Optional[int]							 =      field(
							    default=16  ,						metadata={"""help""": """Number of gradient accumulation steps."""})
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""})
							__lowercase:						Optional[int]							 =      field(default=5_00_00  ,						metadata={"""help""": """Maximum number of training steps."""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=10_24  ,						metadata={"""help""": """Sequence lengths used for training."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Training seed."""})
							__lowercase:						Optional[int]							 =      field(
							    default=10_24  ,						metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}  ,						)
							__lowercase:						Optional[str]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """States path if the training should continue from a checkpoint folder."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """If True the data is pretokenized."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be evaluated."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-valid"""  ,						metadata={"""help""": """Name or path of validation dataset."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size used for evaluation."""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=10_24  ,						metadata={"""help""": """Length of sequences to be evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Random seed used for evaluation."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=snake_case__  ,						metadata={"""help""": """Number of workers used for code evaluation."""})
							__lowercase:						Optional[int]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}  ,						)
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """Sample from the language model's output distribution."""})
							__lowercase:						Optional[float]							 =      field(default=0.2  ,						metadata={"""help""": """Sampling temperature used for generation."""})
							__lowercase:						Optional[int]							 =      field(default=2_56  ,						metadata={"""help""": """Maximum number of newly generated tokens."""})
							__lowercase:						Optional[int]							 =      field(default=0  ,						metadata={"""help""": """Top-k parameter used for generation."""})
							__lowercase:						Optional[float]							 =      field(default=0.9_5  ,						metadata={"""help""": """Top-p parameter used for nucleus sampling."""})
							__lowercase:						Optional[int]							 =      field(default=10  ,						metadata={"""help""": """Number of generations to run in parallel."""})
							__lowercase:						Optional[int]							 =      field(
							    default=2_00  ,						metadata={"""help""": """Number of completions to generate for each sample."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Random seed used for evaluation."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""eval_results.json"""  ,						metadata={"""help""": """Random seed used for evaluation."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""0"""  ,						metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={
							        """help""": (
							            """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
							            """ number corresponds to which GPU device id to run on."""
							        )
							    }  ,						)





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[int]							 =      field(
							    default=snake_case__  ,						metadata={
							        """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
							    }  ,						)
							__lowercase:						Optional[str]							 =      field(
							    default="""transformersbook/codeparrot"""  ,						metadata={"""help""": """Folder or name of dataset to process."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot-clean"""  ,						metadata={"""help""": """Folder to save processed processed dataset."""})
							__lowercase:						Optional[int]							 =      field(
							    default=10_00_00  ,						metadata={"""help""": """Number of files to save per JSON output file."""})
							__lowercase:						Optional[str]							 =      field(default="""content"""  ,						metadata={"""help""": """Column containing text data to process."""})
							__lowercase:						Optional[float]							 =      field(
							    default=10_00  ,						metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=1_00  ,						metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.2_5  ,						metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=1.5  ,						metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.7  ,						metadata={"""help""": """Probability for filtering config, test and uncommon files."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Name or path to the tokenizer."""}  ,						)
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """If True, near-duplicate samples are removed."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.8_5  ,						metadata={"""help""": """Jaccard threshold for near-duplicate samples."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""gpt2"""  ,						metadata={"""help""": """Base tokenizer to build new tokenizer from."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""transformersbook/codeparrot-train"""  ,						metadata={"""help""": """Dataset to train tokenizer on."""})
							__lowercase:						Optional[str]							 =      field(default="""content"""  ,						metadata={"""help""": """Column containing text data to process."""})
							__lowercase:						Optional[int]							 =      field(default=20_00_00  ,						metadata={"""help""": """Number of examples to train tokenizer on."""})
							__lowercase:						Optional[int]							 =      field(
							    default=3_27_68  ,						metadata={"""help""": """Number of examples to train the tokenizer on."""})
							__lowercase:						Optional[str]							 =      field(default="""codeparrot"""  ,						metadata={"""help""": """Name of new tokenizer."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """Push saved tokenizer to the hub."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Name or path to the tokenizer."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-train"""  ,						metadata={"""help""": """Name or path to the dataset to pretokenize."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""tokenized-codeparrot-train"""  ,						metadata={"""help""": """Repo name of the pretokenized data."""})
							__lowercase:						Optional[int]							 =      field(default=snake_case__  ,						metadata={"""help""": """Number of workers used for code evaluation."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""gpt2-large"""  ,						metadata={"""help""": """Configuration to use for model initialization."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Tokenizer attached to model."""})
							__lowercase:						Optional[str]							 =      field(default="""codeparrot"""  ,						metadata={"""help""": """Name of the created model."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """Push saved tokenizer to the hub."""})



