import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class   lowercase      (       a       ):




							def __init__(  self				:   Optional[int]		,							_UpperCamelCase				:   NestedDataStructureLike[PathLike]		,							_UpperCamelCase				:   Optional[NamedSplit] = None		,							_UpperCamelCase				:   Optional[Features] = None		,							_UpperCamelCase				:   str = None		,							_UpperCamelCase				:   bool = False		,							_UpperCamelCase				:   bool = False		,							_UpperCamelCase				:   Optional[str] = None		,							_UpperCamelCase				:   Optional[int] = None		,							**_UpperCamelCase				:   Optional[int]		,							)   ->			Optional[Any]:



												'''simple docstring'''

												super().__init__(
												    _UpperCamelCase		,							split=_UpperCamelCase		,							features=_UpperCamelCase		,							cache_dir=_UpperCamelCase		,							keep_in_memory=_UpperCamelCase		,							streaming=_UpperCamelCase		,							num_proc=_UpperCamelCase		,							**_UpperCamelCase		,							)
												SCREAMING_SNAKE_CASE           =    field
												SCREAMING_SNAKE_CASE           =    path_or_paths if isinstance(_UpperCamelCase		,							_UpperCamelCase ) else {self.split: path_or_paths}
												SCREAMING_SNAKE_CASE           =    Json(
												    cache_dir=_UpperCamelCase		,							data_files=_UpperCamelCase		,							features=_UpperCamelCase		,							field=_UpperCamelCase		,							**_UpperCamelCase		,							)





							def 		__snake_case(  self				:   List[Any] )   ->			Any:



												'''simple docstring'''

												if self.streaming:
																	SCREAMING_SNAKE_CASE           =    self.builder.as_streaming_dataset(split=self.split )
												# Build regular (map-style) dataset
												else:
																	SCREAMING_SNAKE_CASE           =    None
																	SCREAMING_SNAKE_CASE           =    None
																	SCREAMING_SNAKE_CASE           =    None
																	SCREAMING_SNAKE_CASE           =    None

																	self.builder.download_and_prepare(
																	    download_config=_UpperCamelCase		,							download_mode=_UpperCamelCase		,							verification_mode=_UpperCamelCase		,							base_path=_UpperCamelCase		,							num_proc=self.num_proc		,							)
																	SCREAMING_SNAKE_CASE           =    self.builder.as_dataset(
																	    split=self.split		,							verification_mode=_UpperCamelCase		,							in_memory=self.keep_in_memory )
												return dataset



class   lowercase      :




							def __init__(  self				:   Any		,							_UpperCamelCase				:   Dataset		,							_UpperCamelCase				:   Union[PathLike, BinaryIO]		,							_UpperCamelCase				:   Optional[int] = None		,							_UpperCamelCase				:   Optional[int] = None		,							**_UpperCamelCase				:   str		,							)   ->			Union[str, Any]:



												'''simple docstring'''

												if num_proc is not None and num_proc <= 0:
																	raise ValueError(F"num_proc {num_proc} must be an integer > 0." )

												SCREAMING_SNAKE_CASE           =    dataset
												SCREAMING_SNAKE_CASE           =    path_or_buf
												SCREAMING_SNAKE_CASE           =    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
												SCREAMING_SNAKE_CASE           =    num_proc
												SCREAMING_SNAKE_CASE           =    "utf-8"
												SCREAMING_SNAKE_CASE           =    to_json_kwargs




							def 		__snake_case(  self				:   Optional[int] )   ->			int:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    self.to_json_kwargs.pop("path_or_buf"		,							_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    self.to_json_kwargs.pop("orient"		,							"records" )
												SCREAMING_SNAKE_CASE           =    self.to_json_kwargs.pop("lines"		,							True if orient == "records" else False )
												SCREAMING_SNAKE_CASE           =    self.to_json_kwargs.pop("index"		,							False if orient in ["split", "table"] else True )
												SCREAMING_SNAKE_CASE           =    self.to_json_kwargs.pop("compression"		,							_UpperCamelCase )

												if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																	raise NotImplementedError(F"`datasets` currently does not support {compression} compression" )

												if isinstance(self.path_or_buf		,							(str, bytes, os.PathLike) ):
																	with fsspec.open(self.path_or_buf		,							"wb"		,							compression=_UpperCamelCase ) as buffer:
																						SCREAMING_SNAKE_CASE           =    self._write(file_obj=_UpperCamelCase		,							orient=_UpperCamelCase		,							lines=_UpperCamelCase		,							index=_UpperCamelCase		,							**self.to_json_kwargs )
												else:
																	if compression:
																						raise NotImplementedError(
																						    F"The compression parameter is not supported when writing to a buffer, but compression={compression}"
																						    " was passed. Please provide a local path instead." )
																	SCREAMING_SNAKE_CASE           =    self._write(
																	    file_obj=self.path_or_buf		,							orient=_UpperCamelCase		,							lines=_UpperCamelCase		,							index=_UpperCamelCase		,							**self.to_json_kwargs )
												return written




							def 		__snake_case(  self				:   str		,							_UpperCamelCase				:   str )   ->			Optional[Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    args

												SCREAMING_SNAKE_CASE           =    query_table(
												    table=self.dataset.data		,							key=slice(_UpperCamelCase		,							offset + self.batch_size )		,							indices=self.dataset._indices		,							)
												SCREAMING_SNAKE_CASE           =    batch.to_pandas().to_json(
												    path_or_buf=_UpperCamelCase		,							orient=_UpperCamelCase		,							lines=_UpperCamelCase		,							index=_UpperCamelCase		,							**_UpperCamelCase )
												if not json_str.endswith("\n" ):
																	json_str += "\n"
												return json_str.encode(self.encoding )





							def 		__snake_case(  self				:   List[str]		,							_UpperCamelCase				:   BinaryIO		,							_UpperCamelCase				:   Tuple		,							_UpperCamelCase				:   Optional[Any]		,							_UpperCamelCase				:   List[Any]		,							**_UpperCamelCase				:   Optional[int]		,							)   ->			int:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    0

												if self.num_proc is None or self.num_proc == 1:
																	for offset in logging.tqdm(
																	    range(0		,							len(self.dataset )		,							self.batch_size )		,							unit="ba"		,							disable=not logging.is_progress_bar_enabled()		,							desc="Creating json from Arrow format"		,							):
																						SCREAMING_SNAKE_CASE           =    self._batch_json((offset, orient, lines, index, to_json_kwargs) )
																						written += file_obj.write(_UpperCamelCase )
												else:
																	SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    len(self.dataset ), self.batch_size
																	with multiprocessing.Pool(self.num_proc ) as pool:
																						for json_str in logging.tqdm(
																						    pool.imap(
																						        self._batch_json		,							[(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,							_UpperCamelCase		,							_UpperCamelCase )]		,							)		,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,							unit="ba"		,							disable=not logging.is_progress_bar_enabled()		,							desc="Creating json from Arrow format"		,							):
																											written += file_obj.write(_UpperCamelCase )

												return written





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase   :				int     			=						logging.get_logger(__name__)

_lowerCamelCase   :				Union[str, Any]     			=						{
    '''microsoft/unispeech-sat-base-100h-libri-ft''': (
        '''https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'''
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}


class   lowercase      (       a       ):
							lowercase__  :    Tuple									=							"""unispeech-sat"""




							def __init__(  self				:   str		,							_UpperCamelCase				:   Tuple=32		,							_UpperCamelCase				:   Union[str, Any]=768		,							_UpperCamelCase				:   Tuple=12		,							_UpperCamelCase				:   List[str]=12		,							_UpperCamelCase				:   Tuple=3_072		,							_UpperCamelCase				:   List[str]="gelu"		,							_UpperCamelCase				:   Tuple=0.1		,							_UpperCamelCase				:   Any=0.1		,							_UpperCamelCase				:   Union[str, Any]=0.1		,							_UpperCamelCase				:   Dict=0.0		,							_UpperCamelCase				:   Tuple=0.0		,							_UpperCamelCase				:   Tuple=0.1		,							_UpperCamelCase				:   Optional[Any]=0.1		,							_UpperCamelCase				:   Tuple=0.0_2		,							_UpperCamelCase				:   Optional[int]=1e-5		,							_UpperCamelCase				:   Union[str, Any]="group"		,							_UpperCamelCase				:   Optional[int]="gelu"		,							_UpperCamelCase				:   Tuple=(512, 512, 512, 512, 512, 512, 512)		,							_UpperCamelCase				:   List[str]=(5, 2, 2, 2, 2, 2, 2)		,							_UpperCamelCase				:   Optional[int]=(10, 3, 3, 3, 3, 2, 2)		,							_UpperCamelCase				:   Optional[int]=False		,							_UpperCamelCase				:   Dict=128		,							_UpperCamelCase				:   Optional[int]=16		,							_UpperCamelCase				:   Tuple=False		,							_UpperCamelCase				:   Union[str, Any]=True		,							_UpperCamelCase				:   Optional[Any]=0.0_5		,							_UpperCamelCase				:   Union[str, Any]=10		,							_UpperCamelCase				:   Union[str, Any]=2		,							_UpperCamelCase				:   str=0.0		,							_UpperCamelCase				:   List[Any]=10		,							_UpperCamelCase				:   Optional[int]=0		,							_UpperCamelCase				:   Any=320		,							_UpperCamelCase				:   List[Any]=2		,							_UpperCamelCase				:   str=0.1		,							_UpperCamelCase				:   str=100		,							_UpperCamelCase				:   int=256		,							_UpperCamelCase				:   Optional[Any]=256		,							_UpperCamelCase				:   List[Any]=0.1		,							_UpperCamelCase				:   str="mean"		,							_UpperCamelCase				:   int=False		,							_UpperCamelCase				:   Optional[Any]=False		,							_UpperCamelCase				:   Any=256		,							_UpperCamelCase				:   str=(512, 512, 512, 512, 1_500)		,							_UpperCamelCase				:   List[Any]=(5, 3, 3, 1, 1)		,							_UpperCamelCase				:   Union[str, Any]=(1, 2, 3, 1, 1)		,							_UpperCamelCase				:   Any=512		,							_UpperCamelCase				:   str=0		,							_UpperCamelCase				:   int=1		,							_UpperCamelCase				:   Any=2		,							_UpperCamelCase				:   Optional[Any]=504		,							**_UpperCamelCase				:   str		,							)   ->			int:



												'''simple docstring'''

												super().__init__(**_UpperCamelCase		,							pad_token_id=_UpperCamelCase		,							bos_token_id=_UpperCamelCase		,							eos_token_id=_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    hidden_size
												SCREAMING_SNAKE_CASE           =    feat_extract_norm
												SCREAMING_SNAKE_CASE           =    feat_extract_activation
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    conv_bias
												SCREAMING_SNAKE_CASE           =    num_conv_pos_embeddings
												SCREAMING_SNAKE_CASE           =    num_conv_pos_embedding_groups
												SCREAMING_SNAKE_CASE           =    len(self.conv_dim )
												SCREAMING_SNAKE_CASE           =    num_hidden_layers
												SCREAMING_SNAKE_CASE           =    intermediate_size
												SCREAMING_SNAKE_CASE           =    hidden_act
												SCREAMING_SNAKE_CASE           =    num_attention_heads
												SCREAMING_SNAKE_CASE           =    hidden_dropout
												SCREAMING_SNAKE_CASE           =    attention_dropout
												SCREAMING_SNAKE_CASE           =    activation_dropout
												SCREAMING_SNAKE_CASE           =    feat_proj_dropout
												SCREAMING_SNAKE_CASE           =    final_dropout
												SCREAMING_SNAKE_CASE           =    layerdrop
												SCREAMING_SNAKE_CASE           =    layer_norm_eps
												SCREAMING_SNAKE_CASE           =    initializer_range
												SCREAMING_SNAKE_CASE           =    vocab_size
												SCREAMING_SNAKE_CASE           =    num_clusters
												SCREAMING_SNAKE_CASE           =    do_stable_layer_norm
												SCREAMING_SNAKE_CASE           =    use_weighted_layer_sum

												if (
												    (len(self.conv_stride ) != self.num_feat_extract_layers)
												    or (len(self.conv_kernel ) != self.num_feat_extract_layers)
												    or (len(self.conv_dim ) != self.num_feat_extract_layers)
												):
																	raise ValueError(
																	    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
																	    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
																	    F" {len(self.conv_dim )}`, `len(config.conv_stride) = {len(self.conv_stride )}`,"
																	    F" `len(config.conv_kernel) = {len(self.conv_kernel )}`." )

												# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
												SCREAMING_SNAKE_CASE           =    apply_spec_augment
												SCREAMING_SNAKE_CASE           =    mask_time_prob
												SCREAMING_SNAKE_CASE           =    mask_time_length
												SCREAMING_SNAKE_CASE           =    mask_time_min_masks
												SCREAMING_SNAKE_CASE           =    mask_feature_prob
												SCREAMING_SNAKE_CASE           =    mask_feature_length
												SCREAMING_SNAKE_CASE           =    mask_feature_min_masks

												# parameters for pretraining with codevector quantized representations
												SCREAMING_SNAKE_CASE           =    num_codevectors_per_group
												SCREAMING_SNAKE_CASE           =    num_codevector_groups
												SCREAMING_SNAKE_CASE           =    contrastive_logits_temperature
												SCREAMING_SNAKE_CASE           =    feat_quantizer_dropout
												SCREAMING_SNAKE_CASE           =    num_negatives
												SCREAMING_SNAKE_CASE           =    codevector_dim
												SCREAMING_SNAKE_CASE           =    proj_codevector_dim
												SCREAMING_SNAKE_CASE           =    diversity_loss_weight

												# ctc loss
												SCREAMING_SNAKE_CASE           =    ctc_loss_reduction
												SCREAMING_SNAKE_CASE           =    ctc_zero_infinity

												# SequenceClassification-specific parameter. Feel free to ignore for other classes.
												SCREAMING_SNAKE_CASE           =    classifier_proj_size

												# XVector-specific parameters. Feel free to ignore for other classes.
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    list(_UpperCamelCase )
												SCREAMING_SNAKE_CASE           =    xvector_output_dim





							@property
							def 		__snake_case(  self				:   Tuple )   ->			str:



												'''simple docstring'''

												return functools.reduce(operator.mul		,							self.conv_stride		,							1 )




