
'''simple docstring'''



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase = None   ,       _lowercase = None   ,       _lowercase = None   ,       _lowercase = False   ,       _lowercase = False   ,       _lowercase = None   ,       _lowercase = None   ,       **_lowercase   ,       ):

												"""simple docstring"""



												super().__init__(
												    _lowercase   ,       split=_lowercase   ,       features=_lowercase   ,       cache_dir=_lowercase   ,       keep_in_memory=_lowercase   ,       streaming=_lowercase   ,       num_proc=_lowercase   ,       **_lowercase   ,       )
												_lowerCAmelCase  						=			field
												_lowerCAmelCase  						=			path_or_paths if isinstance(_lowercase   ,       _lowercase				) else {self.split: path_or_paths}
												_lowerCAmelCase  						=			Json(
												    cache_dir=_lowercase   ,       data_files=_lowercase   ,       features=_lowercase   ,       field=_lowercase   ,       **_lowercase   ,       )




						def  _lowercase     ( self				):

												"""simple docstring"""



												if self.streaming:
																		_lowerCAmelCase  						=			self.builder.as_streaming_dataset(split=self.split				)
												# Build regular (map-style) dataset
												else:
																		_lowerCAmelCase  						=			None
																		_lowerCAmelCase  						=			None
																		_lowerCAmelCase  						=			None
																		_lowerCAmelCase  						=			None

																		self.builder.download_and_prepare(
																		    download_config=_lowercase   ,       download_mode=_lowercase   ,       verification_mode=_lowercase   ,       base_path=_lowercase   ,       num_proc=self.num_proc   ,       )
																		_lowerCAmelCase  						=			self.builder.as_dataset(
																		    split=self.split   ,       verification_mode=_lowercase   ,       in_memory=self.keep_in_memory				)
												return dataset






class   UpperCAmelCase_					:






						'''simple docstring'''






						def __init__( self   ,       _lowercase   ,       _lowercase   ,       _lowercase = None   ,       _lowercase = None   ,       **_lowercase   ,       ):

												"""simple docstring"""



												if num_proc is not None and num_proc <= 0:
																		raise ValueError(F'num_proc {num_proc} must be an integer > 0.'				)

												_lowerCAmelCase  						=			dataset
												_lowerCAmelCase  						=			path_or_buf
												_lowerCAmelCase  						=			batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
												_lowerCAmelCase  						=			num_proc
												_lowerCAmelCase  						=			"""utf-8"""
												_lowerCAmelCase  						=			to_json_kwargs




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.to_json_kwargs.pop("""path_or_buf"""   ,       _lowercase				)
												_lowerCAmelCase  						=			self.to_json_kwargs.pop("""orient"""   ,       """records"""				)
												_lowerCAmelCase  						=			self.to_json_kwargs.pop("""lines"""   ,       True if orient == """records""" else False				)
												_lowerCAmelCase  						=			self.to_json_kwargs.pop("""index"""   ,       False if orient in ["""split""", """table"""] else True				)
												_lowerCAmelCase  						=			self.to_json_kwargs.pop("""compression"""   ,       _lowercase				)

												if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																		raise NotImplementedError(F'`datasets` currently does not support {compression} compression'				)

												if isinstance(self.path_or_buf   ,       (str, bytes, os.PathLike)				):
																		with fsspec.open(self.path_or_buf   ,       """wb"""   ,       compression=_lowercase				) as buffer:
																								_lowerCAmelCase  						=			self._write(file_obj=_lowercase   ,       orient=_lowercase   ,       lines=_lowercase   ,       index=_lowercase   ,       **self.to_json_kwargs				)
												else:
																		if compression:
																								raise NotImplementedError(
																								    F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
																								    """ was passed. Please provide a local path instead."""				)
																		_lowerCAmelCase  						=			self._write(
																		    file_obj=self.path_or_buf   ,       orient=_lowercase   ,       lines=_lowercase   ,       index=_lowercase   ,       **self.to_json_kwargs				)
												return written




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase						,   _lowerCAmelCase						,   _lowerCAmelCase						,   _lowerCAmelCase						,   _lowerCAmelCase  						=			args

												_lowerCAmelCase  						=			query_table(
												    table=self.dataset.data   ,       key=slice(_lowercase   ,       offset + self.batch_size				)   ,       indices=self.dataset._indices   ,       )
												_lowerCAmelCase  						=			batch.to_pandas().to_json(
												    path_or_buf=_lowercase   ,       orient=_lowercase   ,       lines=_lowercase   ,       index=_lowercase   ,       **_lowercase				)
												if not json_str.endswith("""\n"""				):
																		json_str += "\n"
												return json_str.encode(self.encoding				)




						def  _lowercase     ( self   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       _lowercase   ,       **_lowercase   ,       ):

												"""simple docstring"""



												_lowerCAmelCase  						=			0

												if self.num_proc is None or self.num_proc == 1:
																		for offset in logging.tqdm(
																		    range(0   ,       len(self.dataset				)   ,       self.batch_size				)   ,       unit="""ba"""   ,       disable=not logging.is_progress_bar_enabled()   ,       desc="""Creating json from Arrow format"""   ,       ):
																								_lowerCAmelCase  						=			self._batch_json((offset, orient, lines, index, to_json_kwargs)				)
																								written += file_obj.write(_lowercase				)
												else:
																		_lowerCAmelCase						,   _lowerCAmelCase  						=			len(self.dataset				), self.batch_size
																		with multiprocessing.Pool(self.num_proc				) as pool:
																								for json_str in logging.tqdm(
																								    pool.imap(
																								        self._batch_json   ,       [(offset, orient, lines, index, to_json_kwargs) for offset in range(0   ,       _lowercase   ,       _lowercase				)]   ,       )   ,       total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,       unit="""ba"""   ,       disable=not logging.is_progress_bar_enabled()   ,       desc="""Creating json from Arrow format"""   ,       ):
																														written += file_obj.write(_lowercase				)

												return written


'''simple docstring'''



# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

_lowercase    =		"""2.13.1"""

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("""3.7"""):
						raise ImportWarning(
						    """To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."""
						)

if version.parse(pyarrow.__version__).major < 8:
						raise ImportWarning(
						    """To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"""
						    """If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."""
						)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

_lowercase    =		concatenate_datasets
_lowercase    =		DownloadConfig
_lowercase    =		DownloadManager
_lowercase    =		DownloadMode
_lowercase    =		DownloadConfig
_lowercase    =		DownloadMode
_lowercase    =		DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager
