
from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader




class      __A(    __lowerCamelCase			):

  """simple docstring"""







  def __init__(self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None	,      SCREAMING_SNAKE_CASE_ = None	,      SCREAMING_SNAKE_CASE_ = True	,      SCREAMING_SNAKE_CASE_ = None	,      SCREAMING_SNAKE_CASE_ = False	,      SCREAMING_SNAKE_CASE_ = None	,      SCREAMING_SNAKE_CASE_ = True	,      SCREAMING_SNAKE_CASE_ = "arrow"	,      **SCREAMING_SNAKE_CASE_	,      ):
       super().__init__(
           split=_A	,      features=_A	,      cache_dir=_A	,      keep_in_memory=_A	,      streaming=_A	,      **_A	,      )
       UpperCamelCase__				=    load_from_cache_file
       UpperCamelCase__				=    file_format
       UpperCamelCase__				=    Spark(
           df=_A	,      features=_A	,      cache_dir=_A	,      working_dir=_A	,      **_A	,      )







  def 	UpperCAmelCase_ (self						):
       if self.streaming:
            return self.builder.as_streaming_dataset(split=self.split						)
       UpperCamelCase__				=    None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
       self.builder.download_and_prepare(
           download_mode=_A	,      file_format=self._file_format	,      )
       return self.builder.as_dataset(split=self.split						)







import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows






def 						__magic_name__			(   __a      :	Any						):  # picklable for multiprocessing



     '''simple docstring'''




     return i + 1






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def 						__magic_name__			(   ):



     '''simple docstring'''




     with parallel_backend("""spark"""						):
          assert ParallelBackendConfig.backend_name == "spark"

     UpperCamelCase__				=    [1, 2, 3]
     with pytest.raises(__a						):
          with parallel_backend("""unsupported backend"""						):
               map_nested(__a	,		__a	,		num_proc=2						)

     with pytest.raises(__a						):
          with parallel_backend("""unsupported backend"""						):
               map_nested(__a	,		__a	,		num_proc=-1						)






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("""num_proc"""	,		[2, -1]						)
def 						__magic_name__			(   __a      :	Optional[int]						):



     '''simple docstring'''




     UpperCamelCase__				=    [1, 2]
     UpperCamelCase__				=    {"""a""": 1, """b""": 2}
     UpperCamelCase__				=    {"""a""": [1, 2], """b""": [3, 4]}
     UpperCamelCase__				=    {"""a""": {"""1""": 1}, """b""": 2}
     UpperCamelCase__				=    {"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
     UpperCamelCase__				=    [2, 3]
     UpperCamelCase__				=    {"""a""": 2, """b""": 3}
     UpperCamelCase__				=    {"""a""": [2, 3], """b""": [4, 5]}
     UpperCamelCase__				=    {"""a""": {"""1""": 2}, """b""": 3}
     UpperCamelCase__				=    {"""a""": 2, """b""": 3, """c""": 4, """d""": 5}

     with parallel_backend("""spark"""						):
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa





