


from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)
def _A						(						lowercase	,   lowercase		):







    """simple docstring"""
    a					=[]
    for part_id in partition_order:
        a					=df.where(f'''SPARK_PARTITION_ID() = {part_id}'''		).collect()
        for row_idx, row in enumerate(lowercase		):
            expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict())		)
    return expected_row_ids_and_row_dicts
@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(1_00		).repartition(1		)
    a					=Spark(lowercase		)
    # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
    # that each partition can hold 2 rows.
    spark_builder._repartition_df_if_needed(max_shard_size=16		)
    # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
    assert spark_builder.df.rdd.getNumPartitions() == 50
@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(10		).repartition(2		)
    a					=[1, 0]
    a					=_generate_iterable_examples(lowercase	,   lowercase		)  # Reverse the partitions.
    a					=_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase	,   lowercase		)

    for i, (row_id, row_dict) in enumerate(generate_fn()		):
        a		,				a					=expected_row_ids_and_row_dicts[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(10		).repartition(1		)
    a					=SparkExamplesIterable(lowercase		)
    assert it.n_shards == 1
    for i, (row_id, row_dict) in enumerate(lowercase		):
        assert row_id == f'''0_{i}'''
        assert row_dict == {"id": i}
@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(30		).repartition(3		)
    # Mock the generator so that shuffle reverses the partition indices.
    with patch('''numpy.random.Generator'''		) as generator_mock:
        a					=lambda lowercase		: x.reverse()
        a					=_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase	,   [2, 1, 0]		)

        a					=SparkExamplesIterable(lowercase		).shuffle_data_sources(lowercase		)
        assert shuffled_it.n_shards == 3
        for i, (row_id, row_dict) in enumerate(lowercase		):
            a		,				a					=expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(20		).repartition(4		)

    # Partitions 0 and 2
    a					=SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=0	,   num_workers=2		)
    assert shard_it_a.n_shards == 2
    a					=_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase	,   [0, 2]		)
    for i, (row_id, row_dict) in enumerate(lowercase		):
        a		,				a					=expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict

    # Partitions 1 and 3
    a					=SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=1	,   num_workers=2		)
    assert shard_it_a.n_shards == 2
    a					=_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase	,   [1, 3]		)
    for i, (row_id, row_dict) in enumerate(lowercase		):
        a		,				a					=expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def _A						(						):







    """simple docstring"""
    a					=pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
    a					=spark.range(1_00		).repartition(1		)
    a					=Spark(lowercase		)
    # Choose a small max_shard_size for maximum partitioning.
    spark_builder._repartition_df_if_needed(max_shard_size=1		)
    # The new number of partitions should not be greater than the number of rows.
    assert spark_builder.df.rdd.getNumPartitions() == 1_00



"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase_       :       Union[str, Any]      						=  {"""processing_layoutxlm""": ["""LayoutXLMProcessor"""]}

try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase_       :       Optional[int]      						=  ["""LayoutXLMTokenizer"""]

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCamelCase_       :       Dict      						=  ["""LayoutXLMTokenizerFast"""]

if TYPE_CHECKING:
       from .processing_layoutxlm import LayoutXLMProcessor

       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutxlm import LayoutXLMTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutxlm_fast import LayoutXLMTokenizerFast

else:
       import sys

       lowerCamelCase_       :       Union[str, Any]      						=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)