

import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def     UpperCAmelCase_       (						_A				,		_A    ):





     '''simple docstring'''




     assert isinstance(_A				,		_A    )
     assert dataset.num_rows == 4
     assert dataset.num_columns == 3
     assert dataset.column_names == ["col_1", "col_2", "col_3"]
     for feature, expected_dtype in expected_features.items():
          assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''				,		[False, True]    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(_A				,		cache_dir=_A				,		keep_in_memory=_A    ).read()
     _check_parquet_dataset(_A				,		_A    )
@pytest.mark.parametrize(
    '''features'''				,		[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]				,		)
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     SCREAMING_SNAKE_CASE__   =   features.copy() if features else default_expected_features
     SCREAMING_SNAKE_CASE__   =   (
         Features({feature: Value(_A    ) for feature, dtype in features.items()}    ) if features is not None else None
     )
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(_A				,		features=_A				,		cache_dir=_A    ).read()
     _check_parquet_dataset(_A				,		_A    )
@pytest.mark.parametrize('''split'''				,		[None, NamedSplit('''train'''    ), '''train''', '''test''']    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(_A				,		cache_dir=_A				,		split=_A    ).read()
     _check_parquet_dataset(_A				,		_A    )
     assert dataset.split == split if split else "train"
@pytest.mark.parametrize('''path_type'''				,		[str, list]    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     if issubclass(_A				,		_A    ):
          SCREAMING_SNAKE_CASE__   =   parquet_path
     elif issubclass(_A				,		_A    ):
          SCREAMING_SNAKE_CASE__   =   [parquet_path]
     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(_A				,		cache_dir=_A    ).read()
     _check_parquet_dataset(_A				,		_A    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A=("train",)    ):





     '''simple docstring'''




     assert isinstance(_A				,		_A    )
     for split in splits:
          SCREAMING_SNAKE_CASE__   =   dataset_dict[split]
          assert dataset.num_rows == 4
          assert dataset.num_columns == 3
          assert dataset.column_names == ["col_1", "col_2", "col_3"]
          for feature, expected_dtype in expected_features.items():
               assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''				,		[False, True]    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
          SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(
              {'''train''': parquet_path}				,		cache_dir=_A				,		keep_in_memory=_A    ).read()
     _check_parquet_datasetdict(_A				,		_A    )
@pytest.mark.parametrize(
    '''features'''				,		[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]				,		)
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     SCREAMING_SNAKE_CASE__   =   features.copy() if features else default_expected_features
     SCREAMING_SNAKE_CASE__   =   (
         Features({feature: Value(_A    ) for feature, dtype in features.items()}    ) if features is not None else None
     )
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader({'''train''': parquet_path}				,		features=_A				,		cache_dir=_A    ).read()
     _check_parquet_datasetdict(_A				,		_A    )
@pytest.mark.parametrize('''split'''				,		[None, NamedSplit('''train'''    ), '''train''', '''test''']    )
def     UpperCAmelCase_       (						_A				,		_A				,		_A    ):





     '''simple docstring'''




     if split:
          SCREAMING_SNAKE_CASE__   =   {split: parquet_path}
     else:
          SCREAMING_SNAKE_CASE__   =   '''train'''
          SCREAMING_SNAKE_CASE__   =   {'''train''': parquet_path, '''test''': parquet_path}
     SCREAMING_SNAKE_CASE__   =   tmp_path / '''cache'''
     SCREAMING_SNAKE_CASE__   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(_A				,		cache_dir=_A    ).read()
     _check_parquet_datasetdict(_A				,		_A				,		splits=list(path.keys()    )    )
     assert all(dataset[split].split == split for split in path.keys()    )
def     UpperCAmelCase_       (						_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   ParquetDatasetWriter(_A				,		tmp_path / '''foo.parquet'''    )
     assert writer.write() > 0
     SCREAMING_SNAKE_CASE__   =   pq.ParquetFile(tmp_path / '''foo.parquet'''    )
     SCREAMING_SNAKE_CASE__   =   pf.read()
     assert dataset.data.table == output_table
def     UpperCAmelCase_       (						_A				,		_A    ):





     '''simple docstring'''




     SCREAMING_SNAKE_CASE__   =   str(shared_datadir / '''test_image_rgb.jpg'''    )
     SCREAMING_SNAKE_CASE__   =   {'''image''': [image_path]}
     SCREAMING_SNAKE_CASE__   =   Features({'''image''': Image()}    )
     SCREAMING_SNAKE_CASE__   =   Dataset.from_dict(_A				,		features=_A    )
     SCREAMING_SNAKE_CASE__   =   ParquetDatasetWriter(_A				,		tmp_path / '''foo.parquet'''    )
     assert writer.write() > 0

     SCREAMING_SNAKE_CASE__   =   Dataset.from_parquet(str(tmp_path / '''foo.parquet'''    )    )
     assert dataset.features == reloaded_dataset.features

     SCREAMING_SNAKE_CASE__   =   ParquetDatasetReader(str(tmp_path / '''foo.parquet'''    )				,		streaming=_A    ).read()
     assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    '''feature, expected'''				,		[
        (Features({'''foo''': Value('''int32'''    )}    ), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]				,		)
def     UpperCAmelCase_       (						_A				,		_A    ):





     '''simple docstring'''




     assert get_writer_batch_size(_A    ) == expected





import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)






class 	UpperCAmelCase__		(	A__	):






  """simple docstring"""


  def __init__(							self						:		List[str]     ,       __lowerCamelCase						:		Tuple=0.01     ,       __lowerCamelCase						:		Optional[Any]=1000 )		->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__   =   p_stop
       SCREAMING_SNAKE_CASE__   =   max_length







  def __iter__(							self						:		List[str] )		->							Optional[Any]:
       SCREAMING_SNAKE_CASE__   =   0
       SCREAMING_SNAKE_CASE__   =   False
       while not stop and count < self.max_length:
            yield count
            count += 1
            SCREAMING_SNAKE_CASE__   =   random.random() < self.p_stop






class 	UpperCAmelCase__		(	unittest.TestCase	):






  """simple docstring"""


  def 				lowercase_		(							self						:		List[Any]     ,       __lowerCamelCase						:		Union[str, Any]     ,       __lowerCamelCase						:		str     ,       __lowerCamelCase						:		Any=False     ,       __lowerCamelCase						:		Optional[int]=True )		->							Dict:
       SCREAMING_SNAKE_CASE__   =   [
           BatchSamplerShard(__lowerCamelCase     ,       2     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )
           for i in range(2 )
       ]
       SCREAMING_SNAKE_CASE__   =   [list(__lowerCamelCase ) for batch_sampler_shard in batch_sampler_shards]
       if not split_batches:
            self.assertListEqual([len(__lowerCamelCase ) for shard in batch_sampler_shards]     ,       [len(__lowerCamelCase ) for e in expected] )
       self.assertListEqual(__lowerCamelCase     ,       __lowerCamelCase )






  def 				lowercase_		(							self						:		Any )		->							Optional[Any]:
       # Check the shards when the dataset is a round multiple of total batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       # Expected shouldn't change
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       # Check the shards when the dataset is a round multiple of batch size but not total batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
       # num_processes batch.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
       # num_processes batch.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(20 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(20 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       # Check the shards when the dataset is very small.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[[0, 1, 0]], [[1, 0, 1]]]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase )






  def 				lowercase_		(							self						:		Optional[int] )		->							int:
       # Check the shards when the dataset is a round multiple of batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       # Expected shouldn't change
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size or num_processes.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       # Check the shards when the dataset is very small.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[[0, 1]], [[0, 1]]]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase )






  def 				lowercase_		(							self						:		str )		->							Dict:
       # Check the shards when the dataset is a round multiple of total batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       # Expected shouldn't change
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is a round multiple of batch size but not total batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
       # num_processes batch.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
       # num_processes batch.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(20 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(20 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
           [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is very small.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[[0, 1]], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=3     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase )






  def 				lowercase_		(							self						:		List[str] )		->							Tuple:
       # Check the shards when the dataset is a round multiple of batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(24 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       # Expected shouldn't change
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(22 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is not a round multiple of batch size or num_processes.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(21 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [
           [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
           [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
       ]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       # Check the shards when the dataset is very small.
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[[0, 1]], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(2 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   [[], []]
       self.check_batch_sampler_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       even_batches=__lowerCamelCase )






  def 				lowercase_		(							self						:		Optional[int] )		->							List[str]:
       SCREAMING_SNAKE_CASE__   =   [[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
       SCREAMING_SNAKE_CASE__   =   [BatchSamplerShard(__lowerCamelCase     ,       2     ,       __lowerCamelCase     ,       even_batches=__lowerCamelCase ) for i in range(2 )]

       self.assertEqual(len(batch_sampler_shards[0] )     ,       3 )
       self.assertEqual(len(batch_sampler_shards[1] )     ,       2 )

       self.assertListEqual(list(batch_sampler_shards[0] )     ,       [[0, 1, 2], [5, 6, 7, 8], [12, 13]] )
       self.assertListEqual(list(batch_sampler_shards[1] )     ,       [[3, 4], [9, 10, 11]] )






  def 				lowercase_		(							self						:		Optional[Any]     ,       __lowerCamelCase						:		Any     ,       __lowerCamelCase						:		str     ,       __lowerCamelCase						:		str     ,       __lowerCamelCase						:		Tuple=False     ,       __lowerCamelCase						:		Optional[int]=2     ,       __lowerCamelCase						:		Dict=False )		->							str:
       random.seed(__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   list(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   [
           IterableDatasetShard(
               __lowerCamelCase     ,       batch_size=__lowerCamelCase     ,       drop_last=__lowerCamelCase     ,       num_processes=__lowerCamelCase     ,       process_index=__lowerCamelCase     ,       split_batches=__lowerCamelCase     ,       )
           for i in range(__lowerCamelCase )
       ]
       SCREAMING_SNAKE_CASE__   =   []
       for iterable_dataset_shard in iterable_dataset_shards:
            # Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
            random.seed(__lowerCamelCase )
            iterable_dataset_lists.append(list(__lowerCamelCase ) )

       SCREAMING_SNAKE_CASE__   =   batch_size // num_processes if split_batches else batch_size
       # All iterable dataset shard should have the same length, a round multiple of shard_batch_size
       SCREAMING_SNAKE_CASE__   =   iterable_dataset_lists[0]
       for l in iterable_dataset_lists[1:]:
            self.assertEqual(len(__lowerCamelCase )     ,       len(__lowerCamelCase ) )
            self.assertTrue(len(__lowerCamelCase ) % shard_batch_size == 0 )

       SCREAMING_SNAKE_CASE__   =   []
       for idx in range(0     ,       len(__lowerCamelCase )     ,       __lowerCamelCase ):
            for l in iterable_dataset_lists:
                 observed += l[idx : idx + shard_batch_size]

       if not drop_last:
            while len(__lowerCamelCase ) < len(__lowerCamelCase ):
                 reference += reference
       self.assertListEqual(__lowerCamelCase     ,       reference[: len(__lowerCamelCase )] )






  def 				lowercase_		(							self						:		str )		->							Dict:
       SCREAMING_SNAKE_CASE__   =   42
       SCREAMING_SNAKE_CASE__   =   RandomIterableDataset()

       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )

       # Edge case with a very small dataset
       SCREAMING_SNAKE_CASE__   =   RandomIterableDataset(max_length=2 )

       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )
       self.check_iterable_dataset_shards(__lowerCamelCase     ,       __lowerCamelCase     ,       batch_size=4     ,       drop_last=__lowerCamelCase     ,       split_batches=__lowerCamelCase )






  def 				lowercase_		(							self						:		Optional[Any] )		->							Any:
       SCREAMING_SNAKE_CASE__   =   BatchSampler(range(16 )     ,       batch_size=4     ,       drop_last=__lowerCamelCase )
       SCREAMING_SNAKE_CASE__   =   SkipBatchSampler(__lowerCamelCase     ,       2 )
       self.assertListEqual(list(__lowerCamelCase )     ,       [[8, 9, 10, 11], [12, 13, 14, 15]] )






  def 				lowercase_		(							self						:		Union[str, Any] )		->							str:
       SCREAMING_SNAKE_CASE__   =   SkipDataLoader(list(range(16 ) )     ,       batch_size=4     ,       skip_batches=2 )
       self.assertListEqual([t.tolist() for t in dataloader]     ,       [[8, 9, 10, 11], [12, 13, 14, 15]] )






  def 				lowercase_		(							self						:		Dict )		->							List[Any]:
       SCREAMING_SNAKE_CASE__   =   DataLoader(list(range(16 ) )     ,       batch_size=4 )
       SCREAMING_SNAKE_CASE__   =   skip_first_batches(__lowerCamelCase     ,       num_batches=2 )
       self.assertListEqual([t.tolist() for t in new_dataloader]     ,       [[8, 9, 10, 11], [12, 13, 14, 15]] )






  def 				lowercase_		(							self						:		List[Any] )		->							List[str]:
       SCREAMING_SNAKE_CASE__   =   DataLoaderShard(list(range(16 ) )     ,       batch_size=4 )
       for idx, _ in enumerate(__lowerCamelCase ):
            self.assertEqual(dataloader.end_of_dataloader     ,       idx == 3 )

       # Test it also works on the second iteration
       for idx, _ in enumerate(__lowerCamelCase ):
            self.assertEqual(dataloader.end_of_dataloader     ,       idx == 3 )







  def 				lowercase_		(							self						:		Union[str, Any] )		->							str:
       Accelerator()
       SCREAMING_SNAKE_CASE__   =   DataLoaderDispatcher(range(16 )     ,       batch_size=4 )
       for idx, _ in enumerate(__lowerCamelCase ):
            self.assertEqual(dataloader.end_of_dataloader     ,       idx == 3 )

       # Test it also works on the second iteration
       for idx, _ in enumerate(__lowerCamelCase ):
            self.assertEqual(dataloader.end_of_dataloader     ,       idx == 3 )


