

import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)







class 			lowercase    (	lowercase_      ):




				def __init__( self				,       snake_case=0.01				,       snake_case=1000	):
											snake_case_          =   p_stop
											snake_case_          =   max_length




				def __iter__( self	):
											snake_case_          =   0
											snake_case_          =   False
											while not stop and count < self.max_length:
																		yield count
																		count += 1
																		snake_case_          =   random.random() < self.p_stop







class 			lowercase    (	unittest.TestCase      ):




				def    a							( self				,       snake_case				,       snake_case				,       snake_case=False				,       snake_case=True	):
											snake_case_          =   [
											    BatchSamplerShard(snake_case				,       2				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)
											    for i in range(2	)
											]
											snake_case_          =   [list(snake_case	) for batch_sampler_shard in batch_sampler_shards]
											if not split_batches:
																		self.assertListEqual([len(snake_case	) for shard in batch_sampler_shards]				,       [len(snake_case	) for e in expected]	)
											self.assertListEqual(snake_case				,       snake_case	)




				def    a							( self	):
											# Check the shards when the dataset is a round multiple of total batch size.
											snake_case_          =   BatchSampler(range(24	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											snake_case_          =   BatchSampler(range(24	)				,       batch_size=3				,       drop_last=snake_case	)
											# Expected shouldn't change
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											# Check the shards when the dataset is a round multiple of batch size but not total batch size.
											snake_case_          =   BatchSampler(range(21	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											snake_case_          =   BatchSampler(range(21	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
											# num_processes batch.
											snake_case_          =   BatchSampler(range(22	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											snake_case_          =   BatchSampler(range(22	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
											# num_processes batch.
											snake_case_          =   BatchSampler(range(20	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											snake_case_          =   BatchSampler(range(20	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											# Check the shards when the dataset is very small.
											snake_case_          =   BatchSampler(range(2	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [[[0, 1, 0]], [[1, 0, 1]]]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)

											snake_case_          =   BatchSampler(range(2	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [[], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case	)




				def    a							( self	):
											# Check the shards when the dataset is a round multiple of batch size.
											snake_case_          =   BatchSampler(range(24	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											snake_case_          =   BatchSampler(range(24	)				,       batch_size=4				,       drop_last=snake_case	)
											# Expected shouldn't change
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size.
											snake_case_          =   BatchSampler(range(22	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											snake_case_          =   BatchSampler(range(22	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size or num_processes.
											snake_case_          =   BatchSampler(range(21	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											snake_case_          =   BatchSampler(range(21	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											# Check the shards when the dataset is very small.
											snake_case_          =   BatchSampler(range(2	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [[[0, 1]], [[0, 1]]]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)

											snake_case_          =   BatchSampler(range(2	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [[], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case	)




				def    a							( self	):
											# Check the shards when the dataset is a round multiple of total batch size.
											snake_case_          =   BatchSampler(range(24	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(24	)				,       batch_size=3				,       drop_last=snake_case	)
											# Expected shouldn't change
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is a round multiple of batch size but not total batch size.
											snake_case_          =   BatchSampler(range(21	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(21	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
											# num_processes batch.
											snake_case_          =   BatchSampler(range(22	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(22	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
											# num_processes batch.
											snake_case_          =   BatchSampler(range(20	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(20	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
											    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is very small.
											snake_case_          =   BatchSampler(range(2	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [[[0, 1]], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(2	)				,       batch_size=3				,       drop_last=snake_case	)
											snake_case_          =   [[], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       even_batches=snake_case	)




				def    a							( self	):
											# Check the shards when the dataset is a round multiple of batch size.
											snake_case_          =   BatchSampler(range(24	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(24	)				,       batch_size=4				,       drop_last=snake_case	)
											# Expected shouldn't change
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size.
											snake_case_          =   BatchSampler(range(22	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(22	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is not a round multiple of batch size or num_processes.
											snake_case_          =   BatchSampler(range(21	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(21	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [
											    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
											    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
											]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											# Check the shards when the dataset is very small.
											snake_case_          =   BatchSampler(range(2	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [[[0, 1]], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)

											snake_case_          =   BatchSampler(range(2	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   [[], []]
											self.check_batch_sampler_shards(snake_case				,       snake_case				,       split_batches=snake_case				,       even_batches=snake_case	)




				def    a							( self	):
											snake_case_          =   [[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
											snake_case_          =   [BatchSamplerShard(snake_case				,       2				,       snake_case				,       even_batches=snake_case	) for i in range(2	)]

											self.assertEqual(len(batch_sampler_shards[0]	)				,       3	)
											self.assertEqual(len(batch_sampler_shards[1]	)				,       2	)

											self.assertListEqual(list(batch_sampler_shards[0]	)				,       [[0, 1, 2], [5, 6, 7, 8], [12, 13]]	)
											self.assertListEqual(list(batch_sampler_shards[1]	)				,       [[3, 4], [9, 10, 11]]	)




				def    a							( self				,       snake_case				,       snake_case				,       snake_case				,       snake_case=False				,       snake_case=2				,       snake_case=False	):
											random.seed(snake_case	)
											snake_case_          =   list(snake_case	)

											snake_case_          =   [
											    IterableDatasetShard(
											        snake_case				,       batch_size=snake_case				,       drop_last=snake_case				,       num_processes=snake_case				,       process_index=snake_case				,       split_batches=snake_case				,       )
											    for i in range(snake_case	)
											]
											snake_case_          =   []
											for iterable_dataset_shard in iterable_dataset_shards:
																		# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
																		random.seed(snake_case	)
																		iterable_dataset_lists.append(list(snake_case	)	)

											snake_case_          =   batch_size // num_processes if split_batches else batch_size
											# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
											snake_case_          =   iterable_dataset_lists[0]
											for l in iterable_dataset_lists[1:]:
																		self.assertEqual(len(snake_case	)				,       len(snake_case	)	)
																		self.assertTrue(len(snake_case	) % shard_batch_size == 0	)

											snake_case_          =   []
											for idx in range(0				,       len(snake_case	)				,       snake_case	):
																		for l in iterable_dataset_lists:
																									observed += l[idx : idx + shard_batch_size]

											if not drop_last:
																		while len(snake_case	) < len(snake_case	):
																									reference += reference
											self.assertListEqual(snake_case				,       reference[: len(snake_case	)]	)




				def    a							( self	):
											snake_case_          =   42
											snake_case_          =   RandomIterableDataset()

											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)

											# Edge case with a very small dataset
											snake_case_          =   RandomIterableDataset(max_length=2	)

											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)
											self.check_iterable_dataset_shards(snake_case				,       snake_case				,       batch_size=4				,       drop_last=snake_case				,       split_batches=snake_case	)




				def    a							( self	):
											snake_case_          =   BatchSampler(range(16	)				,       batch_size=4				,       drop_last=snake_case	)
											snake_case_          =   SkipBatchSampler(snake_case				,       2	)
											self.assertListEqual(list(snake_case	)				,       [[8, 9, 10, 11], [12, 13, 14, 15]]	)




				def    a							( self	):
											snake_case_          =   SkipDataLoader(list(range(16	)	)				,       batch_size=4				,       skip_batches=2	)
											self.assertListEqual([t.tolist() for t in dataloader]				,       [[8, 9, 10, 11], [12, 13, 14, 15]]	)




				def    a							( self	):
											snake_case_          =   DataLoader(list(range(16	)	)				,       batch_size=4	)
											snake_case_          =   skip_first_batches(snake_case				,       num_batches=2	)
											self.assertListEqual([t.tolist() for t in new_dataloader]				,       [[8, 9, 10, 11], [12, 13, 14, 15]]	)




				def    a							( self	):
											snake_case_          =   DataLoaderShard(list(range(16	)	)				,       batch_size=4	)
											for idx, _ in enumerate(snake_case	):
																		self.assertEqual(dataloader.end_of_dataloader				,       idx == 3	)

											# Test it also works on the second iteration
											for idx, _ in enumerate(snake_case	):
																		self.assertEqual(dataloader.end_of_dataloader				,       idx == 3	)




				def    a							( self	):
											Accelerator()
											snake_case_          =   DataLoaderDispatcher(range(16	)				,       batch_size=4	)
											for idx, _ in enumerate(snake_case	):
																		self.assertEqual(dataloader.end_of_dataloader				,       idx == 3	)

											# Test it also works on the second iteration
											for idx, _ in enumerate(snake_case	):
																		self.assertEqual(dataloader.end_of_dataloader				,       idx == 3	)







import requests

_UpperCAmelCase		:       Union[str, Any]						    =	"""https://newsapi.org/v1/articles?source=bbc-news&sortBy=top&apiKey="""





def  __lowerCamelCase     (			UpperCamelCase__				):







							'''simple docstring'''






							snake_case_          =   requests.get(_NEWS_API + bbc_news_api_key				).json()
							# each article in the list is a dict
							for i, article in enumerate(bbc_news_page['articles']       ,							1				):
														print(F'''{i}.) {article["title"]}'''				)


if __name__ == "__main__":
					fetch_bbc_news(bbc_news_api_key="""<Your BBC News API key goes here>""")




