





'''simple docstring'''




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase__   :				Tuple      =   datasets.utils.logging.get_logger(__name__)


@dataclass
class 			SCREAMING_SNAKE_CASE   (datasets.BuilderConfig							):
     lowerCAmelCase		      =	None
     lowerCAmelCase		      =	"utf-8"
     lowerCAmelCase		      =	None
     lowerCAmelCase		      =	None
     lowerCAmelCase		      =	True  # deprecated
     lowerCAmelCase		      =	None  # deprecated
     lowerCAmelCase		      =	10 << 20  # 10MB
     lowerCAmelCase		      =	None




class 			SCREAMING_SNAKE_CASE   (datasets.ArrowBasedBuilder							):
     lowerCAmelCase		      =	JsonConfig


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.config.block_size is not None:
               logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead')
               __A						:			Union[str, Any]								=	self.config.block_size
          if self.config.use_threads is not True:
               logger.warning(
                   'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.')
          if self.config.newlines_in_values is not None:
               raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported')
          return datasets.DatasetInfo(features=self.config.features)


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          if not self.config.data_files:
               raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
          __A						:			Optional[int]								=	dl_manager.download_and_extract(self.config.data_files)
          if isinstance(_UpperCAmelCase						,     (str, list, tuple)):
               __A						:			Optional[Any]								=	data_files
               if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):
                    __A						:			Any								=	[files]
               __A						:			Union[str, Any]								=	[dl_manager.iter_files(_UpperCAmelCase) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,     gen_kwargs={'files': files})]
          __A						:			Optional[int]								=	[]
          for split_name, files in data_files.items():
               if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):
                    __A						:			Optional[int]								=	[files]
               __A						:			Union[str, Any]								=	[dl_manager.iter_files(_UpperCAmelCase) for file in files]
               splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,     gen_kwargs={'files': files}))
          return splits


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          if self.config.features is not None:
               # adding missing columns
               for column_name in set(self.config.features) - set(pa_table.column_names):
                    __A						:			Any								=	self.config.features.arrow_schema.field(_UpperCAmelCase).type
                    __A						:			Tuple								=	pa_table.append_column(_UpperCAmelCase						,     pa.array([None] * len(_UpperCAmelCase)						,     type=_UpperCAmelCase))
               # more expensive cast to support nested structures with keys in a different order
               # allows str <-> int/float or str to Audio for example
               __A						:			str								=	table_cast(_UpperCAmelCase						,     self.config.features.arrow_schema)
          return pa_table


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase)):
               # If the file is one json object and if we need to look at the list of items in one specific field
               if self.config.field is not None:
                    with open(_UpperCAmelCase						,     encoding=self.config.encoding						,     errors=self.config.encoding_errors) as f:
                         __A						:			str								=	json.load(_UpperCAmelCase)

                    # We keep only the field we are interested in
                    __A						:			List[Any]								=	dataset[self.config.field]

                    # We accept two format: a list of dicts or a dict of lists
                    if isinstance(_UpperCAmelCase						,     (list, tuple)):
                         __A						:			Tuple								=	set().union(*[row.keys() for row in dataset])
                         __A						:			str								=	{col: [row.get(_UpperCAmelCase) for row in dataset] for col in keys}
                    else:
                         __A						:			List[Any]								=	dataset
                    __A						:			Optional[Any]								=	pa.Table.from_pydict(_UpperCAmelCase)
                    yield file_idx, self._cast_table(_UpperCAmelCase)

               # If the file has one json object per line
               else:
                    with open(_UpperCAmelCase						,     'rb') as f:
                         __A						:			Dict								=	0
                         # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                         # Set a default minimum value of 16kB if the chunk size is really small
                         __A						:			List[str]								=	max(self.config.chunksize // 32						,     16 << 10)
                         __A						:			int								=	(
                             self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
                         )
                         while True:
                              __A						:			Optional[int]								=	f.read(self.config.chunksize)
                              if not batch:
                                   break
                              # Finish current line
                              try:
                                   batch += f.readline()
                              except (AttributeError, io.UnsupportedOperation):
                                   batch += readline(_UpperCAmelCase)
                              # PyArrow only accepts utf-8 encoded bytes
                              if self.config.encoding != "utf-8":
                                   __A						:			Union[str, Any]								=	batch.decode(self.config.encoding						,     errors=_UpperCAmelCase).encode('utf-8')
                              try:
                                   while True:
                                        try:
                                             __A						:			str								=	paj.read_json(
                                                 io.BytesIO(_UpperCAmelCase)						,     read_options=paj.ReadOptions(block_size=_UpperCAmelCase))
                                             break
                                        except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                             if (
                                                 isinstance(_UpperCAmelCase						,     pa.ArrowInvalid)
                                                 and "straddling" not in str(_UpperCAmelCase)
                                                 or block_size > len(_UpperCAmelCase)
                                             ):
                                                  raise
                                             else:
                                                  # Increase the block size in case it was too small.
                                                  # The block size will be reset for the next file.
                                                  logger.debug(
                                                      F'Batch of {len(_UpperCAmelCase)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.')
                                                  block_size *= 2
                              except pa.ArrowInvalid as e:
                                   try:
                                        with open(
                                            _UpperCAmelCase						,     encoding=self.config.encoding						,     errors=self.config.encoding_errors) as f:
                                             __A						:			List[str]								=	json.load(_UpperCAmelCase)
                                   except json.JSONDecodeError:
                                        logger.error(F'Failed to read file \'{file}\' with error {type(_UpperCAmelCase)}: {e}')
                                        raise e
                                   # If possible, parse the file as a list of json objects and exit the loop
                                   if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):  # list is the only sequence type supported in JSON
                                        try:
                                             __A						:			Tuple								=	set().union(*[row.keys() for row in dataset])
                                             __A						:			Any								=	{col: [row.get(_UpperCAmelCase) for row in dataset] for col in keys}
                                             __A						:			Tuple								=	pa.Table.from_pydict(_UpperCAmelCase)
                                        except (pa.ArrowInvalid, AttributeError) as e:
                                             logger.error(F'Failed to read file \'{file}\' with error {type(_UpperCAmelCase)}: {e}')
                                             raise ValueError(F'Not able to read records in the JSON file at {file}.') from None
                                        yield file_idx, self._cast_table(_UpperCAmelCase)
                                        break
                                   else:
                                        logger.error(F'Failed to read file \'{file}\' with error {type(_UpperCAmelCase)}: {e}')
                                        raise ValueError(
                                            F'Not able to read records in the JSON file at {file}. '
                                            F'You should probably indicate the field of the JSON file containing your records. '
                                            F'This JSON file contain the following fields: {str(list(dataset.keys()))}. '
                                            F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. ') from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                              yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase)
                              batch_idx += 1






'''simple docstring'''




import PIL.Image
import PIL.ImageOps
from packaging import version
from PIL import Image


if version.parse(version.parse(PIL.__version__).base_version) >= version.parse('''9.1.0'''):
  lowercase__   :				Dict      =   {
      '''linear''': PIL.Image.Resampling.BILINEAR,
      '''bilinear''': PIL.Image.Resampling.BILINEAR,
      '''bicubic''': PIL.Image.Resampling.BICUBIC,
      '''lanczos''': PIL.Image.Resampling.LANCZOS,
      '''nearest''': PIL.Image.Resampling.NEAREST,
  }
else:
  lowercase__   :				Any      =   {
      '''linear''': PIL.Image.LINEAR,
      '''bilinear''': PIL.Image.BILINEAR,
      '''bicubic''': PIL.Image.BICUBIC,
      '''lanczos''': PIL.Image.LANCZOS,
      '''nearest''': PIL.Image.NEAREST,
  }






def 	_lowerCAmelCase      (  __snake_case  :			Any      )					->					Optional[Any]:
     __A						:			Dict								=	(images / 2 + 0.5).clamp(0			,							1      )
     __A						:			str								=	images.cpu().permute(0			,							2			,							3			,							1      ).float().numpy()
     __A						:			Dict								=	numpy_to_pil(__snake_case      )
     return images







def 	_lowerCAmelCase      (  __snake_case  :			List[Any]      )					->					Optional[Any]:
     if images.ndim == 3:
          __A						:			List[Any]								=	images[None, ...]
     __A						:			List[str]								=	(images * 2_55).round().astype('uint8'      )
     if images.shape[-1] == 1:
          # special case for grayscale (single channel) images
          __A						:			str								=	[Image.fromarray(image.squeeze()			,							mode='L'      ) for image in images]
     else:
          __A						:			str								=	[Image.fromarray(__snake_case      ) for image in images]

     return pil_images