


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase_	   =     datasets.utils.logging.get_logger(__name__)


@dataclass
class 	A_ (	datasets.BuilderConfig   ):




      '''simple docstring'''


      __snake_case 	=       None
      __snake_case 	=       "utf-8"
      __snake_case 	=       None
      __snake_case 	=       None
      __snake_case 	=       True  # deprecated
      __snake_case 	=       None  # deprecated
      __snake_case 	=       10 << 20  # 10MB
      __snake_case 	=       None



class 	A_ (	datasets.ArrowBasedBuilder   ):




      '''simple docstring'''


      __snake_case 	=       JsonConfig




      def 					_snake_case     (    self:	int    ):
       if self.config.block_size is not None:
        logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'    )
        __lowerCamelCase		:							Any		 =						self.config.block_size
       if self.config.use_threads is not True:
        logger.warning(
            'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'    )
       if self.config.newlines_in_values is not None:
        raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'    )
       return datasets.DatasetInfo(features=self.config.features    )




      def 					_snake_case     (    self:	List[Any]						,				a:	List[str]    ):
       if not self.config.data_files:
        raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'    )
       __lowerCamelCase		:							Optional[int]		 =						dl_manager.download_and_extract(self.config.data_files    )
       if isinstance(lowerCAmelCase__						,				(str, list, tuple)    ):
        __lowerCamelCase		:							str		 =						data_files
        if isinstance(lowerCAmelCase__						,				lowerCAmelCase__    ):
         __lowerCamelCase		:							Union[str, Any]		 =						[files]
        __lowerCamelCase		:							str		 =						[dl_manager.iter_files(lowerCAmelCase__    ) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,				gen_kwargs={'files': files}    )]
       __lowerCamelCase		:							List[Any]		 =						[]
       for split_name, files in data_files.items():
        if isinstance(lowerCAmelCase__						,				lowerCAmelCase__    ):
         __lowerCamelCase		:							int		 =						[files]
        __lowerCamelCase		:							List[Any]		 =						[dl_manager.iter_files(lowerCAmelCase__    ) for file in files]
        splits.append(datasets.SplitGenerator(name=lowerCAmelCase__						,				gen_kwargs={'files': files}    )    )
       return splits




      def 					_snake_case     (    self:	str						,				a:	List[Any]    ):
       if self.config.features is not None:
        # adding missing columns
        for column_name in set(self.config.features    ) - set(pa_table.column_names    ):
         __lowerCamelCase		:							Tuple		 =						self.config.features.arrow_schema.field(lowerCAmelCase__    ).type
         __lowerCamelCase		:							Dict		 =						pa_table.append_column(lowerCAmelCase__						,				pa.array([None] * len(lowerCAmelCase__    )						,				type=lowerCAmelCase__    )    )
        # more expensive cast to support nested structures with keys in a different order
        # allows str <-> int/float or str to Audio for example
        __lowerCamelCase		:							Union[str, Any]		 =						table_cast(lowerCAmelCase__						,				self.config.features.arrow_schema    )
       return pa_table




      def 					_snake_case     (    self:	Optional[int]						,				a:	Optional[int]    ):
       for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__    )    ):
        # If the file is one json object and if we need to look at the list of items in one specific field
        if self.config.field is not None:
         with open(lowerCAmelCase__						,				encoding=self.config.encoding						,				errors=self.config.encoding_errors    ) as f:
          __lowerCamelCase		:							Dict		 =						json.load(lowerCAmelCase__    )

         # We keep only the field we are interested in
         __lowerCamelCase		:							List[Any]		 =						dataset[self.config.field]

         # We accept two format: a list of dicts or a dict of lists
         if isinstance(lowerCAmelCase__						,				(list, tuple)    ):
          __lowerCamelCase		:							Tuple		 =						set().union(*[row.keys() for row in dataset]    )
          __lowerCamelCase		:							Any		 =						{col: [row.get(lowerCAmelCase__    ) for row in dataset] for col in keys}
         else:
          __lowerCamelCase		:							Union[str, Any]		 =						dataset
         __lowerCamelCase		:							Optional[Any]		 =						pa.Table.from_pydict(lowerCAmelCase__    )
         yield file_idx, self._cast_table(lowerCAmelCase__    )

        # If the file has one json object per line
        else:
         with open(lowerCAmelCase__						,				'rb'    ) as f:
          __lowerCamelCase		:							Union[str, Any]		 =						0
          # Use block_size equal to the chunk size divided by 32 to leverage multithreading
          # Set a default minimum value of 16kB if the chunk size is really small
          __lowerCamelCase		:							Optional[Any]		 =						max(self.config.chunksize // 32						,				16 << 10    )
          __lowerCamelCase		:							Union[str, Any]		 =						(
              self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
          )
          while True:
           __lowerCamelCase		:							Optional[Any]		 =						f.read(self.config.chunksize    )
           if not batch:
            break
           # Finish current line
           try:
            batch += f.readline()
           except (AttributeError, io.UnsupportedOperation):
            batch += readline(lowerCAmelCase__    )
           # PyArrow only accepts utf-8 encoded bytes
           if self.config.encoding != "utf-8":
            __lowerCamelCase		:							Union[str, Any]		 =						batch.decode(self.config.encoding						,				errors=lowerCAmelCase__    ).encode('utf-8'    )
           try:
            while True:
             try:
              __lowerCamelCase		:							Tuple		 =						paj.read_json(
                  io.BytesIO(lowerCAmelCase__    )						,				read_options=paj.ReadOptions(block_size=lowerCAmelCase__    )    )
              break
             except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
              if (
                  isinstance(lowerCAmelCase__						,				pa.ArrowInvalid    )
                  and "straddling" not in str(lowerCAmelCase__    )
                  or block_size > len(lowerCAmelCase__    )
              ):
               raise
              else:
               # Increase the block size in case it was too small.
               # The block size will be reset for the next file.
               logger.debug(
                   F'Batch of {len(lowerCAmelCase__    )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'    )
               block_size *= 2
           except pa.ArrowInvalid as e:
            try:
             with open(
                 lowerCAmelCase__						,				encoding=self.config.encoding						,				errors=self.config.encoding_errors    ) as f:
              __lowerCamelCase		:							Union[str, Any]		 =						json.load(lowerCAmelCase__    )
            except json.JSONDecodeError:
             logger.error(F'Failed to read file \'{file}\' with error {type(lowerCAmelCase__    )}: {e}'    )
             raise e
            # If possible, parse the file as a list of json objects and exit the loop
            if isinstance(lowerCAmelCase__						,				lowerCAmelCase__    ):  # list is the only sequence type supported in JSON
             try:
              __lowerCamelCase		:							Dict		 =						set().union(*[row.keys() for row in dataset]    )
              __lowerCamelCase		:							Dict		 =						{col: [row.get(lowerCAmelCase__    ) for row in dataset] for col in keys}
              __lowerCamelCase		:							Optional[Any]		 =						pa.Table.from_pydict(lowerCAmelCase__    )
             except (pa.ArrowInvalid, AttributeError) as e:
              logger.error(F'Failed to read file \'{file}\' with error {type(lowerCAmelCase__    )}: {e}'    )
              raise ValueError(F'Not able to read records in the JSON file at {file}.'    ) from None
             yield file_idx, self._cast_table(lowerCAmelCase__    )
             break
            else:
             logger.error(F'Failed to read file \'{file}\' with error {type(lowerCAmelCase__    )}: {e}'    )
             raise ValueError(
                 F'Not able to read records in the JSON file at {file}. '
                 F'You should probably indicate the field of the JSON file containing your records. '
                 F'This JSON file contain the following fields: {str(list(dataset.keys()    )    )}. '
                 F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '    ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
           yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__    )
           batch_idx += 1





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowercase_	   =     {
    'configuration_convnext': ['CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ConvNextConfig', 'ConvNextOnnxConfig']
}

try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_	   =     ['ConvNextFeatureExtractor']
       lowercase_	   =     ['ConvNextImageProcessor']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_	   =     [
           'CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'ConvNextForImageClassification',
           'ConvNextModel',
           'ConvNextPreTrainedModel',
           'ConvNextBackbone',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowercase_	   =     [
           'TFConvNextForImageClassification',
           'TFConvNextModel',
           'TFConvNextPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_convnext import CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvNextConfig, ConvNextOnnxConfig

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_convnext import ConvNextFeatureExtractor
              from .image_processing_convnext import ConvNextImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_convnext import (
                  CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  ConvNextBackbone,
                  ConvNextForImageClassification,
                  ConvNextModel,
                  ConvNextPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_convnext import TFConvNextForImageClassification, TFConvNextModel, TFConvNextPreTrainedModel


else:
       import sys

       lowercase_	   =     _LazyModule(__name__, globals()['__file__'], _import_structure)

