





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE_						       =	datasets.utils.logging.get_logger(__name__)

@dataclass
class 		UpperCamelCase__			(				datasets.BuilderConfig       ):



     '''simple docstring'''

     __snake_case   :     Optional[datasets.Features]		     =      None
     __snake_case   :     str		     =      "utf-8"
     __snake_case   :     Optional[str]		     =      None
     __snake_case   :     Optional[str]		     =      None
     __snake_case   :     bool		     =      True  # deprecated
     __snake_case   :     Optional[int]		     =      None  # deprecated
     __snake_case   :     int		     =      10 << 20  # 10MB
     __snake_case   :     Optional[bool]		     =      None

class 		UpperCamelCase__			(				datasets.ArrowBasedBuilder       ):



     '''simple docstring'''

     __snake_case   :     str		     =      JsonConfig






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Union[str, Any]:




           '''simple docstring'''


           if self.config.block_size is not None:
                 logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""	)
                 SCREAMING_SNAKE_CASE		=						self.config.block_size
           if self.config.use_threads is not True:
                 logger.warning(
                     """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""	)
           if self.config.newlines_in_values is not None:
                 raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""	)
           return datasets.DatasetInfo(features=self.config.features	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   List[Any]	)					->		Optional[int]:




           '''simple docstring'''


           if not self.config.data_files:
                 raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
           SCREAMING_SNAKE_CASE		=						dl_manager.download_and_extract(self.config.data_files	)
           if isinstance(lowerCamelCase__						,(str, list, tuple)	):
                 SCREAMING_SNAKE_CASE		=						data_files
                 if isinstance(lowerCamelCase__						,lowerCamelCase__	):
                       SCREAMING_SNAKE_CASE		=						[files]
                 SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__	) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,gen_kwargs={"""files""": files}	)]
           SCREAMING_SNAKE_CASE		=						[]
           for split_name, files in data_files.items():
                 if isinstance(lowerCamelCase__						,lowerCamelCase__	):
                       SCREAMING_SNAKE_CASE		=						[files]
                 SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__	) for file in files]
                 splits.append(datasets.SplitGenerator(name=lowerCamelCase__						,gen_kwargs={"""files""": files}	)	)
           return splits






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]						,lowerCamelCase__						:   pa.Table	)					->		pa.Table:




           '''simple docstring'''


           if self.config.features is not None:
                 # adding missing columns
                 for column_name in set(self.config.features	) - set(pa_table.column_names	):
                       SCREAMING_SNAKE_CASE		=						self.config.features.arrow_schema.field(lowerCamelCase__	).type
                       SCREAMING_SNAKE_CASE		=						pa_table.append_column(lowerCamelCase__						,pa.array([None] * len(lowerCamelCase__	)						,type=lowerCamelCase__	)	)
                 # more expensive cast to support nested structures with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 SCREAMING_SNAKE_CASE		=						table_cast(lowerCamelCase__						,self.config.features.arrow_schema	)
           return pa_table






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   List[str]	)					->		List[str]:




           '''simple docstring'''


           for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__	)	):
                 # If the file is one json object and if we need to look at the list of items in one specific field
                 if self.config.field is not None:
                       with open(lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors	) as f:
                             SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__	)

                       # We keep only the field we are interested in
                       SCREAMING_SNAKE_CASE		=						dataset[self.config.field]

                       # We accept two format: a list of dicts or a dict of lists
                       if isinstance(lowerCamelCase__						,(list, tuple)	):
                             SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]	)
                             SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__	) for row in dataset] for col in keys}
                       else:
                             SCREAMING_SNAKE_CASE		=						dataset
                       SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__	)
                       yield file_idx, self._cast_table(lowerCamelCase__	)

                 # If the file has one json object per line
                 else:
                       with open(lowerCamelCase__						,"""rb"""	) as f:
                             SCREAMING_SNAKE_CASE		=						0
                             # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                             # Set a default minimum value of 16kB if the chunk size is really small
                             SCREAMING_SNAKE_CASE		=						max(self.config.chunksize // 32						,16 << 10	)
                             SCREAMING_SNAKE_CASE		=						(
                                 self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                             )
                             while True:
                                   SCREAMING_SNAKE_CASE		=						f.read(self.config.chunksize	)
                                   if not batch:
                                         break
                                   # Finish current line
                                   try:
                                         batch += f.readline()
                                   except (AttributeError, io.UnsupportedOperation):
                                         batch += readline(lowerCamelCase__	)
                                   # PyArrow only accepts utf-8 encoded bytes
                                   if self.config.encoding != "utf-8":
                                         SCREAMING_SNAKE_CASE		=						batch.decode(self.config.encoding						,errors=lowerCamelCase__	).encode("""utf-8"""	)
                                   try:
                                         while True:
                                               try:
                                                     SCREAMING_SNAKE_CASE		=						paj.read_json(
                                                         io.BytesIO(lowerCamelCase__	)						,read_options=paj.ReadOptions(block_size=lowerCamelCase__	)	)
                                                     break
                                               except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                     if (
                                                         isinstance(lowerCamelCase__						,pa.ArrowInvalid	)
                                                         and "straddling" not in str(lowerCamelCase__	)
                                                         or block_size > len(lowerCamelCase__	)
                                                     ):
                                                           raise
                                                     else:
                                                           # Increase the block size in case it was too small.
                                                           # The block size will be reset for the next file.
                                                           logger.debug(
                                                               F"""Batch of {len(lowerCamelCase__	)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""	)
                                                           block_size *= 2
                                   except pa.ArrowInvalid as e:
                                         try:
                                               with open(
                                                   lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors	) as f:
                                                     SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__	)
                                         except json.JSONDecodeError:
                                               logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
                                               raise e
                                         # If possible, parse the file as a list of json objects and exit the loop
                                         if isinstance(lowerCamelCase__						,lowerCamelCase__	):  # list is the only sequence type supported in JSON
                                               try:
                                                     SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]	)
                                                     SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__	) for row in dataset] for col in keys}
                                                     SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__	)
                                               except (pa.ArrowInvalid, AttributeError) as e:
                                                     logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
                                                     raise ValueError(F"""Not able to read records in the JSON file at {file}."""	) from None
                                               yield file_idx, self._cast_table(lowerCamelCase__	)
                                               break
                                         else:
                                               logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
                                               raise ValueError(
                                                   F"""Not able to read records in the JSON file at {file}. """
                                                   F"""You should probably indicate the field of the JSON file containing your records. """
                                                   F"""This JSON file contain the following fields: {str(list(dataset.keys()	)	)}. """
                                                   F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """	) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                   yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__	)
                                   batch_idx += 1









import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict

@dataclass
class 		UpperCamelCase__			(				lowerCAmelCase_       ):



     '''simple docstring'''

     __snake_case   :     int		     =      ComputeEnvironment.AMAZON_SAGEMAKER
     __snake_case   :     List[Any]		     =      True
     __snake_case   :     Optional[int]		     =      "ml.p3.2xlarge"
     __snake_case   :     List[str]		     =      "accelerate_sagemaker_execution_role"
     __snake_case   :     Tuple		     =      "hf-sm"
     __snake_case   :     Any		     =      "us-east-1"
     __snake_case   :     Union[str, Any]		     =      1
     __snake_case   :     Dict		     =      "accelerate-sagemaker-1"
     __snake_case   :     Tuple		     =      "1.6"
     __snake_case   :     List[str]		     =      "4.4"
     __snake_case   :     str		     =      "train.py"
     __snake_case   :     List[str]		     =      [
         "--model_name_or_path",
         "bert",
         "--do_train",
         "False",
         "--epochs",
         "3",
         "--learning_rate",
         "5e-5",
         "--max_steps",
         "50.5",
     ]
     __snake_case   :     Optional[int]		     =      [
         "--model_name_or_path",
         "bert",
         "--do_train",
         "--do_test",
         "False",
         "--do_predict",
         "--epochs",
         "3",
         "--learning_rate",
         "5e-5",
         "--max_steps",
         "50.5",
     ]

class 		UpperCamelCase__			(				unittest.TestCase       ):



     '''simple docstring'''

     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						_convert_nargs_to_dict(MockLaunchConfig.success_training_script_args	)
           assert isinstance(converted_args["""model_name_or_path"""]						,lowerCamelCase__	)
           assert isinstance(converted_args["""do_train"""]						,lowerCamelCase__	)
           assert isinstance(converted_args["""epochs"""]						,lowerCamelCase__	)
           assert isinstance(converted_args["""learning_rate"""]						,lowerCamelCase__	)
           assert isinstance(converted_args["""max_steps"""]						,lowerCamelCase__	)

           with pytest.raises(lowerCamelCase__	):
                 _convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args	)


