




"""simple docstring"""


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_A													=      datasets.utils.logging.get_logger(__name__)


@dataclass
class _lowerCamelCase    (					datasets.BuilderConfig						):
   _lowerCamelCase    :Optional[datasets.Features]     				=     None
   _lowerCamelCase    :str     				=     "utf-8"
   _lowerCamelCase    :Optional[str]     				=     None
   _lowerCamelCase    :Optional[str]     				=     None
   _lowerCamelCase    :bool     				=     True  # deprecated
   _lowerCamelCase    :Optional[int]     				=     None  # deprecated
   _lowerCamelCase    :int     				=     10 << 20  # 10MB
   _lowerCamelCase    :Optional[bool]     				=     None






class _lowerCamelCase    (					datasets.ArrowBasedBuilder						):
   _lowerCamelCase    :str     				=     JsonConfig






   def 	_lowerCAmelCase   (		self   )			->						Tuple:

         """simple docstring"""
         if self.config.block_size is not None:
               logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""   )
               lowerCAmelCase__      :			str   =	self.config.block_size
         if self.config.use_threads is not True:
               logger.warning(
                   """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""   )
         if self.config.newlines_in_values is not None:
               raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""   )
         return datasets.DatasetInfo(features=self.config.features   )






   def 	_lowerCAmelCase   (		self   ,     UpperCamelCase   )			->						str:

         """simple docstring"""
         if not self.config.data_files:
               raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
         lowerCAmelCase__      :			Optional[Any]   =	dl_manager.download_and_extract(self.config.data_files   )
         if isinstance(UpperCamelCase   ,     (str, list, tuple)   ):
               lowerCAmelCase__      :			List[Any]   =	data_files
               if isinstance(UpperCamelCase   ,     UpperCamelCase   ):
                     lowerCAmelCase__      :			Optional[Any]   =	[files]
               lowerCAmelCase__      :			Optional[int]   =	[dl_manager.iter_files(UpperCamelCase   ) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,     gen_kwargs={"""files""": files}   )]
         lowerCAmelCase__      :			Dict   =	[]
         for split_name, files in data_files.items():
               if isinstance(UpperCamelCase   ,     UpperCamelCase   ):
                     lowerCAmelCase__      :			int   =	[files]
               lowerCAmelCase__      :			Union[str, Any]   =	[dl_manager.iter_files(UpperCamelCase   ) for file in files]
               splits.append(datasets.SplitGenerator(name=UpperCamelCase   ,     gen_kwargs={"""files""": files}   )   )
         return splits






   def 	_lowerCAmelCase   (		self   ,     UpperCamelCase   )			->						pa.Table:

         """simple docstring"""
         if self.config.features is not None:
               # adding missing columns
               for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
                     lowerCAmelCase__      :			str   =	self.config.features.arrow_schema.field(UpperCamelCase   ).type
                     lowerCAmelCase__      :			Dict   =	pa_table.append_column(UpperCamelCase   ,     pa.array([None] * len(UpperCamelCase   )   ,     type=UpperCamelCase   )   )
               # more expensive cast to support nested structures with keys in a different order
               # allows str <-> int/float or str to Audio for example
               lowerCAmelCase__      :			Any   =	table_cast(UpperCamelCase   ,     self.config.features.arrow_schema   )
         return pa_table






   def 	_lowerCAmelCase   (		self   ,     UpperCamelCase   )			->						Tuple:

         """simple docstring"""
         for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase   )   ):
               # If the file is one json object and if we need to look at the list of items in one specific field
               if self.config.field is not None:
                     with open(UpperCamelCase   ,     encoding=self.config.encoding   ,     errors=self.config.encoding_errors   ) as f:
                           lowerCAmelCase__      :			List[Any]   =	json.load(UpperCamelCase   )

                     # We keep only the field we are interested in
                     lowerCAmelCase__      :			str   =	dataset[self.config.field]

                     # We accept two format: a list of dicts or a dict of lists
                     if isinstance(UpperCamelCase   ,     (list, tuple)   ):
                           lowerCAmelCase__      :			Tuple   =	set().union(*[row.keys() for row in dataset]   )
                           lowerCAmelCase__      :			str   =	{col: [row.get(UpperCamelCase   ) for row in dataset] for col in keys}
                     else:
                           lowerCAmelCase__      :			List[str]   =	dataset
                     lowerCAmelCase__      :			str   =	pa.Table.from_pydict(UpperCamelCase   )
                     yield file_idx, self._cast_table(UpperCamelCase   )

               # If the file has one json object per line
               else:
                     with open(UpperCamelCase   ,     """rb"""   ) as f:
                           lowerCAmelCase__      :			Dict   =	0
                           # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                           # Set a default minimum value of 16kB if the chunk size is really small
                           lowerCAmelCase__      :			str   =	max(self.config.chunksize // 32   ,     16 << 10   )
                           lowerCAmelCase__      :			Any   =	(
                               self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                           )
                           while True:
                                 lowerCAmelCase__      :			Union[str, Any]   =	f.read(self.config.chunksize   )
                                 if not batch:
                                       break
                                 # Finish current line
                                 try:
                                       batch += f.readline()
                                 except (AttributeError, io.UnsupportedOperation):
                                       batch += readline(UpperCamelCase   )
                                 # PyArrow only accepts utf-8 encoded bytes
                                 if self.config.encoding != "utf-8":
                                       lowerCAmelCase__      :			Tuple   =	batch.decode(self.config.encoding   ,     errors=UpperCamelCase   ).encode("""utf-8"""   )
                                 try:
                                       while True:
                                             try:
                                                   lowerCAmelCase__      :			Optional[Any]   =	paj.read_json(
                                                       io.BytesIO(UpperCamelCase   )   ,     read_options=paj.ReadOptions(block_size=UpperCamelCase   )   )
                                                   break
                                             except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                   if (
                                                       isinstance(UpperCamelCase   ,     pa.ArrowInvalid   )
                                                       and "straddling" not in str(UpperCamelCase   )
                                                       or block_size > len(UpperCamelCase   )
                                                   ):
                                                         raise
                                                   else:
                                                         # Increase the block size in case it was too small.
                                                         # The block size will be reset for the next file.
                                                         logger.debug(
                                                             f"""Batch of {len(UpperCamelCase   )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""   )
                                                         block_size *= 2
                                 except pa.ArrowInvalid as e:
                                       try:
                                             with open(
                                                 UpperCamelCase   ,     encoding=self.config.encoding   ,     errors=self.config.encoding_errors   ) as f:
                                                   lowerCAmelCase__      :			Dict   =	json.load(UpperCamelCase   )
                                       except json.JSONDecodeError:
                                             logger.error(f"""Failed to read file '{file}' with error {type(UpperCamelCase   )}: {e}"""   )
                                             raise e
                                       # If possible, parse the file as a list of json objects and exit the loop
                                       if isinstance(UpperCamelCase   ,     UpperCamelCase   ):  # list is the only sequence type supported in JSON
                                             try:
                                                   lowerCAmelCase__      :			List[str]   =	set().union(*[row.keys() for row in dataset]   )
                                                   lowerCAmelCase__      :			Union[str, Any]   =	{col: [row.get(UpperCamelCase   ) for row in dataset] for col in keys}
                                                   lowerCAmelCase__      :			Union[str, Any]   =	pa.Table.from_pydict(UpperCamelCase   )
                                             except (pa.ArrowInvalid, AttributeError) as e:
                                                   logger.error(f"""Failed to read file '{file}' with error {type(UpperCamelCase   )}: {e}"""   )
                                                   raise ValueError(f"""Not able to read records in the JSON file at {file}."""   ) from None
                                             yield file_idx, self._cast_table(UpperCamelCase   )
                                             break
                                       else:
                                             logger.error(f"""Failed to read file '{file}' with error {type(UpperCamelCase   )}: {e}"""   )
                                             raise ValueError(
                                                 f"""Not able to read records in the JSON file at {file}. """
                                                 f"""You should probably indicate the field of the JSON file containing your records. """
                                                 f"""This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. """
                                                 f"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                 yield (file_idx, batch_idx), self._cast_table(UpperCamelCase   )
                                 batch_idx += 1











"""simple docstring"""


import unittest

from .lib import (
    Matrix,
    Vector,
    axpy,
    square_zero_matrix,
    unit_basis_vector,
    zero_vector,
)


class _lowerCamelCase    (					unittest.TestCase						):






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Vector([1, 2, 3]   )
         self.assertEqual(x.component(0   )   ,     1   )
         self.assertEqual(x.component(2   )   ,     3   )
         lowerCAmelCase__      :			Union[str, Any]   =	Vector()






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	Vector([0, 0, 0, 0, 0, 1]   )
         self.assertEqual(str(UpperCamelCase   )   ,     """(0,0,0,0,0,1)"""   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	Vector([1, 2, 3, 4]   )
         self.assertEqual(len(UpperCamelCase   )   ,     4   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	Vector([1, 2]   )
         lowerCAmelCase__      :			Optional[int]   =	Vector([1, 2, 3, 4, 5]   )
         lowerCAmelCase__      :			Union[str, Any]   =	Vector([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]   )
         lowerCAmelCase__      :			List[Any]   =	Vector([1, -1, 1, -1, 2, -3, 4, -5]   )
         self.assertAlmostEqual(x.euclidean_length()   ,     2.236   ,     3   )
         self.assertAlmostEqual(y.euclidean_length()   ,     7.416   ,     3   )
         self.assertEqual(z.euclidean_length()   ,     0   )
         self.assertAlmostEqual(w.euclidean_length()   ,     7.616   ,     3   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	Vector([1, 2, 3]   )
         lowerCAmelCase__      :			Optional[Any]   =	Vector([1, 1, 1]   )
         self.assertEqual((x + y).component(0   )   ,     2   )
         self.assertEqual((x + y).component(1   )   ,     3   )
         self.assertEqual((x + y).component(2   )   ,     4   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	Vector([1, 2, 3]   )
         lowerCAmelCase__      :			Dict   =	Vector([1, 1, 1]   )
         self.assertEqual((x - y).component(0   )   ,     0   )
         self.assertEqual((x - y).component(1   )   ,     1   )
         self.assertEqual((x - y).component(2   )   ,     2   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	Vector([1, 2, 3]   )
         lowerCAmelCase__      :			Any   =	Vector([2, -1, 4]   )  # for test of dot product
         lowerCAmelCase__      :			Any   =	Vector([1, -2, -1]   )
         self.assertEqual(str(x * 3.0   )   ,     """(3.0,6.0,9.0)"""   )
         self.assertEqual((a * b)   ,     0   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						None:

         """simple docstring"""
         self.assertEqual(str(zero_vector(10   )   ).count("""0"""   )   ,     10   )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						None:

         """simple docstring"""
         self.assertEqual(str(unit_basis_vector(3   ,     1   )   )   ,     """(0,1,0)"""   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Vector([1, 2, 3]   )
         lowerCAmelCase__      :			Optional[Any]   =	Vector([1, 0, 1]   )
         self.assertEqual(str(axpy(2   ,     UpperCamelCase   ,     UpperCamelCase   )   )   ,     """(3,4,7)"""   )






   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Vector([1, 0, 0, 0, 0, 0]   )
         lowerCAmelCase__      :			Any   =	x.copy()
         self.assertEqual(str(UpperCamelCase   )   ,     str(UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Vector([1, 0, 0]   )
         x.change_component(0   ,     0   )
         x.change_component(1   ,     1   )
         self.assertEqual(str(UpperCamelCase   )   ,     """(0,1,0)"""   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[int]   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         self.assertEqual("""|1,2,3|\n|2,4,5|\n|6,7,8|\n"""   ,     str(UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         lowerCAmelCase__      :			Dict   =	[[-3, -14, -10], [-5, -10, -5], [-2, -1, 0]]
         for x in range(a.height()   ):
               for y in range(a.width()   ):
                     self.assertEqual(minors[x][y]   ,     a.minor(UpperCamelCase   ,     UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         lowerCAmelCase__      :			int   =	[[-3, 14, -10], [5, -10, 5], [-2, 1, 0]]
         for x in range(a.height()   ):
               for y in range(a.width()   ):
                     self.assertEqual(cofactors[x][y]   ,     a.cofactor(UpperCamelCase   ,     UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         self.assertEqual(-5   ,     a.determinant()   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]   ,     3   ,     3   )
         lowerCAmelCase__      :			Tuple   =	Vector([1, 2, 3]   )
         self.assertEqual("""(14,32,50)"""   ,     str(a * x   )   )
         self.assertEqual("""|2,4,6|\n|8,10,12|\n|14,16,18|\n"""   ,     str(a * 2   )   )






   def 	_lowerCAmelCase   (		self       :		str   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Dict   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         a.change_component(0   ,     2   ,     5   )
         self.assertEqual("""|1,2,5|\n|2,4,5|\n|6,7,8|\n"""   ,     str(UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         self.assertEqual(7   ,     a.component(2   ,     1   )   ,     0.01   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         lowerCAmelCase__      :			List[Any]   =	Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]   ,     3   ,     3   )
         self.assertEqual("""|2,4,10|\n|4,8,10|\n|12,14,18|\n"""   ,     str(a + b   )   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						None:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]   ,     3   ,     3   )
         lowerCAmelCase__      :			Dict   =	Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]   ,     3   ,     3   )
         self.assertEqual("""|0,0,-4|\n|0,0,0|\n|0,0,-2|\n"""   ,     str(a - b   )   )






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						None:

         """simple docstring"""
         self.assertEqual(
             """|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n"""   ,     str(square_zero_matrix(5   )   )   ,     )


if __name__ == "__main__":
  unittest.main()





