"""simple docstring"""





from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
  from ..models.auto import AutoTokenizer
class 				lowercase__       :







       '''simple docstring'''


       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  List[str]   )			->						Any:


             '''simple docstring'''




             raise NotImplementedError()







       def 					lowercase__    (							self      :  Tuple   )			->						List[Any]:


             '''simple docstring'''




             raise NotImplementedError()



class 				lowercase__       (   lowerCamelCase__				):







       '''simple docstring'''


       def __init__(							self      :  str		,    _UpperCAmelCase      :  "AutoTokenizer"		,    _UpperCAmelCase      :  bool = False		,    **_UpperCAmelCase      :  Dict   )			->						Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_      			=							tokenizer
             UpperCAmelCase_      			=							skip_prompt
             UpperCAmelCase_      			=							decode_kwargs

             # variables used in the streaming process
             UpperCAmelCase_      			=							[]
             UpperCAmelCase_      			=							0
             UpperCAmelCase_      			=							True



       def 					lowercase__    (							self      :  List[Any]		,    _UpperCAmelCase      :  Optional[Any]   )			->						Optional[Any]:


             '''simple docstring'''




             if len(value.shape   ) > 1 and value.shape[0] > 1:
                   raise ValueError("TextStreamer only supports batch size 1"   )
             elif len(value.shape   ) > 1:
                   UpperCAmelCase_      			=							value[0]

             if self.skip_prompt and self.next_tokens_are_prompt:
                   UpperCAmelCase_      			=							False
                   return

             # Add the new token to the cache and decodes the entire thing.
             self.token_cache.extend(value.tolist()   )
             UpperCAmelCase_      			=							self.tokenizer.decode(self.token_cache		,    **self.decode_kwargs   )

             # After the symbol for a new line, we flush the cache.
             if text.endswith("\n"   ):
                   UpperCAmelCase_      			=							text[self.print_len :]
                   UpperCAmelCase_      			=							[]
                   UpperCAmelCase_      			=							0
             # If the last token is a CJK character, we print the characters.
             elif len(__A   ) > 0 and self._is_chinese_char(ord(text[-1]   )   ):
                   UpperCAmelCase_      			=							text[self.print_len :]
                   self.print_len += len(__A   )
             # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
             # which may change with the subsequent token -- there are probably smarter ways to do this!)
             else:
                   UpperCAmelCase_      			=							text[self.print_len : text.rfind(" "   ) + 1]
                   self.print_len += len(__A   )

             self.on_finalized_text(__A   )



       def 					lowercase__    (							self      :  Dict   )			->						List[Any]:


             '''simple docstring'''




             if len(self.token_cache   ) > 0:
                   UpperCAmelCase_      			=							self.tokenizer.decode(self.token_cache		,    **self.decode_kwargs   )
                   UpperCAmelCase_      			=							text[self.print_len :]
                   UpperCAmelCase_      			=							[]
                   UpperCAmelCase_      			=							0
             else:
                   UpperCAmelCase_      			=							''''''

             UpperCAmelCase_      			=							True
             self.on_finalized_text(__A		,    stream_end=__A   )



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  str		,    _UpperCAmelCase      :  bool = False   )			->						Tuple:


             '''simple docstring'''




             print(__A		,    flush=__A		,    end="" if not stream_end else None   )







       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  str   )			->						str:


             '''simple docstring'''




             if (
                 (cp >= 0x4e_00 and cp <= 0x9f_ff)
                 or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
                 or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
                 or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
                 or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
                 or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
                 or (cp >= 0xf9_00 and cp <= 0xfa_ff)
                 or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
             ):  #
                   return True

             return False



class 				lowercase__       (   lowerCamelCase__				):







       '''simple docstring'''


       def __init__(							self      :  Optional[int]		,    _UpperCAmelCase      :  "AutoTokenizer"		,    _UpperCAmelCase      :  bool = False		,    _UpperCAmelCase      :  Optional[float] = None		,    **_UpperCAmelCase      :  int   )			->						str:


             '''simple docstring'''




             super().__init__(__A		,    __A		,    **__A   )
             UpperCAmelCase_      			=							Queue()
             UpperCAmelCase_      			=							None
             UpperCAmelCase_      			=							timeout



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  str		,    _UpperCAmelCase      :  bool = False   )			->						List[Any]:


             '''simple docstring'''




             self.text_queue.put(__A		,    timeout=self.timeout   )
             if stream_end:
                   self.text_queue.put(self.stop_signal		,    timeout=self.timeout   )



       def __iter__(							self      :  int   )			->						Tuple:


             '''simple docstring'''




             return self







       def 					lowercase__    (							self      :  List[Any]   )			->						Tuple:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.text_queue.get(timeout=self.timeout   )
             if value == self.stop_signal:
                   raise StopIteration()
             else:
                   return value









"""simple docstring"""





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCamelCase       	=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 				lowercase__       (   datasets.BuilderConfig				):







       '''simple docstring'''


       UpperCamelCase						=					None
       UpperCamelCase						=					"utf-8"
       UpperCamelCase						=					None
       UpperCamelCase						=					None
       UpperCamelCase						=					True  # deprecated
       UpperCamelCase						=					None  # deprecated
       UpperCamelCase						=					10 << 20  # 10MB
       UpperCamelCase						=					None



class 				lowercase__       (   datasets.ArrowBasedBuilder				):







       '''simple docstring'''


       UpperCamelCase						=					JsonConfig



       def 					lowercase__    (							self      :  List[Any]   )			->						Dict:


             '''simple docstring'''




             if self.config.block_size is not None:
                   logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"   )
                   UpperCAmelCase_      			=							self.config.block_size
             if self.config.use_threads is not True:
                   logger.warning(
                       "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."   )
             if self.config.newlines_in_values is not None:
                   raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"   )
             return datasets.DatasetInfo(features=self.config.features   )



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  Optional[int]   )			->						Optional[int]:


             '''simple docstring'''




             if not self.config.data_files:
                   raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
             UpperCAmelCase_      			=							dl_manager.download_and_extract(self.config.data_files   )
             if isinstance(_UpperCAmelCase		,    (str, list, tuple)   ):
                   UpperCAmelCase_      			=							data_files
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,    gen_kwargs={"files": files}   )]
             UpperCAmelCase_      			=							[]
             for split_name, files in data_files.items():
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   splits.append(datasets.SplitGenerator(name=_UpperCAmelCase		,    gen_kwargs={"files": files}   )   )
             return splits



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  pa.Table   )			->						pa.Table:


             '''simple docstring'''




             if self.config.features is not None:
                   # adding missing columns
                   for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
                         UpperCAmelCase_      			=							self.config.features.arrow_schema.field(_UpperCAmelCase   ).type
                         UpperCAmelCase_      			=							pa_table.append_column(_UpperCAmelCase		,    pa.array([None] * len(_UpperCAmelCase   )		,    type=_UpperCAmelCase   )   )
                   # more expensive cast to support nested structures with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   UpperCAmelCase_      			=							table_cast(_UpperCAmelCase		,    self.config.features.arrow_schema   )
             return pa_table







       def 					lowercase__    (							self      :  Optional[int]		,    _UpperCAmelCase      :  List[Any]   )			->						str:


             '''simple docstring'''




             for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase   )   ):
                   # If the file is one json object and if we need to look at the list of items in one specific field
                   if self.config.field is not None:
                         with open(_UpperCAmelCase		,    encoding=self.config.encoding		,    errors=self.config.encoding_errors   ) as f:
                               UpperCAmelCase_      			=							json.load(_UpperCAmelCase   )

                         # We keep only the field we are interested in
                         UpperCAmelCase_      			=							dataset[self.config.field]

                         # We accept two format: a list of dicts or a dict of lists
                         if isinstance(_UpperCAmelCase		,    (list, tuple)   ):
                               UpperCAmelCase_      			=							set().union(*[row.keys() for row in dataset]   )
                               UpperCAmelCase_      			=							{col: [row.get(_UpperCAmelCase   ) for row in dataset] for col in keys}
                         else:
                               UpperCAmelCase_      			=							dataset
                         UpperCAmelCase_      			=							pa.Table.from_pydict(_UpperCAmelCase   )
                         yield file_idx, self._cast_table(_UpperCAmelCase   )

                   # If the file has one json object per line
                   else:
                         with open(_UpperCAmelCase		,    "rb"   ) as f:
                               UpperCAmelCase_      			=							0
                               # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                               # Set a default minimum value of 16kB if the chunk size is really small
                               UpperCAmelCase_      			=							max(self.config.chunksize // 32		,    16 << 10   )
                               UpperCAmelCase_      			=							(
                                   self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
                               )
                               while True:
                                     UpperCAmelCase_      			=							f.read(self.config.chunksize   )
                                     if not batch:
                                           break
                                     # Finish current line
                                     try:
                                           batch += f.readline()
                                     except (AttributeError, io.UnsupportedOperation):
                                           batch += readline(_UpperCAmelCase   )
                                     # PyArrow only accepts utf-8 encoded bytes
                                     if self.config.encoding != "utf-8":
                                           UpperCAmelCase_      			=							batch.decode(self.config.encoding		,    errors=_UpperCAmelCase   ).encode("utf-8"   )
                                     try:
                                           while True:
                                                 try:
                                                       UpperCAmelCase_      			=							paj.read_json(
                                                           io.BytesIO(_UpperCAmelCase   )		,    read_options=paj.ReadOptions(block_size=_UpperCAmelCase   )   )
                                                       break
                                                 except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                       if (
                                                           isinstance(_UpperCAmelCase		,    pa.ArrowInvalid   )
                                                           and "straddling" not in str(_UpperCAmelCase   )
                                                           or block_size > len(_UpperCAmelCase   )
                                                       ):
                                                             raise
                                                       else:
                                                             # Increase the block size in case it was too small.
                                                             # The block size will be reset for the next file.
                                                             logger.debug(
                                                                 F"""Batch of {len(_UpperCAmelCase   )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""   )
                                                             block_size *= 2
                                     except pa.ArrowInvalid as e:
                                           try:
                                                 with open(
                                                     _UpperCAmelCase		,    encoding=self.config.encoding		,    errors=self.config.encoding_errors   ) as f:
                                                       UpperCAmelCase_      			=							json.load(_UpperCAmelCase   )
                                           except json.JSONDecodeError:
                                                 logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                 raise e
                                           # If possible, parse the file as a list of json objects and exit the loop
                                           if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):  # list is the only sequence type supported in JSON
                                                 try:
                                                       UpperCAmelCase_      			=							set().union(*[row.keys() for row in dataset]   )
                                                       UpperCAmelCase_      			=							{col: [row.get(_UpperCAmelCase   ) for row in dataset] for col in keys}
                                                       UpperCAmelCase_      			=							pa.Table.from_pydict(_UpperCAmelCase   )
                                                 except (pa.ArrowInvalid, AttributeError) as e:
                                                       logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                       raise ValueError(F"""Not able to read records in the JSON file at {file}."""   ) from None
                                                 yield file_idx, self._cast_table(_UpperCAmelCase   )
                                                 break
                                           else:
                                                 logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                 raise ValueError(
                                                     F"""Not able to read records in the JSON file at {file}. """
                                                     F"""You should probably indicate the field of the JSON file containing your records. """
                                                     F"""This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. """
                                                     F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase   )
                                     batch_idx += 1




