





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE_						       =	datasets.utils.logging.get_logger(__name__)

@dataclass
class 		UpperCamelCase__			(				datasets.BuilderConfig       ):



					'''simple docstring'''

					__snake_case   :     Optional[datasets.Features]		     =      None
					__snake_case   :     str		     =      "utf-8"
					__snake_case   :     Optional[str]		     =      None
					__snake_case   :     Optional[str]		     =      None
					__snake_case   :     bool		     =      True  # deprecated
					__snake_case   :     Optional[int]		     =      None  # deprecated
					__snake_case   :     int		     =      10 << 20  # 10MB
					__snake_case   :     Optional[bool]		     =      None

class 		UpperCamelCase__			(				datasets.ArrowBasedBuilder       ):



					'''simple docstring'''

					__snake_case   :     str		     =      JsonConfig






					def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Union[str, Any]:




											'''simple docstring'''


											if self.config.block_size is not None:
																	logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""	)
																	SCREAMING_SNAKE_CASE		=						self.config.block_size
											if self.config.use_threads is not True:
																	logger.warning(
																	    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""	)
											if self.config.newlines_in_values is not None:
																	raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""	)
											return datasets.DatasetInfo(features=self.config.features	)






					def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   List[Any]	)					->		Optional[int]:




											'''simple docstring'''


											if not self.config.data_files:
																	raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
											SCREAMING_SNAKE_CASE		=						dl_manager.download_and_extract(self.config.data_files	)
											if isinstance(lowerCamelCase__						,(str, list, tuple)	):
																	SCREAMING_SNAKE_CASE		=						data_files
																	if isinstance(lowerCamelCase__						,lowerCamelCase__	):
																							SCREAMING_SNAKE_CASE		=						[files]
																	SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__	) for file in files]
																	return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,gen_kwargs={"""files""": files}	)]
											SCREAMING_SNAKE_CASE		=						[]
											for split_name, files in data_files.items():
																	if isinstance(lowerCamelCase__						,lowerCamelCase__	):
																							SCREAMING_SNAKE_CASE		=						[files]
																	SCREAMING_SNAKE_CASE		=						[dl_manager.iter_files(lowerCamelCase__	) for file in files]
																	splits.append(datasets.SplitGenerator(name=lowerCamelCase__						,gen_kwargs={"""files""": files}	)	)
											return splits






					def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]						,lowerCamelCase__						:   pa.Table	)					->		pa.Table:




											'''simple docstring'''


											if self.config.features is not None:
																	# adding missing columns
																	for column_name in set(self.config.features	) - set(pa_table.column_names	):
																							SCREAMING_SNAKE_CASE		=						self.config.features.arrow_schema.field(lowerCamelCase__	).type
																							SCREAMING_SNAKE_CASE		=						pa_table.append_column(lowerCamelCase__						,pa.array([None] * len(lowerCamelCase__	)						,type=lowerCamelCase__	)	)
																	# more expensive cast to support nested structures with keys in a different order
																	# allows str <-> int/float or str to Audio for example
																	SCREAMING_SNAKE_CASE		=						table_cast(lowerCamelCase__						,self.config.features.arrow_schema	)
											return pa_table






					def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   List[str]	)					->		List[str]:




											'''simple docstring'''


											for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__	)	):
																	# If the file is one json object and if we need to look at the list of items in one specific field
																	if self.config.field is not None:
																							with open(lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors	) as f:
																													SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__	)

																							# We keep only the field we are interested in
																							SCREAMING_SNAKE_CASE		=						dataset[self.config.field]

																							# We accept two format: a list of dicts or a dict of lists
																							if isinstance(lowerCamelCase__						,(list, tuple)	):
																													SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]	)
																													SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__	) for row in dataset] for col in keys}
																							else:
																													SCREAMING_SNAKE_CASE		=						dataset
																							SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__	)
																							yield file_idx, self._cast_table(lowerCamelCase__	)

																	# If the file has one json object per line
																	else:
																							with open(lowerCamelCase__						,"""rb"""	) as f:
																													SCREAMING_SNAKE_CASE		=						0
																													# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																													# Set a default minimum value of 16kB if the chunk size is really small
																													SCREAMING_SNAKE_CASE		=						max(self.config.chunksize // 32						,16 << 10	)
																													SCREAMING_SNAKE_CASE		=						(
																													    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																													)
																													while True:
																																			SCREAMING_SNAKE_CASE		=						f.read(self.config.chunksize	)
																																			if not batch:
																																									break
																																			# Finish current line
																																			try:
																																									batch += f.readline()
																																			except (AttributeError, io.UnsupportedOperation):
																																									batch += readline(lowerCamelCase__	)
																																			# PyArrow only accepts utf-8 encoded bytes
																																			if self.config.encoding != "utf-8":
																																									SCREAMING_SNAKE_CASE		=						batch.decode(self.config.encoding						,errors=lowerCamelCase__	).encode("""utf-8"""	)
																																			try:
																																									while True:
																																															try:
																																																					SCREAMING_SNAKE_CASE		=						paj.read_json(
																																																					    io.BytesIO(lowerCamelCase__	)						,read_options=paj.ReadOptions(block_size=lowerCamelCase__	)	)
																																																					break
																																															except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																					if (
																																																					    isinstance(lowerCamelCase__						,pa.ArrowInvalid	)
																																																					    and "straddling" not in str(lowerCamelCase__	)
																																																					    or block_size > len(lowerCamelCase__	)
																																																					):
																																																											raise
																																																					else:
																																																											# Increase the block size in case it was too small.
																																																											# The block size will be reset for the next file.
																																																											logger.debug(
																																																											    F"""Batch of {len(lowerCamelCase__	)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""	)
																																																											block_size *= 2
																																			except pa.ArrowInvalid as e:
																																									try:
																																															with open(
																																															    lowerCamelCase__						,encoding=self.config.encoding						,errors=self.config.encoding_errors	) as f:
																																																					SCREAMING_SNAKE_CASE		=						json.load(lowerCamelCase__	)
																																									except json.JSONDecodeError:
																																															logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
																																															raise e
																																									# If possible, parse the file as a list of json objects and exit the loop
																																									if isinstance(lowerCamelCase__						,lowerCamelCase__	):  # list is the only sequence type supported in JSON
																																															try:
																																																					SCREAMING_SNAKE_CASE		=						set().union(*[row.keys() for row in dataset]	)
																																																					SCREAMING_SNAKE_CASE		=						{col: [row.get(lowerCamelCase__	) for row in dataset] for col in keys}
																																																					SCREAMING_SNAKE_CASE		=						pa.Table.from_pydict(lowerCamelCase__	)
																																															except (pa.ArrowInvalid, AttributeError) as e:
																																																					logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
																																																					raise ValueError(F"""Not able to read records in the JSON file at {file}."""	) from None
																																															yield file_idx, self._cast_table(lowerCamelCase__	)
																																															break
																																									else:
																																															logger.error(F"""Failed to read file '{file}' with error {type(lowerCamelCase__	)}: {e}"""	)
																																															raise ValueError(
																																															    F"""Not able to read records in the JSON file at {file}. """
																																															    F"""You should probably indicate the field of the JSON file containing your records. """
																																															    F"""This JSON file contain the following fields: {str(list(dataset.keys()	)	)}. """
																																															    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """	) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																			yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__	)
																																			batch_idx += 1









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


SCREAMING_SNAKE_CASE_						       =	{
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      SCREAMING_SNAKE_CASE_						       =	[
          """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """MegatronBertForCausalLM""",
          """MegatronBertForMaskedLM""",
          """MegatronBertForMultipleChoice""",
          """MegatronBertForNextSentencePrediction""",
          """MegatronBertForPreTraining""",
          """MegatronBertForQuestionAnswering""",
          """MegatronBertForSequenceClassification""",
          """MegatronBertForTokenClassification""",
          """MegatronBertModel""",
          """MegatronBertPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_megatron_bert import (
                MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                MegatronBertForCausalLM,
                MegatronBertForMaskedLM,
                MegatronBertForMultipleChoice,
                MegatronBertForNextSentencePrediction,
                MegatronBertForPreTraining,
                MegatronBertForQuestionAnswering,
                MegatronBertForSequenceClassification,
                MegatronBertForTokenClassification,
                MegatronBertModel,
                MegatronBertPreTrainedModel,
            )

else:
      import sys

      SCREAMING_SNAKE_CASE_						       =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


