
'''simple docstring'''



import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_lowercase    =		datasets.utils.logging.get_logger(__name__)




@dataclass
class   UpperCAmelCase_					(		datasets.BuilderConfig			):






						'''simple docstring'''






						_lowercase		:				int									=       1_0_0_0_0
						_lowercase		:				Optional[List[str]]									=       None
						_lowercase		:				Optional[datasets.Features]									=       None






class   UpperCAmelCase_					(		datasets.ArrowBasedBuilder			):






						'''simple docstring'''






						_lowercase		:				List[str]									=       ParquetConfig




						def  _lowercase     ( self				):

												"""simple docstring"""



												return datasets.DatasetInfo(features=self.config.features				)




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												if not self.config.data_files:
																		raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'				)
												_lowerCAmelCase  						=			dl_manager.download_and_extract(self.config.data_files				)
												if isinstance(_lowercase   ,       (str, list, tuple)				):
																		_lowerCAmelCase  						=			data_files
																		if isinstance(_lowercase   ,       _lowercase				):
																								_lowerCAmelCase  						=			[files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		_lowerCAmelCase  						=			[dl_manager.iter_files(_lowercase				) for file in files]
																		return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,       gen_kwargs={"""files""": files}				)]
												_lowerCAmelCase  						=			[]
												for split_name, files in data_files.items():
																		if isinstance(_lowercase   ,       _lowercase				):
																								_lowerCAmelCase  						=			[files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		_lowerCAmelCase  						=			[dl_manager.iter_files(_lowercase				) for file in files]
																		# Infer features is they are stoed in the arrow schema
																		if self.info.features is None:
																								for file in itertools.chain.from_iterable(_lowercase				):
																														with open(_lowercase   ,       """rb"""				) as f:
																																				_lowerCAmelCase  						=			datasets.Features.from_arrow_schema(pq.read_schema(_lowercase				)				)
																														break
																		splits.append(datasets.SplitGenerator(name=_lowercase   ,       gen_kwargs={"""files""": files}				)				)
												return splits




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												if self.info.features is not None:
																		# more expensive cast to support nested features with keys in a different order
																		# allows str <-> int/float or str to Audio for example
																		_lowerCAmelCase  						=			table_cast(_lowercase   ,       self.info.features.arrow_schema				)
												return pa_table




						def  _lowercase     ( self   ,       _lowercase				):

												"""simple docstring"""



												_lowerCAmelCase  						=			self.info.features.arrow_schema if self.info.features is not None else None
												if self.info.features is not None and self.config.columns is not None:
																		if sorted(field.name for field in schema				) != sorted(self.config.columns				):
																								raise ValueError(
																								    F'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''				)
												for file_idx, file in enumerate(itertools.chain.from_iterable(_lowercase				)				):
																		with open(_lowercase   ,       """rb"""				) as f:
																								_lowerCAmelCase  						=			pq.ParquetFile(_lowercase				)
																								try:
																														for batch_idx, record_batch in enumerate(
																														    parquet_file.iter_batches(batch_size=self.config.batch_size   ,       columns=self.config.columns				)				):
																																				_lowerCAmelCase  						=			pa.Table.from_batches([record_batch]				)
																																				# Uncomment for debugging (will print the Arrow table size and elements)
																																				# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																				# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																				yield F'{file_idx}_{batch_idx}', self._cast_table(_lowercase				)
																								except ValueError as e:
																														logger.error(F'Failed to read file \'{file}\' with error {type(_lowercase				)}: {e}'				)
																														raise


'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase    =		{
    """configuration_table_transformer""": [
        """TABLE_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """TableTransformerConfig""",
        """TableTransformerOnnxConfig""",
    ]
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		[
						    """TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TableTransformerForObjectDetection""",
						    """TableTransformerModel""",
						    """TableTransformerPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_table_transformer import (
						    TABLE_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    TableTransformerConfig,
						    TableTransformerOnnxConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_table_transformer import (
												    TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TableTransformerForObjectDetection,
												    TableTransformerModel,
												    TableTransformerPreTrainedModel,
												)

else:
						import sys

						_lowercase    =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
