



"""simple docstring"""





import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowerCamelCase       	=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 				lowercase__       (   datasets.BuilderConfig				):







       '''simple docstring'''


       UpperCamelCase						=					1_00_00
       UpperCamelCase						=					None
       UpperCamelCase						=					None



class 				lowercase__       (   datasets.ArrowBasedBuilder				):







       '''simple docstring'''


       UpperCamelCase						=					ParquetConfig



       def 					lowercase__    (							self      :  int   )			->						List[Any]:


             '''simple docstring'''




             return datasets.DatasetInfo(features=self.config.features   )



       def 					lowercase__    (							self      :  Dict		,    _UpperCAmelCase      :  str   )			->						Tuple:


             '''simple docstring'''




             if not self.config.data_files:
                   raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
             UpperCAmelCase_      			=							dl_manager.download_and_extract(self.config.data_files   )
             if isinstance(_UpperCAmelCase		,    (str, list, tuple)   ):
                   UpperCAmelCase_      			=							data_files
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,    gen_kwargs={"files": files}   )]
             UpperCAmelCase_      			=							[]
             for split_name, files in data_files.items():
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   # Infer features is they are stoed in the arrow schema
                   if self.info.features is None:
                         for file in itertools.chain.from_iterable(_UpperCAmelCase   ):
                               with open(_UpperCAmelCase		,    "rb"   ) as f:
                                     UpperCAmelCase_      			=							datasets.Features.from_arrow_schema(pq.read_schema(_UpperCAmelCase   )   )
                               break
                   splits.append(datasets.SplitGenerator(name=_UpperCAmelCase		,    gen_kwargs={"files": files}   )   )
             return splits



       def 					lowercase__    (							self      :  int		,    _UpperCAmelCase      :  pa.Table   )			->						pa.Table:


             '''simple docstring'''




             if self.info.features is not None:
                   # more expensive cast to support nested features with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   UpperCAmelCase_      			=							table_cast(_UpperCAmelCase		,    self.info.features.arrow_schema   )
             return pa_table







       def 					lowercase__    (							self      :  Optional[Any]		,    _UpperCAmelCase      :  Optional[Any]   )			->						List[Any]:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.info.features.arrow_schema if self.info.features is not None else None
             if self.info.features is not None and self.config.columns is not None:
                   if sorted(field.name for field in schema   ) != sorted(self.config.columns   ):
                         raise ValueError(
                             F"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"""   )
             for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase   )   ):
                   with open(_UpperCAmelCase		,    "rb"   ) as f:
                         UpperCAmelCase_      			=							pq.ParquetFile(_UpperCAmelCase   )
                         try:
                               for batch_idx, record_batch in enumerate(
                                   parquet_file.iter_batches(batch_size=self.config.batch_size		,    columns=self.config.columns   )   ):
                                     UpperCAmelCase_      			=							pa.Table.from_batches([record_batch]   )
                                     # Uncomment for debugging (will print the Arrow table size and elements)
                                     # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                     # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield F"""{file_idx}_{batch_idx}""", self._cast_table(_UpperCAmelCase   )
                         except ValueError as e:
                               logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                               raise









"""simple docstring"""





import warnings

from ...utils import logging
from .image_processing_flava import FlavaImageProcessor


lowerCamelCase       	=						logging.get_logger(__name__)
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       def __init__(							self      :  Union[str, Any]		,    *_UpperCAmelCase      :  List[Any]		,    **_UpperCAmelCase      :  Union[str, Any]   )			->						None:


             '''simple docstring'''




             warnings.warn(
                 "The class FlavaFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
                 " use FlavaImageProcessor instead."		,    _UpperCAmelCase		,    )
             super().__init__(*_UpperCAmelCase		,    **_UpperCAmelCase   )




