





'''simple docstring'''






import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


A__    :       List[Any]              =datasets.utils.logging.get_logger(__name__)




@dataclass
class 			UpperCAmelCase					(      datasets.BuilderConfig       ):
      _lowercase:      int											=       10000
      _lowercase:      Optional[List[str]]											=       None
      _lowercase:      Optional[datasets.Features]											=       None




class 			UpperCAmelCase					(      datasets.ArrowBasedBuilder       ):
      _lowercase:      Dict											=       ParquetConfig




      def     lowercase__   (      self      :			List[str]  )    ->				List[Any]:
            return datasets.DatasetInfo(features=self.config.features  )




      def     lowercase__   (      self      :			Optional[int]      ,      __snake_case      :			Optional[int]  )    ->				Any:
            if not self.config.data_files:
                  raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"  )
            _lowerCAmelCase												=							dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__snake_case      ,      (str, list, tuple)  ):
                  _lowerCAmelCase												=							data_files
                  if isinstance(__snake_case      ,      __snake_case  ):
                        _lowerCAmelCase												=							[files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  _lowerCAmelCase												=							[dl_manager.iter_files(__snake_case  ) for file in files]
                  return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,      gen_kwargs={"""files""": files}  )]
            _lowerCAmelCase												=							[]
            for split_name, files in data_files.items():
                  if isinstance(__snake_case      ,      __snake_case  ):
                        _lowerCAmelCase												=							[files]
                  # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                  _lowerCAmelCase												=							[dl_manager.iter_files(__snake_case  ) for file in files]
                  # Infer features is they are stoed in the arrow schema
                  if self.info.features is None:
                        for file in itertools.chain.from_iterable(__snake_case  ):
                              with open(__snake_case      ,      """rb"""  ) as f:
                                    _lowerCAmelCase												=							datasets.Features.from_arrow_schema(pq.read_schema(__snake_case  )  )
                              break
                  splits.append(datasets.SplitGenerator(name=__snake_case      ,      gen_kwargs={"""files""": files}  )  )
            return splits




      def     lowercase__   (      self      :			int      ,      __snake_case      :			pa.Table  )    ->				pa.Table:
            if self.info.features is not None:
                  # more expensive cast to support nested features with keys in a different order
                  # allows str <-> int/float or str to Audio for example
                  _lowerCAmelCase												=							table_cast(__snake_case      ,      self.info.features.arrow_schema  )
            return pa_table




      def     lowercase__   (      self      :			Optional[Any]      ,      __snake_case      :			Dict  )    ->				Optional[Any]:
            _lowerCAmelCase												=							self.info.features.arrow_schema if self.info.features is not None else None
            if self.info.features is not None and self.config.columns is not None:
                  if sorted(field.name for field in schema  ) != sorted(self.config.columns  ):
                        raise ValueError(
                            f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"  )
            for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case  )  ):
                  with open(__snake_case      ,      """rb"""  ) as f:
                        _lowerCAmelCase												=							pq.ParquetFile(__snake_case  )
                        try:
                              for batch_idx, record_batch in enumerate(
                                  parquet_file.iter_batches(batch_size=self.config.batch_size      ,      columns=self.config.columns  )  ):
                                    _lowerCAmelCase												=							pa.Table.from_batches([record_batch]  )
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield f"{file_idx}_{batch_idx}", self._cast_table(__snake_case  )
                        except ValueError as e:
                              logger.error(f"Failed to read file '{file}' with error {type(__snake_case  )}: {e}"  )
                              raise













'''simple docstring'''






import unittest

from transformers import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
      from PIL import Image
else:




      class 			UpperCAmelCase					:




            @staticmethod
            def     lowercase__   (      *__snake_case      :			Optional[Any]      ,      **__snake_case      :			Any  )    ->				Tuple:
                  pass




@is_pipeline_test
@require_torch
@require_vision
class 			UpperCAmelCase					(      unittest.TestCase       ):
      _lowercase:      Union[str, Any]											=       MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING




      def     lowercase__   (      self      :			List[str]      ,      __snake_case      :			Optional[Any]      ,      __snake_case      :			str      ,      __snake_case      :			List[str]  )    ->				int:
            _lowerCAmelCase												=							pipeline("""visual-question-answering"""      ,      model="""hf-internal-testing/tiny-vilt-random-vqa"""  )
            _lowerCAmelCase												=							[
                {
                    """image""": Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""  ),
                    """question""": """How many cats are there?""",
                },
                {
                    """image""": """./tests/fixtures/tests_samples/COCO/000000039769.png""",
                    """question""": """How many cats are there?""",
                },
            ]
            return vqa_pipeline, examples




      def     lowercase__   (      self      :			Any      ,      __snake_case      :			List[Any]      ,      __snake_case      :			List[Any]  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							vqa_pipeline(__snake_case      ,      top_k=1  )
            self.assertEqual(
                __snake_case      ,      [
                    [{"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}],
                    [{"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}],
                ]      ,      )




      @require_torch
      def     lowercase__   (      self      :			str  )    ->				int:
            _lowerCAmelCase												=							pipeline("""visual-question-answering"""      ,      model="""hf-internal-testing/tiny-vilt-random-vqa"""  )
            _lowerCAmelCase												=							"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
            _lowerCAmelCase												=							"""How many cats are there?"""

            _lowerCAmelCase												=							vqa_pipeline(image=__snake_case      ,      question="""How many cats are there?"""      ,      top_k=2  )
            self.assertEqual(
                __snake_case      ,      [{"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}, {"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}]  )

            _lowerCAmelCase												=							vqa_pipeline({"""image""": image, """question""": question}      ,      top_k=2  )
            self.assertEqual(
                __snake_case      ,      [{"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}, {"""score""": ANY(__snake_case  ), """answer""": ANY(__snake_case  )}]  )




      @slow
      @require_torch
      def     lowercase__   (      self      :			List[Any]  )    ->				List[str]:
            _lowerCAmelCase												=							pipeline("""visual-question-answering"""      ,      model="""dandelin/vilt-b32-finetuned-vqa"""  )
            _lowerCAmelCase												=							"""./tests/fixtures/tests_samples/COCO/000000039769.png"""
            _lowerCAmelCase												=							"""How many cats are there?"""

            _lowerCAmelCase												=							vqa_pipeline(image=__snake_case      ,      question=__snake_case      ,      top_k=2  )
            self.assertEqual(
                nested_simplify(__snake_case      ,      decimals=4  )      ,      [{"""score""": 0.87_99, """answer""": """2"""}, {"""score""": 0.2_96, """answer""": """1"""}]  )

            _lowerCAmelCase												=							vqa_pipeline({"""image""": image, """question""": question}      ,      top_k=2  )
            self.assertEqual(
                nested_simplify(__snake_case      ,      decimals=4  )      ,      [{"""score""": 0.87_99, """answer""": """2"""}, {"""score""": 0.2_96, """answer""": """1"""}]  )

            _lowerCAmelCase												=							vqa_pipeline(
                [{"""image""": image, """question""": question}, {"""image""": image, """question""": question}]      ,      top_k=2  )
            self.assertEqual(
                nested_simplify(__snake_case      ,      decimals=4  )      ,      [[{"""score""": 0.87_99, """answer""": """2"""}, {"""score""": 0.2_96, """answer""": """1"""}]] * 2      ,      )




      @require_tf
      @unittest.skip("""Visual question answering not implemented in TF"""  )
      def     lowercase__   (      self      :			List[str]  )    ->				Union[str, Any]:
            pass






