


# this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.:
#   python ./utils/get_modified_files.py utils src tests examples
#
# it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered
# since the output of this script is fed into Makefile commands it doesn't print a newline after the results

import re
import subprocess
import sys


lowercase_	   =     subprocess.check_output('git merge-base main HEAD'.split()).decode('utf-8')
lowercase_	   =     (
    subprocess.check_output(F"""git diff --diff-filter=d --name-only {fork_point_sha}""".split()).decode('utf-8').split()
)

lowercase_	   =     '|'.join(sys.argv[1:])
lowercase_	   =     re.compile(RF"""^({joined_dirs}).*?\.py$""")

lowercase_	   =     [x for x in modified_files if regex.match(x)]
print(' '.join(relevant_modified_files), end='')





import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							List[str]		 =						np.inf

 def set_batch_size(SCREAMING_SNAKE_CASE__						) -> None:
  nonlocal batch_size
  if isinstance(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
   __lowerCamelCase		:							Union[str, Any]		 =						min(SCREAMING_SNAKE_CASE__      ,	config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS						)
  elif isinstance(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
   __lowerCamelCase		:							Optional[Any]		 =						min(SCREAMING_SNAKE_CASE__      ,	config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS						)
  elif isinstance(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						) and feature.dtype == "binary":
   __lowerCamelCase		:							List[str]		 =						min(SCREAMING_SNAKE_CASE__      ,	config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS						)

 _visit(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						)

 return None if batch_size is np.inf else batch_size


class 	A_ (	__UpperCamelCase   ):




      '''simple docstring'''


      def __init__(    self:	Tuple						,				a:	NestedDataStructureLike[PathLike]						,				a:	Optional[NamedSplit] = None						,				a:	Optional[Features] = None						,				a:	str = None						,				a:	bool = False						,				a:	bool = False						,				a:	Optional[int] = None						,				**a:	Optional[Any]						,				):
       super().__init__(
           a						,				split=a						,				features=a						,				cache_dir=a						,				keep_in_memory=a						,				streaming=a						,				num_proc=a						,				**a						,				)
       __lowerCamelCase		:							List[Any]		 =						path_or_paths if isinstance(a						,				a    ) else {self.split: path_or_paths}
       __lowerCamelCase		:							Optional[Any]		 =						_PACKAGED_DATASETS_MODULES['parquet'][1]
       __lowerCamelCase		:							List[str]		 =						Parquet(
           cache_dir=a						,				data_files=a						,				features=a						,				hash=a						,				**a						,				)




      def 					_snake_case     (    self:	List[str]    ):
       # Build iterable dataset
       if self.streaming:
        __lowerCamelCase		:							str		 =						self.builder.as_streaming_dataset(split=self.split    )
       # Build regular (map-style) dataset
       else:
        __lowerCamelCase		:							str		 =						None
        __lowerCamelCase		:							Optional[Any]		 =						None
        __lowerCamelCase		:							Union[str, Any]		 =						None
        __lowerCamelCase		:							int		 =						None

        self.builder.download_and_prepare(
            download_config=a						,				download_mode=a						,				verification_mode=a						,				base_path=a						,				num_proc=self.num_proc						,				)
        __lowerCamelCase		:							Tuple		 =						self.builder.as_dataset(
            split=self.split						,				verification_mode=a						,				in_memory=self.keep_in_memory    )
       return dataset



class 	A_ :




      '''simple docstring'''


      def __init__(    self:	Optional[int]						,				a:	Dataset						,				a:	Union[PathLike, BinaryIO]						,				a:	Optional[int] = None						,				**a:	List[Any]						,				):
       __lowerCamelCase		:							Optional[int]		 =						dataset
       __lowerCamelCase		:							List[Any]		 =						path_or_buf
       __lowerCamelCase		:							List[str]		 =						batch_size or get_writer_batch_size(dataset.features    )
       __lowerCamelCase		:							List[Any]		 =						parquet_writer_kwargs




      def 					_snake_case     (    self:	Union[str, Any]    ):
       __lowerCamelCase		:							Optional[int]		 =						self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

       if isinstance(self.path_or_buf						,				(str, bytes, os.PathLike)    ):
        with open(self.path_or_buf						,				'wb+'    ) as buffer:
         __lowerCamelCase		:							Optional[int]		 =						self._write(file_obj=a						,				batch_size=a						,				**self.parquet_writer_kwargs    )
       else:
        __lowerCamelCase		:							Any		 =						self._write(file_obj=self.path_or_buf						,				batch_size=a						,				**self.parquet_writer_kwargs    )
       return written




      def 					_snake_case     (    self:	Optional[int]						,				a:	BinaryIO						,				a:	int						,				**a:	str    ):
       __lowerCamelCase		:							Dict		 =						0
       __lowerCamelCase		:							Union[str, Any]		 =						parquet_writer_kwargs.pop('path_or_buf'						,				a    )
       __lowerCamelCase		:							str		 =						self.dataset.features.arrow_schema

       __lowerCamelCase		:							Any		 =						pq.ParquetWriter(a						,				schema=a						,				**a    )

       for offset in logging.tqdm(
           range(0						,				len(self.dataset    )						,				a    )						,				unit='ba'						,				disable=not logging.is_progress_bar_enabled()						,				desc='Creating parquet from Arrow format'						,				):
        __lowerCamelCase		:							Any		 =						query_table(
            table=self.dataset._data						,				key=slice(a						,				offset + batch_size    )						,				indices=self.dataset._indices if self.dataset._indices is not None else None						,				)
        writer.write_table(a    )
        written += batch.nbytes
       writer.close()
       return written

