




import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCamelCase__			:       Tuple         =		"\\n    Text data.\n    Second line of data."

lowerCamelCase__			:       List[Any]         =		"file"

@pytest.fixture(scope='session'							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[int]							) ->			Optional[int]:
      SCREAMING_SNAKE_CASE_     						=      tmp_path_factory.mktemp('data'							) / (FILE_PATH + '.zstd')
      SCREAMING_SNAKE_CASE_     						=      bytes(__UpperCAmelCase      ,      'utf-8'							)
      with zstd.open(__UpperCAmelCase      ,      'wb'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture
def  UpperCAmelCase_							(				__UpperCAmelCase						:     List[Any]							) ->			Any:





      with open(os.path.join(tmpfs.local_root_dir      ,      __UpperCAmelCase							)      ,      'w'							) as f:
            f.write(__UpperCAmelCase							)
      return FILE_PATH

@pytest.mark.parametrize('compression_format'      ,      ['gzip', 'xz', 'zstd']							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     Tuple      ,      __UpperCAmelCase						:     Union[str, Any]      ,      __UpperCAmelCase						:     Tuple      ,      __UpperCAmelCase						:     int      ,      __UpperCAmelCase						:     str      ,      __UpperCAmelCase						:     Tuple							) ->			Dict:
      SCREAMING_SNAKE_CASE_     						=      {'gzip': gz_file, 'xz': xz_file, 'zstd': zstd_path}
      SCREAMING_SNAKE_CASE_     						=      input_paths[compression_format]
      SCREAMING_SNAKE_CASE_     						=      tmp_path / 'cache'
      SCREAMING_SNAKE_CASE_     						=      DownloadConfig(cache_dir=__UpperCAmelCase      ,      extract_compressed_file=__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_     						=      cached_path(__UpperCAmelCase      ,      download_config=__UpperCAmelCase							)
      with open(__UpperCAmelCase							) as f:
            SCREAMING_SNAKE_CASE_     						=      f.read()
      with open(__UpperCAmelCase							) as f:
            SCREAMING_SNAKE_CASE_     						=      f.read()
      assert extracted_file_content == expected_file_content

@pytest.mark.parametrize('default_extracted'      ,      [True, False]							)
@pytest.mark.parametrize('default_cache_dir'      ,      [True, False]							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[Any]      ,      __UpperCAmelCase						:     int      ,      __UpperCAmelCase						:     Union[str, Any]      ,      __UpperCAmelCase						:     List[Any]      ,      __UpperCAmelCase						:     Tuple							) ->			Optional[Any]:
      SCREAMING_SNAKE_CASE_     						=      'custom_cache'
      SCREAMING_SNAKE_CASE_     						=      'custom_extracted_dir'
      SCREAMING_SNAKE_CASE_     						=      tmp_path / 'custom_extracted_path'
      if default_extracted:
            SCREAMING_SNAKE_CASE_     						=      ('downloads' if default_cache_dir else custom_cache_dir, 'extracted')
      else:
            monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_DIR'      ,      __UpperCAmelCase							)
            monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH'      ,      str(__UpperCAmelCase							)							)
            SCREAMING_SNAKE_CASE_     						=      custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

      SCREAMING_SNAKE_CASE_     						=      xz_file
      SCREAMING_SNAKE_CASE_     						=      (
          DownloadConfig(extract_compressed_file=__UpperCAmelCase							)
          if default_cache_dir
          else DownloadConfig(cache_dir=tmp_path / custom_cache_dir      ,      extract_compressed_file=__UpperCAmelCase							)
      )
      SCREAMING_SNAKE_CASE_     						=      cached_path(__UpperCAmelCase      ,      download_config=__UpperCAmelCase							)
      assert Path(__UpperCAmelCase							).parent.parts[-2:] == expected

def  UpperCAmelCase_							(				__UpperCAmelCase						:     int							) ->			Optional[Any]:
      SCREAMING_SNAKE_CASE_     						=      str(Path(__UpperCAmelCase							).resolve()							)
      assert cached_path(__UpperCAmelCase							) == text_file
      # relative path
      SCREAMING_SNAKE_CASE_     						=      str(Path(__UpperCAmelCase							).resolve().relative_to(Path(os.getcwd()							)							)							)
      assert cached_path(__UpperCAmelCase							) == text_file

def  UpperCAmelCase_							(				__UpperCAmelCase						:     List[Any]							) ->			Dict:
      SCREAMING_SNAKE_CASE_     						=      str(tmp_path.resolve() / '__missing_file__.txt'							)
      with pytest.raises(__UpperCAmelCase							):
            cached_path(__UpperCAmelCase							)
      # relative path
      SCREAMING_SNAKE_CASE_     						=      './__missing_file__.txt'
      with pytest.raises(__UpperCAmelCase							):
            cached_path(__UpperCAmelCase							)

def  UpperCAmelCase_							(				__UpperCAmelCase						:     List[Any]							) ->			Union[str, Any]:
      SCREAMING_SNAKE_CASE_     						=      get_from_cache(f"tmp://{tmpfs_file}"							)
      with open(__UpperCAmelCase							) as f:
            SCREAMING_SNAKE_CASE_     						=      f.read()
      assert output_file_content == FILE_CONTENT

@patch('datasets.config.HF_DATASETS_OFFLINE'      ,      __UpperCAmelCase							)
def  UpperCAmelCase_							(				) ->			Optional[Any]:





      with pytest.raises(__UpperCAmelCase							):
            cached_path('https://huggingface.co'							)

@patch('datasets.config.HF_DATASETS_OFFLINE'      ,      __UpperCAmelCase							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     List[str]							) ->			int:
      SCREAMING_SNAKE_CASE_     						=      tmp_path_factory.mktemp('data'							) / 'file.html'
      with pytest.raises(__UpperCAmelCase							):
            http_get('https://huggingface.co'      ,      temp_file=__UpperCAmelCase							)
      with pytest.raises(__UpperCAmelCase							):
            http_head('https://huggingface.co'							)

@patch('datasets.config.HF_DATASETS_OFFLINE'      ,      __UpperCAmelCase							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     Tuple							) ->			List[Any]:
      SCREAMING_SNAKE_CASE_     						=      tmp_path_factory.mktemp('data'							) / 'file.html'
      with pytest.raises(__UpperCAmelCase							):
            ftp_get('ftp://huggingface.co'      ,      temp_file=__UpperCAmelCase							)
      with pytest.raises(__UpperCAmelCase							):
            ftp_head('ftp://huggingface.co'							)





@patch('datasets.config.HF_DATASETS_OFFLINE'      ,      __UpperCAmelCase							)
def  UpperCAmelCase_							(				__UpperCAmelCase						:     Any							) ->			int:
      SCREAMING_SNAKE_CASE_     						=      tmp_path_factory.mktemp('data'							) / 'file.html'
      with pytest.raises(__UpperCAmelCase							):
            fsspec_get('s3://huggingface.co'      ,      temp_file=__UpperCAmelCase							)
      with pytest.raises(__UpperCAmelCase							):
            fsspec_head('s3://huggingface.co'							)





from packaging import version

from .import_utils import is_accelerate_available


if is_accelerate_available():
   import accelerate

def  UpperCAmelCase_							(				__UpperCAmelCase						:     Optional[int]							) ->			int:
      if not is_accelerate_available():
            return method
      SCREAMING_SNAKE_CASE_     						=      version.parse(accelerate.__version__							).base_version
      if version.parse(__UpperCAmelCase							) < version.parse('0.17.0'							):
            return method

      def wrapper(self						:     Optional[int]      ,      *__UpperCAmelCase						:     Optional[Any]      ,      **__UpperCAmelCase						:     Optional[Any]							):
            if hasattr(self      ,      '_hf_hook'							) and hasattr(self._hf_hook      ,      'pre_forward'							):
                  self._hf_hook.pre_forward(self							)
            return method(self      ,      *__UpperCAmelCase      ,      **__UpperCAmelCase							)

      return wrapper