





'''simple docstring'''




import importlib
import os

import fsspec
import pytest
from fsspec import register_implementation
from fsspec.registry import _registry as _fsspec_registry

from datasets.filesystems import COMPRESSION_FILESYSTEMS, HfFileSystem, extract_path_from_uri, is_remote_filesystem

from .utils import require_lza, require_zstandard






def 	_lowerCAmelCase      (  __snake_case  :			Tuple      )					->					Dict:
     assert "mock" in _fsspec_registry
     assert "bz2" in _fsspec_registry






def 	_lowerCAmelCase      (  )					->					Union[str, Any]:
     assert "mock" not in _fsspec_registry
     assert "bz2" in _fsspec_registry






def 	_lowerCAmelCase      (  )					->					str:
     __A						:			int								=	'mock-s3-bucket'
     __A						:			Any								=	f's3://{mock_bucket}'
     __A						:			Tuple								=	extract_path_from_uri(__snake_case      )
     assert dataset_path.startswith('s3://'      ) is False

     __A						:			str								=	'./local/path'
     __A						:			List[str]								=	extract_path_from_uri(__snake_case      )
     assert dataset_path == new_dataset_path






def 	_lowerCAmelCase      (  __snake_case  :			Optional[Any]      )					->					Dict:
     __A						:			Optional[int]								=	is_remote_filesystem(__snake_case      )
     assert is_remote is True

     __A						:			Any								=	fsspec.filesystem('file'      )

     __A						:			List[str]								=	is_remote_filesystem(__snake_case      )
     assert is_remote is False






@pytest.mark.parametrize('compression_fs_class'			,							__snake_case      )
def 	_lowerCAmelCase      (  __snake_case  :			int			,							__snake_case  :			Optional[int]			,							__snake_case  :			Dict			,							__snake_case  :			Union[str, Any]			,							__snake_case  :			Any			,							__snake_case  :			List[Any]			,							__snake_case  :			List[str]      )					->					Tuple:
     __A						:			Optional[int]								=	{'gzip': gz_file, 'xz': xz_file, 'zstd': zstd_file, 'bz2': bza_file, 'lz4': lza_file}
     __A						:			List[Any]								=	input_paths[compression_fs_class.protocol]
     if input_path is None:
          __A						:			Optional[int]								=	f'for \'{compression_fs_class.protocol}\' compression protocol, '
          if compression_fs_class.protocol == "lz4":
               reason += require_lza.kwargs["reason"]
          elif compression_fs_class.protocol == "zstd":
               reason += require_zstandard.kwargs["reason"]
          pytest.skip(__snake_case      )
     __A						:			str								=	fsspec.filesystem(compression_fs_class.protocol			,							fo=__snake_case      )
     assert isinstance(__snake_case			,							__snake_case      )
     __A						:			int								=	os.path.basename(__snake_case      )
     __A						:			Any								=	expected_filename[: expected_filename.rindex('.'      )]
     assert fs.glob('*'      ) == [expected_filename]
     with fs.open(__snake_case			,							'r'			,							encoding='utf-8'      ) as f, open(__snake_case			,							encoding='utf-8'      ) as expected_file:
          assert f.read() == expected_file.read()






@pytest.mark.parametrize('protocol'			,							['zip', 'gzip']      )
def 	_lowerCAmelCase      (  __snake_case  :			List[str]			,							__snake_case  :			List[str]			,							__snake_case  :			Any      )					->					Union[str, Any]:
     __A						:			Tuple								=	{'zip': zip_jsonl_path, 'gzip': jsonl_gz_path}
     __A						:			Optional[Any]								=	compressed_file_paths[protocol]
     __A						:			Any								=	'dataset.jsonl'
     __A						:			Any								=	f'{protocol}://{member_file_path}::{compressed_file_path}'
     __A			,*__A						:			Optional[Any]								=	fsspec.get_fs_token_paths(__snake_case      )
     assert fs.isfile(__snake_case      )
     assert not fs.isfile('non_existing_' + member_file_path      )






@pytest.mark.integration
def 	_lowerCAmelCase      (  __snake_case  :			int			,							__snake_case  :			Optional[int]			,							__snake_case  :			Union[str, Any]			,							__snake_case  :			Optional[int]      )					->					Tuple:
     __A						:			Tuple								=	hf_api.dataset_info(__snake_case			,							token=__snake_case      )
     __A						:			Optional[Any]								=	HfFileSystem(repo_info=__snake_case			,							token=__snake_case      )
     assert sorted(hffs.glob('*'      )      ) == [".gitattributes", "data"]
     assert hffs.isdir('data'      )
     assert hffs.isfile('.gitattributes'      ) and hffs.isfile('data/text_data.txt'      )
     with open(__snake_case      ) as f:
          assert hffs.open('data/text_data.txt'			,							'r'      ).read() == f.read()







def 	_lowerCAmelCase      (  )					->					Union[str, Any]:
     __A						:			Optional[int]								=	'bz2'

     # Import module
     import datasets.filesystems

     # Overwrite protocol and reload
     register_implementation(__snake_case			,							__snake_case			,							clobber=__snake_case      )
     with pytest.warns(__snake_case      ) as warning_info:
          importlib.reload(datasets.filesystems      )

     assert len(__snake_case      ) == 1
     assert (
         str(warning_info[0].message      )
         == f'A filesystem protocol was already set for {protocol} and will be overwritten.'
     )






'''simple docstring'''




from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch


@require_torch
@require_tf
class 			SCREAMING_SNAKE_CASE   (a__							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	SMALL_MODEL_IDENTIFIER
          __A						:			Any								=	'pt'
          __A						:			str								=	'tf'


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Any								=	AutoModel.from_pretrained(self.test_model)
          model_pt.save_pretrained(_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	TFAutoModel.from_pretrained(self.test_model						,     from_pt=_UpperCAmelCase)
          model_tf.save_pretrained(_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	'mock_framework'

          # Framework provided - return whatever the user provides
          __A						:			Union[str, Any]								=	FeaturesManager.determine_framework(self.test_model						,     _UpperCAmelCase)
          self.assertEqual(_UpperCAmelCase						,     _UpperCAmelCase)

          # Local checkpoint and framework provided - return provided framework
          # PyTorch checkpoint
          with TemporaryDirectory() as local_pt_ckpt:
               self._setup_pt_ckpt(_UpperCAmelCase)
               __A						:			List[str]								=	FeaturesManager.determine_framework(_UpperCAmelCase						,     _UpperCAmelCase)
               self.assertEqual(_UpperCAmelCase						,     _UpperCAmelCase)

          # TensorFlow checkpoint
          with TemporaryDirectory() as local_tf_ckpt:
               self._setup_tf_ckpt(_UpperCAmelCase)
               __A						:			Tuple								=	FeaturesManager.determine_framework(_UpperCAmelCase						,     _UpperCAmelCase)
               self.assertEqual(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with TemporaryDirectory() as local_pt_ckpt:
               self._setup_pt_ckpt(_UpperCAmelCase)
               __A						:			Optional[int]								=	FeaturesManager.determine_framework(_UpperCAmelCase)
               self.assertEqual(_UpperCAmelCase						,     self.framework_pt)

          # TensorFlow checkpoint
          with TemporaryDirectory() as local_tf_ckpt:
               self._setup_tf_ckpt(_UpperCAmelCase)
               __A						:			List[str]								=	FeaturesManager.determine_framework(_UpperCAmelCase)
               self.assertEqual(_UpperCAmelCase						,     self.framework_tf)

          # Invalid local checkpoint
          with TemporaryDirectory() as local_invalid_ckpt:
               with self.assertRaises(_UpperCAmelCase):
                    __A						:			Tuple								=	FeaturesManager.determine_framework(_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	MagicMock(return_value=_UpperCAmelCase)
          with patch('transformers.onnx.features.is_tf_available'						,     _UpperCAmelCase):
               __A						:			Union[str, Any]								=	FeaturesManager.determine_framework(self.test_model)
               self.assertEqual(_UpperCAmelCase						,     self.framework_pt)

          # PyTorch not in environment -> use TensorFlow
          __A						:			List[str]								=	MagicMock(return_value=_UpperCAmelCase)
          with patch('transformers.onnx.features.is_torch_available'						,     _UpperCAmelCase):
               __A						:			List[Any]								=	FeaturesManager.determine_framework(self.test_model)
               self.assertEqual(_UpperCAmelCase						,     self.framework_tf)

          # Both in environment -> use PyTorch
          __A						:			Any								=	MagicMock(return_value=_UpperCAmelCase)
          __A						:			Dict								=	MagicMock(return_value=_UpperCAmelCase)
          with patch('transformers.onnx.features.is_tf_available'						,     _UpperCAmelCase), patch(
              'transformers.onnx.features.is_torch_available'						,     _UpperCAmelCase):
               __A						:			int								=	FeaturesManager.determine_framework(self.test_model)
               self.assertEqual(_UpperCAmelCase						,     self.framework_pt)

          # Both not in environment -> raise error
          __A						:			List[str]								=	MagicMock(return_value=_UpperCAmelCase)
          __A						:			Tuple								=	MagicMock(return_value=_UpperCAmelCase)
          with patch('transformers.onnx.features.is_tf_available'						,     _UpperCAmelCase), patch(
              'transformers.onnx.features.is_torch_available'						,     _UpperCAmelCase):
               with self.assertRaises(_UpperCAmelCase):
                    __A						:			int								=	FeaturesManager.determine_framework(self.test_model)