
"""simple docstring"""







import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


lowerCamelCase__        =		"""<<<<<<< This should probably be modified because it mentions: """

lowerCamelCase__        =		"""=======
>>>>>>>
"""

lowerCamelCase__        =		[
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

lowerCamelCase__        =		[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R"""tfds\.core""", R"""datasets"""),
    (R"""tf\.io\.gfile\.GFile""", R"""open"""),
    (R"""tf\.([\w\d]+)""", R"""datasets.Value('\1')"""),
    (R"""tfds\.features\.Text\(\)""", R"""datasets.Value('string')"""),
    (R"""tfds\.features\.Text\(""", R"""datasets.Value('string'),"""),
    (R"""features\s*=\s*tfds.features.FeaturesDict\(""", R"""features=datasets.Features("""),
    (R"""tfds\.features\.FeaturesDict\(""", R"""dict("""),
    (R"""The TensorFlow Datasets Authors""", R"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (R"""tfds\.""", R"""datasets."""),
    (R"""dl_manager\.manual_dir""", R"""self.config.data_dir"""),
    (R"""self\.builder_config""", R"""self.config"""),
]
def    __lowerCAmelCase    (_UpperCamelCase		):
	return ConvertCommand(args.tfds_path	,  args.datasets_directory		)




class 			A__	(     _lowerCamelCase):

		@staticmethod
		def 							__lowerCamelCase							(				_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						List[str]						      =				parser.add_parser(
			    'convert'	,		help='Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'	,		)
			train_parser.add_argument(
			    '--tfds_path'	,		type=_SCREAMING_SNAKE_CASE	,		required=_SCREAMING_SNAKE_CASE	,		help='Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'	,		)
			train_parser.add_argument(
			    '--datasets_directory'	,		type=_SCREAMING_SNAKE_CASE	,		required=_SCREAMING_SNAKE_CASE	,		help='Path to the HuggingFace Datasets folder.'			)
			train_parser.set_defaults(func=_SCREAMING_SNAKE_CASE			)

		def __init__(				self	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		*_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Tuple						      =				get_logger('datasets-cli/converting'			)

			__lowerCAmelCase      :						str						      =				tfds_path
			__lowerCAmelCase      :						int						      =				datasets_directory







		def 							__lowerCamelCase							(				self			):
			if os.path.isdir(self._tfds_path			):
				__lowerCAmelCase      :						int						      =				os.path.abspath(self._tfds_path			)
			elif os.path.isfile(self._tfds_path			):
				__lowerCAmelCase      :						List[str]						      =				os.path.dirname(self._tfds_path			)
			else:
				raise ValueError('--tfds_path is neither a directory nor a file. Please check path.'			)

			__lowerCAmelCase      :						str						      =				os.path.abspath(self._datasets_directory			)

			self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}"			)

			__lowerCAmelCase      :						str						      =				[]
			__lowerCAmelCase      :						Optional[int]						      =				[]
			__lowerCAmelCase      :						List[Any]						      =				{}

			if os.path.isdir(self._tfds_path			):
				__lowerCAmelCase      :						Any						      =				os.listdir(_SCREAMING_SNAKE_CASE			)
			else:
				__lowerCAmelCase      :						Any						      =				[os.path.basename(self._tfds_path			)]

			for f_name in file_names:
				self._logger.info(f"Looking at file {f_name}"			)
				__lowerCAmelCase      :						str						      =				os.path.join(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
				__lowerCAmelCase      :						Optional[int]						      =				os.path.join(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

				if not os.path.isfile(_SCREAMING_SNAKE_CASE			) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
					self._logger.info('Skipping file'			)
					continue

				with open(_SCREAMING_SNAKE_CASE	,		encoding='utf-8'			) as f:
					__lowerCAmelCase      :						Tuple						      =				f.readlines()

				__lowerCAmelCase      :						Tuple						      =				[]
				__lowerCAmelCase      :						Optional[int]						      =				False
				__lowerCAmelCase      :						Tuple						      =				False
				__lowerCAmelCase      :						Any						      =				[]
				for line in lines:
					__lowerCAmelCase      :						Optional[int]						      =				line

					# Convert imports
					if "import tensorflow.compat.v2 as tf" in out_line:
						continue
					elif "@tfds.core" in out_line:
						continue
					elif "builder=self" in out_line:
						continue
					elif "import tensorflow_datasets.public_api as tfds" in out_line:
						__lowerCAmelCase      :						Dict						      =				'import datasets\n'
					elif "import tensorflow" in out_line:
						# order is important here
						__lowerCAmelCase      :						List[str]						      =				''
						continue
					elif "from absl import logging" in out_line:
						__lowerCAmelCase      :						Tuple						      =				'from datasets import logging\n'
					elif "getLogger" in out_line:
						__lowerCAmelCase      :						int						      =				out_line.replace('getLogger'	,		'get_logger'			)
					elif any(expression in out_line for expression in TO_HIGHLIGHT			):
						__lowerCAmelCase      :						Union[str, Any]						      =				True
						__lowerCAmelCase      :						Optional[int]						      =				list(filter(lambda _SCREAMING_SNAKE_CASE			: e in out_line	,		_SCREAMING_SNAKE_CASE			)			)
						out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(_SCREAMING_SNAKE_CASE			) + '\n'			)
						out_lines.append(_SCREAMING_SNAKE_CASE			)
						out_lines.append(_SCREAMING_SNAKE_CASE			)
						continue
					else:
						for pattern, replacement in TO_CONVERT:
							__lowerCAmelCase      :						List[str]						      =				re.sub(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

                # Take care of saving utilities (to later move them together with main script)
					if "tensorflow_datasets" in out_line:
						__lowerCAmelCase      :						Optional[Any]						      =				re.match(R'from\stensorflow_datasets.*import\s([^\.\r\n]+)'	,		_SCREAMING_SNAKE_CASE			)
						tfds_imports.extend(imp.strip() for imp in match.group(1			).split(','			)			)
						__lowerCAmelCase      :						Tuple						      =				'from . import ' + match.group(1			)

					# Check we have not forget anything
					if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
						raise ValueError(f"Error converting {out_line.strip()}"			)

					if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
						__lowerCAmelCase      :						Tuple						      =				True
					out_lines.append(_SCREAMING_SNAKE_CASE			)

				if is_builder or "wmt" in f_name:
					# We create a new directory for each dataset
					__lowerCAmelCase      :						Union[str, Any]						      =				f_name.replace('.py'	,		''			)
					__lowerCAmelCase      :						Union[str, Any]						      =				os.path.join(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase      :						List[str]						      =				os.path.join(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
					os.makedirs(_SCREAMING_SNAKE_CASE	,		exist_ok=_SCREAMING_SNAKE_CASE			)
					self._logger.info(f"Adding directory {output_dir}"			)
					imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}			)
				else:
					# Utilities will be moved at the end
					utils_files.append(_SCREAMING_SNAKE_CASE			)

				if needs_manual_update:
					with_manual_update.append(_SCREAMING_SNAKE_CASE			)

				with open(_SCREAMING_SNAKE_CASE	,		'w'	,		encoding='utf-8'			) as f:
					f.writelines(_SCREAMING_SNAKE_CASE			)
				self._logger.info(f"Converted in {output_file}"			)

			for utils_file in utils_files:
				try:
					__lowerCAmelCase      :						int						      =				os.path.basename(_SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase      :						Any						      =				imports_to_builder_map[f_name.replace('.py'	,		''			)]
					self._logger.info(f"Moving {dest_folder} to {utils_file}"			)
					shutil.copy(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
				except KeyError:
					self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually."			)

			if with_manual_update:
				for file_path in with_manual_update:
					self._logger.warning(
					    f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."			)

"""simple docstring"""







import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


lowerCamelCase__        =		"""hf-internal-testing/tiny-random-bert"""
lowerCamelCase__        =		os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
lowerCamelCase__        =		"""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""




class 			A__	(     unittest.TestCase):

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Union[str, Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
			# Should have downloaded the file in here
			self.assertTrue(os.path.isdir(_SCREAMING_SNAKE_CASE			)			)
			# Cache should contain at least those three subfolders:
			for subfolder in ["blobs", "refs", "snapshots"]:
				self.assertTrue(os.path.isdir(os.path.join(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)			)			)
			with open(os.path.join(_SCREAMING_SNAKE_CASE	,		'refs'	,		'main'			)			) as f:
				__lowerCAmelCase      :						List[Any]						      =				f.read()
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		os.path.join(_SCREAMING_SNAKE_CASE	,		'snapshots'	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)			)
			self.assertTrue(os.path.isfile(_SCREAMING_SNAKE_CASE			)			)

			# File is cached at the same place the second time.
			__lowerCAmelCase      :						Union[str, Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

			# Using a specific revision to test the full commit hash.
			__lowerCAmelCase      :						Union[str, Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		revision='9b8c223'			)
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		os.path.join(_SCREAMING_SNAKE_CASE	,		'snapshots'	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)			)

		def 							__lowerCamelCase							(				self			):
			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'is not a valid model identifier'			):
				__lowerCAmelCase      :						Optional[Any]						      =				cached_file('tiny-random-bert'	,		_SCREAMING_SNAKE_CASE			)

			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'is not a valid git identifier'			):
				__lowerCAmelCase      :						str						      =				cached_file(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		revision='aaaa'			)

			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'does not appear to have a file named'			):
				__lowerCAmelCase      :						Optional[Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		'conf'			)

		def 							__lowerCamelCase							(				self			):
			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'does not appear to have a file named'			):
				__lowerCAmelCase      :						Optional[int]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		'conf'			)

			with open(os.path.join(_SCREAMING_SNAKE_CASE	,		'refs'	,		'main'			)			) as f:
				__lowerCAmelCase      :						Tuple						      =				f.read()
			self.assertTrue(os.path.isfile(os.path.join(_SCREAMING_SNAKE_CASE	,		'.no_exist'	,		_SCREAMING_SNAKE_CASE	,		'conf'			)			)			)

			__lowerCAmelCase      :						List[Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		'conf'	,		_raise_exceptions_for_missing_entries=_SCREAMING_SNAKE_CASE			)
			self.assertIsNone(_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						List[str]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		'conf'	,		local_files_only=_SCREAMING_SNAKE_CASE	,		_raise_exceptions_for_missing_entries=_SCREAMING_SNAKE_CASE			)
			self.assertIsNone(_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						List[Any]						      =				mock.Mock()
			__lowerCAmelCase      :						Tuple						      =				5_00
			__lowerCAmelCase      :						List[Any]						      =				{}
			__lowerCAmelCase      :						Dict						      =				HTTPError
			__lowerCAmelCase      :						str						      =				{}

			# Under the mock environment we get a 500 error when trying to reach the tokenizer.
			with mock.patch('requests.Session.request'	,		return_value=_SCREAMING_SNAKE_CASE			) as mock_head:
				__lowerCAmelCase      :						Optional[Any]						      =				cached_file(_SCREAMING_SNAKE_CASE	,		'conf'	,		_raise_exceptions_for_connection_errors=_SCREAMING_SNAKE_CASE			)
				self.assertIsNone(_SCREAMING_SNAKE_CASE			)
				# This check we did call the fake head request
				mock_head.assert_called()

		def 							__lowerCamelCase							(				self			):
			self.assertTrue(has_file('hf-internal-testing/tiny-bert-pt-only'	,		_SCREAMING_SNAKE_CASE			)			)
			self.assertFalse(has_file('hf-internal-testing/tiny-bert-pt-only'	,		_SCREAMING_SNAKE_CASE			)			)
			self.assertFalse(has_file('hf-internal-testing/tiny-bert-pt-only'	,		_SCREAMING_SNAKE_CASE			)			)

		def 							__lowerCamelCase							(				self			):
			# `get_file_from_repo` returns None if the file does not exist
			self.assertIsNone(get_file_from_repo('bert-base-cased'	,		'ahah.txt'			)			)

			# The function raises if the repository does not exist.
			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'is not a valid model identifier'			):
				get_file_from_repo('bert-base-case'	,		_SCREAMING_SNAKE_CASE			)

			# The function raises if the revision does not exist.
			with self.assertRaisesRegex(_SCREAMING_SNAKE_CASE	,		'is not a valid git identifier'			):
				get_file_from_repo('bert-base-cased'	,		_SCREAMING_SNAKE_CASE	,		revision='ahaha'			)

			__lowerCAmelCase      :						Union[str, Any]						      =				get_file_from_repo('bert-base-cased'	,		_SCREAMING_SNAKE_CASE			)
			# The name is the cached name which is not very easy to test, so instead we load the content.
			__lowerCAmelCase      :						List[Any]						      =				json.loads(open(_SCREAMING_SNAKE_CASE	,		'r'			).read()			)
			self.assertEqual(config['hidden_size']	,		7_68			)







		def 							__lowerCamelCase							(				self			):
			with tempfile.TemporaryDirectory() as tmp_dir:
				__lowerCAmelCase      :						str						      =				Path(_SCREAMING_SNAKE_CASE			) / 'a.txt'
				filename.touch()
				self.assertEqual(get_file_from_repo(_SCREAMING_SNAKE_CASE	,		'a.txt'			)	,		str(_SCREAMING_SNAKE_CASE			)			)

				self.assertIsNone(get_file_from_repo(_SCREAMING_SNAKE_CASE	,		'b.txt'			)			)