



"""simple docstring"""


import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


snake_case_    					=	"""<<<<<<< This should probably be modified because it mentions: """

snake_case_    					=	"""=======
>>>>>>>
"""

snake_case_    					=	[
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

snake_case_    					=	[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R"""tfds\.core""", R"""datasets"""),
    (R"""tf\.io\.gfile\.GFile""", R"""open"""),
    (R"""tf\.([\w\d]+)""", R"""datasets.Value('\1')"""),
    (R"""tfds\.features\.Text\(\)""", R"""datasets.Value('string')"""),
    (R"""tfds\.features\.Text\(""", R"""datasets.Value('string'),"""),
    (R"""features\s*=\s*tfds.features.FeaturesDict\(""", R"""features=datasets.Features("""),
    (R"""tfds\.features\.FeaturesDict\(""", R"""dict("""),
    (R"""The TensorFlow Datasets Authors""", R"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (R"""tfds\.""", R"""datasets."""),
    (R"""dl_manager\.manual_dir""", R"""self.config.data_dir"""),
    (R"""self\.builder_config""", R"""self.config"""),
]

def 			_lowerCAmelCase				(       lowercase_						):
			return ConvertCommand(args.tfds_path		,     args.datasets_directory						)







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						@staticmethod
						def      UpperCAmelCase__					( lowercase_			:ArgumentParser )    ->      int:
									UpperCAmelCase					=			parser.add_parser(
									    'convert'       ,  help='Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'       ,  )
									train_parser.add_argument(
									    '--tfds_path'       ,  type=lowercase_       ,  required=lowercase_       ,  help='Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'       ,  )
									train_parser.add_argument(
									    '--datasets_directory'       ,  type=lowercase_       ,  required=lowercase_       ,  help='Path to the HuggingFace Datasets folder.' )
									train_parser.set_defaults(func=lowercase_ )






						def __init__( self			:Optional[Any]       ,  lowercase_			:str       ,  lowercase_			:str       ,  *lowercase_			:Tuple )    ->      Optional[Any]:
									UpperCAmelCase					=			get_logger('datasets-cli/converting' )

									UpperCAmelCase					=			tfds_path
									UpperCAmelCase					=			datasets_directory






						def      UpperCAmelCase__					( self			:Dict )    ->      Any:
									if os.path.isdir(self._tfds_path ):
												UpperCAmelCase					=			os.path.abspath(self._tfds_path )
									elif os.path.isfile(self._tfds_path ):
												UpperCAmelCase					=			os.path.dirname(self._tfds_path )
									else:
												raise ValueError('--tfds_path is neither a directory nor a file. Please check path.' )

									UpperCAmelCase					=			os.path.abspath(self._datasets_directory )

									self._logger.info(f"""Converting datasets from {abs_tfds_path} to {abs_datasets_path}""" )

									UpperCAmelCase					=			[]
									UpperCAmelCase					=			[]
									UpperCAmelCase					=			{}

									if os.path.isdir(self._tfds_path ):
												UpperCAmelCase					=			os.listdir(lowercase_ )
									else:
												UpperCAmelCase					=			[os.path.basename(self._tfds_path )]

									for f_name in file_names:
												self._logger.info(f"""Looking at file {f_name}""" )
												UpperCAmelCase					=			os.path.join(lowercase_       ,  lowercase_ )
												UpperCAmelCase					=			os.path.join(lowercase_       ,  lowercase_ )

												if not os.path.isfile(lowercase_ ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
															self._logger.info('Skipping file' )
															continue

												with open(lowercase_       ,  encoding='utf-8' ) as f:
															UpperCAmelCase					=			f.readlines()

												UpperCAmelCase					=			[]
												UpperCAmelCase					=			False
												UpperCAmelCase					=			False
												UpperCAmelCase					=			[]
												for line in lines:
															UpperCAmelCase					=			line

															# Convert imports
															if "import tensorflow.compat.v2 as tf" in out_line:
																		continue
															elif "@tfds.core" in out_line:
																		continue
															elif "builder=self" in out_line:
																		continue
															elif "import tensorflow_datasets.public_api as tfds" in out_line:
																		UpperCAmelCase					=			'import datasets\n'
															elif "import tensorflow" in out_line:
																		# order is important here
																		UpperCAmelCase					=			''
																		continue
															elif "from absl import logging" in out_line:
																		UpperCAmelCase					=			'from datasets import logging\n'
															elif "getLogger" in out_line:
																		UpperCAmelCase					=			out_line.replace('getLogger'       ,  'get_logger' )
															elif any(expression in out_line for expression in TO_HIGHLIGHT ):
																		UpperCAmelCase					=			True
																		UpperCAmelCase					=			list(filter(lambda lowercase_ : e in out_line       ,  lowercase_ ) )
																		out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(lowercase_ ) + '\n' )
																		out_lines.append(lowercase_ )
																		out_lines.append(lowercase_ )
																		continue
															else:
																		for pattern, replacement in TO_CONVERT:
																					UpperCAmelCase					=			re.sub(lowercase_       ,  lowercase_       ,  lowercase_ )

                # Take care of saving utilities (to later move them together with main script)
															if "tensorflow_datasets" in out_line:
																		UpperCAmelCase					=			re.match(R'from\stensorflow_datasets.*import\s([^\.\r\n]+)'       ,  lowercase_ )
																		tfds_imports.extend(imp.strip() for imp in match.group(1 ).split(',' ) )
																		UpperCAmelCase					=			'from . import ' + match.group(1 )

															# Check we have not forget anything
															if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
																		raise ValueError(f"""Error converting {out_line.strip()}""" )

															if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
																		UpperCAmelCase					=			True
															out_lines.append(lowercase_ )

												if is_builder or "wmt" in f_name:
															# We create a new directory for each dataset
															UpperCAmelCase					=			f_name.replace('.py'       ,  '' )
															UpperCAmelCase					=			os.path.join(lowercase_       ,  lowercase_ )
															UpperCAmelCase					=			os.path.join(lowercase_       ,  lowercase_ )
															os.makedirs(lowercase_       ,  exist_ok=lowercase_ )
															self._logger.info(f"""Adding directory {output_dir}""" )
															imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
												else:
															# Utilities will be moved at the end
															utils_files.append(lowercase_ )

												if needs_manual_update:
															with_manual_update.append(lowercase_ )

												with open(lowercase_       ,  'w'       ,  encoding='utf-8' ) as f:
															f.writelines(lowercase_ )
												self._logger.info(f"""Converted in {output_file}""" )

									for utils_file in utils_files:
												try:
															UpperCAmelCase					=			os.path.basename(lowercase_ )
															UpperCAmelCase					=			imports_to_builder_map[f_name.replace('.py'       ,  '' )]
															self._logger.info(f"""Moving {dest_folder} to {utils_file}""" )
															shutil.copy(lowercase_       ,  lowercase_ )
												except KeyError:
															self._logger.error(f"""Cannot find destination folder for {utils_file}. Please copy manually.""" )

									if with_manual_update:
												for file_path in with_manual_update:
															self._logger.warning(
															    f"""You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'.""" )










"""simple docstring"""


import logging
import os
import threading
import time


try:
				import warnings
except ImportError:
				snake_case_    					=	None

try:
				import msvcrt
except ImportError:
				snake_case_    					=	None

try:
				import fcntl
except ImportError:
				snake_case_    					=	None


# Backward compatibility
# ------------------------------------------------
try:
				TimeoutError
except NameError:
				snake_case_    					=	OSError


# Data
# ------------------------------------------------
snake_case_    					=	[
    """Timeout""",
    """BaseFileLock""",
    """WindowsFileLock""",
    """UnixFileLock""",
    """SoftFileLock""",
    """FileLock""",
]

snake_case_    					=	"""3.0.12"""


snake_case_    					=	None

def 			_lowerCAmelCase				(       ):
						global _logger
						UpperCAmelCase					=			_logger or logging.getLogger(__name__						)
						return _logger







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						def __init__( self			:int       ,  lowercase_			:List[Any] )    ->      Optional[int]:
												UpperCAmelCase					=			lock_file
												return None






						def __str__( self			:Dict )    ->      Optional[int]:
												UpperCAmelCase					=			f"""The file lock '{self.lock_file}' could not be acquired."""
												return temp







class 				A_      :



						"""simple docstring"""






						def __init__( self			:List[Any]       ,  lowercase_			:Dict )    ->      Dict:
												UpperCAmelCase					=			lock
												return None






						def __enter__( self			:Optional[int] )    ->      Optional[int]:
												return self.lock






						def __exit__( self			:Optional[int]       ,  lowercase_			:Tuple       ,  lowercase_			:Any       ,  lowercase_			:Union[str, Any] )    ->      List[Any]:
												self.lock.release()
												return None







class 				A_      :



						"""simple docstring"""






						def __init__( self			:List[str]       ,  lowercase_			:str       ,  lowercase_			:str=-1       ,  lowercase_			:List[Any]=None )    ->      Dict:
												UpperCAmelCase					=			max_filename_length if max_filename_length is not None else 2_55
												# Hash the filename if it's too long
												UpperCAmelCase					=			self.hash_filename_if_too_long(lowercase_       ,  lowercase_ )
												# The path to the lock file.
												UpperCAmelCase					=			lock_file

												# The file descriptor for the *_lock_file* as it is returned by the
												# os.open() function.
												# This file lock is only NOT None, if the object currently holds the
												# lock.
												UpperCAmelCase					=			None

												# The default timeout value.
												UpperCAmelCase					=			timeout

												# We use this lock primarily for the lock counter.
												UpperCAmelCase					=			threading.Lock()

												# The lock counter is used for implementing the nested locking
												# mechanism. Whenever the lock is acquired, the counter is increased and
												# the lock is only released, when this value is 0 again.
												UpperCAmelCase					=			0
												return None






						@property
						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      int:
												return self._lock_file






						@property
						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      Any:
												return self._timeout






						@timeout.setter
						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  lowercase_			:Any )    ->      Tuple:
												UpperCAmelCase					=			float(lowercase_ )
												return None






						def      UpperCAmelCase__					( self			:Any )    ->      List[str]:
												raise NotImplementedError()






						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      List[str]:
												raise NotImplementedError()






						@property
						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Union[str, Any]:
												return self._lock_file_fd is not None






						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  lowercase_			:Dict=None       ,  lowercase_			:int=0.05 )    ->      Union[str, Any]:
												# Use the default timeout, if no timeout is provided.
												if timeout is None:
																		UpperCAmelCase					=			self.timeout

												# Increment the number right at the beginning.
												# We can still undo it, if something fails.
												with self._thread_lock:
																		self._lock_counter += 1

												UpperCAmelCase					=			id(self )
												UpperCAmelCase					=			self._lock_file
												UpperCAmelCase					=			time.time()
												try:
																		while True:
																								with self._thread_lock:
																														if not self.is_locked:
																																				logger().debug(f"""Attempting to acquire lock {lock_id} on {lock_filename}""" )
																																				self._acquire()

																								if self.is_locked:
																														logger().debug(f"""Lock {lock_id} acquired on {lock_filename}""" )
																														break
																								elif timeout >= 0 and time.time() - start_time > timeout:
																														logger().debug(f"""Timeout on acquiring lock {lock_id} on {lock_filename}""" )
																														raise Timeout(self._lock_file )
																								else:
																														logger().debug(
																														    f"""Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...""" )
																														time.sleep(lowercase_ )
												except:  # noqa
																		# Something did go wrong, so decrement the counter.
																		with self._thread_lock:
																								UpperCAmelCase					=			max(0       ,  self._lock_counter - 1 )

																		raise
												return _Acquire_ReturnProxy(lock=self )






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:List[str]=False )    ->      int:
												with self._thread_lock:
																		if self.is_locked:
																								self._lock_counter -= 1

																								if self._lock_counter == 0 or force:
																														UpperCAmelCase					=			id(self )
																														UpperCAmelCase					=			self._lock_file

																														logger().debug(f"""Attempting to release lock {lock_id} on {lock_filename}""" )
																														self._release()
																														UpperCAmelCase					=			0
																														logger().debug(f"""Lock {lock_id} released on {lock_filename}""" )

												return None






						def __enter__( self			:Any )    ->      Tuple:
												self.acquire()
												return self






						def __exit__( self			:Dict       ,  lowercase_			:Optional[int]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:Union[str, Any] )    ->      Any:
												self.release()
												return None






						def __del__( self			:Any )    ->      int:
												self.release(force=lowercase_ )
												return None






						def      UpperCAmelCase__					( self			:List[str]       ,  lowercase_			:str       ,  lowercase_			:int )    ->      str:
												UpperCAmelCase					=			os.path.basename(lowercase_ )
												if len(lowercase_ ) > max_length and max_length > 0:
																		UpperCAmelCase					=			os.path.dirname(lowercase_ )
																		UpperCAmelCase					=			str(hash(lowercase_ ) )
																		UpperCAmelCase					=			filename[: max_length - len(lowercase_ ) - 8] + '...' + hashed_filename + '.lock'
																		return os.path.join(lowercase_       ,  lowercase_ )
												else:
																		return path







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						def __init__( self			:Tuple       ,  lowercase_			:Tuple       ,  lowercase_			:List[Any]=-1       ,  lowercase_			:Union[str, Any]=None )    ->      Optional[Any]:
												from .file_utils import relative_to_absolute_path

												super().__init__(lowercase_       ,  timeout=lowercase_       ,  max_filename_length=lowercase_ )
												UpperCAmelCase					=			'\\\\?\\' + relative_to_absolute_path(self.lock_file )






						def      UpperCAmelCase__					( self			:List[str] )    ->      str:
												UpperCAmelCase					=			os.O_RDWR | os.O_CREAT | os.O_TRUNC

												try:
																		UpperCAmelCase					=			os.open(self._lock_file       ,  lowercase_ )
												except OSError:
																		pass
												else:
																		try:
																								msvcrt.locking(lowercase_       ,  msvcrt.LK_NBLCK       ,  1 )
																		except OSError:
																								os.close(lowercase_ )
																		else:
																								UpperCAmelCase					=			fd
												return None






						def      UpperCAmelCase__					( self			:int )    ->      Union[str, Any]:
												UpperCAmelCase					=			self._lock_file_fd
												UpperCAmelCase					=			None
												msvcrt.locking(lowercase_       ,  msvcrt.LK_UNLCK       ,  1 )
												os.close(lowercase_ )

												try:
																		os.remove(self._lock_file )
												# Probably another instance of the application
												# that acquired the file lock.
												except OSError:
																		pass
												return None







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						def __init__( self			:Optional[Any]       ,  lowercase_			:str       ,  lowercase_			:List[str]=-1       ,  lowercase_			:str=None )    ->      str:
												UpperCAmelCase					=			os.statvfs(os.path.dirname(lowercase_ ) ).f_namemax
												super().__init__(lowercase_       ,  timeout=lowercase_       ,  max_filename_length=lowercase_ )






						def      UpperCAmelCase__					( self			:str )    ->      Dict:
												UpperCAmelCase					=			os.O_RDWR | os.O_CREAT | os.O_TRUNC
												UpperCAmelCase					=			os.open(self._lock_file       ,  lowercase_ )

												try:
																		fcntl.flock(lowercase_       ,  fcntl.LOCK_EX | fcntl.LOCK_NB )
												except OSError:
																		os.close(lowercase_ )
												else:
																		UpperCAmelCase					=			fd
												return None






						def      UpperCAmelCase__					( self			:List[Any] )    ->      List[Any]:
												# Do not remove the lockfile:
												#
												#   https://github.com/benediktschmitt/py-filelock/issues/31
												#   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
												UpperCAmelCase					=			self._lock_file_fd
												UpperCAmelCase					=			None
												fcntl.flock(lowercase_       ,  fcntl.LOCK_UN )
												os.close(lowercase_ )
												return None







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""






						def      UpperCAmelCase__					( self			:List[Any] )    ->      List[str]:
												UpperCAmelCase					=			os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
												try:
																		UpperCAmelCase					=			os.open(self._lock_file       ,  lowercase_ )
												except OSError:
																		pass
												else:
																		UpperCAmelCase					=			fd
												return None






						def      UpperCAmelCase__					( self			:Optional[int] )    ->      List[Any]:
												os.close(self._lock_file_fd )
												UpperCAmelCase					=			None

												try:
																		os.remove(self._lock_file )
												# The file is already deleted and that's what we want.
												except OSError:
																		pass
												return None


snake_case_    					=	None

if msvcrt:
				snake_case_    					=	WindowsFileLock
elif fcntl:
				snake_case_    					=	UnixFileLock
else:
				snake_case_    					=	SoftFileLock

				if warnings is not None:
								warnings.warn("""only soft file lock is available""")





