




"""simple docstring"""


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_A													=      logging.get_logger(__name__)


_A													=      {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}



def 	lowercase_				(			__UpperCAmelCase    ,		__UpperCAmelCase    ,		__UpperCAmelCase    ,		__UpperCAmelCase				)						->    Optional[Any]:





      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}."""				)

      if tokenizer_name is None:
            lowerCAmelCase__      :			Dict   =	TOKENIZER_CLASSES
      else:
            lowerCAmelCase__      :			Optional[int]   =	{tokenizer_name: getattr(__UpperCAmelCase    ,		tokenizer_name + """Fast"""				)}

      logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""				)

      for tokenizer_name in tokenizer_names:
            lowerCAmelCase__      :			List[Any]   =	TOKENIZER_CLASSES[tokenizer_name]

            lowerCAmelCase__      :			Any   =	True
            if checkpoint_name is None:
                  lowerCAmelCase__      :			str   =	list(tokenizer_class.max_model_input_sizes.keys()				)
            else:
                  lowerCAmelCase__      :			List[Any]   =	[checkpoint_name]

            logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""				)

            for checkpoint in checkpoint_names:
                  logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""				)

                  # Load tokenizer
                  lowerCAmelCase__      :			Dict   =	tokenizer_class.from_pretrained(__UpperCAmelCase    ,		force_download=__UpperCAmelCase				)

                  # Save fast tokenizer
                  logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""				)

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        lowerCAmelCase__ ,   lowerCAmelCase__      :			List[Any]   =	checkpoint.split("""/"""				)
                        lowerCAmelCase__      :			Dict   =	os.path.join(__UpperCAmelCase    ,		__UpperCAmelCase				)
                  elif add_prefix:
                        lowerCAmelCase__      :			int   =	checkpoint
                        lowerCAmelCase__      :			Dict   =	dump_path
                  else:
                        lowerCAmelCase__      :			Optional[int]   =	None
                        lowerCAmelCase__      :			List[Any]   =	dump_path

                  logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
                        lowerCAmelCase__      :			str   =	list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
                        lowerCAmelCase__      :			int   =	file_path.split(__UpperCAmelCase				)[-1][0]
                        if next_char == "/":
                              lowerCAmelCase__      :			List[Any]   =	os.path.join(__UpperCAmelCase    ,		__UpperCAmelCase				)
                              lowerCAmelCase__      :			Tuple   =	None

                        logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

                  lowerCAmelCase__      :			List[str]   =	tokenizer.save_pretrained(
                      __UpperCAmelCase    ,		legacy_format=__UpperCAmelCase    ,		filename_prefix=__UpperCAmelCase				)
                  logger.info(f"""=> File names {file_names}"""				)

                  for file_name in file_names:
                        if not file_name.endswith("""tokenizer.json"""				):
                              os.remove(__UpperCAmelCase				)
                              logger.info(f"""=> removing {file_name}"""				)


if __name__ == "__main__":
 _A													=      argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
 )
 parser.add_argument(
     """--tokenizer_name""",
     default=None,
     type=str,
     help=(
         f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
         """download and convert all the checkpoints from AWS."""
     ),
 )
 parser.add_argument(
     """--checkpoint_name""",
     default=None,
     type=str,
     help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
 )
 parser.add_argument(
     """--force_download""",
     action="""store_true""",
     help="""Re-download checkpoints.""",
 )
 _A													=      parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)











"""simple docstring"""


import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, ClassLabel, Features
from .base import TaskTemplate


@dataclass(frozen=a_						)
class _lowerCamelCase    (					a_						):
   _lowerCamelCase    :str     				=     field(default="audio-classification"       ,	metadata={"include_in_asdict_even_if_is_default": True}						)
   _lowerCamelCase    :ClassVar[Features]     				=     Features({"audio": Audio()}						)
   _lowerCamelCase    :ClassVar[Features]     				=     Features({"labels": ClassLabel}						)
   _lowerCamelCase    :str     				=     "audio"
   _lowerCamelCase    :str     				=     "labels"






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		Optional[int]   )			->						Union[str, Any]:

         """simple docstring"""
         if self.label_column not in features:
               raise ValueError(f"""Column {self.label_column} is not present in features."""   )
         if not isinstance(features[self.label_column]   ,     UpperCamelCase   ):
               raise ValueError(f"""Column {self.label_column} is not a ClassLabel."""   )
         lowerCAmelCase__      :			str   =	copy.deepcopy(self   )
         lowerCAmelCase__      :			Optional[int]   =	self.label_schema.copy()
         lowerCAmelCase__      :			List[Any]   =	features[self.label_column]
         lowerCAmelCase__      :			Optional[int]   =	label_schema
         return task_template






   @property
   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						Dict[str, str]:

         """simple docstring"""
         return {
             self.audio_column: "audio",
             self.label_column: "labels",
         }





