




import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase    :		Union[str, Any]			   =       logging.get_logger(__name__)


lowerCamelCase    :		str			   =       {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}

def       _SCREAMING_SNAKE_CASE      (							lowercase     :	Any     ,							lowercase     :	str     ,							lowercase     :	int     ,							lowercase     :	List[Any]					):


      '''simple docstring'''



      if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
            raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()					)}."""					)

      if tokenizer_name is None:
            lowerCamelCase_							=   TOKENIZER_CLASSES
      else:
            lowerCamelCase_							=   {tokenizer_name: getattr(lowercase     ,							tokenizer_name + 'Fast'					)}

      logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""					)

      for tokenizer_name in tokenizer_names:
            lowerCamelCase_							=   TOKENIZER_CLASSES[tokenizer_name]

            lowerCamelCase_							=   True
            if checkpoint_name is None:
                  lowerCamelCase_							=   list(tokenizer_class.max_model_input_sizes.keys()					)
            else:
                  lowerCamelCase_							=   [checkpoint_name]

            logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""					)

            for checkpoint in checkpoint_names:
                  logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""					)

                  # Load tokenizer
                  lowerCamelCase_							=   tokenizer_class.from_pretrained(lowercase     ,							force_download=lowercase					)

                  # Save fast tokenizer
                  logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""					)

                  # For organization names we create sub-directories
                  if "/" in checkpoint:
                        lowerCamelCase_					,    lowerCamelCase_							=   checkpoint.split('/'					)
                        lowerCamelCase_							=   os.path.join(lowercase     ,							lowercase					)
                  elif add_prefix:
                        lowerCamelCase_							=   checkpoint
                        lowerCamelCase_							=   dump_path
                  else:
                        lowerCamelCase_							=   None
                        lowerCamelCase_							=   dump_path

                  logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""					)

                  if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()					)[0]:
                        lowerCamelCase_							=   list(tokenizer.pretrained_vocab_files_map.values()					)[0][checkpoint]
                        lowerCamelCase_							=   file_path.split(lowercase					)[-1][0]
                        if next_char == "/":
                              lowerCamelCase_							=   os.path.join(lowercase     ,							lowercase					)
                              lowerCamelCase_							=   None

                        logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""					)

                  lowerCamelCase_							=   tokenizer.save_pretrained(
                      lowercase     ,							legacy_format=lowercase     ,							filename_prefix=lowercase					)
                  logger.info(f"""=> File names {file_names}"""					)

                  for file_name in file_names:
                        if not file_name.endswith('tokenizer.json'					):
                              os.remove(lowercase					)
                              logger.info(f"""=> removing {file_name}"""					)


if __name__ == "__main__":
 lowerCamelCase    :		Tuple			   =       argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
 )
 parser.add_argument(
     "--tokenizer_name",
     default=None,
     type=str,
     help=(
         F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
         "download and convert all the checkpoints from AWS."
     ),
 )
 parser.add_argument(
     "--checkpoint_name",
     default=None,
     type=str,
     help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
 )
 parser.add_argument(
     "--force_download",
     action="store_true",
     help="Re-download checkpoints.",
 )
 lowerCamelCase    :		Union[str, Any]			   =       parser.parse_args()

 convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)









import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
lowerCamelCase    :		int			   =       logging.getLogger()

lowerCamelCase    :		Tuple			   =       logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 def     a__			(    self :  List[str]						,							A_ :  Optional[Any]  )       ->     int:

       """simple docstring"""






       os.makedirs(A_						,							exist_ok=A_  )
       lowerCamelCase_							=   {'source': 'What is love ?', 'target': 'life'}
       lowerCamelCase_							=   {'train': 12, 'val': 2, 'test': 2}
       for split in ["train", "test", "val"]:
             for field in ["source", "target"]:
                   lowerCamelCase_							=   '\n'.join([contents[field]] * n_lines[split]  )
                   with open(os.path.join(A_						,							f"""{split}.{field}"""  )						,							'w'  ) as f:
                         f.write(A_  )

 def     a__			(    self :  Optional[Any]						,							A_ :  int						,							A_ :  str = "pytorch"  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   self.get_auto_remove_tmp_dir()
       lowerCamelCase_							=   os.path.join(A_						,							'output'  )
       lowerCamelCase_							=   os.path.join(A_						,							'data'  )
       self._create_dummy_data(data_dir=A_  )

       lowerCamelCase_							=   f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

       if gpus > 0:
             testargs.append(f"""--gpus={gpus}"""  )
             if is_apex_available():
                   testargs.append('--fp16'  )
       else:
             testargs.append('--gpus=0'  )
             testargs.append('--distributed_backend=ddp_cpu'  )
             testargs.append('--num_processes=2'  )

       lowerCamelCase_							=   [sys.executable, str(Path(finetune_rag.__file__  ).resolve()  )] + testargs
       execute_subprocess_async(A_						,							env=self.get_env()  )

       lowerCamelCase_							=   os.path.join(A_						,							'metrics.json'  )
       with open(A_  ) as f:
             lowerCamelCase_							=   json.load(A_  )
       return result

 @require_torch_gpu
 def     a__			(    self :  List[Any]  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )

 @require_torch_multi_gpu
 def     a__			(    self :  List[Any]  )       ->     Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=2  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )

 @require_torch_gpu
 @require_ray
 def     a__			(    self :  Tuple  )       ->     int:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1						,							distributed_retriever='ray'  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )





 @require_torch_multi_gpu
 @require_ray
 def     a__			(    self :  Dict  )       ->     Dict:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1						,							distributed_retriever='ray'  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )



