





import argparse
import re
from typing import Dict

import torch
from datasets import Audio, Dataset, load_dataset, load_metric

from transformers import AutoFeatureExtractor, pipeline






def lowerCamelCase__   (							snake_case_     :  Optional[Any]	,				snake_case_     :  str	)				->    List[str]:
   __snake_case								=       args.log_outputs
   __snake_case								=       '_'.join(args.dataset.split('''/'''	) + [args.config, args.split]	)

   # load metric
   __snake_case								=       load_metric('''wer'''	)
   __snake_case								=       load_metric('''cer'''	)

   # compute metrics
   __snake_case								=       wer.compute(references=result['''target''']	,				predictions=result['''prediction''']	)
   __snake_case								=       cer.compute(references=result['''target''']	,				predictions=result['''prediction''']	)

   # print & log results
   __snake_case								=       f"""WER: {wer_result}\nCER: {cer_result}"""
   print(_UpperCAmelCase	)

   with open(f"""{dataset_id}_eval_results.txt"""	,				'''w'''	) as f:
      f.write(_UpperCAmelCase	)

   # log all results in text file. Possibly interesting for analysis
   if log_outputs is not None:
      __snake_case								=       f"""log_{dataset_id}_predictions.txt"""
      __snake_case								=       f"""log_{dataset_id}_targets.txt"""

      with open(_UpperCAmelCase	,				'''w'''	) as p, open(_UpperCAmelCase	,				'''w'''	) as t:
         # mapping function to write output
         def write_to_file(snake_case_     :  Any	,				snake_case_     :  Any	):
            p.write(f"""{i}""" + '''\n'''	)
            p.write(batch['''prediction'''] + '''\n'''	)
            t.write(f"""{i}""" + '''\n'''	)
            t.write(batch['''target'''] + '''\n'''	)

         result.map(_UpperCAmelCase	,				with_indices=_UpperCAmelCase	)






def lowerCamelCase__   (							snake_case_     :  Optional[Any]	)				->    str:
   __snake_case								=       '[,?.!\-\;\:"“%‘”�—’…–]'  # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training

   __snake_case								=       re.sub(_UpperCAmelCase	,				''''''	,				text.lower()	)

   # In addition, we can normalize the target text, e.g. removing new lines characters etc...
   # note that order is important here!
   __snake_case								=       ['\n\n', '\n', '   ', '  ']

   for t in token_sequences_to_ignore:
      __snake_case								=       ' '.join(text.split(_UpperCAmelCase	)	)

   return text






def lowerCamelCase__   (							snake_case_     :  List[Any]	)				->    Union[str, Any]:
   # load dataset
   __snake_case								=       load_dataset(args.dataset	,				args.config	,				split=args.split	,				use_auth_token=_UpperCAmelCase	)

   # for testing: only process the first two examples as a test
   # dataset = dataset.select(range(10))

   # load processor
   __snake_case								=       AutoFeatureExtractor.from_pretrained(args.model_id	)
   __snake_case								=       feature_extractor.sampling_rate

   # resample audio
   __snake_case								=       dataset.cast_column('''audio'''	,				Audio(sampling_rate=_UpperCAmelCase	)	)

   # load eval pipeline
   if args.device is None:
      __snake_case								=       0 if torch.cuda.is_available() else -1
   __snake_case								=       pipeline('''automatic-speech-recognition'''	,				model=args.model_id	,				device=args.device	)

   # map function to decode audio
   def map_to_pred(snake_case_     :  Tuple	):
      __snake_case								=       asr(
          batch['''audio''']['''array''']	,				chunk_length_s=args.chunk_length_s	,				stride_length_s=args.stride_length_s	)

      __snake_case								=       prediction['text']
      __snake_case								=       normalize_text(batch['''sentence''']	)
      return batch

   # run inference on all examples
   __snake_case								=       dataset.map(_UpperCAmelCase	,				remove_columns=dataset.column_names	)

   # compute and log_results
   # do not change function below
   log_results(_UpperCAmelCase	,				_UpperCAmelCase	)


if __name__ == "__main__":
      snake_case_           =      argparse.ArgumentParser()

      parser.add_argument(
          '--model_id', type=str, required=True, help='Model identifier. Should be loadable with 🤗 Transformers'
      )
      parser.add_argument(
          '--dataset',
          type=str,
          required=True,
          help='Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets',
      )
      parser.add_argument(
          '--config', type=str, required=True, help='Config of the dataset. *E.g.* `\'en\'`  for Common Voice'
      )
      parser.add_argument('--split', type=str, required=True, help='Split of the dataset. *E.g.* `\'test\'`')
      parser.add_argument(
          '--chunk_length_s', type=float, default=None, help='Chunk length in seconds. Defaults to 5 seconds.'
      )
      parser.add_argument(
          '--stride_length_s', type=float, default=None, help='Stride of the audio chunks. Defaults to 1 second.'
      )
      parser.add_argument(
          '--log_outputs', action='store_true', help='If defined, write outputs to log file for analysis.'
      )
      parser.add_argument(
          '--device',
          type=int,
          default=None,
          help='The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.',
      )
      snake_case_           =      parser.parse_args()

      main(args)








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_           =      data_utils.TransfoXLTokenizer
snake_case_           =      data_utils.TransfoXLCorpus
snake_case_           =      data_utils
snake_case_           =      data_utils






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  int	,				snake_case_     :  List[Any]	,				snake_case_     :  int	)				->    Dict:
   if transfo_xl_dataset_file:
      # Convert a pre-processed corpus (see original TensorFlow repo)
      with open(snake_case_	,				'''rb'''	) as fp:
         __snake_case								=       pickle.load(snake_case_	,				encoding='''latin1'''	)
      # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
      print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""	)
      __snake_case								=       corpus.vocab.__dict__
      torch.save(snake_case_	,				snake_case_	)

      __snake_case								=       corpus.__dict__
      corpus_dict_no_vocab.pop('''vocab'''	,				snake_case_	)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + CORPUS_NAME
      print(f"""Save dataset to {pytorch_dataset_dump_path}"""	)
      torch.save(snake_case_	,				snake_case_	)

   if tf_checkpoint_path:
      # Convert a pre-trained TensorFlow model
      __snake_case								=       os.path.abspath(snake_case_	)
      __snake_case								=       os.path.abspath(snake_case_	)

      print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""	)
      # Initialise PyTorch model
      if transfo_xl_config_file == "":
         __snake_case								=       TransfoXLConfig()
      else:
         __snake_case								=       TransfoXLConfig.from_json_file(snake_case_	)
      print(f"""Building PyTorch model from configuration: {config}"""	)
      __snake_case								=       TransfoXLLMHeadModel(snake_case_	)

      __snake_case								=       load_tf_weights_in_transfo_xl(snake_case_	,				snake_case_	,				snake_case_	)
      # Save pytorch-model
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      print(f"""Save PyTorch model to {os.path.abspath(snake_case_	)}"""	)
      torch.save(model.state_dict()	,				snake_case_	)
      print(f"""Save configuration file to {os.path.abspath(snake_case_	)}"""	)
      with open(snake_case_	,				'''w'''	,				encoding='''utf-8'''	) as f:
         f.write(config.to_json_string()	)


if __name__ == "__main__":
    snake_case_           =      argparse.ArgumentParser()
    parser.add_argument(
        '--pytorch_dump_folder_path',
        default=None,
        type=str,
        required=True,
        help='Path to the folder to store the PyTorch model or dataset/vocab.',
    )
    parser.add_argument(
        '--tf_checkpoint_path',
        default='',
        type=str,
        help='An optional path to a TensorFlow checkpoint path to be converted.',
    )
    parser.add_argument(
        '--transfo_xl_config_file',
        default='',
        type=str,
        help=(
            'An optional config json file corresponding to the pre-trained BERT model. \n'
            'This specifies the model architecture.'
        ),
    )
    parser.add_argument(
        '--transfo_xl_dataset_file',
        default='',
        type=str,
        help='An optional dataset file to be converted in a vocabulary.',
    )
    snake_case_           =      parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )

