





import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests






def lowerCamelCase__   (							snake_case_     :  str	,				snake_case_     :  Tuple=None	)				->    Optional[Any]:
   __snake_case								=       None
   if token is not None:
      __snake_case								=       {'''Accept''': '''application/vnd.github+json''', '''Authorization''': f"""Bearer {token}"""}

   __snake_case								=       f"""https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"""
   __snake_case								=       requests.get(snake_case_	,				headers=snake_case_	).json()
   __snake_case								=       {}

   try:
      job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}	)
      __snake_case								=       math.ceil((result['''total_count'''] - 100) / 100	)

      for i in range(snake_case_	):
         __snake_case								=       requests.get(url + f"""&page={i + 2}"""	,				headers=snake_case_	).json()
         job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}	)

      return job_links
   except Exception:
      print(f"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""	)

   return {}






def lowerCamelCase__   (							snake_case_     :  List[Any]	,				snake_case_     :  Any=None	)				->    List[Any]:
   __snake_case								=       None
   if token is not None:
      __snake_case								=       {'''Accept''': '''application/vnd.github+json''', '''Authorization''': f"""Bearer {token}"""}

   __snake_case								=       f"""https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"""
   __snake_case								=       requests.get(snake_case_	,				headers=snake_case_	).json()
   __snake_case								=       {}

   try:
      artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}	)
      __snake_case								=       math.ceil((result['''total_count'''] - 100) / 100	)

      for i in range(snake_case_	):
         __snake_case								=       requests.get(url + f"""&page={i + 2}"""	,				headers=snake_case_	).json()
         artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}	)

      return artifacts
   except Exception:
      print(f"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""	)

   return {}






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  List[str]	,				snake_case_     :  int	,				snake_case_     :  Union[str, Any]	)				->    Union[str, Any]:
   __snake_case								=       None
   if token is not None:
      __snake_case								=       {'''Accept''': '''application/vnd.github+json''', '''Authorization''': f"""Bearer {token}"""}

   __snake_case								=       requests.get(snake_case_	,				headers=snake_case_	,				allow_redirects=snake_case_	)
   __snake_case								=       result.headers['''Location''']
   __snake_case								=       requests.get(snake_case_	,				allow_redirects=snake_case_	)
   __snake_case								=       os.path.join(snake_case_	,				f"""{artifact_name}.zip"""	)
   with open(snake_case_	,				'''wb'''	) as fp:
      fp.write(response.content	)






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  List[Any]=None	)				->    str:
   __snake_case								=       []
   __snake_case								=       []
   __snake_case								=       None

   with zipfile.ZipFile(snake_case_	) as z:
      for filename in z.namelist():
         if not os.path.isdir(snake_case_	):
            # read the file
            if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
               with z.open(snake_case_	) as f:
                  for line in f:
                     __snake_case								=       line.decode('''UTF-8'''	).strip()
                     if filename == "failures_line.txt":
                        try:
                           # `error_line` is the place where `error` occurs
                           __snake_case								=       line[: line.index(''': '''	)]
                           __snake_case								=       line[line.index(''': '''	) + len(''': '''	) :]
                           errors.append([error_line, error]	)
                        except Exception:
                           # skip un-related lines
                           pass
                     elif filename == "summary_short.txt" and line.startswith('''FAILED '''	):
                        # `test` is the test method that failed
                        __snake_case								=       line[len('''FAILED '''	) :]
                        failed_tests.append(snake_case_	)
                     elif filename == "job_name.txt":
                        __snake_case								=       line

   if len(snake_case_	) != len(snake_case_	):
      raise ValueError(
          f"""`errors` and `failed_tests` should have the same number of elements. Got {len(snake_case_	)} for `errors` """
          f"""and {len(snake_case_	)} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"""
          ''' problem.'''	)

   __snake_case								=       None
   if job_name and job_links:
      __snake_case								=       job_links.get(snake_case_	,				snake_case_	)

   # A list with elements of the form (line of error, error, failed test)
   __snake_case								=       [x + [y] + [job_link] for x, y in zip(snake_case_	,				snake_case_	)]

   return result






def lowerCamelCase__   (							snake_case_     :  Dict	,				snake_case_     :  List[Any]=None	)				->    Dict:
   __snake_case								=       []

   __snake_case								=       [os.path.join(snake_case_	,				snake_case_	) for p in os.listdir(snake_case_	) if p.endswith('''.zip'''	)]
   for p in paths:
      errors.extend(get_errors_from_single_artifact(snake_case_	,				job_links=snake_case_	)	)

   return errors






def lowerCamelCase__   (							snake_case_     :  List[Any]	,				snake_case_     :  Optional[Any]=None	)				->    str:
   __snake_case								=       Counter()
   counter.update([x[1] for x in logs]	)
   __snake_case								=       counter.most_common()
   __snake_case								=       {}
   for error, count in counts:
      if error_filter is None or error not in error_filter:
         __snake_case								=       {'''count''': count, '''failed_tests''': [(x[2], x[0]) for x in logs if x[1] == error]}

   __snake_case								=       dict(sorted(r.items()	,				key=lambda snake_case_	: item[1]["count"]	,				reverse=snake_case_	)	)
   return r






def lowerCamelCase__   (							snake_case_     :  List[Any]	)				->    Tuple:
   __snake_case								=       test.split('''::'''	)[0]
   if test.startswith('''tests/models/'''	):
      __snake_case								=       test.split('''/'''	)[2]
   else:
      __snake_case								=       None

   return test






def lowerCamelCase__   (							snake_case_     :  int	,				snake_case_     :  Union[str, Any]=None	)				->    Optional[int]:
   __snake_case								=       [(x[0], x[1], get_model(x[2]	)) for x in logs]
   __snake_case								=       [x for x in logs if x[2] is not None]
   __snake_case								=       {x[2] for x in logs}

   __snake_case								=       {}
   for test in tests:
      __snake_case								=       Counter()
      # count by errors in `test`
      counter.update([x[1] for x in logs if x[2] == test]	)
      __snake_case								=       counter.most_common()
      __snake_case								=       {error: count for error, count in counts if (error_filter is None or error not in error_filter)}
      __snake_case								=       sum(error_counts.values()	)
      if n_errors > 0:
         __snake_case								=       {'''count''': n_errors, '''errors''': error_counts}

   __snake_case								=       dict(sorted(r.items()	,				key=lambda snake_case_	: item[1]["count"]	,				reverse=snake_case_	)	)
   return r






def lowerCamelCase__   (							snake_case_     :  Any	)				->    Union[str, Any]:
   __snake_case								=       '''| no. | error | status |'''
   __snake_case								=       '''|-:|:-|:-|'''
   __snake_case								=       [header, sep]
   for error in reduced_by_error:
      __snake_case								=       reduced_by_error[error]['''count''']
      __snake_case								=       f"""| {count} | {error[:100]} |  |"""
      lines.append(snake_case_	)

   return "\n".join(snake_case_	)






def lowerCamelCase__   (							snake_case_     :  List[Any]	)				->    Tuple:
   __snake_case								=       '''| model | no. of errors | major error | count |'''
   __snake_case								=       '''|-:|-:|-:|-:|'''
   __snake_case								=       [header, sep]
   for model in reduced_by_model:
      __snake_case								=       reduced_by_model[model]['''count''']
      __snake_case	,     __snake_case								=       list(reduced_by_model[model]['''errors'''].items()	)[0]
      __snake_case								=       f"""| {model} | {count} | {error[:60]} | {_count} |"""
      lines.append(snake_case_	)

   return "\n".join(snake_case_	)


if __name__ == "__main__":
      snake_case_           =      argparse.ArgumentParser()
      # Required parameters
      parser.add_argument('--workflow_run_id', type=str, required=True, help='A GitHub Actions workflow run id.')
      parser.add_argument(
          '--output_dir',
          type=str,
          required=True,
          help='Where to store the downloaded artifacts and other result files.',
      )
      parser.add_argument('--token', default=None, type=str, help='A token that has actions:read permission.')
      snake_case_           =      parser.parse_args()

      os.makedirs(args.output_dir, exist_ok=True)

      snake_case_           =      get_job_links(args.workflow_run_id, token=args.token)
      snake_case_           =      {}
      # To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
      # For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
      if _job_links:
            for k, v in _job_links.items():
                  # This is how GitHub actions combine job names.
                  if " / " in k:
                        snake_case_           =      k.find(' / ')
                        snake_case_           =      k[index + len(' / ') :]
                  snake_case_           =      v
      with open(os.path.join(args.output_dir, 'job_links.json'), 'w', encoding='UTF-8') as fp:
            json.dump(job_links, fp, ensure_ascii=False, indent=4)

      snake_case_           =      get_artifacts_links(args.workflow_run_id, token=args.token)
      with open(os.path.join(args.output_dir, 'artifacts.json'), 'w', encoding='UTF-8') as fp:
            json.dump(artifacts, fp, ensure_ascii=False, indent=4)

      for idx, (name, url) in enumerate(artifacts.items()):
            download_artifact(name, url, args.output_dir, args.token)
            # Be gentle to GitHub
            time.sleep(1)

      snake_case_           =      get_all_errors(args.output_dir, job_links=job_links)

      # `e[1]` is the error
      snake_case_           =      Counter()
      counter.update([e[1] for e in errors])

      # print the top 30 most common test errors
      snake_case_           =      counter.most_common(30)
      for item in most_common:
            print(item)

      with open(os.path.join(args.output_dir, 'errors.json'), 'w', encoding='UTF-8') as fp:
            json.dump(errors, fp, ensure_ascii=False, indent=4)

      snake_case_           =      reduce_by_error(errors)
      snake_case_           =      reduce_by_model(errors)

      snake_case_           =      make_github_table(reduced_by_error)
      snake_case_           =      make_github_table_per_model(reduced_by_model)

      with open(os.path.join(args.output_dir, 'reduced_by_error.txt'), 'w', encoding='UTF-8') as fp:
            fp.write(sa)
      with open(os.path.join(args.output_dir, 'reduced_by_model.txt'), 'w', encoding='UTF-8') as fp:
            fp.write(sa)








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_           =      data_utils.TransfoXLTokenizer
snake_case_           =      data_utils.TransfoXLCorpus
snake_case_           =      data_utils
snake_case_           =      data_utils






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  int	,				snake_case_     :  List[Any]	,				snake_case_     :  int	)				->    Dict:
   if transfo_xl_dataset_file:
      # Convert a pre-processed corpus (see original TensorFlow repo)
      with open(snake_case_	,				'''rb'''	) as fp:
         __snake_case								=       pickle.load(snake_case_	,				encoding='''latin1'''	)
      # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
      print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""	)
      __snake_case								=       corpus.vocab.__dict__
      torch.save(snake_case_	,				snake_case_	)

      __snake_case								=       corpus.__dict__
      corpus_dict_no_vocab.pop('''vocab'''	,				snake_case_	)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + CORPUS_NAME
      print(f"""Save dataset to {pytorch_dataset_dump_path}"""	)
      torch.save(snake_case_	,				snake_case_	)

   if tf_checkpoint_path:
      # Convert a pre-trained TensorFlow model
      __snake_case								=       os.path.abspath(snake_case_	)
      __snake_case								=       os.path.abspath(snake_case_	)

      print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""	)
      # Initialise PyTorch model
      if transfo_xl_config_file == "":
         __snake_case								=       TransfoXLConfig()
      else:
         __snake_case								=       TransfoXLConfig.from_json_file(snake_case_	)
      print(f"""Building PyTorch model from configuration: {config}"""	)
      __snake_case								=       TransfoXLLMHeadModel(snake_case_	)

      __snake_case								=       load_tf_weights_in_transfo_xl(snake_case_	,				snake_case_	,				snake_case_	)
      # Save pytorch-model
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      print(f"""Save PyTorch model to {os.path.abspath(snake_case_	)}"""	)
      torch.save(model.state_dict()	,				snake_case_	)
      print(f"""Save configuration file to {os.path.abspath(snake_case_	)}"""	)
      with open(snake_case_	,				'''w'''	,				encoding='''utf-8'''	) as f:
         f.write(config.to_json_string()	)


if __name__ == "__main__":
    snake_case_           =      argparse.ArgumentParser()
    parser.add_argument(
        '--pytorch_dump_folder_path',
        default=None,
        type=str,
        required=True,
        help='Path to the folder to store the PyTorch model or dataset/vocab.',
    )
    parser.add_argument(
        '--tf_checkpoint_path',
        default='',
        type=str,
        help='An optional path to a TensorFlow checkpoint path to be converted.',
    )
    parser.add_argument(
        '--transfo_xl_config_file',
        default='',
        type=str,
        help=(
            'An optional config json file corresponding to the pre-trained BERT model. \n'
            'This specifies the model architecture.'
        ),
    )
    parser.add_argument(
        '--transfo_xl_dataset_file',
        default='',
        type=str,
        help='An optional dataset file to be converted in a vocabulary.',
    )
    snake_case_           =      parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )

