




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase								=					re.compile(R'\b(a|an|the)\b', re.UNICODE)

UpperCamelCase								=					None



def 							_A       (						):
     """simple docstring"""

     lowerCAmelCase__          =			argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."      )
     parser.add_argument("data_file"       ,    metavar="data.json"       ,    help="Input data JSON file."      )
     parser.add_argument("pred_file"       ,    metavar="pred.json"       ,    help="Model predictions."      )
     parser.add_argument(
         "--out-file"       ,    "-o"       ,    metavar="eval.json"       ,    help="Write accuracy metrics to file (default is stdout)."      )
     parser.add_argument(
         "--na-prob-file"       ,    "-n"       ,    metavar="na_prob.json"       ,    help="Model estimates of probability of no answer."      )
     parser.add_argument(
         "--na-prob-thresh"       ,    "-t"       ,    type=lowerCAmelCase_       ,    default=1.0       ,    help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."       ,    )
     parser.add_argument(
         "--out-image-dir"       ,    "-p"       ,    metavar="out_images"       ,    default=lowerCAmelCase_       ,    help="Save precision-recall curves to directory."      )
     parser.add_argument("--verbose"       ,    "-v"       ,    action="store_true"      )
     if len(sys.argv      ) == 1:
          parser.print_help()
          sys.exit(1      )
     return parser.parse_args()



def 							_A       (						lowerCAmelCase_					: Tuple      ):
     """simple docstring"""

     lowerCAmelCase__          =			{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    lowerCAmelCase__          =			bool(qa["answers"]["text"]      )
     return qid_to_has_ans



def 							_A       (						lowerCAmelCase_					: str      ):
     """simple docstring"""

     def remove_articles(lowerCAmelCase_					: Union[str, Any]      ):
          return ARTICLES_REGEX.sub(" "       ,    lowerCAmelCase_      )

     def white_space_fix(lowerCAmelCase_					: Union[str, Any]      ):
          return " ".join(text.split()      )

     def remove_punc(lowerCAmelCase_					: List[str]      ):
          lowerCAmelCase__          =			set(string.punctuation      )
          return "".join(ch for ch in text if ch not in exclude      )

     def lower(lowerCAmelCase_					: Optional[Any]      ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase_      )      )      )      )



def 							_A       (						lowerCAmelCase_					: Dict      ):
     """simple docstring"""

     if not s:
          return []
     return normalize_answer(lowerCAmelCase_      ).split()



def 							_A       (						lowerCAmelCase_					: Tuple       ,    lowerCAmelCase_					: Optional[Any]      ):
     """simple docstring"""

     return int(normalize_answer(lowerCAmelCase_      ) == normalize_answer(lowerCAmelCase_      )      )



def 							_A       (						lowerCAmelCase_					: int       ,    lowerCAmelCase_					: Optional[Any]      ):
     """simple docstring"""

     lowerCAmelCase__          =			get_tokens(lowerCAmelCase_      )
     lowerCAmelCase__          =			get_tokens(lowerCAmelCase_      )
     lowerCAmelCase__          =			collections.Counter(lowerCAmelCase_      ) & collections.Counter(lowerCAmelCase_      )
     lowerCAmelCase__          =			sum(common.values()      )
     if len(lowerCAmelCase_      ) == 0 or len(lowerCAmelCase_      ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks      )
     if num_same == 0:
          return 0
     lowerCAmelCase__          =			1.0 * num_same / len(lowerCAmelCase_      )
     lowerCAmelCase__          =			1.0 * num_same / len(lowerCAmelCase_      )
     lowerCAmelCase__          =			(2 * precision * recall) / (precision + recall)
     return fa



def 							_A       (						lowerCAmelCase_					: Optional[int]       ,    lowerCAmelCase_					: Optional[int]      ):
     """simple docstring"""

     lowerCAmelCase__          =			{}
     lowerCAmelCase__          =			{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    lowerCAmelCase__          =			qa["id"]
                    lowerCAmelCase__          =			[t for t in qa["answers"]["text"] if normalize_answer(lowerCAmelCase_      )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         lowerCAmelCase__          =			[""]
                    if qid not in preds:
                         print(F'Missing prediction for {qid}'      )
                         continue
                    lowerCAmelCase__          =			preds[qid]
                    # Take max over all gold answers
                    lowerCAmelCase__          =			max(compute_exact(lowerCAmelCase_       ,    lowerCAmelCase_      ) for a in gold_answers      )
                    lowerCAmelCase__          =			max(compute_fa(lowerCAmelCase_       ,    lowerCAmelCase_      ) for a in gold_answers      )
     return exact_scores, fa_scores



def 							_A       (						lowerCAmelCase_					: Optional[Any]       ,    lowerCAmelCase_					: List[Any]       ,    lowerCAmelCase_					: Dict       ,    lowerCAmelCase_					: Dict      ):
     """simple docstring"""

     lowerCAmelCase__          =			{}
     for qid, s in scores.items():
          lowerCAmelCase__          =			na_probs[qid] > na_prob_thresh
          if pred_na:
               lowerCAmelCase__          =			float(not qid_to_has_ans[qid]      )
          else:
               lowerCAmelCase__          =			s
     return new_scores



def 							_A       (						lowerCAmelCase_					: int       ,    lowerCAmelCase_					: Union[str, Any]       ,    lowerCAmelCase_					: Optional[Any]=None      ):
     """simple docstring"""

     if not qid_list:
          lowerCAmelCase__          =			len(lowerCAmelCase_      )
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores.values()      ) / total),
                  ("f1", 100.0 * sum(fa_scores.values()      ) / total),
                  ("total", total),
              ]      )
     else:
          lowerCAmelCase__          =			len(lowerCAmelCase_      )
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores[k] for k in qid_list      ) / total),
                  ("f1", 100.0 * sum(fa_scores[k] for k in qid_list      ) / total),
                  ("total", total),
              ]      )



def 							_A       (						lowerCAmelCase_					: Dict       ,    lowerCAmelCase_					: List[Any]       ,    lowerCAmelCase_					: Dict      ):
     """simple docstring"""

     for k in new_eval:
          lowerCAmelCase__          =			new_eval[k]



def 							_A       (						lowerCAmelCase_					: Optional[int]       ,    lowerCAmelCase_					: Dict       ,    lowerCAmelCase_					: List[Any]       ,    lowerCAmelCase_					: Union[str, Any]      ):
     """simple docstring"""

     plt.step(lowerCAmelCase_       ,    lowerCAmelCase_       ,    color="b"       ,    alpha=0.2       ,    where="post"      )
     plt.fill_between(lowerCAmelCase_       ,    lowerCAmelCase_       ,    step="post"       ,    alpha=0.2       ,    color="b"      )
     plt.xlabel("Recall"      )
     plt.ylabel("Precision"      )
     plt.xlim([0.0, 1.05]      )
     plt.ylim([0.0, 1.05]      )
     plt.title(lowerCAmelCase_      )
     plt.savefig(lowerCAmelCase_      )
     plt.clf()



def 							_A       (						lowerCAmelCase_					: int       ,    lowerCAmelCase_					: Union[str, Any]       ,    lowerCAmelCase_					: List[Any]       ,    lowerCAmelCase_					: List[str]       ,    lowerCAmelCase_					: List[str]=None       ,    lowerCAmelCase_					: int=None      ):
     """simple docstring"""

     lowerCAmelCase__          =			sorted(lowerCAmelCase_       ,    key=lambda lowerCAmelCase_      : na_probs[k]      )
     lowerCAmelCase__          =			0.0
     lowerCAmelCase__          =			1.0
     lowerCAmelCase__          =			0.0
     lowerCAmelCase__          =			[1.0]
     lowerCAmelCase__          =			[0.0]
     lowerCAmelCase__          =			0.0
     for i, qid in enumerate(lowerCAmelCase_      ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          lowerCAmelCase__          =			true_pos / float(i + 1      )
          lowerCAmelCase__          =			true_pos / float(lowerCAmelCase_      )
          if i == len(lowerCAmelCase_      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(lowerCAmelCase_      )
               recalls.append(lowerCAmelCase_      )
     if out_image:
          plot_pr_curve(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_      )
     return {"ap": 100.0 * avg_prec}



def 							_A       (						lowerCAmelCase_					: Union[str, Any]       ,    lowerCAmelCase_					: Any       ,    lowerCAmelCase_					: List[str]       ,    lowerCAmelCase_					: Any       ,    lowerCAmelCase_					: Union[str, Any]       ,    lowerCAmelCase_					: List[str]      ):
     """simple docstring"""

     if out_image_dir and not os.path.exists(lowerCAmelCase_      ):
          os.makedirs(lowerCAmelCase_      )
     lowerCAmelCase__          =			sum(1 for v in qid_to_has_ans.values() if v      )
     if num_true_pos == 0:
          return
     lowerCAmelCase__          =			make_precision_recall_eval(
         lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    out_image=os.path.join(lowerCAmelCase_       ,    "pr_exact.png"      )       ,    title="Precision-Recall curve for Exact Match score"       ,    )
     lowerCAmelCase__          =			make_precision_recall_eval(
         lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    out_image=os.path.join(lowerCAmelCase_       ,    "pr_f1.png"      )       ,    title="Precision-Recall curve for F1 score"       ,    )
     lowerCAmelCase__          =			{k: float(lowerCAmelCase_      ) for k, v in qid_to_has_ans.items()}
     lowerCAmelCase__          =			make_precision_recall_eval(
         lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    out_image=os.path.join(lowerCAmelCase_       ,    "pr_oracle.png"      )       ,    title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"       ,    )
     merge_eval(lowerCAmelCase_       ,    lowerCAmelCase_       ,    "pr_exact"      )
     merge_eval(lowerCAmelCase_       ,    lowerCAmelCase_       ,    "pr_f1"      )
     merge_eval(lowerCAmelCase_       ,    lowerCAmelCase_       ,    "pr_oracle"      )



def 							_A       (						lowerCAmelCase_					: Tuple       ,    lowerCAmelCase_					: Tuple       ,    lowerCAmelCase_					: Any       ,    lowerCAmelCase_					: int      ):
     """simple docstring"""

     if not qid_list:
          return
     lowerCAmelCase__          =			[na_probs[k] for k in qid_list]
     lowerCAmelCase__          =			np.ones_like(lowerCAmelCase_      ) / float(len(lowerCAmelCase_      )      )
     plt.hist(lowerCAmelCase_       ,    weights=lowerCAmelCase_       ,    bins=20       ,    range=(0.0, 1.0)      )
     plt.xlabel("Model probability of no-answer"      )
     plt.ylabel("Proportion of dataset"      )
     plt.title(F'Histogram of no-answer probability: {name}'      )
     plt.savefig(os.path.join(lowerCAmelCase_       ,    F'na_prob_hist_{name}.png'      )      )
     plt.clf()



def 							_A       (						lowerCAmelCase_					: Optional[int]       ,    lowerCAmelCase_					: Tuple       ,    lowerCAmelCase_					: Union[str, Any]       ,    lowerCAmelCase_					: Dict      ):
     """simple docstring"""

     lowerCAmelCase__          =			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
     lowerCAmelCase__          =			num_no_ans
     lowerCAmelCase__          =			cur_score
     lowerCAmelCase__          =			0.0
     lowerCAmelCase__          =			sorted(lowerCAmelCase_       ,    key=lambda lowerCAmelCase_      : na_probs[k]      )
     for i, qid in enumerate(lowerCAmelCase_      ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               lowerCAmelCase__          =			scores[qid]
          else:
               if preds[qid]:
                    lowerCAmelCase__          =			-1
               else:
                    lowerCAmelCase__          =			0
          cur_score += diff
          if cur_score > best_score:
               lowerCAmelCase__          =			cur_score
               lowerCAmelCase__          =			na_probs[qid]
     return 100.0 * best_score / len(lowerCAmelCase_      ), best_thresh



def 							_A       (						lowerCAmelCase_					: Optional[Any]       ,    lowerCAmelCase_					: Optional[int]       ,    lowerCAmelCase_					: Tuple       ,    lowerCAmelCase_					: Dict       ,    lowerCAmelCase_					: Dict       ,    lowerCAmelCase_					: List[Any]      ):
     """simple docstring"""

     lowerCAmelCase__     ,		lowerCAmelCase__          =			find_best_thresh(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_      )
     lowerCAmelCase__     ,		lowerCAmelCase__          =			find_best_thresh(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_      )
     lowerCAmelCase__          =			best_exact
     lowerCAmelCase__          =			exact_thresh
     lowerCAmelCase__          =			best_fa
     lowerCAmelCase__          =			fa_thresh



def 							_A       (						):
     """simple docstring"""

     with open(OPTS.data_file      ) as f:
          lowerCAmelCase__          =			json.load(lowerCAmelCase_      )
          lowerCAmelCase__          =			dataset_json["data"]
     with open(OPTS.pred_file      ) as f:
          lowerCAmelCase__          =			json.load(lowerCAmelCase_      )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file      ) as f:
               lowerCAmelCase__          =			json.load(lowerCAmelCase_      )
     else:
          lowerCAmelCase__          =			{k: 0.0 for k in preds}
     lowerCAmelCase__          =			make_qid_to_has_ans(lowerCAmelCase_      )  # maps qid to True/False
     lowerCAmelCase__          =			[k for k, v in qid_to_has_ans.items() if v]
     lowerCAmelCase__          =			[k for k, v in qid_to_has_ans.items() if not v]
     lowerCAmelCase__     ,		lowerCAmelCase__          =			get_raw_scores(lowerCAmelCase_       ,    lowerCAmelCase_      )
     lowerCAmelCase__          =			apply_no_ans_threshold(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    OPTS.na_prob_thresh      )
     lowerCAmelCase__          =			apply_no_ans_threshold(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    OPTS.na_prob_thresh      )
     lowerCAmelCase__          =			make_eval_dict(lowerCAmelCase_       ,    lowerCAmelCase_      )
     if has_ans_qids:
          lowerCAmelCase__          =			make_eval_dict(lowerCAmelCase_       ,    lowerCAmelCase_       ,    qid_list=lowerCAmelCase_      )
          merge_eval(lowerCAmelCase_       ,    lowerCAmelCase_       ,    "HasAns"      )
     if no_ans_qids:
          lowerCAmelCase__          =			make_eval_dict(lowerCAmelCase_       ,    lowerCAmelCase_       ,    qid_list=lowerCAmelCase_      )
          merge_eval(lowerCAmelCase_       ,    lowerCAmelCase_       ,    "NoAns"      )
     if OPTS.na_prob_file:
          find_all_best_thresh(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_      )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    lowerCAmelCase_       ,    OPTS.out_image_dir      )
          histogram_na_prob(lowerCAmelCase_       ,    lowerCAmelCase_       ,    OPTS.out_image_dir       ,    "hasAns"      )
          histogram_na_prob(lowerCAmelCase_       ,    lowerCAmelCase_       ,    OPTS.out_image_dir       ,    "noAns"      )
     if OPTS.out_file:
          with open(OPTS.out_file       ,    "w"      ) as f:
               json.dump(lowerCAmelCase_       ,    lowerCAmelCase_      )
     else:
          print(json.dumps(lowerCAmelCase_       ,    indent=2      )      )


if __name__ == "__main__":
     UpperCamelCase								=					parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('Agg')
          import matplotlib.pyplot as plt
     main()






from maths.prime_factors import prime_factors



def 							_A       (						lowerCAmelCase_					: int      ):
     """simple docstring"""

     if not isinstance(lowerCAmelCase_       ,    lowerCAmelCase_      ):
          lowerCAmelCase__          =			F'Input value of [number={number}] must be an integer'
          raise TypeError(lowerCAmelCase_      )
     if number < 1:
          raise ValueError("Input must be a positive integer"      )
     return -1 if len(prime_factors(lowerCAmelCase_      )      ) % 2 else 1


if __name__ == "__main__":
     import doctest

     doctest.testmod()
