




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				:      List[Any]	   = re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				:      Optional[int]	   = None

def  SCREAMING_SNAKE_CASE							(       )      ->					Any:
     a     =				argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.")
     parser.add_argument("data_file"		, metavar="data.json"		, help="Input data JSON file.")
     parser.add_argument("pred_file"		, metavar="pred.json"		, help="Model predictions.")
     parser.add_argument(
         "--out-file"		, "-o"		, metavar="eval.json"		, help="Write accuracy metrics to file (default is stdout).")
     parser.add_argument(
         "--na-prob-file"		, "-n"		, metavar="na_prob.json"		, help="Model estimates of probability of no answer.")
     parser.add_argument(
         "--na-prob-thresh"		, "-t"		, type=__UpperCamelCase		, default=1.0		, help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."		, )
     parser.add_argument(
         "--out-image-dir"		, "-p"		, metavar="out_images"		, default=__UpperCamelCase		, help="Save precision-recall curves to directory.")
     parser.add_argument("--verbose"		, "-v"		, action="store_true")
     if len(sys.argv) == 1:
          parser.print_help()
          sys.exit(1)
     return parser.parse_args()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase)      ->					int:
     a     =				{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a     =				bool(qa["answers"]["text"])
     return qid_to_has_ans

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase)      ->					List[Any]:

     def remove_articles(__UpperCamelCase):
          return ARTICLES_REGEX.sub(" "		, __UpperCamelCase)

     def white_space_fix(__UpperCamelCase):
          return " ".join(text.split())

     def remove_punc(__UpperCamelCase):
          a     =				set(string.punctuation)
          return "".join(ch for ch in text if ch not in exclude)

     def lower(__UpperCamelCase):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__UpperCamelCase))))

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase)      ->					Tuple:
     if not s:
          return []
     return normalize_answer(__UpperCamelCase).split()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase)      ->					str:
     return int(normalize_answer(__UpperCamelCase) == normalize_answer(__UpperCamelCase))

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase)      ->					str:
     a     =				get_tokens(__UpperCamelCase)
     a     =				get_tokens(__UpperCamelCase)
     a     =				collections.Counter(__UpperCamelCase) & collections.Counter(__UpperCamelCase)
     a     =				sum(common.values())
     if len(__UpperCamelCase) == 0 or len(__UpperCamelCase) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks)
     if num_same == 0:
          return 0
     a     =				1.0 * num_same / len(__UpperCamelCase)
     a     =				1.0 * num_same / len(__UpperCamelCase)
     a     =				(2 * precision * recall) / (precision + recall)
     return fa

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase)      ->					str:
     a     =				{}
     a     =				{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a     =				qa["id"]
                    a     =				[t for t in qa["answers"]["text"] if normalize_answer(__UpperCamelCase)]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a     =				[""]
                    if qid not in preds:
                         print(f'''Missing prediction for {qid}''')
                         continue
                    a     =				preds[qid]
                    # Take max over all gold answers
                    a     =				max(compute_exact(__UpperCamelCase		, __UpperCamelCase) for a in gold_answers)
                    a     =				max(compute_fa(__UpperCamelCase		, __UpperCamelCase) for a in gold_answers)
     return exact_scores, fa_scores

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					List[Any]:
     a     =				{}
     for qid, s in scores.items():
          a     =				na_probs[qid] > na_prob_thresh
          if pred_na:
               a     =				float(not qid_to_has_ans[qid])
          else:
               a     =				s
     return new_scores

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase=None)      ->					str:
     if not qid_list:
          a     =				len(__UpperCamelCase)
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores.values()) / total),
                  ("f1", 100.0 * sum(fa_scores.values()) / total),
                  ("total", total),
              ])
     else:
          a     =				len(__UpperCamelCase)
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total),
                  ("f1", 100.0 * sum(fa_scores[k] for k in qid_list) / total),
                  ("total", total),
              ])

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					List[str]:
     for k in new_eval:
          a     =				new_eval[k]

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					List[str]:
     plt.step(__UpperCamelCase		, __UpperCamelCase		, color="b"		, alpha=0.2		, where="post")
     plt.fill_between(__UpperCamelCase		, __UpperCamelCase		, step="post"		, alpha=0.2		, color="b")
     plt.xlabel("Recall")
     plt.ylabel("Precision")
     plt.xlim([0.0, 1.05])
     plt.ylim([0.0, 1.05])
     plt.title(__UpperCamelCase)
     plt.savefig(__UpperCamelCase)
     plt.clf()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase=None		, __UpperCamelCase=None)      ->					int:
     a     =				sorted(__UpperCamelCase		, key=lambda __UpperCamelCase: na_probs[k])
     a     =				0.0
     a     =				1.0
     a     =				0.0
     a     =				[1.0]
     a     =				[0.0]
     a     =				0.0
     for i, qid in enumerate(__UpperCamelCase):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a     =				true_pos / float(i + 1)
          a     =				true_pos / float(__UpperCamelCase)
          if i == len(__UpperCamelCase) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__UpperCamelCase)
               recalls.append(__UpperCamelCase)
     if out_image:
          plot_pr_curve(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)
     return {"ap": 100.0 * avg_prec}

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					int:
     if out_image_dir and not os.path.exists(__UpperCamelCase):
          os.makedirs(__UpperCamelCase)
     a     =				sum(1 for v in qid_to_has_ans.values() if v)
     if num_true_pos == 0:
          return
     a     =				make_precision_recall_eval(
         __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, out_image=os.path.join(__UpperCamelCase		, "pr_exact.png")		, title="Precision-Recall curve for Exact Match score"		, )
     a     =				make_precision_recall_eval(
         __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, out_image=os.path.join(__UpperCamelCase		, "pr_f1.png")		, title="Precision-Recall curve for F1 score"		, )
     a     =				{k: float(__UpperCamelCase) for k, v in qid_to_has_ans.items()}
     a     =				make_precision_recall_eval(
         __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, out_image=os.path.join(__UpperCamelCase		, "pr_oracle.png")		, title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"		, )
     merge_eval(__UpperCamelCase		, __UpperCamelCase		, "pr_exact")
     merge_eval(__UpperCamelCase		, __UpperCamelCase		, "pr_f1")
     merge_eval(__UpperCamelCase		, __UpperCamelCase		, "pr_oracle")

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					Optional[Any]:
     if not qid_list:
          return
     a     =				[na_probs[k] for k in qid_list]
     a     =				np.ones_like(__UpperCamelCase) / float(len(__UpperCamelCase))
     plt.hist(__UpperCamelCase		, weights=__UpperCamelCase		, bins=20		, range=(0.0, 1.0))
     plt.xlabel("Model probability of no-answer")
     plt.ylabel("Proportion of dataset")
     plt.title(f'''Histogram of no-answer probability: {name}''')
     plt.savefig(os.path.join(__UpperCamelCase		, f'''na_prob_hist_{name}.png'''))
     plt.clf()

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					Optional[int]:
     a     =				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
     a     =				num_no_ans
     a     =				cur_score
     a     =				0.0
     a     =				sorted(__UpperCamelCase		, key=lambda __UpperCamelCase: na_probs[k])
     for i, qid in enumerate(__UpperCamelCase):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a     =				scores[qid]
          else:
               if preds[qid]:
                    a     =				-1
               else:
                    a     =				0
          cur_score += diff
          if cur_score > best_score:
               a     =				cur_score
               a     =				na_probs[qid]
     return 100.0 * best_score / len(__UpperCamelCase), best_thresh

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)      ->					Any:
     a  ,				a     =				find_best_thresh(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)
     a  ,				a     =				find_best_thresh(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)
     a     =				best_exact
     a     =				exact_thresh
     a     =				best_fa
     a     =				fa_thresh

def  SCREAMING_SNAKE_CASE							(       )      ->					Union[str, Any]:
     with open(OPTS.data_file) as f:
          a     =				json.load(__UpperCamelCase)
          a     =				dataset_json["data"]
     with open(OPTS.pred_file) as f:
          a     =				json.load(__UpperCamelCase)
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file) as f:
               a     =				json.load(__UpperCamelCase)
     else:
          a     =				{k: 0.0 for k in preds}
     a     =				make_qid_to_has_ans(__UpperCamelCase)  # maps qid to True/False
     a     =				[k for k, v in qid_to_has_ans.items() if v]
     a     =				[k for k, v in qid_to_has_ans.items() if not v]
     a  ,				a     =				get_raw_scores(__UpperCamelCase		, __UpperCamelCase)
     a     =				apply_no_ans_threshold(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, OPTS.na_prob_thresh)
     a     =				apply_no_ans_threshold(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, OPTS.na_prob_thresh)
     a     =				make_eval_dict(__UpperCamelCase		, __UpperCamelCase)
     if has_ans_qids:
          a     =				make_eval_dict(__UpperCamelCase		, __UpperCamelCase		, qid_list=__UpperCamelCase)
          merge_eval(__UpperCamelCase		, __UpperCamelCase		, "HasAns")
     if no_ans_qids:
          a     =				make_eval_dict(__UpperCamelCase		, __UpperCamelCase		, qid_list=__UpperCamelCase)
          merge_eval(__UpperCamelCase		, __UpperCamelCase		, "NoAns")
     if OPTS.na_prob_file:
          find_all_best_thresh(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase)
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, __UpperCamelCase		, OPTS.out_image_dir)
          histogram_na_prob(__UpperCamelCase		, __UpperCamelCase		, OPTS.out_image_dir		, "hasAns")
          histogram_na_prob(__UpperCamelCase		, __UpperCamelCase		, OPTS.out_image_dir		, "noAns")
     if OPTS.out_file:
          with open(OPTS.out_file		, "w") as f:
               json.dump(__UpperCamelCase		, __UpperCamelCase)
     else:
          print(json.dumps(__UpperCamelCase		, indent=2))


if __name__ == "__main__":
   lowercase__				:      List[str]	   = parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use("Agg")
      import matplotlib.pyplot as plt
   main()







import math
import sys

def  SCREAMING_SNAKE_CASE							(       __UpperCamelCase)      ->					int:
     if number != int(__UpperCamelCase):
          raise ValueError("the value of input must be a natural number")
     if number < 0:
          raise ValueError("the value of input must not be a negative number")
     if number == 0:
          return 1
     a     =				[-1] * (number + 1)
     a     =				0
     for i in range(1		, number + 1):
          a     =				sys.maxsize
          a     =				int(math.sqrt(__UpperCamelCase))
          for j in range(1		, root + 1):
               a     =				1 + answers[i - (j**2)]
               a     =				min(__UpperCamelCase		, __UpperCamelCase)
          a     =				answer
     return answers[number]


if __name__ == "__main__":
   import doctest

   doctest.testmod()

