def 		__lowerCamelCase							(UpperCAmelCase__       :							list					):
					for i in range(len(UpperCAmelCase__					) - 1 ,	0 ,	-1					):
										SCREAMING_SNAKE_CASE           =    False

										for j in range(UpperCAmelCase__ ,	0 ,	-1					):
															if unsorted[j] < unsorted[j - 1]:
																				SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    unsorted[j - 1], unsorted[j]
																				SCREAMING_SNAKE_CASE           =    True

										for j in range(UpperCAmelCase__					):
															if unsorted[j] > unsorted[j + 1]:
																				SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    unsorted[j + 1], unsorted[j]
																				SCREAMING_SNAKE_CASE           =    True

										if not swapped:
															break
					return unsorted


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							_lowerCamelCase   :				Tuple     			=						input('''Enter numbers separated by a comma:\n''').strip()
							_lowerCamelCase   :				List[Any]     			=						[int(item) for item in user_input.split(''',''')]
							print(f"""{cocktail_shaker_sort(unsorted) = }""")





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCamelCase   :				Dict     			=						re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

_lowerCamelCase   :				Optional[int]     			=						None





def 		__lowerCamelCase							():
					SCREAMING_SNAKE_CASE           =    argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."					)
					parser.add_argument("data_file" ,	metavar="data.json" ,	help="Input data JSON file."					)
					parser.add_argument("pred_file" ,	metavar="pred.json" ,	help="Model predictions."					)
					parser.add_argument(
					    "--out-file" ,	"-o" ,	metavar="eval.json" ,	help="Write accuracy metrics to file (default is stdout)."					)
					parser.add_argument(
					    "--na-prob-file" ,	"-n" ,	metavar="na_prob.json" ,	help="Model estimates of probability of no answer."					)
					parser.add_argument(
					    "--na-prob-thresh" ,	"-t" ,	type=UpperCAmelCase__ ,	default=1.0 ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)." ,	)
					parser.add_argument(
					    "--out-image-dir" ,	"-p" ,	metavar="out_images" ,	default=UpperCAmelCase__ ,	help="Save precision-recall curves to directory."					)
					parser.add_argument("--verbose" ,	"-v" ,	action="store_true"					)
					if len(sys.argv					) == 1:
										parser.print_help()
										sys.exit(1					)
					return parser.parse_args()





def 		__lowerCamelCase							(UpperCAmelCase__       :							Optional[int]					):
					SCREAMING_SNAKE_CASE           =    {}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				SCREAMING_SNAKE_CASE           =    bool(qa["answers"]["text"]					)
					return qid_to_has_ans





def 		__lowerCamelCase							(UpperCAmelCase__       :							Union[str, Any]					):

					def remove_articles(UpperCAmelCase__       :							List[str]					):
										return ARTICLES_REGEX.sub(" " ,	UpperCAmelCase__					)

					def white_space_fix(UpperCAmelCase__       :							Dict					):
										return " ".join(text.split()					)

					def remove_punc(UpperCAmelCase__       :							str					):
										SCREAMING_SNAKE_CASE           =    set(string.punctuation					)
										return "".join(ch for ch in text if ch not in exclude					)

					def lower(UpperCAmelCase__       :							Dict					):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase__					)					)					)					)





def 		__lowerCamelCase							(UpperCAmelCase__       :							List[str]					):
					if not s:
										return []
					return normalize_answer(UpperCAmelCase__					).split()





def 		__lowerCamelCase							(UpperCAmelCase__       :							Dict ,	UpperCAmelCase__       :							List[str]					):
					return int(normalize_answer(UpperCAmelCase__					) == normalize_answer(UpperCAmelCase__					)					)





def 		__lowerCamelCase							(UpperCAmelCase__       :							Any ,	UpperCAmelCase__       :							str					):
					SCREAMING_SNAKE_CASE           =    get_tokens(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    get_tokens(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    collections.Counter(UpperCAmelCase__					) & collections.Counter(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    sum(common.values()					)
					if len(UpperCAmelCase__					) == 0 or len(UpperCAmelCase__					) == 0:
										# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
										return int(gold_toks == pred_toks					)
					if num_same == 0:
										return 0
					SCREAMING_SNAKE_CASE           =    1.0 * num_same / len(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    1.0 * num_same / len(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    (2 * precision * recall) / (precision + recall)
					return fa





def 		__lowerCamelCase							(UpperCAmelCase__       :							Optional[int] ,	UpperCAmelCase__       :							Tuple					):
					SCREAMING_SNAKE_CASE           =    {}
					SCREAMING_SNAKE_CASE           =    {}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				SCREAMING_SNAKE_CASE           =    qa["id"]
																				SCREAMING_SNAKE_CASE           =    [t for t in qa["answers"]["text"] if normalize_answer(UpperCAmelCase__					)]
																				if not gold_answers:
																									# For unanswerable questions, only correct answer is empty string
																									SCREAMING_SNAKE_CASE           =    [""]
																				if qid not in preds:
																									print(F"Missing prediction for {qid}"					)
																									continue
																				SCREAMING_SNAKE_CASE           =    preds[qid]
																				# Take max over all gold answers
																				SCREAMING_SNAKE_CASE           =    max(compute_exact(UpperCAmelCase__ ,	UpperCAmelCase__					) for a in gold_answers					)
																				SCREAMING_SNAKE_CASE           =    max(compute_fa(UpperCAmelCase__ ,	UpperCAmelCase__					) for a in gold_answers					)
					return exact_scores, fa_scores





def 		__lowerCamelCase							(UpperCAmelCase__       :							Union[str, Any] ,	UpperCAmelCase__       :							Tuple ,	UpperCAmelCase__       :							Dict ,	UpperCAmelCase__       :							str					):
					SCREAMING_SNAKE_CASE           =    {}
					for qid, s in scores.items():
										SCREAMING_SNAKE_CASE           =    na_probs[qid] > na_prob_thresh
										if pred_na:
															SCREAMING_SNAKE_CASE           =    float(not qid_to_has_ans[qid]					)
										else:
															SCREAMING_SNAKE_CASE           =    s
					return new_scores





def 		__lowerCamelCase							(UpperCAmelCase__       :							List[str] ,	UpperCAmelCase__       :							str ,	UpperCAmelCase__       :							Dict=None					):
					if not qid_list:
										SCREAMING_SNAKE_CASE           =    len(UpperCAmelCase__					)
										return collections.OrderedDict(
										    [
										        ("exact", 100.0 * sum(exact_scores.values()					) / total),
										        ("f1", 100.0 * sum(fa_scores.values()					) / total),
										        ("total", total),
										    ]					)
					else:
										SCREAMING_SNAKE_CASE           =    len(UpperCAmelCase__					)
										return collections.OrderedDict(
										    [
										        ("exact", 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
										        ("f1", 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
										        ("total", total),
										    ]					)





def 		__lowerCamelCase							(UpperCAmelCase__       :							Union[str, Any] ,	UpperCAmelCase__       :							List[Any] ,	UpperCAmelCase__       :							List[str]					):
					for k in new_eval:
										SCREAMING_SNAKE_CASE           =    new_eval[k]





def 		__lowerCamelCase							(UpperCAmelCase__       :							Optional[int] ,	UpperCAmelCase__       :							int ,	UpperCAmelCase__       :							List[Any] ,	UpperCAmelCase__       :							List[str]					):
					plt.step(UpperCAmelCase__ ,	UpperCAmelCase__ ,	color="b" ,	alpha=0.2 ,	where="post"					)
					plt.fill_between(UpperCAmelCase__ ,	UpperCAmelCase__ ,	step="post" ,	alpha=0.2 ,	color="b"					)
					plt.xlabel("Recall"					)
					plt.ylabel("Precision"					)
					plt.xlim([0.0, 1.05]					)
					plt.ylim([0.0, 1.05]					)
					plt.title(UpperCAmelCase__					)
					plt.savefig(UpperCAmelCase__					)
					plt.clf()





def 		__lowerCamelCase							(UpperCAmelCase__       :							List[str] ,	UpperCAmelCase__       :							Union[str, Any] ,	UpperCAmelCase__       :							Tuple ,	UpperCAmelCase__       :							Optional[Any] ,	UpperCAmelCase__       :							int=None ,	UpperCAmelCase__       :							str=None					):
					SCREAMING_SNAKE_CASE           =    sorted(UpperCAmelCase__ ,	key=lambda UpperCAmelCase__					: na_probs[k]					)
					SCREAMING_SNAKE_CASE           =    0.0
					SCREAMING_SNAKE_CASE           =    1.0
					SCREAMING_SNAKE_CASE           =    0.0
					SCREAMING_SNAKE_CASE           =    [1.0]
					SCREAMING_SNAKE_CASE           =    [0.0]
					SCREAMING_SNAKE_CASE           =    0.0
					for i, qid in enumerate(UpperCAmelCase__					):
										if qid_to_has_ans[qid]:
															true_pos += scores[qid]
										SCREAMING_SNAKE_CASE           =    true_pos / float(i + 1					)
										SCREAMING_SNAKE_CASE           =    true_pos / float(UpperCAmelCase__					)
										if i == len(UpperCAmelCase__					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
															# i.e., if we can put a threshold after this point
															avg_prec += cur_p * (cur_r - recalls[-1])
															precisions.append(UpperCAmelCase__					)
															recalls.append(UpperCAmelCase__					)
					if out_image:
										plot_pr_curve(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__					)
					return {"ap": 100.0 * avg_prec}





def 		__lowerCamelCase							(UpperCAmelCase__       :							Dict ,	UpperCAmelCase__       :							Union[str, Any] ,	UpperCAmelCase__       :							Optional[Any] ,	UpperCAmelCase__       :							List[str] ,	UpperCAmelCase__       :							Any ,	UpperCAmelCase__       :							List[str]					):
					if out_image_dir and not os.path.exists(UpperCAmelCase__					):
										os.makedirs(UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    sum(1 for v in qid_to_has_ans.values() if v					)
					if num_true_pos == 0:
										return
					SCREAMING_SNAKE_CASE           =    make_precision_recall_eval(
					    UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	out_image=os.path.join(UpperCAmelCase__ ,	"pr_exact.png"					) ,	title="Precision-Recall curve for Exact Match score" ,	)
					SCREAMING_SNAKE_CASE           =    make_precision_recall_eval(
					    UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	out_image=os.path.join(UpperCAmelCase__ ,	"pr_f1.png"					) ,	title="Precision-Recall curve for F1 score" ,	)
					SCREAMING_SNAKE_CASE           =    {k: float(UpperCAmelCase__					) for k, v in qid_to_has_ans.items()}
					SCREAMING_SNAKE_CASE           =    make_precision_recall_eval(
					    UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	out_image=os.path.join(UpperCAmelCase__ ,	"pr_oracle.png"					) ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)" ,	)
					merge_eval(UpperCAmelCase__ ,	UpperCAmelCase__ ,	"pr_exact"					)
					merge_eval(UpperCAmelCase__ ,	UpperCAmelCase__ ,	"pr_f1"					)
					merge_eval(UpperCAmelCase__ ,	UpperCAmelCase__ ,	"pr_oracle"					)





def 		__lowerCamelCase							(UpperCAmelCase__       :							List[Any] ,	UpperCAmelCase__       :							Optional[Any] ,	UpperCAmelCase__       :							List[str] ,	UpperCAmelCase__       :							int					):
					if not qid_list:
										return
					SCREAMING_SNAKE_CASE           =    [na_probs[k] for k in qid_list]
					SCREAMING_SNAKE_CASE           =    np.ones_like(UpperCAmelCase__					) / float(len(UpperCAmelCase__					)					)
					plt.hist(UpperCAmelCase__ ,	weights=UpperCAmelCase__ ,	bins=2_0 ,	range=(0.0, 1.0)					)
					plt.xlabel("Model probability of no-answer"					)
					plt.ylabel("Proportion of dataset"					)
					plt.title(F"Histogram of no-answer probability: {name}"					)
					plt.savefig(os.path.join(UpperCAmelCase__ ,	F"na_prob_hist_{name}.png"					)					)
					plt.clf()





def 		__lowerCamelCase							(UpperCAmelCase__       :							int ,	UpperCAmelCase__       :							Optional[Any] ,	UpperCAmelCase__       :							List[str] ,	UpperCAmelCase__       :							Dict					):
					SCREAMING_SNAKE_CASE           =    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
					SCREAMING_SNAKE_CASE           =    num_no_ans
					SCREAMING_SNAKE_CASE           =    cur_score
					SCREAMING_SNAKE_CASE           =    0.0
					SCREAMING_SNAKE_CASE           =    sorted(UpperCAmelCase__ ,	key=lambda UpperCAmelCase__					: na_probs[k]					)
					for i, qid in enumerate(UpperCAmelCase__					):
										if qid not in scores:
															continue
										if qid_to_has_ans[qid]:
															SCREAMING_SNAKE_CASE           =    scores[qid]
										else:
															if preds[qid]:
																				SCREAMING_SNAKE_CASE           =    -1
															else:
																				SCREAMING_SNAKE_CASE           =    0
										cur_score += diff
										if cur_score > best_score:
															SCREAMING_SNAKE_CASE           =    cur_score
															SCREAMING_SNAKE_CASE           =    na_probs[qid]
					return 100.0 * best_score / len(UpperCAmelCase__					), best_thresh





def 		__lowerCamelCase							(UpperCAmelCase__       :							Union[str, Any] ,	UpperCAmelCase__       :							str ,	UpperCAmelCase__       :							Dict ,	UpperCAmelCase__       :							Optional[int] ,	UpperCAmelCase__       :							int ,	UpperCAmelCase__       :							Optional[Any]					):
					SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    find_best_thresh(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    find_best_thresh(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    best_exact
					SCREAMING_SNAKE_CASE           =    exact_thresh
					SCREAMING_SNAKE_CASE           =    best_fa
					SCREAMING_SNAKE_CASE           =    fa_thresh





def 		__lowerCamelCase							():
					with open(OPTS.data_file					) as f:
										SCREAMING_SNAKE_CASE           =    json.load(UpperCAmelCase__					)
										SCREAMING_SNAKE_CASE           =    dataset_json["data"]
					with open(OPTS.pred_file					) as f:
										SCREAMING_SNAKE_CASE           =    json.load(UpperCAmelCase__					)
					if OPTS.na_prob_file:
										with open(OPTS.na_prob_file					) as f:
															SCREAMING_SNAKE_CASE           =    json.load(UpperCAmelCase__					)
					else:
										SCREAMING_SNAKE_CASE           =    {k: 0.0 for k in preds}
					SCREAMING_SNAKE_CASE           =    make_qid_to_has_ans(UpperCAmelCase__					)  # maps qid to True/False
					SCREAMING_SNAKE_CASE           =    [k for k, v in qid_to_has_ans.items() if v]
					SCREAMING_SNAKE_CASE           =    [k for k, v in qid_to_has_ans.items() if not v]
					SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    get_raw_scores(UpperCAmelCase__ ,	UpperCAmelCase__					)
					SCREAMING_SNAKE_CASE           =    apply_no_ans_threshold(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	OPTS.na_prob_thresh					)
					SCREAMING_SNAKE_CASE           =    apply_no_ans_threshold(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	OPTS.na_prob_thresh					)
					SCREAMING_SNAKE_CASE           =    make_eval_dict(UpperCAmelCase__ ,	UpperCAmelCase__					)
					if has_ans_qids:
										SCREAMING_SNAKE_CASE           =    make_eval_dict(UpperCAmelCase__ ,	UpperCAmelCase__ ,	qid_list=UpperCAmelCase__					)
										merge_eval(UpperCAmelCase__ ,	UpperCAmelCase__ ,	"HasAns"					)
					if no_ans_qids:
										SCREAMING_SNAKE_CASE           =    make_eval_dict(UpperCAmelCase__ ,	UpperCAmelCase__ ,	qid_list=UpperCAmelCase__					)
										merge_eval(UpperCAmelCase__ ,	UpperCAmelCase__ ,	"NoAns"					)
					if OPTS.na_prob_file:
										find_all_best_thresh(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__					)
					if OPTS.na_prob_file and OPTS.out_image_dir:
										run_precision_recall_analysis(UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	UpperCAmelCase__ ,	OPTS.out_image_dir					)
										histogram_na_prob(UpperCAmelCase__ ,	UpperCAmelCase__ ,	OPTS.out_image_dir ,	"hasAns"					)
										histogram_na_prob(UpperCAmelCase__ ,	UpperCAmelCase__ ,	OPTS.out_image_dir ,	"noAns"					)
					if OPTS.out_file:
										with open(OPTS.out_file ,	"w"					) as f:
															json.dump(UpperCAmelCase__ ,	UpperCAmelCase__					)
					else:
										print(json.dumps(UpperCAmelCase__ ,	indent=2					)					)


if __name__ == "__main__":
							_lowerCamelCase   :				Optional[Any]     			=						parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use('''Agg''')
														import matplotlib.pyplot as plt
							main()




