"""simple docstring"""
from typing import Any




class        __snake_case	:



						def __init__(						self   ,      lowercase)     ->      List[Any]:





								'''simple docstring'''





								a__:					Tuple      							=     data
								a__:					Optional[int]      							=     None



						def __repr__(						self)     ->      str:





								'''simple docstring'''





								return f'Node({self.data})'







class        __snake_case	:



						def __init__(						self)     ->      List[str]:





								'''simple docstring'''





								a__:					List[Any]      							=     None



						def __iter__(						self)     ->      Any:





								'''simple docstring'''





								a__:					Any      							=     self.head
								while node:
										yield node.data
										a__:					Optional[Any]      							=     node.next



						def __len__(						self)     ->      int:





								'''simple docstring'''





								return sum(1 for _ in self)



						def __repr__(						self)     ->      str:





								'''simple docstring'''





								return "->".join([str(lowercase) for item in self])



						def __getitem__(						self   ,      lowercase)     ->      Any:





								'''simple docstring'''





								if not 0 <= index < len(self):
										raise ValueError('list index out of range.')
								for i, node in enumerate(self):
										if i == index:
												return node
								return None



						def __setitem__(						self   ,      lowercase   ,      lowercase)     ->      None:





								'''simple docstring'''





								if not 0 <= index < len(self):
										raise ValueError('list index out of range.')
								a__:					Union[str, Any]      							=     self.head
								for _ in range(lowercase):
										a__:					Optional[int]      							=     current.next
								a__:					int      							=     data



						def   lowerCamelCase_       (						self   ,      lowercase)     ->      None:





								'''simple docstring'''





								self.insert_nth(len(self)   ,      lowercase)



						def   lowerCamelCase_       (						self   ,      lowercase)     ->      None:





								'''simple docstring'''





								self.insert_nth(0   ,      lowercase)



						def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase)     ->      None:





								'''simple docstring'''





								if not 0 <= index <= len(self):
										raise IndexError('list index out of range')
								a__:					Optional[int]      							=     Node(lowercase)
								if self.head is None:
										a__:					str      							=     new_node
								elif index == 0:
										a__:					Dict      							=     self.head  # link new_node to head
										a__:					Union[str, Any]      							=     new_node
								else:
										a__:					Dict      							=     self.head
										for _ in range(index - 1):
												a__:					str      							=     temp.next
										a__:					Tuple      							=     temp.next
										a__:					Optional[int]      							=     new_node



						def   lowerCamelCase_       (						self)     ->      None:  # print every node data





								'''simple docstring'''





								print(self)



						def   lowerCamelCase_       (						self)     ->      Any:





								'''simple docstring'''





								return self.delete_nth(0)



						def   lowerCamelCase_       (						self)     ->      Any:  # delete from tail





								'''simple docstring'''





								return self.delete_nth(len(self) - 1)



						def   lowerCamelCase_       (						self   ,      lowercase = 0)     ->      Any:





								'''simple docstring'''





								if not 0 <= index <= len(self) - 1:  # test if index is valid
										raise IndexError('List index out of range.')
								a__:					str      							=     self.head  # default first node
								if index == 0:
										a__:					Optional[Any]      							=     self.head.next
								else:
										a__:					Optional[Any]      							=     self.head
										for _ in range(index - 1):
												a__:					int      							=     temp.next
										a__:					Union[str, Any]      							=     temp.next
										a__:					Optional[int]      							=     temp.next.next
								return delete_node.data



						def   lowerCamelCase_       (						self)     ->      bool:





								'''simple docstring'''





								return self.head is None



						def   lowerCamelCase_       (						self)     ->      None:





								'''simple docstring'''





								a__:					Optional[int]      							=     None
								a__:					List[Any]      							=     self.head

								while current:
										# Store the current node's next node.
										a__:					Tuple      							=     current.next
										# Make the current node's next point backwards
										a__:					Union[str, Any]      							=     prev
										# Make the previous node be the current node
										a__:					Any      							=     current
										# Make the current node the next node (to progress iteration)
										a__:					List[str]      							=     next_node
								# Return prev in order to put the head at the end
								a__:					Any      							=     prev







def 						__a       (     )     ->None:
		a__:					Optional[int]      							=     LinkedList()
		assert linked_list.is_empty() is True
		assert str(_SCREAMING_SNAKE_CASE       ) == ""

		try:
				linked_list.delete_head()
				raise AssertionError  # This should not happen.
		except IndexError:
				assert True  # This should happen.

		try:
				linked_list.delete_tail()
				raise AssertionError  # This should not happen.
		except IndexError:
				assert True  # This should happen.

		for i in range(10       ):
				assert len(_SCREAMING_SNAKE_CASE       ) == i
				linked_list.insert_nth(_SCREAMING_SNAKE_CASE		,					i + 1       )
		assert str(_SCREAMING_SNAKE_CASE       ) == "->".join(str(_SCREAMING_SNAKE_CASE       ) for i in range(1		,					11       )       )

		linked_list.insert_head(0       )
		linked_list.insert_tail(11       )
		assert str(_SCREAMING_SNAKE_CASE       ) == "->".join(str(_SCREAMING_SNAKE_CASE       ) for i in range(0		,					12       )       )

		assert linked_list.delete_head() == 0
		assert linked_list.delete_nth(9       ) == 10
		assert linked_list.delete_tail() == 11
		assert len(_SCREAMING_SNAKE_CASE       ) == 9
		assert str(_SCREAMING_SNAKE_CASE       ) == "->".join(str(_SCREAMING_SNAKE_CASE       ) for i in range(1		,					10       )       )

		assert all(linked_list[i] == i + 1 for i in range(0		,					9       )       ) is True

		for i in range(0		,					9       ):
				a__:					Dict      							=     -i
		assert all(linked_list[i] == -i for i in range(0		,					9       )       ) is True

		linked_list.reverse()
		assert str(_SCREAMING_SNAKE_CASE       ) == "->".join(str(_SCREAMING_SNAKE_CASE       ) for i in range(-8		,					1       )       )







def 						__a       (     )     ->None:
		a__:					int      							=     [
		    -9,
		    100,
		    Node(77345112       ),
		    'dlrow olleH',
		    7,
		    5555,
		    0,
		    -192.55_555,
		    'Hello, world!',
		    77.9,
		    Node(10       ),
		    None,
		    None,
		    12.20,
		]
		a__:					List[Any]      							=     LinkedList()

		for i in test_input:
				linked_list.insert_tail(_SCREAMING_SNAKE_CASE       )

		# Check if it's empty or not
		assert linked_list.is_empty() is False
		assert (
		    str(_SCREAMING_SNAKE_CASE       ) == "-9->100->Node(77345112)->dlrow olleH->7->5555->0->"
		    "-192.55555->Hello, world!->77.9->Node(10)->None->None->12.2"
		)

		# Delete the head
		a__:					Union[str, Any]      							=     linked_list.delete_head()
		assert result == -9
		assert (
		    str(_SCREAMING_SNAKE_CASE       ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
		    "Hello, world!->77.9->Node(10)->None->None->12.2"
		)

		# Delete the tail
		a__:					List[str]      							=     linked_list.delete_tail()
		assert result == 12.2
		assert (
		    str(_SCREAMING_SNAKE_CASE       ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
		    "Hello, world!->77.9->Node(10)->None->None"
		)

		# Delete a node in specific location in linked list
		a__:					Optional[Any]      							=     linked_list.delete_nth(10       )
		assert result is None
		assert (
		    str(_SCREAMING_SNAKE_CASE       ) == "100->Node(77345112)->dlrow olleH->7->5555->0->-192.55555->"
		    "Hello, world!->77.9->Node(10)->None"
		)

		# Add a Node instance to its head
		linked_list.insert_head(Node('Hello again, world!'       )       )
		assert (
		    str(_SCREAMING_SNAKE_CASE       )
		    == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
		    "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None"
		)

		# Add None to its tail
		linked_list.insert_tail(_SCREAMING_SNAKE_CASE       )
		assert (
		    str(_SCREAMING_SNAKE_CASE       )
		    == "Node(Hello again, world!)->100->Node(77345112)->dlrow olleH->"
		    "7->5555->0->-192.55555->Hello, world!->77.9->Node(10)->None->None"
		)

		# Reverse the linked list
		linked_list.reverse()
		assert (
		    str(_SCREAMING_SNAKE_CASE       )
		    == "None->None->Node(10)->77.9->Hello, world!->-192.55555->0->5555->"
		    "7->dlrow olleH->Node(77345112)->100->Node(Hello again, world!)"
		)







def 						__a       (     )     ->Union[str, Any]:
		from doctest import testmod

		testmod()

		a__:					Dict      							=     LinkedList()
		linked_list.insert_head(input('Inserting 1st at head '       ).strip()       )
		linked_list.insert_head(input('Inserting 2nd at head '       ).strip()       )
		print('\nPrint list:'       )
		linked_list.print_list()
		linked_list.insert_tail(input('\nInserting 1st at tail '       ).strip()       )
		linked_list.insert_tail(input('Inserting 2nd at tail '       ).strip()       )
		print('\nPrint list:'       )
		linked_list.print_list()
		print('\nDelete head'       )
		linked_list.delete_head()
		print('Delete tail'       )
		linked_list.delete_tail()
		print('\nPrint list:'       )
		linked_list.print_list()
		print('\nReverse linked list'       )
		linked_list.reverse()
		print('\nPrint list:'       )
		linked_list.print_list()
		print('\nString representation of linked list:'       )
		print(_SCREAMING_SNAKE_CASE       )
		print('\nReading/changing Node data using indexing:'       )
		print(F'Element at Position 1: {linked_list[1]}'       )
		a__:					Any      							=     input('Enter New Value: '       ).strip()
		print('New list:'       )
		print(_SCREAMING_SNAKE_CASE       )
		print(F'length of linked_list is : {len(_SCREAMING_SNAKE_CASE       )}'       )


if __name__ == "__main__":
	main()



"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__ 	=     re.compile(r'\b(a|an|the)\b', re.UNICODE)

lowercase__ 	=     None







def 						__a       (     )     ->List[Any]:
  a__:					Dict      							=     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'       )
  parser.add_argument('data_file'		,					metavar='data.json'		,					help='Input data JSON file.'       )
  parser.add_argument('pred_file'		,					metavar='pred.json'		,					help='Model predictions.'       )
  parser.add_argument(
      '--out-file'		,					'-o'		,					metavar='eval.json'		,					help='Write accuracy metrics to file (default is stdout).'       )
  parser.add_argument(
      '--na-prob-file'		,					'-n'		,					metavar='na_prob.json'		,					help='Model estimates of probability of no answer.'       )
  parser.add_argument(
      '--na-prob-thresh'		,					'-t'		,					type=_SCREAMING_SNAKE_CASE		,					default=1.0		,					help='Predict "" if no-answer probability exceeds this (default = 1.0).'		,					)
  parser.add_argument(
      '--out-image-dir'		,					'-p'		,					metavar='out_images'		,					default=_SCREAMING_SNAKE_CASE		,					help='Save precision-recall curves to directory.'       )
  parser.add_argument('--verbose'		,					'-v'		,					action='store_true'       )
  if len(sys.argv       ) == 1:
    parser.print_help()
    sys.exit(1       )
  return parser.parse_args()







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Union[str, Any]:
  a__:					Optional[int]      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[Any]      							=     bool(qa['answers']['text']       )
  return qid_to_has_ans







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[Any]:

  def remove_articles(_SCREAMING_SNAKE_CASE       ):
    return ARTICLES_REGEX.sub(' '		,					_SCREAMING_SNAKE_CASE       )

  def white_space_fix(_SCREAMING_SNAKE_CASE       ):
    return " ".join(text.split()       )

  def remove_punc(_SCREAMING_SNAKE_CASE       ):
    a__:					Dict      							=     set(string.punctuation       )
    return "".join(ch for ch in text if ch not in exclude       )

  def lower(_SCREAMING_SNAKE_CASE       ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE       )       )       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[int]:
  if not s:
    return []
  return normalize_answer(_SCREAMING_SNAKE_CASE       ).split()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[Any]:
  return int(normalize_answer(_SCREAMING_SNAKE_CASE       ) == normalize_answer(_SCREAMING_SNAKE_CASE       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  a__:					Any      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     collections.Counter(_SCREAMING_SNAKE_CASE       ) & collections.Counter(_SCREAMING_SNAKE_CASE       )
  a__:					Tuple      							=     sum(common.values()       )
  if len(_SCREAMING_SNAKE_CASE       ) == 0 or len(_SCREAMING_SNAKE_CASE       ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks       )
  if num_same == 0:
    return 0
  a__:					Any      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Dict      							=     (2 * precision * recall) / (precision + recall)
  return fa







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Dict:
  a__:					Union[str, Any]      							=     {}
  a__:					Dict      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[int]      							=     qa['id']
        a__:					List[Any]      							=     [t for t in qa['answers']['text'] if normalize_answer(_SCREAMING_SNAKE_CASE       )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          a__:					str      							=     ['']
        if qid not in preds:
          print(F'Missing prediction for {qid}'       )
          continue
        a__:					Any      							=     preds[qid]
        # Take max over all gold answers
        a__:					List[str]      							=     max(compute_exact(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
        a__:					Optional[int]      							=     max(compute_fa(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
  return exact_scores, fa_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Optional[Any]:
  a__:					List[str]      							=     {}
  for qid, s in scores.items():
    a__:					List[Any]      							=     na_probs[qid] > na_prob_thresh
    if pred_na:
      a__:					Optional[int]      							=     float(not qid_to_has_ans[qid]       )
    else:
      a__:					Optional[Any]      							=     s
  return new_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None       )     ->Tuple:
  if not qid_list:
    a__:					str      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores.values()       ) / total),
            ('f1', 100.0 * sum(fa_scores.values()       ) / total),
            ('total', total),
        ]       )
  else:
    a__:					Optional[Any]      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores[k] for k in qid_list       ) / total),
            ('f1', 100.0 * sum(fa_scores[k] for k in qid_list       ) / total),
            ('total', total),
        ]       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  for k in new_eval:
    a__:					List[Any]      							=     new_eval[k]







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  plt.step(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					color='b'		,					alpha=0.2		,					where='post'       )
  plt.fill_between(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					step='post'		,					alpha=0.2		,					color='b'       )
  plt.xlabel('Recall'       )
  plt.ylabel('Precision'       )
  plt.xlim([0.0, 1.05]       )
  plt.ylim([0.0, 1.05]       )
  plt.title(_SCREAMING_SNAKE_CASE       )
  plt.savefig(_SCREAMING_SNAKE_CASE       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None       )     ->List[str]:
  a__:					Optional[int]      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  a__:					Dict      							=     0.0
  a__:					Optional[int]      							=     1.0
  a__:					Tuple      							=     0.0
  a__:					Tuple      							=     [1.0]
  a__:					Optional[Any]      							=     [0.0]
  a__:					Optional[Any]      							=     0.0
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    a__:					Optional[Any]      							=     true_pos / float(i + 1       )
    a__:					int      							=     true_pos / float(_SCREAMING_SNAKE_CASE       )
    if i == len(_SCREAMING_SNAKE_CASE       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(_SCREAMING_SNAKE_CASE       )
      recalls.append(_SCREAMING_SNAKE_CASE       )
  if out_image:
    plot_pr_curve(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  return {"ap": 100.0 * avg_prec}







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->int:
  if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE       ):
    os.makedirs(_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     sum(1 for v in qid_to_has_ans.values() if v       )
  if num_true_pos == 0:
    return
  a__:					Optional[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_exact.png'       )		,					title='Precision-Recall curve for Exact Match score'		,					)
  a__:					List[str]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_f1.png'       )		,					title='Precision-Recall curve for F1 score'		,					)
  a__:					Optional[Any]      							=     {k: float(_SCREAMING_SNAKE_CASE       ) for k, v in qid_to_has_ans.items()}
  a__:					List[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_oracle.png'       )		,					title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'		,					)
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_exact'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_f1'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_oracle'       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  if not qid_list:
    return
  a__:					Any      							=     [na_probs[k] for k in qid_list]
  a__:					List[str]      							=     np.ones_like(_SCREAMING_SNAKE_CASE       ) / float(len(_SCREAMING_SNAKE_CASE       )       )
  plt.hist(_SCREAMING_SNAKE_CASE		,					weights=_SCREAMING_SNAKE_CASE		,					bins=20		,					range=(0.0, 1.0)       )
  plt.xlabel('Model probability of no-answer'       )
  plt.ylabel('Proportion of dataset'       )
  plt.title(F'Histogram of no-answer probability: {name}'       )
  plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE		,					F'na_prob_hist_{name}.png'       )       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__:					str      							=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
  a__:					List[Any]      							=     num_no_ans
  a__:					Union[str, Any]      							=     cur_score
  a__:					Optional[Any]      							=     0.0
  a__:					str      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      a__:					Tuple      							=     scores[qid]
    else:
      if preds[qid]:
        a__:					Optional[Any]      							=     -1
      else:
        a__:					Optional[int]      							=     0
    cur_score += diff
    if cur_score > best_score:
      a__:					Dict      							=     cur_score
      a__:					Optional[int]      							=     na_probs[qid]
  return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE       ), best_thresh







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__							,   a__:					str      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__							,   a__:					Optional[int]      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					List[Any]      							=     best_exact
  a__:					Dict      							=     exact_thresh
  a__:					Optional[int]      							=     best_fa
  a__:					str      							=     fa_thresh







def 						__a       (     )     ->int:
  with open(OPTS.data_file       ) as f:
    a__:					Tuple      							=     json.load(_SCREAMING_SNAKE_CASE       )
    a__:					Union[str, Any]      							=     dataset_json['data']
  with open(OPTS.pred_file       ) as f:
    a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file       ) as f:
      a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  else:
    a__:					Optional[Any]      							=     {k: 0.0 for k in preds}
  a__:					List[Any]      							=     make_qid_to_has_ans(_SCREAMING_SNAKE_CASE       )  # maps qid to True/False
  a__:					Optional[int]      							=     [k for k, v in qid_to_has_ans.items() if v]
  a__:					Union[str, Any]      							=     [k for k, v in qid_to_has_ans.items() if not v]
  a__							,   a__:					Optional[Any]      							=     get_raw_scores(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					Dict      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					str      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if has_ans_qids:
    a__:					List[str]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'HasAns'       )
  if no_ans_qids:
    a__:					Optional[Any]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'NoAns'       )
  if OPTS.na_prob_file:
    find_all_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'hasAns'       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'noAns'       )
  if OPTS.out_file:
    with open(OPTS.out_file		,					'w'       ) as f:
      json.dump(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  else:
    print(json.dumps(_SCREAMING_SNAKE_CASE		,					indent=2       )       )


if __name__ == "__main__":
 lowercase__ 	=     parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
 main()


