






"""simple docstring"""


import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase    :   Dict   			=						get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
class 						snake_case  (  UpperCAmelCase       ,			unittest.TestCase	):
				__magic_name__   					=					XLMProphetNetTokenizer
				__magic_name__   					=					False
				__magic_name__   					=					True





				def     lowerCamelCase__				(							self     : Dict      ):


						'''simple docstring'''




						super().setUp()

						# We have a SentencePiece fixture for testing
						a					:      int    =       XLMProphetNetTokenizer(_lowercase    ,		keep_accents=_lowercase      )
						tokenizer.save_pretrained(self.tmpdirname      )





				def     lowerCamelCase__				(							self     : str      ):


						'''simple docstring'''




						a					:      Optional[Any]    =       '[PAD]'
						a					:      Union[str, Any]    =       0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase      )    ,		_lowercase      )
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase      )    ,		_lowercase      )





				def     lowerCamelCase__				(							self     : Optional[Any]      ):


						'''simple docstring'''




						a					:      int    =       list(self.get_tokenizer().get_vocab().keys()      )

						self.assertEqual(vocab_keys[0]    ,		'[PAD]'      )
						self.assertEqual(vocab_keys[1]    ,		'[CLS]'      )
						self.assertEqual(vocab_keys[-1]    ,		'j'      )
						self.assertEqual(len(_lowercase      )    ,		1_0_1_2      )





				def     lowerCamelCase__				(							self     : str      ):


						'''simple docstring'''




						self.assertEqual(self.get_tokenizer().vocab_size    ,		1_0_1_2      )





				def     lowerCamelCase__				(							self     : Any      ):


						'''simple docstring'''




						a					:      Union[str, Any]    =       XLMProphetNetTokenizer(_lowercase    ,		keep_accents=_lowercase      )

						a					:      Tuple    =       tokenizer.tokenize('This is a test'      )
						self.assertListEqual(_lowercase    ,		['▁This', '▁is', '▁a', '▁t', 'est']      )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(_lowercase      )    ,		[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,		)

						a					:      List[str]    =       tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
						self.assertListEqual(
						    _lowercase    ,		[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]    ,		)
						a					:      Any    =       tokenizer.convert_tokens_to_ids(_lowercase      )
						self.assertListEqual(
						    _lowercase    ,		[
						        value + tokenizer.fairseq_offset
						        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, -9, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, -9, 4]
						    ]    ,		)

						a					:      int    =       tokenizer.convert_ids_to_tokens(_lowercase      )
						self.assertListEqual(
						    _lowercase    ,		[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '[UNK]',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '[UNK]',
						        '.',
						    ]    ,		)





				@cached_property
				def     lowerCamelCase__				(							self     : Tuple      ):


						'''simple docstring'''




						return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'      )





				@slow
				def     lowerCamelCase__				(							self     : Optional[int]      ):


						'''simple docstring'''




						a					:      Optional[Any]    =       'Hello World!'
						a					:      Dict    =       [3_5_3_8_9, 6_6_7_2, 4_9, 2]
						self.assertListEqual(_lowercase    ,		self.big_tokenizer.encode(_lowercase      )      )





				@slow
				def     lowerCamelCase__				(							self     : List[str]      ):


						'''simple docstring'''




						a					:      Optional[int]    =       {'input_ids': [[1_1_0_7_3, 8_2_7_8_3, 1_8, 2_6, 8_2_7_8_3, 5_4_9, 5_1_5_4_0, 2_4_8, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 2_1_5_1_8_6, 1_3_2_5, 1_4_7, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 5_6_3_7_0, 5_3, 1_2_2_0_2_0, 2_0, 1_6_4_7_7, 2_7, 8_7_3_5_5, 4_5_4_8, 2_0, 4_7_2_8, 7_8_3_9_2, 1_7, 1_5_9_9_6_9, 1_8, 2_6, 2_4_4_9_1, 6_2_9, 1_5, 5_3_8, 2_2_7_0_4, 5_4_3_9, 1_5, 2_7_8_8, 2_4_4_9_1, 9_8_8_5, 1_5, 4_3_5_3_4, 6_0_5, 1_5, 8_1_4, 1_8_4_0_3, 3_3_2_0_0, 2_9, 1_5, 4_3_5_3_4, 2_4_4_5_8, 1_2_4_1_0, 1_1_1, 2_4_9_6_6, 8_3_6_6_9, 9_6_3_7, 1_4_4_0_6_8, 2_6, 8_5_0, 2_2_3_4_6, 2_7, 1_4_7, 2_4_9_6_6, 8_3_6_6_9, 8_3_4_9_0, 2_6, 3_9_1_1_3, 7_3_5, 2_7, 6_8_9, 6_5_6, 2_8_0_0, 1_3_3_9, 4_6_0_0, 5_3, 1_2_2_0_2_0, 1_1_5_7_8_5, 3_4, 8_1_6, 1_3_3_9, 4_6_8_8_7, 1_8, 1_4_7, 5_3_9_0_5, 1_9_5_1, 4_2_2_3_8, 4_1_1_7_0, 1_7_7_3_2, 8_3_4, 4_3_6, 1_5, 2_7_5_2_3, 9_8_7_3_3, 2_1_7, 1_4_7, 5_5_4_2, 4_9_8_1, 9_3_0, 1_7_3_4_7, 1_6, 2], [2_0_0_9_1, 6_2_9, 9_4, 8_2_7_8_6, 5_8, 4_9_0, 2_0, 1_5_2_8, 8_4, 5_3_9_0_5, 3_4_4, 8_0_5_9_2, 1_1_0_1_2_8, 1_8_8_2_2, 5_2_6_7, 1_3_0_6, 6_2, 1_5_2_5_3_7, 3_0_8, 7_9_9_7, 4_0_1, 1_2_4_4_2_7, 5_4_9, 3_5_4_4_2, 2_2_5, 1_0_9, 1_5_0_5_5, 2_5_7_4_8, 1_4_7, 7_1_1_9, 4_3_7_1_2, 3_4, 7_6_7, 1_3_5_3_6_6, 1_8, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_9_2, 6_3_7_8_4, 1_1_9_4_6_6, 1_7, 1_4_7_8_0_8, 8_8_2_1_4, 1_8, 6_5_6, 8_1, 3_2, 3_2_9_6, 1_0_2_8_0, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=_lowercase    ,		model_name='microsoft/xprophetnet-large-wiki100-cased'    ,		revision='1acad1643ddd54a44df6a1b797ada8373685d90e'    ,		)












"""simple docstring"""


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCamelCase    :   Optional[Any]   			=						re.compile(r'\b(a|an|the)\b', re.UNICODE)

_UpperCamelCase    :   str   			=						None




def       snake_case  ():







  '''simple docstring'''

  a					:      Any    =       argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'				)
  parser.add_argument('data_file'     ,      metavar='data.json'     ,      help='Input data JSON file.'				)
  parser.add_argument('pred_file'     ,      metavar='pred.json'     ,      help='Model predictions.'				)
  parser.add_argument(
      '--out-file'     ,      '-o'     ,      metavar='eval.json'     ,      help='Write accuracy metrics to file (default is stdout).'				)
  parser.add_argument(
      '--na-prob-file'     ,      '-n'     ,      metavar='na_prob.json'     ,      help='Model estimates of probability of no answer.'				)
  parser.add_argument(
      '--na-prob-thresh'     ,      '-t'     ,      type=A_     ,      default=1.0     ,      help='Predict "" if no-answer probability exceeds this (default = 1.0).'     ,      )
  parser.add_argument(
      '--out-image-dir'     ,      '-p'     ,      metavar='out_images'     ,      default=A_     ,      help='Save precision-recall curves to directory.'				)
  parser.add_argument('--verbose'     ,      '-v'     ,      action='store_true'				)
  if len(sys.argv				) == 1:
    parser.print_help()
    sys.exit(1				)
  return parser.parse_args()




def       snake_case  (A_				:Optional[int]				):







  '''simple docstring'''

  a					:      str    =       {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a					:      Optional[int]    =       bool(qa['answers']['text']				)
  return qid_to_has_ans




def       snake_case  (A_				:List[Any]				):







  '''simple docstring'''

  def remove_articles(A_				:str				):
    return ARTICLES_REGEX.sub(' '     ,      A_				)

  def white_space_fix(A_				:str				):
    return " ".join(text.split()				)

  def remove_punc(A_				:Dict				):
    a					:      int    =       set(string.punctuation				)
    return "".join(ch for ch in text if ch not in exclude				)

  def lower(A_				:Union[str, Any]				):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(A_				)				)				)				)




def       snake_case  (A_				:str				):







  '''simple docstring'''

  if not s:
    return []
  return normalize_answer(A_				).split()




def       snake_case  (A_				:int     ,      A_				:Union[str, Any]				):







  '''simple docstring'''

  return int(normalize_answer(A_				) == normalize_answer(A_				)				)




def       snake_case  (A_				:Optional[int]     ,      A_				:str				):







  '''simple docstring'''

  a					:      int    =       get_tokens(A_				)
  a					:      Tuple    =       get_tokens(A_				)
  a					:      List[Any]    =       collections.Counter(A_				) & collections.Counter(A_				)
  a					:      Dict    =       sum(common.values()				)
  if len(A_				) == 0 or len(A_				) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks				)
  if num_same == 0:
    return 0
  a					:      List[Any]    =       1.0 * num_same / len(A_				)
  a					:      Optional[Any]    =       1.0 * num_same / len(A_				)
  a					:      Union[str, Any]    =       (2 * precision * recall) / (precision + recall)
  return fa




def       snake_case  (A_				:Any     ,      A_				:Dict				):







  '''simple docstring'''

  a					:      Union[str, Any]    =       {}
  a					:      List[Any]    =       {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a					:      str    =       qa['id']
        a					:      Dict    =       [t for t in qa['answers']['text'] if normalize_answer(A_				)]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          a					:      Dict    =       ['']
        if qid not in preds:
          print(f'''Missing prediction for {qid}'''				)
          continue
        a					:      Optional[Any]    =       preds[qid]
        # Take max over all gold answers
        a					:      str    =       max(compute_exact(A_     ,      A_				) for a in gold_answers				)
        a					:      List[Any]    =       max(compute_fa(A_     ,      A_				) for a in gold_answers				)
  return exact_scores, fa_scores




def       snake_case  (A_				:Union[str, Any]     ,      A_				:List[Any]     ,      A_				:List[Any]     ,      A_				:Dict				):







  '''simple docstring'''

  a					:      List[str]    =       {}
  for qid, s in scores.items():
    a					:      Union[str, Any]    =       na_probs[qid] > na_prob_thresh
    if pred_na:
      a					:      int    =       float(not qid_to_has_ans[qid]				)
    else:
      a					:      Union[str, Any]    =       s
  return new_scores




def       snake_case  (A_				:Tuple     ,      A_				:int     ,      A_				:Tuple=None				):







  '''simple docstring'''

  if not qid_list:
    a					:      Optional[int]    =       len(A_				)
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores.values()				) / total),
            ('f1', 100.0 * sum(fa_scores.values()				) / total),
            ('total', total),
        ]				)
  else:
    a					:      List[Any]    =       len(A_				)
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores[k] for k in qid_list				) / total),
            ('f1', 100.0 * sum(fa_scores[k] for k in qid_list				) / total),
            ('total', total),
        ]				)




def       snake_case  (A_				:str     ,      A_				:Dict     ,      A_				:List[Any]				):







  '''simple docstring'''

  for k in new_eval:
    a					:      Union[str, Any]    =       new_eval[k]




def       snake_case  (A_				:Optional[Any]     ,      A_				:Any     ,      A_				:Dict     ,      A_				:Optional[int]				):







  '''simple docstring'''

  plt.step(A_     ,      A_     ,      color='b'     ,      alpha=0.2     ,      where='post'				)
  plt.fill_between(A_     ,      A_     ,      step='post'     ,      alpha=0.2     ,      color='b'				)
  plt.xlabel('Recall'				)
  plt.ylabel('Precision'				)
  plt.xlim([0.0, 1.05]				)
  plt.ylim([0.0, 1.05]				)
  plt.title(A_				)
  plt.savefig(A_				)
  plt.clf()




def       snake_case  (A_				:List[str]     ,      A_				:str     ,      A_				:Any     ,      A_				:Any     ,      A_				:List[Any]=None     ,      A_				:Union[str, Any]=None				):







  '''simple docstring'''

  a					:      Optional[int]    =       sorted(A_     ,      key=lambda A_				: na_probs[k]				)
  a					:      Tuple    =       0.0
  a					:      Tuple    =       1.0
  a					:      Any    =       0.0
  a					:      int    =       [1.0]
  a					:      int    =       [0.0]
  a					:      str    =       0.0
  for i, qid in enumerate(A_				):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    a					:      Tuple    =       true_pos / float(i + 1				)
    a					:      Any    =       true_pos / float(A_				)
    if i == len(A_				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(A_				)
      recalls.append(A_				)
  if out_image:
    plot_pr_curve(A_     ,      A_     ,      A_     ,      A_				)
  return {"ap": 100.0 * avg_prec}




def       snake_case  (A_				:Optional[int]     ,      A_				:Any     ,      A_				:List[Any]     ,      A_				:int     ,      A_				:int     ,      A_				:List[str]				):







  '''simple docstring'''

  if out_image_dir and not os.path.exists(A_				):
    os.makedirs(A_				)
  a					:      List[str]    =       sum(1 for v in qid_to_has_ans.values() if v				)
  if num_true_pos == 0:
    return
  a					:      List[str]    =       make_precision_recall_eval(
      A_     ,      A_     ,      A_     ,      A_     ,      out_image=os.path.join(A_     ,      'pr_exact.png'				)     ,      title='Precision-Recall curve for Exact Match score'     ,      )
  a					:      Optional[Any]    =       make_precision_recall_eval(
      A_     ,      A_     ,      A_     ,      A_     ,      out_image=os.path.join(A_     ,      'pr_f1.png'				)     ,      title='Precision-Recall curve for F1 score'     ,      )
  a					:      Any    =       {k: float(A_				) for k, v in qid_to_has_ans.items()}
  a					:      Optional[int]    =       make_precision_recall_eval(
      A_     ,      A_     ,      A_     ,      A_     ,      out_image=os.path.join(A_     ,      'pr_oracle.png'				)     ,      title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'     ,      )
  merge_eval(A_     ,      A_     ,      'pr_exact'				)
  merge_eval(A_     ,      A_     ,      'pr_f1'				)
  merge_eval(A_     ,      A_     ,      'pr_oracle'				)




def       snake_case  (A_				:List[str]     ,      A_				:List[str]     ,      A_				:List[Any]     ,      A_				:str				):







  '''simple docstring'''

  if not qid_list:
    return
  a					:      List[Any]    =       [na_probs[k] for k in qid_list]
  a					:      List[str]    =       np.ones_like(A_				) / float(len(A_				)				)
  plt.hist(A_     ,      weights=A_     ,      bins=2_0     ,      range=(0.0, 1.0)				)
  plt.xlabel('Model probability of no-answer'				)
  plt.ylabel('Proportion of dataset'				)
  plt.title(f'''Histogram of no-answer probability: {name}'''				)
  plt.savefig(os.path.join(A_     ,      f'''na_prob_hist_{name}.png'''				)				)
  plt.clf()




def       snake_case  (A_				:Tuple     ,      A_				:Tuple     ,      A_				:List[str]     ,      A_				:List[str]				):







  '''simple docstring'''

  a					:      Any    =       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
  a					:      List[str]    =       num_no_ans
  a					:      List[str]    =       cur_score
  a					:      str    =       0.0
  a					:      Union[str, Any]    =       sorted(A_     ,      key=lambda A_				: na_probs[k]				)
  for i, qid in enumerate(A_				):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      a					:      Optional[int]    =       scores[qid]
    else:
      if preds[qid]:
        a					:      Dict    =       -1
      else:
        a					:      Optional[Any]    =       0
    cur_score += diff
    if cur_score > best_score:
      a					:      List[Any]    =       cur_score
      a					:      str    =       na_probs[qid]
  return 100.0 * best_score / len(A_				), best_thresh




def       snake_case  (A_				:List[Any]     ,      A_				:List[Any]     ,      A_				:str     ,      A_				:int     ,      A_				:Optional[Any]     ,      A_				:Union[str, Any]				):







  '''simple docstring'''

  a,					a					:      Any    =       find_best_thresh(A_     ,      A_     ,      A_     ,      A_				)
  a,					a					:      List[Any]    =       find_best_thresh(A_     ,      A_     ,      A_     ,      A_				)
  a					:      Union[str, Any]    =       best_exact
  a					:      List[Any]    =       exact_thresh
  a					:      List[str]    =       best_fa
  a					:      Any    =       fa_thresh




def       snake_case  ():







  '''simple docstring'''

  with open(OPTS.data_file				) as f:
    a					:      List[str]    =       json.load(A_				)
    a					:      Tuple    =       dataset_json['data']
  with open(OPTS.pred_file				) as f:
    a					:      List[Any]    =       json.load(A_				)
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file				) as f:
      a					:      int    =       json.load(A_				)
  else:
    a					:      List[Any]    =       {k: 0.0 for k in preds}
  a					:      List[str]    =       make_qid_to_has_ans(A_				)  # maps qid to True/False
  a					:      Union[str, Any]    =       [k for k, v in qid_to_has_ans.items() if v]
  a					:      Dict    =       [k for k, v in qid_to_has_ans.items() if not v]
  a,					a					:      List[Any]    =       get_raw_scores(A_     ,      A_				)
  a					:      Any    =       apply_no_ans_threshold(A_     ,      A_     ,      A_     ,      OPTS.na_prob_thresh				)
  a					:      Any    =       apply_no_ans_threshold(A_     ,      A_     ,      A_     ,      OPTS.na_prob_thresh				)
  a					:      Union[str, Any]    =       make_eval_dict(A_     ,      A_				)
  if has_ans_qids:
    a					:      Dict    =       make_eval_dict(A_     ,      A_     ,      qid_list=A_				)
    merge_eval(A_     ,      A_     ,      'HasAns'				)
  if no_ans_qids:
    a					:      Tuple    =       make_eval_dict(A_     ,      A_     ,      qid_list=A_				)
    merge_eval(A_     ,      A_     ,      'NoAns'				)
  if OPTS.na_prob_file:
    find_all_best_thresh(A_     ,      A_     ,      A_     ,      A_     ,      A_     ,      A_				)
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(A_     ,      A_     ,      A_     ,      A_     ,      A_     ,      OPTS.out_image_dir				)
    histogram_na_prob(A_     ,      A_     ,      OPTS.out_image_dir     ,      'hasAns'				)
    histogram_na_prob(A_     ,      A_     ,      OPTS.out_image_dir     ,      'noAns'				)
  if OPTS.out_file:
    with open(OPTS.out_file     ,      'w'				) as f:
      json.dump(A_     ,      A_				)
  else:
    print(json.dumps(A_     ,      indent=2				)				)


if __name__ == "__main__":
   _UpperCamelCase    :   Tuple   			=						parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use('Agg')
      import matplotlib.pyplot as plt
   main()




