"""simple docstring"""
import inspect
import unittest

from transformers import RegNetConfig
from transformers.file_utils import cached_property, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import RegNetForImageClassification, RegNetModel
 from transformers.models.regnet.modeling_regnet import REGNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor




class        __snake_case	:



      def __init__(						self   ,      lowercase   ,      lowercase=3   ,      lowercase=32   ,      lowercase=3   ,      lowercase=10   ,      lowercase=[10, 20, 30, 40]   ,      lowercase=[1, 1, 2, 1]   ,      lowercase=True   ,      lowercase=True   ,      lowercase="relu"   ,      lowercase=3   ,      lowercase=None   ,      )     ->      Any:





        '''simple docstring'''





        a__:					Union[str, Any]      							=     parent
        a__:					int      							=     batch_size
        a__:					List[Any]      							=     image_size
        a__:					Any      							=     num_channels
        a__:					Dict      							=     embeddings_size
        a__:					str      							=     hidden_sizes
        a__:					List[Any]      							=     depths
        a__:					Optional[int]      							=     is_training
        a__:					Optional[int]      							=     use_labels
        a__:					Tuple      							=     hidden_act
        a__:					Any      							=     num_labels
        a__:					Union[str, Any]      							=     scope
        a__:					Tuple      							=     len(lowercase)



      def   lowerCamelCase_       (						self)     ->      int:





        '''simple docstring'''





        a__:					Optional[int]      							=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        a__:					str      							=     None
        if self.use_labels:
          a__:					Dict      							=     ids_tensor([self.batch_size]   ,      self.num_labels)

        a__:					int      							=     self.get_config()

        return config, pixel_values, labels



      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        return RegNetConfig(
            num_channels=self.num_channels   ,      embeddings_size=self.embeddings_size   ,      hidden_sizes=self.hidden_sizes   ,      depths=self.depths   ,      hidden_act=self.hidden_act   ,      num_labels=self.num_labels   ,      )



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Any:





        '''simple docstring'''





        a__:					Dict      							=     RegNetModel(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					List[str]      							=     model(lowercase)
        # expected last hidden states: B, C, H // 32, W // 32
        self.parent.assertEqual(
            result.last_hidden_state.shape   ,      (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)   ,      )



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      List[Any]:





        '''simple docstring'''





        a__:					Tuple      							=     self.num_labels
        a__:					Tuple      							=     RegNetForImageClassification(lowercase)
        model.to(lowercase)
        model.eval()
        a__:					Any      							=     model(lowercase   ,      labels=lowercase)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.num_labels))



      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        a__:					int      							=     self.prepare_config_and_inputs()
        a__							,   a__							,   a__:					List[Any]      							=     config_and_inputs
        a__:					Union[str, Any]      							=     {'pixel_values': pixel_values}
        return config, inputs_dict







@require_torch
class        __snake_case	(					__lowerCAmelCase	,				__lowerCAmelCase	,				unittest.TestCase    ):
      a__				=		(RegNetModel, RegNetForImageClassification) if is_torch_available() else ()
      a__				=		(
          {"""feature-extraction""": RegNetModel, """image-classification""": RegNetForImageClassification}
          if is_torch_available()
          else {}
      )

      a__				=		False
      a__				=		False
      a__				=		False
      a__				=		False



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					Dict      							=     RegNetModelTester(self)
        a__:					Dict      							=     ConfigTester(self   ,      config_class=lowercase   ,      has_text_modality=lowercase)



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()



      def   lowerCamelCase_       (						self)     ->      Optional[Any]:





        '''simple docstring'''





        return



      @unittest.skip(reason='RegNet does not use inputs_embeds')
      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        pass



      @unittest.skip(reason='RegNet does not support input and output embeddings')
      def   lowerCamelCase_       (						self)     ->      Optional[Any]:





        '''simple docstring'''





        pass



      def   lowerCamelCase_       (						self)     ->      List[Any]:





        '''simple docstring'''





        a__							,   a__:					str      							=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__:					int      							=     model_class(lowercase)
          a__:					str      							=     inspect.signature(model.forward)
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          a__:					int      							=     [*signature.parameters.keys()]

          a__:					List[str]      							=     ['pixel_values']
          self.assertListEqual(arg_names[:1]   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Optional[Any]:





        '''simple docstring'''





        a__:					Any      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__							,   a__:					int      							=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__:					int      							=     model_class(config=lowercase)
          for name, module in model.named_modules():
            if isinstance(lowercase   ,      (nn.BatchNormad, nn.GroupNorm)):
              self.assertTrue(
                  torch.all(module.weight == 1)   ,      msg=f'Parameter {name} of model {model_class} seems not properly initialized'   ,      )
              self.assertTrue(
                  torch.all(module.bias == 0)   ,      msg=f'Parameter {name} of model {model_class} seems not properly initialized'   ,      )



      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        def check_hidden_states_output(lowercase   ,      lowercase   ,      lowercase):
          a__:					int      							=     model_class(lowercase)
          model.to(lowercase)
          model.eval()

          with torch.no_grad():
            a__:					int      							=     model(**self._prepare_for_class(lowercase   ,      lowercase))

          a__:					Any      							=     outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

          a__:					List[str]      							=     self.model_tester.num_stages
          self.assertEqual(len(lowercase)   ,      expected_num_stages + 1)

          # RegNet's feature maps are of shape (batch_size, num_channels, height, width)
          self.assertListEqual(
              list(hidden_states[0].shape[-2:])   ,      [self.model_tester.image_size // 2, self.model_tester.image_size // 2]   ,      )

        a__							,   a__:					Optional[Any]      							=     self.model_tester.prepare_config_and_inputs_for_common()
        a__:					Dict      							=     ['basic', 'bottleneck']
        for model_class in self.all_model_classes:
          for layer_type in layers_type:
            a__:					Dict      							=     layer_type
            a__:					Dict      							=     True
            check_hidden_states_output(lowercase   ,      lowercase   ,      lowercase)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            a__:					str      							=     True

            check_hidden_states_output(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      List[Any]:





        '''simple docstring'''





        a__:					Optional[Any]      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*lowercase)



      @slow
      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        for model_name in REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          a__:					Tuple      							=     RegNetModel.from_pretrained(lowercase)
          self.assertIsNotNone(lowercase)







def 						__a       (     )     ->Dict:
  a__:					str      							=     Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'       )
  return image




@require_torch
@require_vision
class        __snake_case	(					unittest.TestCase    ):



      @cached_property
      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        return (
            AutoImageProcessor.from_pretrained(REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0])
            if is_vision_available()
            else None
        )



      @slow
      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        a__:					Tuple      							=     RegNetForImageClassification.from_pretrained(REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]).to(lowercase)

        a__:					Tuple      							=     self.default_image_processor
        a__:					str      							=     prepare_img()
        a__:					Optional[int]      							=     image_processor(images=lowercase   ,      return_tensors='pt').to(lowercase)

        # forward pass
        with torch.no_grad():
          a__:					List[Any]      							=     model(**lowercase)

        # verify the logits
        a__:					Tuple      							=     torch.Size((1, 10_00))
        self.assertEqual(outputs.logits.shape   ,      lowercase)

        a__:					Optional[Any]      							=     torch.tensor([-0.4180, -1.5051, -3.4836]).to(lowercase)

        self.assertTrue(torch.allclose(outputs.logits[0, :3]   ,      lowercase   ,      atol=1e-4))



"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__ 	=     re.compile(r'\b(a|an|the)\b', re.UNICODE)

lowercase__ 	=     None







def 						__a       (     )     ->List[Any]:
  a__:					Dict      							=     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'       )
  parser.add_argument('data_file'		,					metavar='data.json'		,					help='Input data JSON file.'       )
  parser.add_argument('pred_file'		,					metavar='pred.json'		,					help='Model predictions.'       )
  parser.add_argument(
      '--out-file'		,					'-o'		,					metavar='eval.json'		,					help='Write accuracy metrics to file (default is stdout).'       )
  parser.add_argument(
      '--na-prob-file'		,					'-n'		,					metavar='na_prob.json'		,					help='Model estimates of probability of no answer.'       )
  parser.add_argument(
      '--na-prob-thresh'		,					'-t'		,					type=_SCREAMING_SNAKE_CASE		,					default=1.0		,					help='Predict "" if no-answer probability exceeds this (default = 1.0).'		,					)
  parser.add_argument(
      '--out-image-dir'		,					'-p'		,					metavar='out_images'		,					default=_SCREAMING_SNAKE_CASE		,					help='Save precision-recall curves to directory.'       )
  parser.add_argument('--verbose'		,					'-v'		,					action='store_true'       )
  if len(sys.argv       ) == 1:
    parser.print_help()
    sys.exit(1       )
  return parser.parse_args()







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Union[str, Any]:
  a__:					Optional[int]      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[Any]      							=     bool(qa['answers']['text']       )
  return qid_to_has_ans







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[Any]:

  def remove_articles(_SCREAMING_SNAKE_CASE       ):
    return ARTICLES_REGEX.sub(' '		,					_SCREAMING_SNAKE_CASE       )

  def white_space_fix(_SCREAMING_SNAKE_CASE       ):
    return " ".join(text.split()       )

  def remove_punc(_SCREAMING_SNAKE_CASE       ):
    a__:					Dict      							=     set(string.punctuation       )
    return "".join(ch for ch in text if ch not in exclude       )

  def lower(_SCREAMING_SNAKE_CASE       ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE       )       )       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[int]:
  if not s:
    return []
  return normalize_answer(_SCREAMING_SNAKE_CASE       ).split()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[Any]:
  return int(normalize_answer(_SCREAMING_SNAKE_CASE       ) == normalize_answer(_SCREAMING_SNAKE_CASE       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  a__:					Any      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     collections.Counter(_SCREAMING_SNAKE_CASE       ) & collections.Counter(_SCREAMING_SNAKE_CASE       )
  a__:					Tuple      							=     sum(common.values()       )
  if len(_SCREAMING_SNAKE_CASE       ) == 0 or len(_SCREAMING_SNAKE_CASE       ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks       )
  if num_same == 0:
    return 0
  a__:					Any      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Dict      							=     (2 * precision * recall) / (precision + recall)
  return fa







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Dict:
  a__:					Union[str, Any]      							=     {}
  a__:					Dict      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[int]      							=     qa['id']
        a__:					List[Any]      							=     [t for t in qa['answers']['text'] if normalize_answer(_SCREAMING_SNAKE_CASE       )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          a__:					str      							=     ['']
        if qid not in preds:
          print(F'Missing prediction for {qid}'       )
          continue
        a__:					Any      							=     preds[qid]
        # Take max over all gold answers
        a__:					List[str]      							=     max(compute_exact(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
        a__:					Optional[int]      							=     max(compute_fa(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
  return exact_scores, fa_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Optional[Any]:
  a__:					List[str]      							=     {}
  for qid, s in scores.items():
    a__:					List[Any]      							=     na_probs[qid] > na_prob_thresh
    if pred_na:
      a__:					Optional[int]      							=     float(not qid_to_has_ans[qid]       )
    else:
      a__:					Optional[Any]      							=     s
  return new_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None       )     ->Tuple:
  if not qid_list:
    a__:					str      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores.values()       ) / total),
            ('f1', 100.0 * sum(fa_scores.values()       ) / total),
            ('total', total),
        ]       )
  else:
    a__:					Optional[Any]      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores[k] for k in qid_list       ) / total),
            ('f1', 100.0 * sum(fa_scores[k] for k in qid_list       ) / total),
            ('total', total),
        ]       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  for k in new_eval:
    a__:					List[Any]      							=     new_eval[k]







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  plt.step(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					color='b'		,					alpha=0.2		,					where='post'       )
  plt.fill_between(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					step='post'		,					alpha=0.2		,					color='b'       )
  plt.xlabel('Recall'       )
  plt.ylabel('Precision'       )
  plt.xlim([0.0, 1.05]       )
  plt.ylim([0.0, 1.05]       )
  plt.title(_SCREAMING_SNAKE_CASE       )
  plt.savefig(_SCREAMING_SNAKE_CASE       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None       )     ->List[str]:
  a__:					Optional[int]      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  a__:					Dict      							=     0.0
  a__:					Optional[int]      							=     1.0
  a__:					Tuple      							=     0.0
  a__:					Tuple      							=     [1.0]
  a__:					Optional[Any]      							=     [0.0]
  a__:					Optional[Any]      							=     0.0
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    a__:					Optional[Any]      							=     true_pos / float(i + 1       )
    a__:					int      							=     true_pos / float(_SCREAMING_SNAKE_CASE       )
    if i == len(_SCREAMING_SNAKE_CASE       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(_SCREAMING_SNAKE_CASE       )
      recalls.append(_SCREAMING_SNAKE_CASE       )
  if out_image:
    plot_pr_curve(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  return {"ap": 100.0 * avg_prec}







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->int:
  if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE       ):
    os.makedirs(_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     sum(1 for v in qid_to_has_ans.values() if v       )
  if num_true_pos == 0:
    return
  a__:					Optional[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_exact.png'       )		,					title='Precision-Recall curve for Exact Match score'		,					)
  a__:					List[str]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_f1.png'       )		,					title='Precision-Recall curve for F1 score'		,					)
  a__:					Optional[Any]      							=     {k: float(_SCREAMING_SNAKE_CASE       ) for k, v in qid_to_has_ans.items()}
  a__:					List[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_oracle.png'       )		,					title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'		,					)
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_exact'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_f1'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_oracle'       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  if not qid_list:
    return
  a__:					Any      							=     [na_probs[k] for k in qid_list]
  a__:					List[str]      							=     np.ones_like(_SCREAMING_SNAKE_CASE       ) / float(len(_SCREAMING_SNAKE_CASE       )       )
  plt.hist(_SCREAMING_SNAKE_CASE		,					weights=_SCREAMING_SNAKE_CASE		,					bins=20		,					range=(0.0, 1.0)       )
  plt.xlabel('Model probability of no-answer'       )
  plt.ylabel('Proportion of dataset'       )
  plt.title(F'Histogram of no-answer probability: {name}'       )
  plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE		,					F'na_prob_hist_{name}.png'       )       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__:					str      							=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
  a__:					List[Any]      							=     num_no_ans
  a__:					Union[str, Any]      							=     cur_score
  a__:					Optional[Any]      							=     0.0
  a__:					str      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      a__:					Tuple      							=     scores[qid]
    else:
      if preds[qid]:
        a__:					Optional[Any]      							=     -1
      else:
        a__:					Optional[int]      							=     0
    cur_score += diff
    if cur_score > best_score:
      a__:					Dict      							=     cur_score
      a__:					Optional[int]      							=     na_probs[qid]
  return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE       ), best_thresh







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__							,   a__:					str      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__							,   a__:					Optional[int]      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					List[Any]      							=     best_exact
  a__:					Dict      							=     exact_thresh
  a__:					Optional[int]      							=     best_fa
  a__:					str      							=     fa_thresh







def 						__a       (     )     ->int:
  with open(OPTS.data_file       ) as f:
    a__:					Tuple      							=     json.load(_SCREAMING_SNAKE_CASE       )
    a__:					Union[str, Any]      							=     dataset_json['data']
  with open(OPTS.pred_file       ) as f:
    a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file       ) as f:
      a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  else:
    a__:					Optional[Any]      							=     {k: 0.0 for k in preds}
  a__:					List[Any]      							=     make_qid_to_has_ans(_SCREAMING_SNAKE_CASE       )  # maps qid to True/False
  a__:					Optional[int]      							=     [k for k, v in qid_to_has_ans.items() if v]
  a__:					Union[str, Any]      							=     [k for k, v in qid_to_has_ans.items() if not v]
  a__							,   a__:					Optional[Any]      							=     get_raw_scores(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					Dict      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					str      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if has_ans_qids:
    a__:					List[str]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'HasAns'       )
  if no_ans_qids:
    a__:					Optional[Any]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'NoAns'       )
  if OPTS.na_prob_file:
    find_all_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'hasAns'       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'noAns'       )
  if OPTS.out_file:
    with open(OPTS.out_file		,					'w'       ) as f:
      json.dump(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  else:
    print(json.dumps(_SCREAMING_SNAKE_CASE		,					indent=2       )       )


if __name__ == "__main__":
 lowercase__ 	=     parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
 main()


