"""simple docstring"""
import inspect
import math
import tempfile
import unittest

import numpy as np

from transformers import ViTMAEConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import ViTMAEForPreTraining, ViTMAEModel
 from transformers.models.vit.modeling_vit import VIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import ViTImageProcessor




class        __snake_case	:



      def __init__(						self   ,      lowercase   ,      lowercase=13   ,      lowercase=30   ,      lowercase=2   ,      lowercase=3   ,      lowercase=True   ,      lowercase=True   ,      lowercase=32   ,      lowercase=5   ,      lowercase=4   ,      lowercase=37   ,      lowercase="gelu"   ,      lowercase=0.1   ,      lowercase=0.1   ,      lowercase=10   ,      lowercase=0.02   ,      lowercase=3   ,      lowercase=0.6   ,      lowercase=None   ,      )     ->      str:





        '''simple docstring'''





        a__:					Any      							=     parent
        a__:					Any      							=     batch_size
        a__:					int      							=     image_size
        a__:					str      							=     patch_size
        a__:					Optional[Any]      							=     num_channels
        a__:					Tuple      							=     is_training
        a__:					Optional[int]      							=     use_labels
        a__:					Tuple      							=     hidden_size
        a__:					List[str]      							=     num_hidden_layers
        a__:					List[str]      							=     num_attention_heads
        a__:					Dict      							=     intermediate_size
        a__:					Any      							=     hidden_act
        a__:					str      							=     hidden_dropout_prob
        a__:					Optional[Any]      							=     attention_probs_dropout_prob
        a__:					Optional[int]      							=     type_sequence_label_size
        a__:					List[str]      							=     initializer_range
        a__:					Dict      							=     mask_ratio
        a__:					Union[str, Any]      							=     scope

        # in ViTMAE, the expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above
        # (we add 1 for the [CLS] token)
        a__:					str      							=     (image_size // patch_size) ** 2
        a__:					Tuple      							=     int(math.ceil((1 - mask_ratio) * (num_patches + 1)))



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					List[str]      							=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        a__:					Union[str, Any]      							=     None
        if self.use_labels:
          a__:					List[str]      							=     ids_tensor([self.batch_size]   ,      self.type_sequence_label_size)

        a__:					str      							=     self.get_config()

        return config, pixel_values, labels



      def   lowerCamelCase_       (						self)     ->      List[Any]:





        '''simple docstring'''





        return ViTMAEConfig(
            image_size=self.image_size   ,      patch_size=self.patch_size   ,      num_channels=self.num_channels   ,      hidden_size=self.hidden_size   ,      num_hidden_layers=self.num_hidden_layers   ,      num_attention_heads=self.num_attention_heads   ,      intermediate_size=self.intermediate_size   ,      hidden_act=self.hidden_act   ,      hidden_dropout_prob=self.hidden_dropout_prob   ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,      is_decoder=lowercase   ,      initializer_range=self.initializer_range   ,      mask_ratio=self.mask_ratio   ,      )



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      List[str]:





        '''simple docstring'''





        a__:					Optional[int]      							=     ViTMAEModel(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					int      							=     model(lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape   ,      (self.batch_size, self.seq_length, self.hidden_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Tuple:





        '''simple docstring'''





        a__:					List[str]      							=     ViTMAEForPreTraining(lowercase)
        model.to(lowercase)
        model.eval()
        a__:					List[str]      							=     model(lowercase)
        a__:					int      							=     (self.image_size // self.patch_size) ** 2
        a__:					Tuple      							=     self.patch_size**2 * self.num_channels
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, num_patches, expected_num_channels))

        # test greyscale images
        a__:					Tuple      							=     1
        a__:					List[str]      							=     ViTMAEForPreTraining(lowercase)
        model.to(lowercase)
        model.eval()
        a__:					Optional[int]      							=     floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        a__:					Union[str, Any]      							=     model(lowercase)
        a__:					Tuple      							=     self.patch_size**2
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, num_patches, expected_num_channels))



      def   lowerCamelCase_       (						self)     ->      int:





        '''simple docstring'''





        a__:					Optional[Any]      							=     self.prepare_config_and_inputs()
        a__							,   a__							,   a__:					str      							=     config_and_inputs
        a__:					List[str]      							=     {'pixel_values': pixel_values}
        return config, inputs_dict







@require_torch
class        __snake_case	(					__lowerCAmelCase	,				__lowerCAmelCase	,				unittest.TestCase    ):
      a__				=		(ViTMAEModel, ViTMAEForPreTraining) if is_torch_available() else ()
      a__				=		{"""feature-extraction""": ViTMAEModel} if is_torch_available() else {}

      a__				=		False
      a__				=		False
      a__				=		False
      a__				=		False



      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        a__:					Dict      							=     ViTMAEModelTester(self)
        a__:					List[Any]      							=     ConfigTester(self   ,      config_class=lowercase   ,      has_text_modality=lowercase   ,      hidden_size=37)



      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        self.config_tester.run_common_tests()



      @unittest.skip(reason='ViTMAE does not use inputs_embeds')
      def   lowerCamelCase_       (						self)     ->      Tuple:





        '''simple docstring'''





        pass



      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        a__							,   a__:					int      							=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__:					Tuple      							=     model_class(lowercase)
          self.assertIsInstance(model.get_input_embeddings()   ,      (nn.Module))
          a__:					Tuple      							=     model.get_output_embeddings()
          self.assertTrue(x is None or isinstance(lowercase   ,      nn.Linear))



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__							,   a__:					List[Any]      							=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__:					Union[str, Any]      							=     model_class(lowercase)
          a__:					List[str]      							=     inspect.signature(model.forward)
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          a__:					Dict      							=     [*signature.parameters.keys()]

          a__:					Dict      							=     ['pixel_values']
          self.assertListEqual(arg_names[:1]   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					int      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowercase)



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__:					Any      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*lowercase)



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Dict:





        '''simple docstring'''





        np.random.seed(2)

        a__:					List[Any]      							=     int((pt_model.config.image_size // pt_model.config.patch_size) ** 2)
        a__:					List[str]      							=     np.random.uniform(size=(self.model_tester.batch_size, num_patches))
        a__:					Any      							=     torch.from_numpy(lowercase)

        # Add `noise` argument.
        # PT inputs will be prepared in `super().check_pt_tf_models()` with this added `noise` argument
        a__:					str      							=     pt_noise

        super().check_pt_tf_models(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      List[Any]:





        '''simple docstring'''





        a__							,   a__:					int      							=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__:					Dict      							=     model_class(lowercase)
          model.to(lowercase)
          model.eval()
          # make random mask reproducible
          torch.manual_seed(2)
          with torch.no_grad():
            a__:					Optional[Any]      							=     model(**self._prepare_for_class(lowercase   ,      lowercase))

          a__:					Any      							=     outputs[0].cpu().numpy()
          a__:					str      							=     0

          with tempfile.TemporaryDirectory() as tmpdirname:
            model.save_pretrained(lowercase)
            a__:					Optional[Any]      							=     model_class.from_pretrained(lowercase)
            model.to(lowercase)
            # make random mask reproducible
            torch.manual_seed(2)
            with torch.no_grad():
              a__:					Any      							=     model(**self._prepare_for_class(lowercase   ,      lowercase))

            # Make sure we don't have nans
            a__:					Optional[int]      							=     after_outputs[0].cpu().numpy()
            a__:					int      							=     0
            a__:					int      							=     np.amax(np.abs(out_a - out_a))
            self.assertLessEqual(lowercase   ,      1e-5)



      @unittest.skip(
          reason='ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load\n    to get deterministic results.')
      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        pass



      @unittest.skip(
          reason='ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load\n    to get deterministic results.')
      def   lowerCamelCase_       (						self)     ->      Optional[Any]:





        '''simple docstring'''





        pass



      @unittest.skip(
          reason='ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load\n    to get deterministic results.')
      def   lowerCamelCase_       (						self)     ->      Tuple:





        '''simple docstring'''





        pass



      @unittest.skip(reason='ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load')
      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        pass



      @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.')
      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        pass



      @slow
      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        for model_name in VIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          a__:					Any      							=     ViTMAEModel.from_pretrained(lowercase)
          self.assertIsNotNone(lowercase)







def 						__a       (     )     ->Dict:
  a__:					Dict      							=     Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'       )
  return image




@require_torch
@require_vision
class        __snake_case	(					unittest.TestCase    ):



      @cached_property
      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        return ViTImageProcessor.from_pretrained('facebook/vit-mae-base') if is_vision_available() else None



      @slow
      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        np.random.seed(2)

        a__:					Optional[Any]      							=     ViTMAEForPreTraining.from_pretrained('facebook/vit-mae-base').to(lowercase)

        a__:					int      							=     self.default_image_processor
        a__:					Optional[Any]      							=     prepare_img()
        a__:					int      							=     image_processor(images=lowercase   ,      return_tensors='pt').to(lowercase)

        # prepare a noise vector that will be also used for testing the TF model
        # (this way we can ensure that the PT and TF models operate on the same inputs)
        a__:					Any      							=     ViTMAEConfig()
        a__:					Tuple      							=     int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2)
        a__:					Union[str, Any]      							=     np.random.uniform(size=(1, num_patches))

        # forward pass
        with torch.no_grad():
          a__:					Dict      							=     model(**lowercase   ,      noise=torch.from_numpy(lowercase).to(device=lowercase))

        # verify the logits
        a__:					Dict      							=     torch.Size((1, 1_96, 7_68))
        self.assertEqual(outputs.logits.shape   ,      lowercase)

        a__:					List[str]      							=     torch.tensor(
            [[-0.0548, -1.7023, -0.9325], [0.3721, -0.5670, -0.2233], [0.8235, -1.3878, -0.3524]])

        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]   ,      expected_slice.to(lowercase)   ,      atol=1e-4))



"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__ 	=     re.compile(r'\b(a|an|the)\b', re.UNICODE)

lowercase__ 	=     None







def 						__a       (     )     ->List[Any]:
  a__:					Dict      							=     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'       )
  parser.add_argument('data_file'		,					metavar='data.json'		,					help='Input data JSON file.'       )
  parser.add_argument('pred_file'		,					metavar='pred.json'		,					help='Model predictions.'       )
  parser.add_argument(
      '--out-file'		,					'-o'		,					metavar='eval.json'		,					help='Write accuracy metrics to file (default is stdout).'       )
  parser.add_argument(
      '--na-prob-file'		,					'-n'		,					metavar='na_prob.json'		,					help='Model estimates of probability of no answer.'       )
  parser.add_argument(
      '--na-prob-thresh'		,					'-t'		,					type=_SCREAMING_SNAKE_CASE		,					default=1.0		,					help='Predict "" if no-answer probability exceeds this (default = 1.0).'		,					)
  parser.add_argument(
      '--out-image-dir'		,					'-p'		,					metavar='out_images'		,					default=_SCREAMING_SNAKE_CASE		,					help='Save precision-recall curves to directory.'       )
  parser.add_argument('--verbose'		,					'-v'		,					action='store_true'       )
  if len(sys.argv       ) == 1:
    parser.print_help()
    sys.exit(1       )
  return parser.parse_args()







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Union[str, Any]:
  a__:					Optional[int]      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[Any]      							=     bool(qa['answers']['text']       )
  return qid_to_has_ans







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[Any]:

  def remove_articles(_SCREAMING_SNAKE_CASE       ):
    return ARTICLES_REGEX.sub(' '		,					_SCREAMING_SNAKE_CASE       )

  def white_space_fix(_SCREAMING_SNAKE_CASE       ):
    return " ".join(text.split()       )

  def remove_punc(_SCREAMING_SNAKE_CASE       ):
    a__:					Dict      							=     set(string.punctuation       )
    return "".join(ch for ch in text if ch not in exclude       )

  def lower(_SCREAMING_SNAKE_CASE       ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE       )       )       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE       )     ->Optional[int]:
  if not s:
    return []
  return normalize_answer(_SCREAMING_SNAKE_CASE       ).split()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[Any]:
  return int(normalize_answer(_SCREAMING_SNAKE_CASE       ) == normalize_answer(_SCREAMING_SNAKE_CASE       )       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  a__:					Any      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     get_tokens(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     collections.Counter(_SCREAMING_SNAKE_CASE       ) & collections.Counter(_SCREAMING_SNAKE_CASE       )
  a__:					Tuple      							=     sum(common.values()       )
  if len(_SCREAMING_SNAKE_CASE       ) == 0 or len(_SCREAMING_SNAKE_CASE       ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks       )
  if num_same == 0:
    return 0
  a__:					Any      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Optional[int]      							=     1.0 * num_same / len(_SCREAMING_SNAKE_CASE       )
  a__:					Dict      							=     (2 * precision * recall) / (precision + recall)
  return fa







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Dict:
  a__:					Union[str, Any]      							=     {}
  a__:					Dict      							=     {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        a__:					Optional[int]      							=     qa['id']
        a__:					List[Any]      							=     [t for t in qa['answers']['text'] if normalize_answer(_SCREAMING_SNAKE_CASE       )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          a__:					str      							=     ['']
        if qid not in preds:
          print(F'Missing prediction for {qid}'       )
          continue
        a__:					Any      							=     preds[qid]
        # Take max over all gold answers
        a__:					List[str]      							=     max(compute_exact(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
        a__:					Optional[int]      							=     max(compute_fa(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       ) for a in gold_answers       )
  return exact_scores, fa_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->Optional[Any]:
  a__:					List[str]      							=     {}
  for qid, s in scores.items():
    a__:					List[Any]      							=     na_probs[qid] > na_prob_thresh
    if pred_na:
      a__:					Optional[int]      							=     float(not qid_to_has_ans[qid]       )
    else:
      a__:					Optional[Any]      							=     s
  return new_scores







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None       )     ->Tuple:
  if not qid_list:
    a__:					str      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores.values()       ) / total),
            ('f1', 100.0 * sum(fa_scores.values()       ) / total),
            ('total', total),
        ]       )
  else:
    a__:					Optional[Any]      							=     len(_SCREAMING_SNAKE_CASE       )
    return collections.OrderedDict(
        [
            ('exact', 100.0 * sum(exact_scores[k] for k in qid_list       ) / total),
            ('f1', 100.0 * sum(fa_scores[k] for k in qid_list       ) / total),
            ('total', total),
        ]       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  for k in new_eval:
    a__:					List[Any]      							=     new_eval[k]







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->str:
  plt.step(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					color='b'		,					alpha=0.2		,					where='post'       )
  plt.fill_between(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					step='post'		,					alpha=0.2		,					color='b'       )
  plt.xlabel('Recall'       )
  plt.ylabel('Precision'       )
  plt.xlim([0.0, 1.05]       )
  plt.ylim([0.0, 1.05]       )
  plt.title(_SCREAMING_SNAKE_CASE       )
  plt.savefig(_SCREAMING_SNAKE_CASE       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE=None		,					_SCREAMING_SNAKE_CASE=None       )     ->List[str]:
  a__:					Optional[int]      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  a__:					Dict      							=     0.0
  a__:					Optional[int]      							=     1.0
  a__:					Tuple      							=     0.0
  a__:					Tuple      							=     [1.0]
  a__:					Optional[Any]      							=     [0.0]
  a__:					Optional[Any]      							=     0.0
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    a__:					Optional[Any]      							=     true_pos / float(i + 1       )
    a__:					int      							=     true_pos / float(_SCREAMING_SNAKE_CASE       )
    if i == len(_SCREAMING_SNAKE_CASE       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(_SCREAMING_SNAKE_CASE       )
      recalls.append(_SCREAMING_SNAKE_CASE       )
  if out_image:
    plot_pr_curve(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  return {"ap": 100.0 * avg_prec}







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->int:
  if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE       ):
    os.makedirs(_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     sum(1 for v in qid_to_has_ans.values() if v       )
  if num_true_pos == 0:
    return
  a__:					Optional[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_exact.png'       )		,					title='Precision-Recall curve for Exact Match score'		,					)
  a__:					List[str]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_f1.png'       )		,					title='Precision-Recall curve for F1 score'		,					)
  a__:					Optional[Any]      							=     {k: float(_SCREAMING_SNAKE_CASE       ) for k, v in qid_to_has_ans.items()}
  a__:					List[Any]      							=     make_precision_recall_eval(
      _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					out_image=os.path.join(_SCREAMING_SNAKE_CASE		,					'pr_oracle.png'       )		,					title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'		,					)
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_exact'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_f1'       )
  merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'pr_oracle'       )







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  if not qid_list:
    return
  a__:					Any      							=     [na_probs[k] for k in qid_list]
  a__:					List[str]      							=     np.ones_like(_SCREAMING_SNAKE_CASE       ) / float(len(_SCREAMING_SNAKE_CASE       )       )
  plt.hist(_SCREAMING_SNAKE_CASE		,					weights=_SCREAMING_SNAKE_CASE		,					bins=20		,					range=(0.0, 1.0)       )
  plt.xlabel('Model probability of no-answer'       )
  plt.ylabel('Proportion of dataset'       )
  plt.title(F'Histogram of no-answer probability: {name}'       )
  plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE		,					F'na_prob_hist_{name}.png'       )       )
  plt.clf()







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__:					str      							=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
  a__:					List[Any]      							=     num_no_ans
  a__:					Union[str, Any]      							=     cur_score
  a__:					Optional[Any]      							=     0.0
  a__:					str      							=     sorted(_SCREAMING_SNAKE_CASE		,					key=lambda _SCREAMING_SNAKE_CASE       : na_probs[k]       )
  for i, qid in enumerate(_SCREAMING_SNAKE_CASE       ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      a__:					Tuple      							=     scores[qid]
    else:
      if preds[qid]:
        a__:					Optional[Any]      							=     -1
      else:
        a__:					Optional[int]      							=     0
    cur_score += diff
    if cur_score > best_score:
      a__:					Dict      							=     cur_score
      a__:					Optional[int]      							=     na_probs[qid]
  return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE       ), best_thresh







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )     ->List[str]:
  a__							,   a__:					str      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__							,   a__:					Optional[int]      							=     find_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					List[Any]      							=     best_exact
  a__:					Dict      							=     exact_thresh
  a__:					Optional[int]      							=     best_fa
  a__:					str      							=     fa_thresh







def 						__a       (     )     ->int:
  with open(OPTS.data_file       ) as f:
    a__:					Tuple      							=     json.load(_SCREAMING_SNAKE_CASE       )
    a__:					Union[str, Any]      							=     dataset_json['data']
  with open(OPTS.pred_file       ) as f:
    a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file       ) as f:
      a__:					Dict      							=     json.load(_SCREAMING_SNAKE_CASE       )
  else:
    a__:					Optional[Any]      							=     {k: 0.0 for k in preds}
  a__:					List[Any]      							=     make_qid_to_has_ans(_SCREAMING_SNAKE_CASE       )  # maps qid to True/False
  a__:					Optional[int]      							=     [k for k, v in qid_to_has_ans.items() if v]
  a__:					Union[str, Any]      							=     [k for k, v in qid_to_has_ans.items() if not v]
  a__							,   a__:					Optional[Any]      							=     get_raw_scores(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  a__:					Any      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					Dict      							=     apply_no_ans_threshold(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.na_prob_thresh       )
  a__:					str      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if has_ans_qids:
    a__:					List[str]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'HasAns'       )
  if no_ans_qids:
    a__:					Optional[Any]      							=     make_eval_dict(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					qid_list=_SCREAMING_SNAKE_CASE       )
    merge_eval(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					'NoAns'       )
  if OPTS.na_prob_file:
    find_all_best_thresh(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'hasAns'       )
    histogram_na_prob(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					OPTS.out_image_dir		,					'noAns'       )
  if OPTS.out_file:
    with open(OPTS.out_file		,					'w'       ) as f:
      json.dump(_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE       )
  else:
    print(json.dumps(_SCREAMING_SNAKE_CASE		,					indent=2       )       )


if __name__ == "__main__":
 lowercase__ 	=     parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
 main()


