






'''simple docstring'''

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__a:			Tuple	       =       """\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

__a:			List[str]	       =       """\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

__a:			Optional[int]	       =       """
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""
def       __UpperCamelCase	(       UpperCAmelCase		,					UpperCAmelCase					):
 return float((preds == labels).mean()					)
def       __UpperCamelCase	(       UpperCAmelCase		,					UpperCAmelCase		,					UpperCAmelCase="binary"					):
 lowercase__    :   Union[str, Any]						= simple_accuracy(UpperCAmelCase		,					UpperCAmelCase					)
 lowercase__    :   Any						= float(fa_score(y_true=UpperCAmelCase		,					y_pred=UpperCAmelCase		,					average=UpperCAmelCase					)					)
 return {
     "accuracy": acc,
     "f1": fa,
 }
def       __UpperCamelCase	(       UpperCAmelCase		,					UpperCAmelCase					):
 lowercase__    :   Optional[Any]						= {}
 for id_pred, label in zip(UpperCAmelCase		,					UpperCAmelCase					):
  lowercase__    :   List[str]						= F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
  lowercase__    :   Optional[Any]						= id_pred['''prediction''']
  if question_id in question_map:
   question_map[question_id].append((pred, label)					)
  else:
   lowercase__    :   Tuple						= [(pred, label)]
 lowercase__    :   Optional[int]						= [], []
 for question, preds_labels in question_map.items():
  lowercase__    :   List[Any]						= zip(*UpperCAmelCase					)
  lowercase__    :   Optional[int]						= fa_score(y_true=UpperCAmelCase		,					y_pred=UpperCAmelCase		,					average='''macro'''					)
  fas.append(UpperCAmelCase					)
  lowercase__    :   Dict						= int(sum(pred == label for pred, label in preds_labels					) == len(UpperCAmelCase					)					)
  ems.append(UpperCAmelCase					)
 lowercase__    :   str						= float(sum(UpperCAmelCase					) / len(UpperCAmelCase					)					)
 lowercase__    :   Optional[int]						= sum(UpperCAmelCase					) / len(UpperCAmelCase					)
 lowercase__    :   str						= float(fa_score(y_true=UpperCAmelCase		,					y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]					)					)
 return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,						_KWARGS_DESCRIPTION )
class       UpperCAmelCase  (						datasets.Metric ):





       '''simple docstring'''



       def _lowerCAmelCase(						self			)  ->  Dict:
        if self.config_name not in [
            "boolq",
            "cb",
            "copa",
            "multirc",
            "record",
            "rte",
            "wic",
            "wsc",
            "wsc.fixed",
            "axb",
            "axg",
        ]:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''			)
        return datasets.MetricInfo(
            description=_DESCRIPTION     ,       citation=_CITATION     ,       inputs_description=_KWARGS_DESCRIPTION     ,       features=datasets.Features(self._get_feature_types()			)     ,       codebase_urls=[]     ,       reference_urls=[]     ,       format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None     ,       )

       def _lowerCAmelCase(						self			)  ->  Optional[Any]:
        if self.config_name == "record":
         return {
             "predictions": {
                 "idx": {
                     "passage": datasets.Value('''int64'''			),
                     "query": datasets.Value('''int64'''			),
                 },
                 "prediction_text": datasets.Value('''string'''			),
             },
             "references": {
                 "idx": {
                     "passage": datasets.Value('''int64'''			),
                     "query": datasets.Value('''int64'''			),
                 },
                 "answers": datasets.Sequence(datasets.Value('''string'''			)			),
             },
         }
        elif self.config_name == "multirc":
         return {
             "predictions": {
                 "idx": {
                     "answer": datasets.Value('''int64'''			),
                     "paragraph": datasets.Value('''int64'''			),
                     "question": datasets.Value('''int64'''			),
                 },
                 "prediction": datasets.Value('''int64'''			),
             },
             "references": datasets.Value('''int64'''			),
         }
        else:
         return {
             "predictions": datasets.Value('''int64'''			),
             "references": datasets.Value('''int64'''			),
         }


       def _lowerCAmelCase(						self     ,       __lowerCAmelCase     ,       __lowerCAmelCase			)  ->  Optional[int]:
        if self.config_name == "axb":
         return {"matthews_correlation": matthews_corrcoef(__lowerCAmelCase     ,       __lowerCAmelCase			)}
        elif self.config_name == "cb":
         return acc_and_fa(__lowerCAmelCase     ,       __lowerCAmelCase     ,       fa_avg='''macro'''			)
        elif self.config_name == "record":
         lowercase__    :   Any						= [
             {
                 '''qas''': [
                     {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                     for ref in references
                 ]
             }
         ]
         lowercase__    :   Tuple						= {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
         return evaluate_record(__lowerCAmelCase     ,       __lowerCAmelCase			)[0]
        elif self.config_name == "multirc":
         return evaluate_multirc(__lowerCAmelCase     ,       __lowerCAmelCase			)
        elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
         return {"accuracy": simple_accuracy(__lowerCAmelCase     ,       __lowerCAmelCase			)}
        else:
         raise KeyError(
             '''You should supply a configuration name selected in '''
             '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''			)


'''simple docstring'''

from collections import UserDict
from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
  from PIL import Image

  from ..image_utils import load_image

if is_torch_available():
  from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING

if is_tf_available():
  from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
  from ..tf_utils import stable_softmax

__a:			Optional[Any]	       =       logging.get_logger(__name__)



@add_end_docstrings(a__ )
class       UpperCAmelCase  (						a__ ):





       '''simple docstring'''



       def __init__(						self     ,       **__lowerCAmelCase			)  ->  int:
        super().__init__(**__lowerCAmelCase			)

        requires_backends(self     ,       '''vision'''			)
        self.check_model_type(
            TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
            if self.framework == '''tf'''
            else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING			)

       def __call__(						self     ,       __lowerCAmelCase     ,       **__lowerCAmelCase			)  ->  List[str]:
        return super().__call__(__lowerCAmelCase     ,       **__lowerCAmelCase			)

       def _lowerCAmelCase(						self     ,       **__lowerCAmelCase			)  ->  Optional[Any]:
        lowercase__    :   str						= {}
        if "candidate_labels" in kwargs:
         lowercase__    :   str						= kwargs['''candidate_labels''']
        if "hypothesis_template" in kwargs:
         lowercase__    :   str						= kwargs['''hypothesis_template''']

        return preprocess_params, {}, {}

       def _lowerCAmelCase(						self     ,       __lowerCAmelCase     ,       __lowerCAmelCase=None     ,       __lowerCAmelCase="This is a photo of {}."			)  ->  Any:
        lowercase__    :   Union[str, Any]						= load_image(__lowerCAmelCase			)
        lowercase__    :   List[str]						= self.image_processor(images=[image]     ,       return_tensors=self.framework			)
        lowercase__    :   Union[str, Any]						= candidate_labels
        lowercase__    :   int						= [hypothesis_template.format(__lowerCAmelCase			) for x in candidate_labels]
        lowercase__    :   Any						= self.tokenizer(__lowerCAmelCase     ,       return_tensors=self.framework     ,       padding=__lowerCAmelCase			)
        lowercase__    :   Any						= [text_inputs]
        return inputs

       def _lowerCAmelCase(						self     ,       __lowerCAmelCase			)  ->  Optional[Any]:
        lowercase__    :   Any						= model_inputs.pop('''candidate_labels'''			)
        lowercase__    :   int						= model_inputs.pop('''text_inputs'''			)
        if isinstance(text_inputs[0]     ,       __lowerCAmelCase			):
         lowercase__    :   Union[str, Any]						= text_inputs[0]
        else:
         # Batching case.
         lowercase__    :   Optional[Any]						= text_inputs[0][0]

        lowercase__    :   Any						= self.model(**__lowerCAmelCase     ,       **__lowerCAmelCase			)

        lowercase__    :   Any						= {
            '''candidate_labels''': candidate_labels,
            '''logits''': outputs.logits_per_image,
        }
        return model_outputs


       def _lowerCAmelCase(						self     ,       __lowerCAmelCase			)  ->  List[str]:
        lowercase__    :   Union[str, Any]						= model_outputs.pop('''candidate_labels'''			)
        lowercase__    :   Optional[int]						= model_outputs['''logits'''][0]
        if self.framework == "pt":
         lowercase__    :   Optional[int]						= logits.softmax(dim=-1			).squeeze(-1			)
         lowercase__    :   Any						= probs.tolist()
         if not isinstance(__lowerCAmelCase     ,       __lowerCAmelCase			):
          lowercase__    :   Dict						= [scores]
        elif self.framework == "tf":
         lowercase__    :   List[Any]						= stable_softmax(__lowerCAmelCase     ,       axis=-1			)
         lowercase__    :   str						= probs.numpy().tolist()
        else:
         raise ValueError(F"""Unsupported framework: {self.framework}"""			)

        lowercase__    :   Optional[int]						= [
            {'''score''': score, '''label''': candidate_label}
            for score, candidate_label in sorted(zip(__lowerCAmelCase     ,       __lowerCAmelCase			)     ,       key=lambda __lowerCAmelCase			: -x[0]			)
        ]
        return result

