


'''simple docstring'''







from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


A				:					Union[str, Any]    						=			'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

A				:					Optional[int]    						=			'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

A				:					Union[str, Any]    						=			'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''






def 				lowerCAmelCase__	(    lowerCamelCase					:      Dict  ,lowerCamelCase					:      Any       ):
       return float((preds == labels).mean()       )






def 				lowerCAmelCase__	(    lowerCamelCase					:      Optional[Any]  ,lowerCamelCase					:      List[Any]  ,lowerCamelCase					:      Any="binary"       ):
       _A					:     Tuple     = simple_accuracy(lowerCamelCase  ,lowerCamelCase       )
       _A					:     List[Any]     = float(fa_score(y_true=lowerCamelCase  ,y_pred=lowerCamelCase  ,average=lowerCamelCase       )       )
       return {
           "accuracy": acc,
           "f1": fa,
       }






def 				lowerCAmelCase__	(    lowerCamelCase					:      Optional[Any]  ,lowerCamelCase					:      Dict       ):
       _A					:     List[str]     = {}
       for id_pred, label in zip(lowerCamelCase  ,lowerCamelCase       ):
              _A					:     Optional[int]     = F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
              _A					:     List[Any]     = id_pred['prediction']
              if question_id in question_map:
                     question_map[question_id].append((pred, label)       )
              else:
                     _A					:     Any     = [(pred, label)]
       _A  ,			_A					:     Any     = [], []
       for question, preds_labels in question_map.items():
              _A  ,			_A					:     Union[str, Any]     = zip(*lowerCamelCase       )
              _A					:     Any     = fa_score(y_true=lowerCamelCase  ,y_pred=lowerCamelCase  ,average='macro'       )
              fas.append(lowerCamelCase       )
              _A					:     Dict     = int(sum(pred == label for pred, label in preds_labels       ) == len(lowerCamelCase       )       )
              ems.append(lowerCamelCase       )
       _A					:     Tuple     = float(sum(lowerCamelCase       ) / len(lowerCamelCase       )       )
       _A					:     Any     = sum(lowerCamelCase       ) / len(lowerCamelCase       )
       _A					:     List[str]     = float(fa_score(y_true=lowerCamelCase  ,y_pred=[id_pred['prediction'] for id_pred in ids_preds]       )       )
       return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    , _KWARGS_DESCRIPTION	)
class 			__lowerCamelCase			(	datasets.Metric	):
    """simple docstring"""


    def        A   (  self      :							List[str]):
           if self.config_name not in [
               "boolq",
               "cb",
               "copa",
               "multirc",
               "record",
               "rte",
               "wic",
               "wsc",
               "wsc.fixed",
               "axb",
               "axg",
           ]:
                  raise KeyError(
                      'You should supply a configuration name selected in '
                      '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]')
           return datasets.MetricInfo(
               description=_DESCRIPTION				,							citation=_CITATION				,							inputs_description=_KWARGS_DESCRIPTION				,							features=datasets.Features(self._get_feature_types())				,							codebase_urls=[]				,							reference_urls=[]				,							format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None				,							)


    def        A   (  self      :							Dict):
           if self.config_name == "record":
                  return {
                      "predictions": {
                          "idx": {
                              "passage": datasets.Value('int64'),
                              "query": datasets.Value('int64'),
                          },
                          "prediction_text": datasets.Value('string'),
                      },
                      "references": {
                          "idx": {
                              "passage": datasets.Value('int64'),
                              "query": datasets.Value('int64'),
                          },
                          "answers": datasets.Sequence(datasets.Value('string')),
                      },
                  }
           elif self.config_name == "multirc":
                  return {
                      "predictions": {
                          "idx": {
                              "answer": datasets.Value('int64'),
                              "paragraph": datasets.Value('int64'),
                              "question": datasets.Value('int64'),
                          },
                          "prediction": datasets.Value('int64'),
                      },
                      "references": datasets.Value('int64'),
                  }
           else:
                  return {
                      "predictions": datasets.Value('int64'),
                      "references": datasets.Value('int64'),
                  }


    def        A   (  self      :							Union[str, Any]				,							SCREAMING_SNAKE_CASE      :							List[Any]				,							SCREAMING_SNAKE_CASE      :							str):
           if self.config_name == "axb":
                  return {"matthews_correlation": matthews_corrcoef(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)}
           elif self.config_name == "cb":
                  return acc_and_fa(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE				,							fa_avg='macro')
           elif self.config_name == "record":
                  _A					:     Optional[int]     = [
                      {
                          'qas': [
                              {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                              for ref in references
                          ]
                      }
                  ]
                  _A					:     int     = {pred['idx']['query']: pred['prediction_text'] for pred in predictions}
                  return evaluate_record(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)[0]
           elif self.config_name == "multirc":
                  return evaluate_multirc(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)
           elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
                  return {"accuracy": simple_accuracy(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)}
           else:
                  raise KeyError(
                      'You should supply a configuration name selected in '
                      '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]')










'''simple docstring'''







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
   from ...onnx.config import PatchingSpec
   from ...tokenization_utils_base import PreTrainedTokenizerBase


A				:					Tuple    						=			logging.get_logger(__name__)

A				:					Tuple    						=			{
    '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json''',
    '''allenai/longformer-large-4096''': '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json''',
    '''allenai/longformer-large-4096-finetuned-triviaqa''': (
        '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json'''
    ),
    '''allenai/longformer-base-4096-extra.pos.embd.only''': (
        '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json'''
    ),
    '''allenai/longformer-large-4096-extra.pos.embd.only''': (
        '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json'''
    ),
}


class 			__lowerCamelCase			(	a_	):
    """simple docstring"""
    a													=   "longformer"


    def __init__(  self      :							Any				,							SCREAMING_SNAKE_CASE      :							Union[List[int], int] = 512				,							SCREAMING_SNAKE_CASE      :							int = 2				,							SCREAMING_SNAKE_CASE      :							int = 1				,							SCREAMING_SNAKE_CASE      :							int = 0				,							SCREAMING_SNAKE_CASE      :							int = 2				,							SCREAMING_SNAKE_CASE      :							int = 30522				,							SCREAMING_SNAKE_CASE      :							int = 768				,							SCREAMING_SNAKE_CASE      :							int = 12				,							SCREAMING_SNAKE_CASE      :							int = 12				,							SCREAMING_SNAKE_CASE      :							int = 3072				,							SCREAMING_SNAKE_CASE      :							str = "gelu"				,							SCREAMING_SNAKE_CASE      :							float = 0.1				,							SCREAMING_SNAKE_CASE      :							float = 0.1				,							SCREAMING_SNAKE_CASE      :							int = 512				,							SCREAMING_SNAKE_CASE      :							int = 2				,							SCREAMING_SNAKE_CASE      :							float = 0.02				,							SCREAMING_SNAKE_CASE      :							float = 1e-12				,							SCREAMING_SNAKE_CASE      :							bool = False				,							**SCREAMING_SNAKE_CASE      :							List[Any]				,							):
           super().__init__(pad_token_id=SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE)

           _A					:     List[Any]     = attention_window
           _A					:     int     = sep_token_id
           _A					:     Tuple     = bos_token_id
           _A					:     Any     = eos_token_id
           _A					:     List[str]     = vocab_size
           _A					:     Any     = hidden_size
           _A					:     Optional[int]     = num_hidden_layers
           _A					:     int     = num_attention_heads
           _A					:     Dict     = hidden_act
           _A					:     List[Any]     = intermediate_size
           _A					:     int     = hidden_dropout_prob
           _A					:     Optional[int]     = attention_probs_dropout_prob
           _A					:     int     = max_position_embeddings
           _A					:     Any     = type_vocab_size
           _A					:     Dict     = initializer_range
           _A					:     Any     = layer_norm_eps
           _A					:     List[Any]     = onnx_export




class 			__lowerCamelCase			(	a_	):
    """simple docstring"""


    def __init__(  self      :							Optional[Any]				,							SCREAMING_SNAKE_CASE      :							"PretrainedConfig"				,							SCREAMING_SNAKE_CASE      :							str = "default"				,							SCREAMING_SNAKE_CASE      :							"List[PatchingSpec]" = None):
           super().__init__(SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE				,							SCREAMING_SNAKE_CASE)
           _A					:     Optional[Any]     = True


    @property
    def        A   (  self      :							List[str]):
           if self.task == "multiple-choice":
                  _A					:     Tuple     = {0: 'batch', 1: 'choice', 2: 'sequence'}
           else:
                  _A					:     List[Any]     = {0: 'batch', 1: 'sequence'}
           return OrderedDict(
               [
                   ('input_ids', dynamic_axis),
                   ('attention_mask', dynamic_axis),
                   ('global_attention_mask', dynamic_axis),
               ])


    @property
    def        A   (  self      :							str):
           _A					:     int     = super().outputs
           if self.task == "default":
                  _A					:     str     = {0: 'batch'}
           return outputs


    @property
    def        A   (  self      :							List[Any]):
           return 1e-4


    @property
    def        A   (  self      :							Dict):
           # needs to be >= 14 to support tril operator
           return max(super().default_onnx_opset				,							14)


    def        A   (  self      :							str				,							SCREAMING_SNAKE_CASE      :							"PreTrainedTokenizerBase"				,							SCREAMING_SNAKE_CASE      :							int = -1				,							SCREAMING_SNAKE_CASE      :							int = -1				,							SCREAMING_SNAKE_CASE      :							bool = False				,							SCREAMING_SNAKE_CASE      :							Optional[TensorType] = None				,							):
           _A					:     Union[str, Any]     = super().generate_dummy_inputs(
               preprocessor=SCREAMING_SNAKE_CASE				,							batch_size=SCREAMING_SNAKE_CASE				,							seq_length=SCREAMING_SNAKE_CASE				,							is_pair=SCREAMING_SNAKE_CASE				,							framework=SCREAMING_SNAKE_CASE)
           import torch

           # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
           # makes the export fail randomly
           _A					:     Tuple     = torch.zeros_like(inputs['input_ids'])
           # make every second token global
           _A					:     Dict     = 1

           return inputs






