


'''simple docstring'''






import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel




class 		a__    :





							'''simple docstring'''






							def __init__(		self ,   lowerCamelCase_ ,   lowerCamelCase_=99 ,   lowerCamelCase_=13 ,   lowerCamelCase_=7 ,   lowerCamelCase_=9 ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=False ,   lowerCamelCase_=32 ,   lowerCamelCase_=5 ,   lowerCamelCase_=4 ,   lowerCamelCase_=37 ,   lowerCamelCase_=8 ,   lowerCamelCase_=0.1 ,   lowerCamelCase_=0.002 ,   lowerCamelCase_=1 ,   lowerCamelCase_=0 ,   lowerCamelCase_=0 ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   )	->							str:
														lowerCAmelCase__	       =     parent
														lowerCAmelCase__	       =     batch_size
														lowerCAmelCase__	       =     encoder_seq_length
														lowerCAmelCase__	       =     decoder_seq_length
														# For common tests
														lowerCAmelCase__	       =     self.decoder_seq_length
														lowerCAmelCase__	       =     is_training
														lowerCAmelCase__	       =     use_attention_mask
														lowerCAmelCase__	       =     use_labels
														lowerCAmelCase__	       =     vocab_size
														lowerCAmelCase__	       =     hidden_size
														lowerCAmelCase__	       =     num_hidden_layers
														lowerCAmelCase__	       =     num_attention_heads
														lowerCAmelCase__	       =     d_ff
														lowerCAmelCase__	       =     relative_attention_num_buckets
														lowerCAmelCase__	       =     dropout_rate
														lowerCAmelCase__	       =     initializer_factor
														lowerCAmelCase__	       =     eos_token_id
														lowerCAmelCase__	       =     pad_token_id
														lowerCAmelCase__	       =     decoder_start_token_id
														lowerCAmelCase__	       =     None
														lowerCAmelCase__	       =     decoder_layers







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														return TaConfig.from_pretrained('''google/umt5-base'''       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   )	->							List[str]:
														if attention_mask is None:
																					lowerCAmelCase__	       =     input_ids.ne(config.pad_token_id       )
														if decoder_attention_mask is None:
																					lowerCAmelCase__	       =     decoder_input_ids.ne(config.pad_token_id       )
														if head_mask is None:
																					lowerCAmelCase__	       =     torch.ones(config.num_hidden_layers ,   config.num_attention_heads ,   device=lowerCamelCase_       )
														if decoder_head_mask is None:
																					lowerCAmelCase__	       =     torch.ones(config.num_decoder_layers ,   config.num_attention_heads ,   device=lowerCamelCase_       )
														if cross_attn_head_mask is None:
																					lowerCAmelCase__	       =     torch.ones(
																					    config.num_decoder_layers ,   config.num_attention_heads ,   device=lowerCamelCase_       )
														return {
														    "input_ids": input_ids,
														    "decoder_input_ids": decoder_input_ids,
														    "attention_mask": attention_mask,
														    "decoder_attention_mask": decoder_attention_mask,
														    "head_mask": head_mask,
														    "decoder_head_mask": decoder_head_mask,
														    "cross_attn_head_mask": cross_attn_head_mask,
														}







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														lowerCAmelCase__	       =     ids_tensor([self.batch_size, self.encoder_seq_length] ,   self.vocab_size       )
														lowerCAmelCase__	       =     ids_tensor([self.batch_size, self.decoder_seq_length] ,   self.vocab_size       )

														# we need to clamp the input ids here to avoid having pad token in between
														# this is because for NllbMoe the position_ids are prepared such that
														# all pad tokens have pos id = 2 and rest are between 2..seq_length
														# and the seq_length here is seq_length - num_pad_tokens
														# but when using past, there is no way of knowing if the past input ids had
														# pad tokens in them, which results in incorrect seq_lenth and which in turn results in
														# position_ids being off by num_pad_tokens in past input
														lowerCAmelCase__	       =     input_ids.clamp(self.pad_token_id + 1       )
														lowerCAmelCase__	       =     decoder_input_ids.clamp(self.pad_token_id + 1       )

														lowerCAmelCase__	       =     self.get_config()
														lowerCAmelCase__	       =     config.num_attention_heads
														lowerCAmelCase__	       =     self.prepare_inputs_dict(lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_       )
														return config, input_dict







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														lowerCAmelCase__       , lowerCAmelCase__	       =     self.prepare_config_and_inputs()
														return config, inputs_dict







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														return TaConfig(
														    vocab_size=1_66 ,   d_model=self.hidden_size ,   d_ff=self.d_ff ,   d_kv=self.hidden_size // self.num_attention_heads ,   num_layers=self.num_hidden_layers ,   num_decoder_layers=self.decoder_layers ,   num_heads=self.num_attention_heads ,   relative_attention_num_buckets=self.relative_attention_num_buckets ,   dropout_rate=self.dropout_rate ,   initializer_factor=self.initializer_factor ,   eos_token_id=self.eos_token_id ,   bos_token_id=self.pad_token_id ,   pad_token_id=self.pad_token_id ,   decoder_start_token_id=self.decoder_start_token_id ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														return TaConfig(
														    vocab_size=self.vocab_size ,   d_model=self.hidden_size ,   d_ff=self.d_ff ,   d_kv=self.hidden_size // self.num_attention_heads ,   num_layers=self.num_hidden_layers ,   num_decoder_layers=self.decoder_layers ,   num_heads=self.num_attention_heads ,   relative_attention_num_buckets=self.relative_attention_num_buckets ,   dropout_rate=self.dropout_rate ,   initializer_factor=self.initializer_factor ,   eos_token_id=self.eos_token_id ,   bos_token_id=self.pad_token_id ,   pad_token_id=self.pad_token_id ,   decoder_start_token_id=self.decoder_start_token_id ,   )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							Dict:
														lowerCAmelCase__	       =     UMTaModel(config=lowerCamelCase_       )
														model.to(lowerCamelCase_       )
														model.eval()
														lowerCAmelCase__	       =     model(
														    input_ids=lowerCamelCase_ ,   decoder_input_ids=lowerCamelCase_ ,   attention_mask=lowerCamelCase_ ,   decoder_attention_mask=lowerCamelCase_ ,   )
														lowerCAmelCase__	       =     model(input_ids=lowerCamelCase_ ,   decoder_input_ids=lowerCamelCase_       )
														lowerCAmelCase__	       =     result.last_hidden_state
														lowerCAmelCase__	       =     result.past_key_values
														lowerCAmelCase__	       =     result.encoder_last_hidden_state

														self.parent.assertEqual(encoder_output.size() ,   (self.batch_size, self.encoder_seq_length, self.hidden_size)       )
														self.parent.assertEqual(decoder_output.size() ,   (self.batch_size, self.decoder_seq_length, self.hidden_size)       )
														# There should be `num_layers` key value embeddings stored in decoder_past
														self.parent.assertEqual(len(lowerCamelCase_       ) ,   config.num_layers       )
														# There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
														self.parent.assertEqual(len(decoder_past[0]       ) ,   4       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							int:
														lowerCAmelCase__	       =     UMTaModel(config=lowerCamelCase_       ).get_decoder().to(lowerCamelCase_       ).eval()
														# first forward pass
														lowerCAmelCase__	       =     model(lowerCamelCase_ ,   use_cache=lowerCamelCase_       )
														lowerCAmelCase__	       =     model(lowerCamelCase_       )
														lowerCAmelCase__	       =     model(lowerCamelCase_ ,   use_cache=lowerCamelCase_       )

														self.parent.assertTrue(len(lowerCamelCase_       ) == len(lowerCamelCase_       )       )
														self.parent.assertTrue(len(lowerCamelCase_       ) == len(lowerCamelCase_       ) + 1       )

														lowerCAmelCase__       , lowerCAmelCase__	       =     outputs.to_tuple()

														# create hypothetical next token and extent to next_input_ids
														lowerCAmelCase__	       =     ids_tensor((self.batch_size, 1) ,   config.vocab_size       )

														# append to next input_ids and
														lowerCAmelCase__	       =     torch.cat([input_ids, next_tokens] ,   dim=-1       )

														lowerCAmelCase__	       =     model(lowerCamelCase_       )['''last_hidden_state''']
														lowerCAmelCase__	       =     model(lowerCamelCase_ ,   past_key_values=lowerCamelCase_       )['''last_hidden_state''']

														# select random slice
														lowerCAmelCase__	       =     ids_tensor((1,) ,   output_from_past.shape[-1]       ).item()
														lowerCAmelCase__	       =     output_from_no_past[:, -1, random_slice_idx].detach()
														lowerCAmelCase__	       =     output_from_past[:, 0, random_slice_idx].detach()

														# test that outputs are equal for slice
														self.parent.assertTrue(torch.allclose(lowerCamelCase_ ,   lowerCamelCase_ ,   atol=1e-3       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							List[str]:
														lowerCAmelCase__	       =     UMTaModel(config=lowerCamelCase_       ).to(lowerCamelCase_       ).half().eval()
														lowerCAmelCase__	       =     model(**lowerCamelCase_       )['''last_hidden_state''']
														self.parent.assertFalse(torch.isnan(lowerCamelCase_       ).any().item()       )




@require_torch
class 		a__    (							a__    ,    a__    ,    a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       Tuple     						=				(
							    (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
							)
							lowercase__      :       List[Any]     						=				(UMTaForConditionalGeneration,) if is_torch_available() else ()
							lowercase__      :       Dict     						=				(
							    {
							        "conversational": UMTaForConditionalGeneration,
							        "feature-extraction": UMTaModel,
							        "summarization": UMTaForConditionalGeneration,
							        "text2text-generation": UMTaForConditionalGeneration,
							        "translation": UMTaForConditionalGeneration,
							        "question-answering": UMTaForQuestionAnswering,
							    }
							    if is_torch_available()
							    else {}
							)
							lowercase__      :       Optional[int]     						=				True
							lowercase__      :       Tuple     						=				False
							lowercase__      :       Optional[int]     						=				False
							lowercase__      :       Optional[Any]     						=				True
							lowercase__      :       Optional[int]     						=				True
							# The small UMT5 model needs higher percentages for CPU/MP tests
							lowercase__      :       int     						=				[0.8, 0.9]







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     UMTaModelTester(self       )







							@unittest.skip('''Test has a segmentation fault on torch 1.8.0'''       )
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														lowerCAmelCase__	       =     UMTaModel(config_and_inputs[0]       ).to(lowerCamelCase_       )
														with tempfile.TemporaryDirectory() as tmpdirname:
																					torch.onnx.export(
																					    lowerCamelCase_ ,   (config_and_inputs[1], config_and_inputs[3], config_and_inputs[2]) ,   F"""{tmpdirname}/t5_test.onnx""" ,   export_params=lowerCamelCase_ ,   opset_version=9 ,   input_names=['''input_ids''', '''decoder_input_ids'''] ,   )







							@unittest.skipIf(torch_device == '''cpu''' ,   '''Cant do half precision'''       )
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model_fpaa_forward(*lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     ['''encoder_attentions''', '''decoder_attentions''', '''cross_attentions''']
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														lowerCAmelCase__	       =     config_and_inputs[0]
														lowerCAmelCase__	       =     UMTaForConditionalGeneration(lowerCamelCase_       ).eval()
														model.to(lowerCamelCase_       )

														lowerCAmelCase__	       =     {
														    '''head_mask''': torch.zeros(config.num_layers ,   config.num_heads ,   device=lowerCamelCase_       ),
														    '''decoder_head_mask''': torch.zeros(config.num_decoder_layers ,   config.num_heads ,   device=lowerCamelCase_       ),
														    '''cross_attn_head_mask''': torch.zeros(config.num_decoder_layers ,   config.num_heads ,   device=lowerCamelCase_       ),
														}

														for attn_name, (name, mask) in zip(lowerCamelCase_ ,   head_masking.items()       ):
																					lowerCAmelCase__	       =     {name: mask}
																					# Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
																					if name == "head_mask":
																												lowerCAmelCase__	       =     torch.ones(
																												    config.num_decoder_layers ,   config.num_heads ,   device=lowerCamelCase_       )

																					lowerCAmelCase__	       =     model.generate(
																					    config_and_inputs[1]['''input_ids'''] ,   num_beams=1 ,   max_length=3 ,   output_attentions=lowerCamelCase_ ,   return_dict_in_generate=lowerCamelCase_ ,   **lowerCamelCase_ ,   )
																					# We check the state of decoder_attentions and cross_attentions just from the last step
																					lowerCAmelCase__	       =     out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
																					self.assertEqual(sum([w.sum().item() for w in attn_weights]       ) ,   0.0       )







							@unittest.skip('''Does not work on the tiny model as we keep hitting edge cases.'''       )
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							int:
														pass




@require_torch
@require_sentencepiece
@require_tokenizers
class 		a__    (							unittest.TestCase  ):





							'''simple docstring'''






							@slow
							@unittest.skip(
							    '''Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged'''       )
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														lowerCAmelCase__	       =     UMTaForConditionalGeneration.from_pretrained('''google/umt5-small''' ,   return_dict=lowerCamelCase_       ).to(lowerCamelCase_       )
														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''google/umt5-small''' ,   use_fast=lowerCamelCase_ ,   legacy=lowerCamelCase_       )
														lowerCAmelCase__	       =     [
														    '''Bonjour monsieur <extra_id_0> bien <extra_id_1>.''',
														    '''No se como puedo <extra_id_0>.''',
														    '''This is the reason why we <extra_id_0> them.''',
														    '''The <extra_id_0> walks in <extra_id_1>, seats''',
														    '''A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.''',
														]
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   return_tensors='''pt''' ,   padding=lowerCamelCase_       ).input_ids
														# fmt: off
														lowerCAmelCase__	       =     torch.tensor(
														    [
														        [ 3_85_30, 21_07_03, 25_62_99, 14_10, 25_62_98, 2_74, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
														        [   8_26, 3_21, 6_71, 2_59_22, 25_62_99, 2_74, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
														        [  14_60, 3_39, 3_12, 1_90_14, 1_06_20, 7_58, 25_62_99, 23_55,2_74, 1, 0, 0, 0, 0, 0, 0,0, 0],
														        [   5_17, 25_62_99, 1_48_69, 2_81, 3_01, 25_62_98, 2_75, 11_99_83,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
														        [   3_20, 25_62_99, 1_48_69, 2_81, 22_34, 2_89, 22_75, 3_33,6_13_91, 2_89, 25_62_98, 5_43, 25_62_97, 16_87_14, 3_29, 25_62_96,2_74, 1],
														    ]       )
														# fmt: on
														torch.testing.assert_allclose(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     model.generate(input_ids.to(lowerCamelCase_       )       )
														lowerCAmelCase__	       =     [
														    '''<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>''',
														    '''<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
														    '''<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
														    '''<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
														    '''<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
														]
														lowerCAmelCase__	       =     tokenizer.batch_decode(lowerCamelCase_       )
														self.assertEqual(lowerCamelCase_ ,   lowerCamelCase_       )



'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__UpperCAmelCase       =							'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

__UpperCAmelCase       =							'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

__UpperCAmelCase       =							'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''


def  _snake_case       (					A			,   A	)						->   List[Any]:
							return float((preds == labels).mean()	)


def  _snake_case       (					A			,   A			,   A="binary"	)						->   int:
							lowerCAmelCase__	       =     simple_accuracy(A			,   A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=A			,   average=A	)	)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}


def  _snake_case       (					A			,   A	)						->   List[Any]:
							lowerCAmelCase__	       =     {}
							for id_pred, label in zip(A			,   A	):
														lowerCAmelCase__	       =     F"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
														lowerCAmelCase__	       =     id_pred['''prediction''']
														if question_id in question_map:
																					question_map[question_id].append((pred, label)	)
														else:
																					lowerCAmelCase__	       =     [(pred, label)]
							lowerCAmelCase__       , lowerCAmelCase__	       =     [], []
							for question, preds_labels in question_map.items():
														lowerCAmelCase__       , lowerCAmelCase__	       =     zip(*A	)
														lowerCAmelCase__	       =     fa_score(y_true=A			,   y_pred=A			,   average='''macro'''	)
														fas.append(A	)
														lowerCAmelCase__	       =     int(sum(pred == label for pred, label in preds_labels	) == len(A	)	)
														ems.append(A	)
							lowerCAmelCase__	       =     float(sum(A	) / len(A	)	)
							lowerCAmelCase__	       =     sum(A	) / len(A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]	)	)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,    _KWARGS_DESCRIPTION  )
class 		a__    (							datasets.Metric  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION ,   citation=_CITATION ,   inputs_description=_KWARGS_DESCRIPTION ,   features=datasets.Features(self._get_feature_types()       ) ,   codebase_urls=[] ,   reference_urls=[] ,   format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "prediction_text": datasets.Value('''string'''       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value('''string'''       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value('''int64'''       ),
																					            "paragraph": datasets.Value('''int64'''       ),
																					            "question": datasets.Value('''int64'''       ),
																					        },
																					        "prediction": datasets.Value('''int64'''       ),
																					    },
																					    "references": datasets.Value('''int64'''       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value('''int64'''       ),
																					    "references": datasets.Value('''int64'''       ),
																					}







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_       )	->							Dict:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase_ ,   lowerCamelCase_       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase_ ,   lowerCamelCase_ ,   fa_avg='''macro'''       )
														elif self.config_name == "record":
																					lowerCAmelCase__	       =     [
																					    {
																					        '''qas''': [
																					            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
																					            for ref in references
																					        ]
																					    }
																					]
																					lowerCAmelCase__	       =     {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
																					return evaluate_record(lowerCamelCase_ ,   lowerCamelCase_       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase_ ,   lowerCamelCase_       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase_ ,   lowerCamelCase_       )}
														else:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )