


'''simple docstring'''






import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    FlaubertForMultipleChoice,
		    FlaubertForQuestionAnswering,
		    FlaubertForQuestionAnsweringSimple,
		    FlaubertForSequenceClassification,
		    FlaubertForTokenClassification,
		    FlaubertModel,
		    FlaubertWithLMHeadModel,
		)
		from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class 		a__    (							_lowerCAmelCase  ):





							'''simple docstring'''






							def __init__(		self ,   lowerCamelCase_ ,   lowerCamelCase_=13 ,   lowerCamelCase_=7 ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=False ,   lowerCamelCase_=False ,   lowerCamelCase_=False ,   lowerCamelCase_=2 ,   lowerCamelCase_=99 ,   lowerCamelCase_=0 ,   lowerCamelCase_=32 ,   lowerCamelCase_=5 ,   lowerCamelCase_=4 ,   lowerCamelCase_=0.1 ,   lowerCamelCase_=0.1 ,   lowerCamelCase_=5_12 ,   lowerCamelCase_=12 ,   lowerCamelCase_=2 ,   lowerCamelCase_=0.02 ,   lowerCamelCase_=3 ,   lowerCamelCase_=4 ,   lowerCamelCase_="last" ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   )	->							Optional[int]:
														lowerCAmelCase__	       =     parent
														lowerCAmelCase__	       =     batch_size
														lowerCAmelCase__	       =     seq_length
														lowerCAmelCase__	       =     is_training
														lowerCAmelCase__	       =     use_input_lengths
														lowerCAmelCase__	       =     use_token_type_ids
														lowerCAmelCase__	       =     use_labels
														lowerCAmelCase__	       =     gelu_activation
														lowerCAmelCase__	       =     sinusoidal_embeddings
														lowerCAmelCase__	       =     causal
														lowerCAmelCase__	       =     asm
														lowerCAmelCase__	       =     n_langs
														lowerCAmelCase__	       =     vocab_size
														lowerCAmelCase__	       =     n_special
														lowerCAmelCase__	       =     hidden_size
														lowerCAmelCase__	       =     num_hidden_layers
														lowerCAmelCase__	       =     num_attention_heads
														lowerCAmelCase__	       =     hidden_dropout_prob
														lowerCAmelCase__	       =     attention_probs_dropout_prob
														lowerCAmelCase__	       =     max_position_embeddings
														lowerCAmelCase__	       =     type_vocab_size
														lowerCAmelCase__	       =     type_sequence_label_size
														lowerCAmelCase__	       =     initializer_range
														lowerCAmelCase__	       =     num_labels
														lowerCAmelCase__	       =     num_choices
														lowerCAmelCase__	       =     summary_type
														lowerCAmelCase__	       =     use_proj
														lowerCAmelCase__	       =     scope







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														lowerCAmelCase__	       =     ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size       )
														lowerCAmelCase__	       =     random_attention_mask([self.batch_size, self.seq_length]       )

														lowerCAmelCase__	       =     None
														if self.use_input_lengths:
																					lowerCAmelCase__	       =     (
																					    ids_tensor([self.batch_size] ,   vocab_size=2       ) + self.seq_length - 2
																					)  # small variation of seq_length

														lowerCAmelCase__	       =     None
														if self.use_token_type_ids:
																					lowerCAmelCase__	       =     ids_tensor([self.batch_size, self.seq_length] ,   self.n_langs       )

														lowerCAmelCase__	       =     None
														lowerCAmelCase__	       =     None
														lowerCAmelCase__	       =     None
														if self.use_labels:
																					lowerCAmelCase__	       =     ids_tensor([self.batch_size] ,   self.type_sequence_label_size       )
																					lowerCAmelCase__	       =     ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels       )
																					lowerCAmelCase__	       =     ids_tensor([self.batch_size] ,   2       ).float()
																					lowerCAmelCase__	       =     ids_tensor([self.batch_size] ,   self.num_choices       )

														lowerCAmelCase__	       =     self.get_config()

														return (
														    config,
														    input_ids,
														    token_type_ids,
														    input_lengths,
														    sequence_labels,
														    token_labels,
														    is_impossible_labels,
														    choice_labels,
														    input_mask,
														)







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														return FlaubertConfig(
														    vocab_size=self.vocab_size ,   n_special=self.n_special ,   emb_dim=self.hidden_size ,   n_layers=self.num_hidden_layers ,   n_heads=self.num_attention_heads ,   dropout=self.hidden_dropout_prob ,   attention_dropout=self.attention_probs_dropout_prob ,   gelu_activation=self.gelu_activation ,   sinusoidal_embeddings=self.sinusoidal_embeddings ,   asm=self.asm ,   causal=self.causal ,   n_langs=self.n_langs ,   max_position_embeddings=self.max_position_embeddings ,   initializer_range=self.initializer_range ,   summary_type=self.summary_type ,   use_proj=self.use_proj ,   )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							int:
														lowerCAmelCase__	       =     FlaubertModel(config=SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()
														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   lengths=SCREAMING_SNAKE_CASE_ ,   langs=SCREAMING_SNAKE_CASE_       )
														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   langs=SCREAMING_SNAKE_CASE_       )
														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_       )
														self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							List[Any]:
														lowerCAmelCase__	       =     FlaubertWithLMHeadModel(SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   token_type_ids=SCREAMING_SNAKE_CASE_ ,   labels=SCREAMING_SNAKE_CASE_       )
														self.parent.assertEqual(result.loss.shape ,   ()       )
														self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							Optional[int]:
														lowerCAmelCase__	       =     FlaubertForQuestionAnsweringSimple(SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_       )

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   start_positions=SCREAMING_SNAKE_CASE_ ,   end_positions=SCREAMING_SNAKE_CASE_       )
														self.parent.assertEqual(result.start_logits.shape ,   (self.batch_size, self.seq_length)       )
														self.parent.assertEqual(result.end_logits.shape ,   (self.batch_size, self.seq_length)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							Optional[int]:
														lowerCAmelCase__	       =     FlaubertForQuestionAnswering(SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_       )

														lowerCAmelCase__	       =     model(
														    SCREAMING_SNAKE_CASE_ ,   start_positions=SCREAMING_SNAKE_CASE_ ,   end_positions=SCREAMING_SNAKE_CASE_ ,   cls_index=SCREAMING_SNAKE_CASE_ ,   is_impossible=SCREAMING_SNAKE_CASE_ ,   p_mask=SCREAMING_SNAKE_CASE_ ,   )

														lowerCAmelCase__	       =     model(
														    SCREAMING_SNAKE_CASE_ ,   start_positions=SCREAMING_SNAKE_CASE_ ,   end_positions=SCREAMING_SNAKE_CASE_ ,   cls_index=SCREAMING_SNAKE_CASE_ ,   is_impossible=SCREAMING_SNAKE_CASE_ ,   )

														(lowerCAmelCase__    )	       =     result_with_labels.to_tuple()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   start_positions=SCREAMING_SNAKE_CASE_ ,   end_positions=SCREAMING_SNAKE_CASE_       )

														(lowerCAmelCase__    )	       =     result_with_labels.to_tuple()

														self.parent.assertEqual(result_with_labels.loss.shape ,   ()       )
														self.parent.assertEqual(result.start_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top)       )
														self.parent.assertEqual(result.start_top_index.shape ,   (self.batch_size, model.config.start_n_top)       )
														self.parent.assertEqual(
														    result.end_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)       )
														self.parent.assertEqual(
														    result.end_top_index.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)       )
														self.parent.assertEqual(result.cls_logits.shape ,   (self.batch_size,)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							List[Any]:
														lowerCAmelCase__	       =     FlaubertForSequenceClassification(SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_       )
														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   labels=SCREAMING_SNAKE_CASE_       )

														self.parent.assertEqual(result.loss.shape ,   ()       )
														self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							List[Any]:
														lowerCAmelCase__	       =     self.num_labels
														lowerCAmelCase__	       =     FlaubertForTokenClassification(SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()

														lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_ ,   attention_mask=SCREAMING_SNAKE_CASE_ ,   labels=SCREAMING_SNAKE_CASE_       )
														self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels)       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   )	->							int:
														lowerCAmelCase__	       =     self.num_choices
														lowerCAmelCase__	       =     FlaubertForMultipleChoice(config=SCREAMING_SNAKE_CASE_       )
														model.to(SCREAMING_SNAKE_CASE_       )
														model.eval()
														lowerCAmelCase__	       =     input_ids.unsqueeze(1       ).expand(-1 ,   self.num_choices ,   -1       ).contiguous()
														lowerCAmelCase__	       =     token_type_ids.unsqueeze(1       ).expand(-1 ,   self.num_choices ,   -1       ).contiguous()
														lowerCAmelCase__	       =     input_mask.unsqueeze(1       ).expand(-1 ,   self.num_choices ,   -1       ).contiguous()
														lowerCAmelCase__	       =     model(
														    SCREAMING_SNAKE_CASE_ ,   attention_mask=SCREAMING_SNAKE_CASE_ ,   token_type_ids=SCREAMING_SNAKE_CASE_ ,   labels=SCREAMING_SNAKE_CASE_ ,   )
														self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_choices)       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														lowerCAmelCase__	       =     self.prepare_config_and_inputs()
														(
														    lowerCAmelCase__    
														)	       =     config_and_inputs
														lowerCAmelCase__	       =     {
														    """input_ids""": input_ids,
														    """token_type_ids""": token_type_ids,
														    """lengths""": input_lengths,
														    """attention_mask""": input_mask,
														}
														return config, inputs_dict




@require_torch
class 		a__    (							_lowerCAmelCase    ,    _lowerCAmelCase    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       Union[str, Any]     						=				(
							    (
							        FlaubertModel,
							        FlaubertWithLMHeadModel,
							        FlaubertForQuestionAnswering,
							        FlaubertForQuestionAnsweringSimple,
							        FlaubertForSequenceClassification,
							        FlaubertForTokenClassification,
							        FlaubertForMultipleChoice,
							    )
							    if is_torch_available()
							    else ()
							)
							lowercase__      :       Dict     						=				(
							    {
							        "feature-extraction": FlaubertModel,
							        "fill-mask": FlaubertWithLMHeadModel,
							        "question-answering": FlaubertForQuestionAnsweringSimple,
							        "text-classification": FlaubertForSequenceClassification,
							        "token-classification": FlaubertForTokenClassification,
							        "zero-shot": FlaubertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_       )	->							List[Any]:
														if (
														    pipeline_test_casse_name == "QAPipelineTests"
														    and tokenizer_name is not None
														    and not tokenizer_name.endswith('''Fast'''       )
														):
																					# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																					# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																					# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																					return True

														return False







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_=False       )	->							List[Any]:
														lowerCAmelCase__	       =     super()._prepare_for_class(SCREAMING_SNAKE_CASE_ ,   SCREAMING_SNAKE_CASE_ ,   return_labels=SCREAMING_SNAKE_CASE_       )

														if return_labels:
																					if model_class.__name__ == "FlaubertForQuestionAnswering":
																												lowerCAmelCase__	       =     torch.zeros(
																												    self.model_tester.batch_size ,   dtype=torch.long ,   device=SCREAMING_SNAKE_CASE_       )
																												lowerCAmelCase__	       =     torch.zeros(
																												    self.model_tester.batch_size ,   dtype=torch.long ,   device=SCREAMING_SNAKE_CASE_       )

														return inputs_dict







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     FlaubertModelTester(self       )
														lowerCAmelCase__	       =     ConfigTester(self ,   config_class=SCREAMING_SNAKE_CASE_ ,   emb_dim=37       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														self.config_tester.run_common_tests()







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_model(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_lm_head(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_simple_qa(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_qa(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_sequence_classif(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_token_classif(*SCREAMING_SNAKE_CASE_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_flaubert_multiple_choice(*SCREAMING_SNAKE_CASE_       )







							@slow
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					lowerCAmelCase__	       =     FlaubertModel.from_pretrained(SCREAMING_SNAKE_CASE_       )
																					self.assertIsNotNone(SCREAMING_SNAKE_CASE_       )







							@slow
							@require_torch_gpu
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							int:
														lowerCAmelCase__	       =     self.model_tester.prepare_config_and_inputs_for_common()
														for model_class in self.all_model_classes:
																					# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
																					if model_class == FlaubertForMultipleChoice:
																												return

																					lowerCAmelCase__	       =     True
																					lowerCAmelCase__	       =     model_class(config=SCREAMING_SNAKE_CASE_       )

																					lowerCAmelCase__	       =     self._prepare_for_class(SCREAMING_SNAKE_CASE_ ,   SCREAMING_SNAKE_CASE_       )
																					lowerCAmelCase__	       =     torch.jit.trace(
																					    SCREAMING_SNAKE_CASE_ ,   (inputs_dict['''input_ids'''].to('''cpu'''       ), inputs_dict['''attention_mask'''].to('''cpu'''       ))       )

																					with tempfile.TemporaryDirectory() as tmp:
																												torch.jit.save(SCREAMING_SNAKE_CASE_ ,   os.path.join(SCREAMING_SNAKE_CASE_ ,   '''traced_model.pt'''       )       )
																												lowerCAmelCase__	       =     torch.jit.load(os.path.join(SCREAMING_SNAKE_CASE_ ,   '''traced_model.pt'''       ) ,   map_location=SCREAMING_SNAKE_CASE_       )
																												loaded(inputs_dict['''input_ids'''].to(SCREAMING_SNAKE_CASE_       ) ,   inputs_dict['''attention_mask'''].to(SCREAMING_SNAKE_CASE_       )       )




@require_torch
class 		a__    (							unittest.TestCase  ):





							'''simple docstring'''






							@slow
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														lowerCAmelCase__	       =     FlaubertModel.from_pretrained('''flaubert/flaubert_base_cased'''       )
														lowerCAmelCase__	       =     torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]       )
														with torch.no_grad():
																					lowerCAmelCase__	       =     model(SCREAMING_SNAKE_CASE_       )[0]
														lowerCAmelCase__	       =     torch.Size((1, 11, 7_68)       )
														self.assertEqual(output.shape ,   SCREAMING_SNAKE_CASE_       )
														lowerCAmelCase__	       =     torch.tensor(
														    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]       )

														self.assertTrue(torch.allclose(output[:, :3, :3] ,   SCREAMING_SNAKE_CASE_ ,   atol=1e-4       )       )



'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__UpperCAmelCase       =							'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

__UpperCAmelCase       =							'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

__UpperCAmelCase       =							'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''


def  _snake_case       (					A			,   A	)						->   List[Any]:
							return float((preds == labels).mean()	)


def  _snake_case       (					A			,   A			,   A="binary"	)						->   int:
							lowerCAmelCase__	       =     simple_accuracy(A			,   A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=A			,   average=A	)	)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}


def  _snake_case       (					A			,   A	)						->   List[Any]:
							lowerCAmelCase__	       =     {}
							for id_pred, label in zip(A			,   A	):
														lowerCAmelCase__	       =     F"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
														lowerCAmelCase__	       =     id_pred['''prediction''']
														if question_id in question_map:
																					question_map[question_id].append((pred, label)	)
														else:
																					lowerCAmelCase__	       =     [(pred, label)]
							lowerCAmelCase__       , lowerCAmelCase__	       =     [], []
							for question, preds_labels in question_map.items():
														lowerCAmelCase__       , lowerCAmelCase__	       =     zip(*A	)
														lowerCAmelCase__	       =     fa_score(y_true=A			,   y_pred=A			,   average='''macro'''	)
														fas.append(A	)
														lowerCAmelCase__	       =     int(sum(pred == label for pred, label in preds_labels	) == len(A	)	)
														ems.append(A	)
							lowerCAmelCase__	       =     float(sum(A	) / len(A	)	)
							lowerCAmelCase__	       =     sum(A	) / len(A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]	)	)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,    _KWARGS_DESCRIPTION  )
class 		a__    (							datasets.Metric  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION ,   citation=_CITATION ,   inputs_description=_KWARGS_DESCRIPTION ,   features=datasets.Features(self._get_feature_types()       ) ,   codebase_urls=[] ,   reference_urls=[] ,   format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "prediction_text": datasets.Value('''string'''       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value('''string'''       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value('''int64'''       ),
																					            "paragraph": datasets.Value('''int64'''       ),
																					            "question": datasets.Value('''int64'''       ),
																					        },
																					        "prediction": datasets.Value('''int64'''       ),
																					    },
																					    "references": datasets.Value('''int64'''       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value('''int64'''       ),
																					    "references": datasets.Value('''int64'''       ),
																					}







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_       )	->							Dict:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase_ ,   lowerCamelCase_       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase_ ,   lowerCamelCase_ ,   fa_avg='''macro'''       )
														elif self.config_name == "record":
																					lowerCAmelCase__	       =     [
																					    {
																					        '''qas''': [
																					            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
																					            for ref in references
																					        ]
																					    }
																					]
																					lowerCAmelCase__	       =     {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
																					return evaluate_record(lowerCamelCase_ ,   lowerCamelCase_       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase_ ,   lowerCamelCase_       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase_ ,   lowerCamelCase_       )}
														else:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )