


'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       List[str]     						=				CodeGenTokenizer
							lowercase__      :       int     						=				CodeGenTokenizerFast
							lowercase__      :       int     						=				True
							lowercase__      :       List[Any]     						=				{"add_prefix_space": True}
							lowercase__      :       Dict     						=				False







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														lowerCAmelCase__	       =     [
														    '''l''',
														    '''o''',
														    '''w''',
														    '''e''',
														    '''r''',
														    '''s''',
														    '''t''',
														    '''i''',
														    '''d''',
														    '''n''',
														    '''\u0120''',
														    '''\u0120l''',
														    '''\u0120n''',
														    '''\u0120lo''',
														    '''\u0120low''',
														    '''er''',
														    '''\u0120lowest''',
														    '''\u0120newer''',
														    '''\u0120wider''',
														    '''<unk>''',
														    '''<|endoftext|>''',
														]
														lowerCAmelCase__	       =     dict(zip(lowerCamelCase_ ,   range(len(lowerCamelCase_       )       )       )       )
														lowerCAmelCase__	       =     ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
														lowerCAmelCase__	       =     {'''unk_token''': '''<unk>'''}

														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''merges_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write(json.dumps(lowerCamelCase_       ) + '''\n'''       )
														with open(self.merges_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							List[str]:
														kwargs.update(self.special_tokens_map       )
														return CodeGenTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							List[str]:
														kwargs.update(self.special_tokens_map       )
														return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							List[str]:
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     '''lower newer'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     CodeGenTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map       )
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokens + [tokenizer.unk_token]
														lowerCAmelCase__	       =     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														if not self.test_rust_tokenizer:
																					return

														lowerCAmelCase__	       =     self.get_tokenizer()
														lowerCAmelCase__	       =     self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''lower newer'''

														# Testing tokenization
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing conversion to ids without special tokens
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing conversion to ids with special tokens
														lowerCAmelCase__	       =     self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing the unknown token
														lowerCAmelCase__	       =     tokens + [rust_tokenizer.unk_token]
														lowerCAmelCase__	       =     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   *lowerCamelCase_ ,   **lowerCamelCase_       )	->							str:
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_=15       )	->							List[Any]:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												# Simple input
																												lowerCAmelCase__	       =     '''This is a simple input'''
																												lowerCAmelCase__	       =     ['''This is a simple input 1''', '''This is a simple input 2''']
																												lowerCAmelCase__	       =     ('''This is a simple input''', '''This is a pair''')
																												lowerCAmelCase__	       =     [
																												    ('''This is a simple input 1''', '''This is a simple input 2'''),
																												    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																												]

																												# Simple input tests
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Simple input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Simple input
																												self.assertRaises(
																												    lowerCamelCase_ ,   tokenizer_r.batch_encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length''' ,   )

																												# Pair input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Pair input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Pair input
																												self.assertRaises(
																												    lowerCamelCase_ ,   tokenizer_r.batch_encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length''' ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														lowerCAmelCase__	       =     CodeGenTokenizer.from_pretrained(self.tmpdirname ,   pad_token='''<pad>'''       )

														# Simple input
														lowerCAmelCase__	       =     '''This is a simple input'''
														lowerCAmelCase__	       =     ['''This is a simple input looooooooong''', '''This is a simple input''']
														lowerCAmelCase__	       =     ('''This is a simple input''', '''This is a pair''')
														lowerCAmelCase__	       =     [
														    ('''This is a simple input loooooong''', '''This is a simple input'''),
														    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
														]

														lowerCAmelCase__	       =     tokenizer.pad_token_id

														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding='''max_length''' ,   max_length=30 ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncate=lowerCamelCase_ ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(*lowerCamelCase_ ,   padding='''max_length''' ,   max_length=60 ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncate=lowerCamelCase_ ,   return_tensors='''np'''       )

														# s
														# test single string max_length padding
														self.assertEqual(out_s['''input_ids'''].shape[-1] ,   30       )
														self.assertTrue(pad_token_id in out_s['''input_ids''']       )
														self.assertTrue(0 in out_s['''attention_mask''']       )

														# s2
														# test automatic padding
														self.assertEqual(out_sa['''input_ids'''].shape[-1] ,   33       )
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]       )
														self.assertFalse(0 in out_sa['''attention_mask'''][0]       )
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]       )
														self.assertTrue(0 in out_sa['''attention_mask'''][1]       )

														# p
														# test single pair max_length padding
														self.assertEqual(out_p['''input_ids'''].shape[-1] ,   60       )
														self.assertTrue(pad_token_id in out_p['''input_ids''']       )
														self.assertTrue(0 in out_p['''attention_mask''']       )

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa['''input_ids'''].shape[-1] ,   52       )
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]       )
														self.assertFalse(0 in out_pa['''attention_mask'''][0]       )
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]       )
														self.assertTrue(0 in out_pa['''attention_mask'''][1]       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							int:
														lowerCAmelCase__	       =     '''$$$'''
														lowerCAmelCase__	       =     CodeGenTokenizer.from_pretrained(self.tmpdirname ,   bos_token=lowerCamelCase_ ,   add_bos_token=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''This is a simple input'''
														lowerCAmelCase__	       =     ['''This is a simple input 1''', '''This is a simple input 2''']

														lowerCAmelCase__	       =     tokenizer.bos_token_id

														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_       )

														self.assertEqual(out_s.input_ids[0] ,   lowerCamelCase_       )
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

														lowerCAmelCase__	       =     tokenizer.decode(out_s.input_ids       )
														lowerCAmelCase__	       =     tokenizer.batch_decode(out_sa.input_ids       )

														self.assertEqual(decode_s.split()[0] ,   lowerCamelCase_       )
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )







							@slow
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''       )

														lowerCAmelCase__	       =     '''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
														lowerCAmelCase__	       =     '''\nif len_a > len_b:      result = a\nelse:      result = b'''

														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_       )
														lowerCAmelCase__	       =     ['''^#''', re.escape('''<|endoftext|>'''       ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
														lowerCAmelCase__	       =     tokenizer.decode(lowerCamelCase_ ,   truncate_before_pattern=lowerCamelCase_       )
														self.assertEqual(lowerCamelCase_ ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														pass



'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__UpperCAmelCase       =							'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

__UpperCAmelCase       =							'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

__UpperCAmelCase       =							'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''


def  _snake_case       (					A			,   A	)						->   List[Any]:
							return float((preds == labels).mean()	)


def  _snake_case       (					A			,   A			,   A="binary"	)						->   int:
							lowerCAmelCase__	       =     simple_accuracy(A			,   A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=A			,   average=A	)	)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}


def  _snake_case       (					A			,   A	)						->   List[Any]:
							lowerCAmelCase__	       =     {}
							for id_pred, label in zip(A			,   A	):
														lowerCAmelCase__	       =     F"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
														lowerCAmelCase__	       =     id_pred['''prediction''']
														if question_id in question_map:
																					question_map[question_id].append((pred, label)	)
														else:
																					lowerCAmelCase__	       =     [(pred, label)]
							lowerCAmelCase__       , lowerCAmelCase__	       =     [], []
							for question, preds_labels in question_map.items():
														lowerCAmelCase__       , lowerCAmelCase__	       =     zip(*A	)
														lowerCAmelCase__	       =     fa_score(y_true=A			,   y_pred=A			,   average='''macro'''	)
														fas.append(A	)
														lowerCAmelCase__	       =     int(sum(pred == label for pred, label in preds_labels	) == len(A	)	)
														ems.append(A	)
							lowerCAmelCase__	       =     float(sum(A	) / len(A	)	)
							lowerCAmelCase__	       =     sum(A	) / len(A	)
							lowerCAmelCase__	       =     float(fa_score(y_true=A			,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]	)	)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,    _KWARGS_DESCRIPTION  )
class 		a__    (							datasets.Metric  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION ,   citation=_CITATION ,   inputs_description=_KWARGS_DESCRIPTION ,   features=datasets.Features(self._get_feature_types()       ) ,   codebase_urls=[] ,   reference_urls=[] ,   format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "prediction_text": datasets.Value('''string'''       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value('''int64'''       ),
																					            "query": datasets.Value('''int64'''       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value('''string'''       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value('''int64'''       ),
																					            "paragraph": datasets.Value('''int64'''       ),
																					            "question": datasets.Value('''int64'''       ),
																					        },
																					        "prediction": datasets.Value('''int64'''       ),
																					    },
																					    "references": datasets.Value('''int64'''       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value('''int64'''       ),
																					    "references": datasets.Value('''int64'''       ),
																					}







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_       )	->							Dict:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase_ ,   lowerCamelCase_       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase_ ,   lowerCamelCase_ ,   fa_avg='''macro'''       )
														elif self.config_name == "record":
																					lowerCAmelCase__	       =     [
																					    {
																					        '''qas''': [
																					            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
																					            for ref in references
																					        ]
																					    }
																					]
																					lowerCAmelCase__	       =     {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
																					return evaluate_record(lowerCamelCase_ ,   lowerCamelCase_       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase_ ,   lowerCamelCase_       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase_ ,   lowerCamelCase_       )}
														else:
																					raise KeyError(
																					    '''You should supply a configuration name selected in '''
																					    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''       )