


"""simple docstring"""







import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY






@is_pipeline_test
class 						_A  (  unittest.TestCase  ):







    """simple docstring"""




    UpperCAmelCase	:							Optional[int]     =	MODEL_FOR_MASKED_LM_MAPPING
    UpperCAmelCase	:							List[Any]     =	TF_MODEL_FOR_MASKED_LM_MAPPING






    def      __snake_case							( self    :   Union[str, Any]):
      super().tearDown()
      # clean-up as much as possible GPU memory occupied by PyTorch
      gc.collect()
      if is_torch_available():
        import torch

        torch.cuda.empty_cache()






    @require_tf
    def      __snake_case							( self    :   Optional[Any]):
      a						:		Any											=      pipeline(task="fill-mask"    ,			model="sshleifer/tiny-distilroberta-base"    ,			top_k=2    ,			framework="tf")
      a						:		Any											=      unmasker("My name is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {"sequence": "My name is grouped", "score": 2.1e-05, "token": 38015, "token_str": " grouped"},
              {"sequence": "My name is accuser", "score": 2.1e-05, "token": 25506, "token_str": " accuser"},
          ]    ,			)

      a						:		int											=      unmasker("The largest city in France is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {
                  "sequence": "The largest city in France is grouped",
                  "score": 2.1e-05,
                  "token": 38015,
                  "token_str": " grouped",
              },
              {
                  "sequence": "The largest city in France is accuser",
                  "score": 2.1e-05,
                  "token": 25506,
                  "token_str": " accuser",
              },
          ]    ,			)

      a						:		Union[str, Any]											=      unmasker("My name is <mask>"    ,			targets=[" Patrick", " Clara", " Teven"]    ,			top_k=3)
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
              {"sequence": "My name is Patrick", "score": 2e-05, "token": 3499, "token_str": " Patrick"},
              {"sequence": "My name is Te", "score": 1.9e-05, "token": 2941, "token_str": " Te"},
          ]    ,			)






    @require_torch
    def      __snake_case							( self    :   List[str]):
      a						:		Any											=      pipeline(task="fill-mask"    ,			model="sshleifer/tiny-distilroberta-base"    ,			top_k=2    ,			framework="pt")

      a						:		List[str]											=      unmasker("My name is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {"sequence": "My name is Maul", "score": 2.2e-05, "token": 35676, "token_str": " Maul"},
              {"sequence": "My name isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
          ]    ,			)

      a						:		Any											=      unmasker("The largest city in France is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {
                  "sequence": "The largest city in France is Maul",
                  "score": 2.2e-05,
                  "token": 35676,
                  "token_str": " Maul",
              },
              {"sequence": "The largest city in France isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
          ]    ,			)

      a						:		Any											=      unmasker("My name is <mask>"    ,			targets=[" Patrick", " Clara", " Teven"]    ,			top_k=3)
      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              {"sequence": "My name is Patrick", "score": 2.1e-05, "token": 3499, "token_str": " Patrick"},
              {"sequence": "My name is Te", "score": 2e-05, "token": 2941, "token_str": " Te"},
              {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
          ]    ,			)

      a						:		Any											=      unmasker("My name is <mask> <mask>"    ,			top_k=2)

      self.assertEqual(
          nested_simplify(__UpperCAmelCase    ,			decimals=6)    ,			[
              [
                  {
                      "score": 2.2e-05,
                      "token": 35676,
                      "token_str": " Maul",
                      "sequence": "<s>My name is Maul<mask></s>",
                  },
                  {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
              ],
              [
                  {
                      "score": 2.2e-05,
                      "token": 35676,
                      "token_str": " Maul",
                      "sequence": "<s>My name is<mask> Maul</s>",
                  },
                  {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
              ],
          ]    ,			)






    @require_torch_gpu
    def      __snake_case							( self    :   Optional[Any]):
      a						:		str											=      pipeline("fill-mask"    ,			model="hf-internal-testing/tiny-random-distilbert"    ,			device=0    ,			framework="pt")

      # convert model to fp16
      pipe.model.half()

      a						:		Optional[int]											=      pipe("Paris is the [MASK] of France.")
      # We actually don't care about the result, we just want to make sure
      # it works, meaning the float16 tensor got casted back to float32
      # for postprocessing.
      self.assertIsInstance(__UpperCAmelCase    ,			__UpperCAmelCase)






    @slow
    @require_torch
    def      __snake_case							( self    :   str):
      a						:		Union[str, Any]											=      pipeline(task="fill-mask"    ,			model="distilroberta-base"    ,			top_k=2    ,			framework="pt")
      self.run_large_test(__UpperCAmelCase)






    @slow
    @require_tf
    def      __snake_case							( self    :   Dict):
      a						:		List[Any]											=      pipeline(task="fill-mask"    ,			model="distilroberta-base"    ,			top_k=2    ,			framework="tf")
      self.run_large_test(__UpperCAmelCase)






    def      __snake_case							( self    :   Dict    ,			__UpperCAmelCase    :   List[Any]):
      a						:		str											=      unmasker("My name is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase)    ,			[
              {"sequence": "My name is John", "score": 0.008, "token": 610, "token_str": " John"},
              {"sequence": "My name is Chris", "score": 0.007, "token": 1573, "token_str": " Chris"},
          ]    ,			)
      a						:		Optional[Any]											=      unmasker("The largest city in France is <mask>")
      self.assertEqual(
          nested_simplify(__UpperCAmelCase)    ,			[
              {
                  "sequence": "The largest city in France is Paris",
                  "score": 0.251,
                  "token": 2201,
                  "token_str": " Paris",
              },
              {
                  "sequence": "The largest city in France is Lyon",
                  "score": 0.214,
                  "token": 12790,
                  "token_str": " Lyon",
              },
          ]    ,			)

      a						:		int											=      unmasker("My name is <mask>"    ,			targets=[" Patrick", " Clara", " Teven"]    ,			top_k=3)
      self.assertEqual(
          nested_simplify(__UpperCAmelCase)    ,			[
              {"sequence": "My name is Patrick", "score": 0.005, "token": 3499, "token_str": " Patrick"},
              {"sequence": "My name is Clara", "score": 0.000, "token": 13606, "token_str": " Clara"},
              {"sequence": "My name is Te", "score": 0.000, "token": 2941, "token_str": " Te"},
          ]    ,			)






    @require_torch
    def      __snake_case							( self    :   Tuple):
      a						:		List[str]											=      pipeline(task="fill-mask"    ,			model="sshleifer/tiny-distilroberta-base"    ,			framework="pt")
      a						:		List[Any]											=      None
      a						:		Union[str, Any]											=      None
      self.run_pipeline_test(__UpperCAmelCase    ,			[])






    @require_tf
    def      __snake_case							( self    :   Optional[int]):
      a						:		Dict											=      pipeline(task="fill-mask"    ,			model="sshleifer/tiny-distilroberta-base"    ,			framework="tf")
      a						:		List[str]											=      None
      a						:		int											=      None
      self.run_pipeline_test(__UpperCAmelCase    ,			[])






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   List[Any]    ,			__UpperCAmelCase    :   Dict    ,			__UpperCAmelCase    :   Optional[int]):
      if tokenizer is None or tokenizer.mask_token_id is None:
        self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)")

      a						:		Dict											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)
      a						:		int											=      [
          f'''This is another {tokenizer.mask_token} test''',
      ]
      return fill_masker, examples






    def      __snake_case							( self    :   Dict    ,			__UpperCAmelCase    :   str    ,			__UpperCAmelCase    :   Tuple):
      a						:		Union[str, Any]											=      fill_masker.tokenizer
      a						:		Optional[int]											=      fill_masker.model

      a						:		List[Any]											=      fill_masker(
          f'''This is a {tokenizer.mask_token}'''    ,			)
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)

      a						:		Optional[Any]											=      fill_masker([f'''This is a {tokenizer.mask_token}'''])
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)

      a						:		Optional[int]											=      fill_masker([f'''This is a {tokenizer.mask_token}''', f'''Another {tokenizer.mask_token} great test.'''])
      self.assertEqual(
          __UpperCAmelCase    ,			[
              [
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              ],
              [
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              ],
          ]    ,			)

      with self.assertRaises(__UpperCAmelCase):
        fill_masker([None])
      # No mask_token is not supported
      with self.assertRaises(__UpperCAmelCase):
        fill_masker("This is")

      self.run_test_top_k(__UpperCAmelCase    ,			__UpperCAmelCase)
      self.run_test_targets(__UpperCAmelCase    ,			__UpperCAmelCase)
      self.run_test_top_k_targets(__UpperCAmelCase    ,			__UpperCAmelCase)
      self.fill_mask_with_duplicate_targets_and_top_k(__UpperCAmelCase    ,			__UpperCAmelCase)
      self.fill_mask_with_multiple_masks(__UpperCAmelCase    ,			__UpperCAmelCase)






    def      __snake_case							( self    :   List[Any]    ,			__UpperCAmelCase    :   List[str]    ,			__UpperCAmelCase    :   str):
      a						:		Optional[int]											=      tokenizer.get_vocab()
      a						:		Tuple											=      sorted(vocab.keys())[:2]
      # Pipeline argument
      a						:		Dict											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase    ,			targets=__UpperCAmelCase)
      a						:		Union[str, Any]											=      fill_masker(f'''This is a {tokenizer.mask_token}''')
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)
      a						:		Union[str, Any]											=      {vocab[el] for el in targets}
      self.assertEqual({el["token"] for el in outputs}    ,			__UpperCAmelCase)
      a						:		str											=      [tokenizer.decode([x]) for x in target_ids]
      self.assertEqual({el["token_str"] for el in outputs}    ,			set(__UpperCAmelCase))

      # Call argument
      a						:		Optional[int]											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)
      a						:		int											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets=__UpperCAmelCase)
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)
      a						:		Dict											=      {vocab[el] for el in targets}
      self.assertEqual({el["token"] for el in outputs}    ,			__UpperCAmelCase)
      a						:		Dict											=      [tokenizer.decode([x]) for x in target_ids]
      self.assertEqual({el["token_str"] for el in outputs}    ,			set(__UpperCAmelCase))

      # Score equivalence
      a						:		str											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets=__UpperCAmelCase)
      a						:		int											=      [top_mask["token_str"] for top_mask in outputs]
      a						:		Union[str, Any]											=      [top_mask["score"] for top_mask in outputs]

      # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
      if set(__UpperCAmelCase) == set(__UpperCAmelCase):
        a						:		Any											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets=__UpperCAmelCase)
        a						:		Optional[int]											=      [top_mask["score"] for top_mask in unmasked_targets]
        self.assertEqual(nested_simplify(__UpperCAmelCase)    ,			nested_simplify(__UpperCAmelCase))

      # Raises with invalid
      with self.assertRaises(__UpperCAmelCase):
        a						:		Union[str, Any]											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets=[])
      # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
      if "" not in tokenizer.get_vocab():
        with self.assertRaises(__UpperCAmelCase):
          a						:		Optional[Any]											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets=[""])
        with self.assertRaises(__UpperCAmelCase):
          a						:		List[Any]											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			targets="")






    def      __snake_case							( self    :   Tuple    ,			__UpperCAmelCase    :   int    ,			__UpperCAmelCase    :   Optional[int]):
      a						:		int											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase    ,			top_k=2)
      a						:		List[Any]											=      fill_masker(f'''This is a {tokenizer.mask_token}''')
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)

      a						:		List[Any]											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)
      a						:		Dict											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			top_k=2)
      self.assertEqual(
          __UpperCAmelCase    ,			[
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
          ]    ,			)
      self.assertEqual(nested_simplify(__UpperCAmelCase)    ,			nested_simplify(__UpperCAmelCase))






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   List[Any]    ,			__UpperCAmelCase    :   Tuple):
      a						:		Optional[int]											=      tokenizer.get_vocab()
      a						:		List[str]											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)

      # top_k=2, ntargets=3
      a						:		int											=      sorted(vocab.keys())[:3]
      a						:		int											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			top_k=2    ,			targets=__UpperCAmelCase)

      # If we use the most probably targets, and filter differently, we should still
      # have the same results
      a						:		str											=      [el["token_str"] for el in sorted(__UpperCAmelCase    ,			key=lambda __UpperCAmelCase: x["score"]    ,			reverse=__UpperCAmelCase)]
      # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
      if set(__UpperCAmelCase).issubset(__UpperCAmelCase):
        a						:		List[str]											=      fill_masker(f'''This is a {tokenizer.mask_token}'''    ,			top_k=3    ,			targets=__UpperCAmelCase)
        # They should yield exactly the same result
        self.assertEqual(nested_simplify(__UpperCAmelCase)    ,			nested_simplify(__UpperCAmelCase))






    def      __snake_case							( self    :   List[Any]    ,			__UpperCAmelCase    :   Any    ,			__UpperCAmelCase    :   Any):
      a						:		List[str]											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)
      a						:		List[str]											=      tokenizer.get_vocab()
      # String duplicates + id duplicates
      a						:		Dict											=      sorted(vocab.keys())[:3]
      a						:		List[Any]											=      [targets[0], targets[1], targets[0], targets[2], targets[1]]
      a						:		Optional[int]											=      fill_masker(f'''My name is {tokenizer.mask_token}'''    ,			targets=__UpperCAmelCase    ,			top_k=10)

      # The target list contains duplicates, so we can't output more
      # than them
      self.assertEqual(len(__UpperCAmelCase)    ,			3)






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   str    ,			__UpperCAmelCase    :   List[str]):
      a						:		Dict											=      FillMaskPipeline(model=__UpperCAmelCase    ,			tokenizer=__UpperCAmelCase)

      a						:		Optional[int]											=      fill_masker(
          f'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''    ,			top_k=2)
      self.assertEqual(
          __UpperCAmelCase    ,			[
              [
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              ],
              [
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              ],
              [
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
                  {"sequence": ANY(__UpperCAmelCase), "score": ANY(__UpperCAmelCase), "token": ANY(__UpperCAmelCase), "token_str": ANY(__UpperCAmelCase)},
              ],
          ]    ,			)










"""simple docstring"""







import datasets


__lowercase    =				"""\
@InProceedings{conneau2018xnli,
  author = \"Conneau, Alexis
                 and Rinott, Ruty
                 and Lample, Guillaume
                 and Williams, Adina
                 and Bowman, Samuel R.
                 and Schwenk, Holger
                 and Stoyanov, Veselin\",
  title = \"XNLI: Evaluating Cross-lingual Sentence Representations\",
  booktitle = \"Proceedings of the 2018 Conference on Empirical Methods
               in Natural Language Processing\",
  year = \"2018\",
  publisher = \"Association for Computational Linguistics\",
  location = \"Brussels, Belgium\",
}
"""

__lowercase    =				"""\
XNLI is a subset of a few thousand examples from MNLI which has been translated
into a 14 different languages (some low-ish resource). As with MNLI, the goal is
to predict textual entailment (does sentence A imply/contradict/neither sentence
B) and is a classification task (given two sentences, predict one of three
labels).
"""

__lowercase    =				"""
Computes XNLI score which is just simple accuracy.
Args:
    predictions: Predicted labels.
    references: Ground truth labels.
Returns:
    'accuracy': accuracy
Examples:

    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> xnli_metric = datasets.load_metric(\"xnli\")
    >>> results = xnli_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}
"""


def 					lowercase      (    A_  ,       A_	)->				List[str]:
  '''simple docstring'''







  return (preds == labels).mean()







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,_KWARGS_DESCRIPTION  )
class 						_A  (  datasets.Metric  ):







    """simple docstring"""




    def      __snake_case							( self    :   List[str]):
      return datasets.MetricInfo(
          description=_DESCRIPTION    ,			citation=_CITATION    ,			inputs_description=_KWARGS_DESCRIPTION    ,			features=datasets.Features(
              {
                  "predictions": datasets.Value("int64" if self.config_name != "sts-b" else "float32"),
                  "references": datasets.Value("int64" if self.config_name != "sts-b" else "float32"),
              })    ,			codebase_urls=[]    ,			reference_urls=[]    ,			format="numpy"    ,			)






    def      __snake_case							( self    :   str    ,			__UpperCAmelCase    :   List[Any]    ,			__UpperCAmelCase    :   Tuple):
      return {"accuracy": simple_accuracy(__UpperCAmelCase    ,			__UpperCAmelCase)}






