



'''simple docstring'''





import datasets

from .evaluate import evaluate


_lowerCAmelCase										=     '''\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
'''

_lowerCAmelCase										=     '''
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
'''

_lowerCAmelCase										=     '''
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the CUAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
    \'aupr\': Area Under the Precision-Recall curve
    \'prec_at_80_recall\': Precision at 80% recall
    \'prec_at_90_recall\': Precision at 90% recall
Examples:
    >>> predictions = [{\'prediction_text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\'], \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> references = [{\'answers\': {\'answer_start\': [143, 49], \'text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\']}, \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> cuad_metric = datasets.load_metric("cuad")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0, \'aupr\': 0.0, \'prec_at_80_recall\': 1.0, \'prec_at_90_recall\': 1.0}
'''






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,			_KWARGS_DESCRIPTION					)
class 	lowerCAmelCase_(    datasets.Metric					):
  '''simple docstring'''




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      return datasets.MetricInfo(
          description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(
              {
                  """predictions""": {
                      """id""": datasets.Value("""string"""				),
                      """prediction_text""": datasets.features.Sequence(datasets.Value("""string"""				)				),
                  },
                  """references""": {
                      """id""": datasets.Value("""string"""				),
                      """answers""": datasets.features.Sequence(
                          {
                              """text""": datasets.Value("""string"""				),
                              """answer_start""": datasets.Value("""int32"""				),
                          }				),
                  },
              }				)  ,codebase_urls=["""https://www.atticusprojectai.org/cuad"""]  ,reference_urls=["""https://www.atticusprojectai.org/cuad"""]  ,)






  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		str:
      lowerCAmelCase__			:  Optional[Any]											= {prediction["""id"""]: prediction["""prediction_text"""] for prediction in predictions}
      lowerCAmelCase__			:  List[str]											= [
          {
              """paragraphs""": [
                  {
                      """qas""": [
                          {
                              """answers""": [{"""text""": answer_text} for answer_text in ref["""answers"""]["""text"""]],
                              """id""": ref["""id"""],
                          }
                          for ref in references
                      ]
                  }
              ]
          }
      ]
      lowerCAmelCase__			:  Union[str, Any]											= evaluate(dataset=__UpperCAmelCase  ,predictions=__UpperCAmelCase				)
      return score







'''simple docstring'''





import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

 from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
 _lowerCAmelCase										=     False

if is_vision_available():
 from PIL import Image

 from transformers import PixaStructImageProcessor






class 	lowerCAmelCase_(    unittest.TestCase					):
  '''simple docstring'''




  def __init__(   self  ,__UpperCAmelCase  ,__UpperCAmelCase=7  ,__UpperCAmelCase=3  ,__UpperCAmelCase=18  ,__UpperCAmelCase=30  ,__UpperCAmelCase=400  ,__UpperCAmelCase=None  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=None  ,)   ->		Optional[Any]:
      lowerCAmelCase__			:  Optional[Any]											= size if size is not None else {"""height""": 20, """width""": 20}
      lowerCAmelCase__			:  List[Any]											= parent
      lowerCAmelCase__			:  List[str]											= batch_size
      lowerCAmelCase__			:  Tuple											= num_channels
      lowerCAmelCase__			:  Optional[int]											= image_size
      lowerCAmelCase__			:  Optional[Any]											= min_resolution
      lowerCAmelCase__			:  Tuple											= max_resolution
      lowerCAmelCase__			:  List[Any]											= size
      lowerCAmelCase__			:  List[str]											= do_normalize
      lowerCAmelCase__			:  Optional[int]											= do_convert_rgb
      lowerCAmelCase__			:  str											= [512, 1024, 2048, 4096]
      lowerCAmelCase__			:  int											= patch_size if patch_size is not None else {"""height""": 16, """width""": 16}




  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}






  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  Optional[Any]											= """https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg"""
      lowerCAmelCase__			:  Union[str, Any]											= Image.open(requests.get(__UpperCAmelCase  ,stream=__UpperCAmelCase				).raw				).convert("""RGB"""				)
      return raw_image







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11						,			reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''						,			)
@require_torch
@require_vision
class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_						,			unittest.TestCase					):
  '''simple docstring'''




  __lowercase							:				Union[str, Any]           =				PixaStructImageProcessor if is_vision_available() else None




  def 							UpperCAmelCase_     (   self				)   ->		List[str]:
      lowerCAmelCase__			:  Union[str, Any]											= PixaStructImageProcessingTester(self				)




  @property
  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      return self.image_processor_tester.prepare_image_processor_dict()




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      lowerCAmelCase__			:  Dict											= self.image_processing_class(**self.image_processor_dict				)
      self.assertTrue(hasattr(__UpperCAmelCase  ,"""do_normalize"""				)				)
      self.assertTrue(hasattr(__UpperCAmelCase  ,"""do_convert_rgb"""				)				)




  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      lowerCAmelCase__			:  List[Any]											= self.image_processor_tester.prepare_dummy_image()

      lowerCAmelCase__			:  List[Any]											= self.image_processing_class(**self.image_processor_dict				)
      lowerCAmelCase__			:  str											= 2048

      lowerCAmelCase__			:  Tuple											= image_processor(__UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				)
      self.assertTrue(torch.allclose(inputs.flattened_patches.mean()  ,torch.tensor(0.0_6_0_6				)  ,atol=1E-3  ,rtol=1E-3				)				)




  def 							UpperCAmelCase_     (   self				)   ->		str:
      # Initialize image_processor
      lowerCAmelCase__			:  int											= self.image_processing_class(**self.image_processor_dict				)
      # create random PIL images
      lowerCAmelCase__			:  int											= prepare_image_inputs(self.image_processor_tester  ,equal_resolution=__UpperCAmelCase				)
      for image in image_inputs:
          self.assertIsInstance(__UpperCAmelCase  ,Image.Image				)

      # Test not batched input
      lowerCAmelCase__			:  List[str]											= (
          (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
          * self.image_processor_tester.num_channels
      ) + 2

      for max_patch in self.image_processor_tester.max_patches:
          # Test not batched input
          lowerCAmelCase__			:  str											= image_processor(
              image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(1, max_patch, expected_hidden_dim)  ,)

          # Test batched
          lowerCAmelCase__			:  Any											= image_processor(
              __UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)  ,)




  def 							UpperCAmelCase_     (   self				)   ->		Any:
      # Initialize image_processor
      lowerCAmelCase__			:  Optional[int]											= self.image_processing_class(**self.image_processor_dict				)
      # create random PIL images
      lowerCAmelCase__			:  Optional[int]											= prepare_image_inputs(self.image_processor_tester  ,equal_resolution=__UpperCAmelCase				)
      for image in image_inputs:
          self.assertIsInstance(__UpperCAmelCase  ,Image.Image				)

      # Test not batched input
      lowerCAmelCase__			:  Tuple											= (
          (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
          * self.image_processor_tester.num_channels
      ) + 2

      lowerCAmelCase__			:  Optional[int]											= True

      for max_patch in self.image_processor_tester.max_patches:
          # Test not batched input
          with self.assertRaises(__UpperCAmelCase				):
              lowerCAmelCase__			:  Any											= image_processor(
                  image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches

          lowerCAmelCase__			:  Optional[Any]											= """Hello"""

          lowerCAmelCase__			:  List[str]											= image_processor(
              image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase  ,header_text=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(1, max_patch, expected_hidden_dim)  ,)

          # Test batched
          lowerCAmelCase__			:  str											= image_processor(
              __UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase  ,header_text=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)  ,)




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      # Initialize image_processor
      lowerCAmelCase__			:  Optional[Any]											= self.image_processing_class(**self.image_processor_dict				)
      # create random numpy tensors
      lowerCAmelCase__			:  List[str]											= prepare_image_inputs(self.image_processor_tester  ,equal_resolution=__UpperCAmelCase  ,numpify=__UpperCAmelCase				)
      for image in image_inputs:
          self.assertIsInstance(__UpperCAmelCase  ,np.ndarray				)

      lowerCAmelCase__			:  Tuple											= (
          (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
          * self.image_processor_tester.num_channels
      ) + 2

      for max_patch in self.image_processor_tester.max_patches:
          # Test not batched input
          lowerCAmelCase__			:  Any											= image_processor(
              image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(1, max_patch, expected_hidden_dim)  ,)

          # Test batched
          lowerCAmelCase__			:  int											= image_processor(
              __UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)  ,)






  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      # Initialize image_processor
      lowerCAmelCase__			:  Tuple											= self.image_processing_class(**self.image_processor_dict				)
      # create random PyTorch tensors
      lowerCAmelCase__			:  int											= prepare_image_inputs(self.image_processor_tester  ,equal_resolution=__UpperCAmelCase  ,torchify=__UpperCAmelCase				)
      for image in image_inputs:
          self.assertIsInstance(__UpperCAmelCase  ,torch.Tensor				)

      # Test not batched input
      lowerCAmelCase__			:  Optional[int]											= (
          (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
          * self.image_processor_tester.num_channels
      ) + 2

      for max_patch in self.image_processor_tester.max_patches:
          # Test not batched input
          lowerCAmelCase__			:  Dict											= image_processor(
              image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(1, max_patch, expected_hidden_dim)  ,)

          # Test batched
          lowerCAmelCase__			:  Union[str, Any]											= image_processor(
              __UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)  ,)







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11						,			reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''						,			)
@require_torch
@require_vision
class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_						,			unittest.TestCase					):
  '''simple docstring'''




  __lowercase							:				Optional[int]           =				PixaStructImageProcessor if is_vision_available() else None




  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      lowerCAmelCase__			:  Dict											= PixaStructImageProcessingTester(self  ,num_channels=4				)
      lowerCAmelCase__			:  str											= 3




  @property
  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      return self.image_processor_tester.prepare_image_processor_dict()




  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  Union[str, Any]											= self.image_processing_class(**self.image_processor_dict				)
      self.assertTrue(hasattr(__UpperCAmelCase  ,"""do_normalize"""				)				)
      self.assertTrue(hasattr(__UpperCAmelCase  ,"""do_convert_rgb"""				)				)






  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      # Initialize image_processor
      lowerCAmelCase__			:  Optional[Any]											= self.image_processing_class(**self.image_processor_dict				)
      # create random PIL images
      lowerCAmelCase__			:  Dict											= prepare_image_inputs(self.image_processor_tester  ,equal_resolution=__UpperCAmelCase				)
      for image in image_inputs:
          self.assertIsInstance(__UpperCAmelCase  ,Image.Image				)

      # Test not batched input
      lowerCAmelCase__			:  Dict											= (
          (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
          * (self.image_processor_tester.num_channels - 1)
      ) + 2

      for max_patch in self.image_processor_tester.max_patches:
          # Test not batched input
          lowerCAmelCase__			:  int											= image_processor(
              image_inputs[0]  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(1, max_patch, expected_hidden_dim)  ,)

          # Test batched
          lowerCAmelCase__			:  Dict											= image_processor(
              __UpperCAmelCase  ,return_tensors="""pt"""  ,max_patches=__UpperCAmelCase				).flattened_patches
          self.assertEqual(
              encoded_images.shape  ,(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)  ,)


