





import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
      import PIL.Image

      from .features import FeatureType


_UpperCAmelCase             =							None
_UpperCAmelCase             =							'<' if sys.byteorder == 'little' else '>'
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
_UpperCAmelCase             =							[
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]






@dataclass
class        snake_case_    :
  A_      							=   True
  A_      							=   None
  # Automatically constructed
  A_      							=   'PIL.Image.Image'
  A_      							=   pa.struct({'bytes': pa.binary(), 'path': pa.string()}						)
  A_      							=   field(default='Image'					,init=_A					,repr=_A						)






  def __call__(							self			:					str      )->Tuple:






   '''simple docstring'''

   return self.pa_type






  def 				UpperCAmelCase__						(							self			:					Tuple       ,  _snake_case			:					Union[str, bytes, dict, np.ndarray, "PIL.Image.Image"]      )->List[str]:






   '''simple docstring'''

   if config.PIL_AVAILABLE:
    import PIL.Image
   else:
    raise ImportError("""To support encoding images, please install 'Pillow'."""      )

   if isinstance(__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE      ):
    __lowerCAmelCase					:	List[str]      =       np.array(__SCREAMING_SNAKE_CASE      )

   if isinstance(__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE      ):
    return {"path": value, "bytes": None}
   elif isinstance(__SCREAMING_SNAKE_CASE       ,  __SCREAMING_SNAKE_CASE      ):
    return {"path": None, "bytes": value}
   elif isinstance(__SCREAMING_SNAKE_CASE       ,  np.ndarray      ):
    # convert the image array to PNG/TIFF bytes
    return encode_np_array(__SCREAMING_SNAKE_CASE      )
   elif isinstance(__SCREAMING_SNAKE_CASE       ,  PIL.Image.Image      ):
    # convert the PIL image to bytes (default format is PNG/TIFF)
    return encode_pil_image(__SCREAMING_SNAKE_CASE      )
   elif value.get("""path"""      ) is not None and os.path.isfile(value["""path"""]      ):
    # we set "bytes": None to not duplicate the data if they're already available locally
    return {"bytes": None, "path": value.get("""path"""      )}
   elif value.get("""bytes"""      ) is not None or value.get("""path"""      ) is not None:
    # store the image bytes, and path is used to infer the image format using the file extension
    return {"bytes": value.get("""bytes"""      ), "path": value.get("""path"""      )}
   else:
    raise ValueError(
        F'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''      )






  def 				UpperCAmelCase__						(							self			:					int       ,  _snake_case			:					dict       ,  _snake_case			:					Union[str, Any]=None      )->Union[str, Any]:






   '''simple docstring'''

   if not self.decode:
    raise RuntimeError("""Decoding is disabled for this feature. Please use Image(decode=True) instead."""      )

   if config.PIL_AVAILABLE:
    import PIL.Image
   else:
    raise ImportError("""To support decoding images, please install 'Pillow'."""      )

   if token_per_repo_id is None:
    __lowerCAmelCase					:	int      =       {}

   __lowerCAmelCase ,					__lowerCAmelCase					:	Optional[int]      =       value["""path"""], value["""bytes"""]
   if bytes_ is None:
    if path is None:
     raise ValueError(F'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''      )
    else:
     if is_local_path(__SCREAMING_SNAKE_CASE      ):
      __lowerCAmelCase					:	Union[str, Any]      =       PIL.Image.open(__SCREAMING_SNAKE_CASE      )
     else:
      __lowerCAmelCase					:	str      =       path.split("""::"""      )[-1]
      try:
       __lowerCAmelCase					:	Dict      =       string_to_dict(__SCREAMING_SNAKE_CASE       ,  config.HUB_DATASETS_URL      )["""repo_id"""]
       __lowerCAmelCase					:	Any      =       token_per_repo_id.get(__SCREAMING_SNAKE_CASE      )
      except ValueError:
       __lowerCAmelCase					:	int      =       None
      with xopen(__SCREAMING_SNAKE_CASE       ,  """rb"""       ,  use_auth_token=__SCREAMING_SNAKE_CASE      ) as f:
       __lowerCAmelCase					:	Optional[int]      =       BytesIO(f.read()      )
      __lowerCAmelCase					:	int      =       PIL.Image.open(bytes_      )
   else:
    __lowerCAmelCase					:	int      =       PIL.Image.open(BytesIO(bytes_      )      )
   image.load()  # to avoid "Too many open files" errors
   return image






  def 				UpperCAmelCase__						(							self			:					Optional[Any]      )->int:






   '''simple docstring'''

   from .features import Value

   return (
       self
       if self.decode
       else {
           "bytes": Value("""binary"""      ),
           "path": Value("""string"""      ),
       }
   )






  def 				UpperCAmelCase__						(							self			:					Optional[int]       ,  _snake_case			:					Union[pa.StringArray, pa.StructArray, pa.ListArray]      )->Any:






   '''simple docstring'''

   if pa.types.is_string(storage.type      ):
    __lowerCAmelCase					:	Optional[Any]      =       pa.array([None] * len(__SCREAMING_SNAKE_CASE      )       ,  type=pa.binary()      )
    __lowerCAmelCase					:	Union[str, Any]      =       pa.StructArray.from_arrays([bytes_array, storage]       ,  ["""bytes""", """path"""]       ,  mask=storage.is_null()      )
   elif pa.types.is_binary(storage.type      ):
    __lowerCAmelCase					:	List[Any]      =       pa.array([None] * len(__SCREAMING_SNAKE_CASE      )       ,  type=pa.string()      )
    __lowerCAmelCase					:	Tuple      =       pa.StructArray.from_arrays([storage, path_array]       ,  ["""bytes""", """path"""]       ,  mask=storage.is_null()      )
   elif pa.types.is_struct(storage.type      ):
    if storage.type.get_field_index("""bytes"""      ) >= 0:
     __lowerCAmelCase					:	Tuple      =       storage.field("""bytes"""      )
    else:
     __lowerCAmelCase					:	Tuple      =       pa.array([None] * len(__SCREAMING_SNAKE_CASE      )       ,  type=pa.binary()      )
    if storage.type.get_field_index("""path"""      ) >= 0:
     __lowerCAmelCase					:	List[str]      =       storage.field("""path"""      )
    else:
     __lowerCAmelCase					:	List[str]      =       pa.array([None] * len(__SCREAMING_SNAKE_CASE      )       ,  type=pa.string()      )
    __lowerCAmelCase					:	Optional[int]      =       pa.StructArray.from_arrays([bytes_array, path_array]       ,  ["""bytes""", """path"""]       ,  mask=storage.is_null()      )
   elif pa.types.is_list(storage.type      ):
    __lowerCAmelCase					:	Optional[int]      =       pa.array(
        [encode_np_array(np.array(__SCREAMING_SNAKE_CASE      )      )["""bytes"""] if arr is not None else None for arr in storage.to_pylist()]       ,  type=pa.binary()       ,  )
    __lowerCAmelCase					:	Dict      =       pa.array([None] * len(__SCREAMING_SNAKE_CASE      )       ,  type=pa.string()      )
    __lowerCAmelCase					:	Tuple      =       pa.StructArray.from_arrays(
        [bytes_array, path_array]       ,  ["""bytes""", """path"""]       ,  mask=bytes_array.is_null()      )
   return array_cast(__SCREAMING_SNAKE_CASE       ,  self.pa_type      )






  def 				UpperCAmelCase__						(							self			:					Any       ,  _snake_case			:					pa.StructArray      )->Optional[int]:






   '''simple docstring'''

   @no_op_if_value_is_null
   def path_to_bytes(_snake_case			:					str      ):
    with xopen(__SCREAMING_SNAKE_CASE       ,  """rb"""      ) as f:
     __lowerCAmelCase					:	str      =       f.read()
    return bytes_

   __lowerCAmelCase					:	Optional[Any]      =       pa.array(
       [
           (path_to_bytes(x["""path"""]      ) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
           for x in storage.to_pylist()
       ]       ,  type=pa.binary()       ,  )
   __lowerCAmelCase					:	Any      =       pa.array(
       [os.path.basename(__SCREAMING_SNAKE_CASE      ) if path is not None else None for path in storage.field("""path"""      ).to_pylist()]       ,  type=pa.string()       ,  )
   __lowerCAmelCase					:	Tuple      =       pa.StructArray.from_arrays([bytes_array, path_array]       ,  ["""bytes""", """path"""]       ,  mask=bytes_array.is_null()      )
   return array_cast(__SCREAMING_SNAKE_CASE       ,  self.pa_type      )






def 	_SCREAMING_SNAKE_CASE		(      )      ->   Any:
 if config.PIL_AVAILABLE:
  import PIL.Image
 else:
  raise ImportError("""To support encoding images, please install 'Pillow'."""   )

 global _IMAGE_COMPRESSION_FORMATS
 if _IMAGE_COMPRESSION_FORMATS is None:
  PIL.Image.init()
  __lowerCAmelCase					:	Any      =       list(set(PIL.Image.OPEN.keys()   ) & set(PIL.Image.SAVE.keys()   )   )
 return _IMAGE_COMPRESSION_FORMATS






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Any   )      ->   Union[str, Any]:
 __lowerCAmelCase					:	Optional[int]      =       BytesIO()
 if image.format in list_image_compression_formats():
  __lowerCAmelCase					:	List[Any]      =       image.format
 else:
  __lowerCAmelCase					:	str      =       """PNG""" if image.mode in ["""1""", """L""", """LA""", """RGB""", """RGBA"""] else """TIFF"""
 image.save(lowercase__   ,   format=lowercase__   )
 return buffer.getvalue()






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:List[str]   )      ->   List[Any]:
 if hasattr(lowercase__   ,   """filename"""   ) and image.filename != "":
  return {"path": image.filename, "bytes": None}
 else:
  return {"path": None, "bytes": image_to_bytes(lowercase__   )}






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:str   )      ->   List[Any]:
 if config.PIL_AVAILABLE:
  import PIL.Image
 else:
  raise ImportError("""To support encoding images, please install 'Pillow'."""   )

 __lowerCAmelCase					:	Optional[Any]      =       array.dtype
 __lowerCAmelCase					:	Optional[int]      =       dtype.byteorder if dtype.byteorder != """=""" else _NATIVE_BYTEORDER
 __lowerCAmelCase					:	int      =       dtype.kind
 __lowerCAmelCase					:	List[str]      =       dtype.itemsize

 __lowerCAmelCase					:	Tuple      =       None

 # Multi-channel array case (only np.dtype("|u1") is allowed)
 if array.shape[2:]:
  __lowerCAmelCase					:	Any      =       np.dtype("""|u1"""   )
  if dtype_kind not in ["u", "i"]:
   raise TypeError(
       F'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.'''   )
  if dtype is not dest_dtype:
   warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''   )
    # Exact match
 elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
  __lowerCAmelCase					:	Union[str, Any]      =       dtype
 else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
  while dtype_itemsize >= 1:
   __lowerCAmelCase					:	Union[str, Any]      =       dtype_byteorder + dtype_kind + str(lowercase__   )
   __lowerCAmelCase					:	Dict      =       np.dtype(lowercase__   )
   if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
    warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''   )
    break
   else:
    dtype_itemsize //= 2

 if dest_dtype is None:
  raise TypeError(
      F'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}'''   )
 __lowerCAmelCase					:	List[str]      =       PIL.Image.fromarray(array.astype(lowercase__   )   )
 return {"path": None, "bytes": image_to_bytes(lowercase__   )}







def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Optional[int]   )      ->   Any:
 if config.PIL_AVAILABLE:
  import PIL.Image
 else:
  raise ImportError("""To support encoding images, please install 'Pillow'."""   )

 if objs:
  __lowerCAmelCase ,					__lowerCAmelCase					:	Optional[int]      =       first_non_null_value(lowercase__   )
  if isinstance(lowercase__   ,   lowercase__   ):
   return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
  if isinstance(lowercase__   ,   np.ndarray   ):
   __lowerCAmelCase					:	Any      =       no_op_if_value_is_null(lowercase__   )
   return [obj_to_image_dict_func(lowercase__   ) for obj in objs]
  elif isinstance(lowercase__   ,   PIL.Image.Image   ):
   __lowerCAmelCase					:	Optional[int]      =       no_op_if_value_is_null(lowercase__   )
   return [obj_to_image_dict_func(lowercase__   ) for obj in objs]
  else:
   return objs
 else:
  return objs










import sacrebleu as scb
from packaging import version
from sacrebleu import TER

import datasets


_UpperCAmelCase             =							'\\n@inproceedings{snover-etal-2006-study,\n    title = "A Study of Translation Edit Rate with Targeted Human Annotation",\n    author = "Snover, Matthew  and\n      Dorr, Bonnie  and\n      Schwartz, Rich  and\n      Micciulla, Linnea  and\n      Makhoul, John",\n    booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers",\n    month = aug # " 8-12",\n    year = "2006",\n    address = "Cambridge, Massachusetts, USA",\n    publisher = "Association for Machine Translation in the Americas",\n    url = "https://aclanthology.org/2006.amta-papers.25",\n    pages = "223--231",\n}\n@inproceedings{post-2018-call,\n    title = "A Call for Clarity in Reporting {BLEU} Scores",\n    author = "Post, Matt",\n    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",\n    month = oct,\n    year = "2018",\n    address = "Belgium, Brussels",\n    publisher = "Association for Computational Linguistics",\n    url = "https://www.aclweb.org/anthology/W18-6319",\n    pages = "186--191",\n}\n'

_UpperCAmelCase             =							'\\nTER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a\nhypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu\n(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found\nhere: https://github.com/jhclark/tercom.\n\nThe implementation here is slightly different from sacrebleu in terms of the required input format. The length of\nthe references and hypotheses lists need to be the same, so you may need to transpose your references compared to\nsacrebleu\'s required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534\n\nSee the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information.\n'

_UpperCAmelCase             =							'\nProduces TER scores alongside the number of edits and reference length.\n\nArgs:\n    predictions (list of str): The system stream (a sequence of segments).\n    references (list of list of str): A list of one or more reference streams (each a sequence of segments).\n    normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.\n    ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.\n    support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters,\n                                    as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana.\n                                    Only applies if `normalized = True`. Defaults to `False`.\n    case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`.\n\nReturns:\n    \'score\' (float): TER score (num_edits / sum_ref_lengths * 100)\n    \'num_edits\' (int): The cumulative number of edits\n    \'ref_length\' (float): The cumulative average reference length\n\nExamples:\n    Example 1:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?",\n        ...                     "What did the TER metric user say to the developer?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],\n        ...             ["Your jokes are...", "...TERrible"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 150.0, \'num_edits\': 15, \'ref_length\': 10.0}\n\n    Example 2:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 62.5, \'num_edits\': 5, \'ref_length\': 8.0}\n\n    Example 3:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         normalized=True,\n        ...                         case_sensitive=True)\n        >>> print(results)\n        {\'score\': 57.14285714285714, \'num_edits\': 6, \'ref_length\': 10.5}\n\n    Example 4:\n        >>> predictions = ["does this sentence match??",\n        ...                     "what about this sentence?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         ignore_punct=True,\n        ...                         case_sensitive=False)\n        >>> print(results)\n        {\'score\': 0.0, \'num_edits\': 0, \'ref_length\': 8.0}\n\n    Example 5:\n        >>> predictions = ["does this sentence match??",\n        ...                    "what about this sentence?",\n        ...                    "What did the TER metric user say to the developer?"]\n        >>> references = [["does this sentence match", "does this sentence match!?!"],\n        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],\n        ...             ["Your jokes are...", "...TERrible"]]\n        >>> ter = datasets.load_metric("ter")\n        >>> results = ter.compute(predictions=predictions,\n        ...                         references=references,\n        ...                         ignore_punct=True,\n        ...                         case_sensitive=False)\n        >>> print(results)\n        {\'score\': 100.0, \'num_edits\': 10, \'ref_length\': 10.0}\n'






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,_KWARGS_DESCRIPTION						)
class        snake_case_    (		datasets.Metric						):






  def 				UpperCAmelCase__						(							self			:					Tuple      )->str:






   '''simple docstring'''

   if version.parse(scb.__version__      ) < version.parse("""1.4.12"""      ):
    raise ImportWarning(
        """To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"""
        """You can install it with `pip install \"sacrebleu>=1.4.12\"`."""      )
   return datasets.MetricInfo(
       description=_DESCRIPTION       ,  citation=_CITATION       ,  homepage="""http://www.cs.umd.edu/~snover/tercom/"""       ,  inputs_description=_KWARGS_DESCRIPTION       ,  features=datasets.Features(
           {
               """predictions""": datasets.Value("""string"""       ,  id="""sequence"""      ),
               """references""": datasets.Sequence(datasets.Value("""string"""       ,  id="""sequence"""      )       ,  id="""references"""      ),
           }      )       ,  codebase_urls=["""https://github.com/mjpost/sacreBLEU#ter"""]       ,  reference_urls=[
           """https://github.com/jhclark/tercom""",
       ]       ,  )






  def 				UpperCAmelCase__						(							self			:					Dict       ,  _snake_case			:					List[Any]       ,  _snake_case			:					int       ,  _snake_case			:					bool = False       ,  _snake_case			:					bool = False       ,  _snake_case			:					bool = False       ,  _snake_case			:					bool = False       ,  )->str:






   '''simple docstring'''

   __lowerCAmelCase					:	List[str]      =       len(references[0]      )
   if any(len(_snake_case      ) != references_per_prediction for refs in references      ):
    raise ValueError("""Sacrebleu requires the same number of references for each prediction"""      )
   __lowerCAmelCase					:	str      =       [[refs[i] for refs in references] for i in range(_snake_case      )]

   __lowerCAmelCase					:	Tuple      =       TER(
       normalized=_snake_case       ,  no_punct=_snake_case       ,  asian_support=_snake_case       ,  case_sensitive=_snake_case       ,  )
   __lowerCAmelCase					:	List[Any]      =       sb_ter.corpus_score(_snake_case       ,  _snake_case      )

   return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length}