





import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

snake_case_           =      logging.get_logger(__name__)






def lowerCamelCase__   (							snake_case_     :  Tuple	,				snake_case_     :  Optional[Any]	)				->    Tuple:







   def run_func(snake_case_     :  Any	):
      @wraps(_A	)
      def run_in_eager_mode(*snake_case_     :  str	,				**snake_case_     :  int	):
         return func(*_A	,				**_A	)

      @wraps(_A	)
      @tf.function(experimental_compile=_A	)
      def run_in_graph_mode(*snake_case_     :  Any	,				**snake_case_     :  Union[str, Any]	):
         return func(*_A	,				**_A	)

      if do_eager_mode is True:
         if use_xla is not False:
            raise ValueError(
                '''Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'''	)
         return run_in_eager_mode
      else:
         return run_in_graph_mode

   return run_func






def lowerCamelCase__   (							snake_case_     :  str	,				snake_case_     :  int	,				snake_case_     :  Any	)				->    Any:
   __snake_case								=       random.Random()
   __snake_case								=       [rng.randint(0	,				vocab_size - 1	) for i in range(batch_size * sequence_length	)]
   return tf.constant(_A	,				shape=(batch_size, sequence_length)	,				dtype=tf.intaa	)







class 		SCREAMING_SNAKE_CASE__    (	SCREAMING_SNAKE_CASE_  ):
      A_						:    List[Any]							=   42
      A_						:    str							=   42
      A_						:    List[Any]							=   'TensorFlow'


      @property
      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         return tf.__version__


      def        a			(self	:				List[Any]    ,      a__	:				Optional[Any]    ,      a__	:				Dict    ,      a__	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       self.args.strategy
         if strategy is None:
            raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''				)
         __snake_case								=       self._prepare_inference_func(snake_case__    ,      snake_case__    ,      snake_case__				)
         return self._measure_speed(_inference				)


      def        a			(self	:				List[Any]    ,      a__	:				Dict    ,      a__	:				List[str]    ,      a__	:				Tuple				):





         """simple docstring"""



         __snake_case								=       self.args.strategy
         if strategy is None:
            raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''				)
         __snake_case								=       self._prepare_train_func(snake_case__    ,      snake_case__    ,      snake_case__				)
         return self._measure_speed(_train				)


      def        a			(self	:				Union[str, Any]    ,      a__	:				Optional[int]    ,      a__	:				str    ,      a__	:				List[str]				):





         """simple docstring"""



         if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]    ,      snake_case__				)
         __snake_case								=       self.args.strategy
         if strategy is None:
            raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''				)
         __snake_case								=       self._prepare_inference_func(snake_case__    ,      snake_case__    ,      snake_case__				)
         return self._measure_memory(_inference				)


      def        a			(self	:				Union[str, Any]    ,      a__	:				Dict    ,      a__	:				Tuple    ,      a__	:				Optional[int]				):





         """simple docstring"""



         if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]    ,      snake_case__				)
         __snake_case								=       self.args.strategy
         if strategy is None:
            raise ValueError('''A device strategy has to be initialized before using TensorFlow.'''				)

         __snake_case								=       self._prepare_train_func(snake_case__    ,      snake_case__    ,      snake_case__				)
         return self._measure_memory(_train				)


      def        a			(self	:				Any    ,      a__	:				Any    ,      a__	:				Union[str, Any]    ,      a__	:				List[str]				):





         """simple docstring"""



         __snake_case								=       self.config_dict[model_name]

         if self.args.fpaa:
            raise NotImplementedError('''Mixed precision is currently not supported.'''				)

         __snake_case								=       (
             hasattr(snake_case__    ,      '''architectures'''				)
             and isinstance(config.architectures    ,      snake_case__				)
             and len(config.architectures				) > 0
         )
         if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
               __snake_case								=       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
               __snake_case								=       __import__('''transformers'''    ,      fromlist=[model_class]				)
               __snake_case								=       getattr(snake_case__    ,      snake_case__				)
               __snake_case								=       model_cls(snake_case__				)
            except ImportError:
               raise ImportError(
                   f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                   ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''				)
         else:
            __snake_case								=       TF_MODEL_MAPPING[config.__class__](snake_case__				)

         # encoder-decoder has vocab size saved differently
         __snake_case								=       config.vocab_size if hasattr(snake_case__    ,      '''vocab_size'''				) else config.encoder.vocab_size
         __snake_case								=       random_input_ids(snake_case__    ,      snake_case__    ,      snake_case__				)

         @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla				)
         def encoder_decoder_forward():
            return model(snake_case__    ,      decoder_input_ids=snake_case__    ,      training=snake_case__				)

         @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla				)
         def encoder_forward():
            return model(snake_case__    ,      training=snake_case__				)

         __snake_case								=       encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

         return _inference


      def        a			(self	:				Dict    ,      a__	:				Any    ,      a__	:				List[str]    ,      a__	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       self.config_dict[model_name]

         if self.args.eager_mode is not False:
            raise ValueError('''Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'''				)

         if self.args.fpaa:
            raise NotImplementedError('''Mixed precision is currently not supported.'''				)

         __snake_case								=       (
             hasattr(snake_case__    ,      '''architectures'''				)
             and isinstance(config.architectures    ,      snake_case__				)
             and len(config.architectures				) > 0
         )
         if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
               __snake_case								=       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
               __snake_case								=       __import__('''transformers'''    ,      fromlist=[model_class]				)
               __snake_case								=       getattr(snake_case__    ,      snake_case__				)
               __snake_case								=       model_cls(snake_case__				)
            except ImportError:
               raise ImportError(
                   f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                   ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''				)
         else:
            __snake_case								=       TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](snake_case__				)

         # encoder-decoder has vocab size saved differently
         __snake_case								=       config.vocab_size if hasattr(snake_case__    ,      '''vocab_size'''				) else config.encoder.vocab_size
         __snake_case								=       random_input_ids(snake_case__    ,      snake_case__    ,      snake_case__				)

         @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla				)
         def encoder_decoder_train():
            __snake_case								=       model(snake_case__    ,      decoder_input_ids=snake_case__    ,      labels=snake_case__    ,      training=snake_case__				)[0]
            __snake_case								=       tf.gradients(snake_case__    ,      model.trainable_variables				)
            return gradients

         @run_with_tf_optimizations(self.args.eager_mode    ,      self.args.use_xla				)
         def encoder_train():
            __snake_case								=       model(snake_case__    ,      labels=snake_case__    ,      training=snake_case__				)[0]
            __snake_case								=       tf.gradients(snake_case__    ,      model.trainable_variables				)
            return gradients

         __snake_case								=       encoder_decoder_train if config.is_encoder_decoder else encoder_train

         return _train


      def        a			(self	:				Tuple    ,      a__	:				Dict				):





         """simple docstring"""



         with self.args.strategy.scope():
            try:
               if self.args.is_tpu or self.args.use_xla:
                  # run additional 10 times to stabilize compilation for tpu
                  logger.info('''Do inference on TPU. Running model 5 times to stabilize compilation'''				)
                  timeit.repeat(snake_case__    ,      repeat=1    ,      number=5				)

               # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
               __snake_case								=       timeit.repeat(
                   snake_case__    ,      repeat=self.args.repeat    ,      number=10    ,      )

               return min(snake_case__				) / 1_0.0
            except ResourceExhaustedError as e:
               self.print_fn(f"""Doesn\'t fit on GPU. {e}"""				)


      def        a			(self	:				Tuple    ,      a__	:				Any				):





         """simple docstring"""



         logger.info(
             '''Note that TensorFlow allocates more memory than '''
             '''it might need to speed up computation. '''
             '''The memory reported here corresponds to the memory '''
             '''reported by `nvidia-smi`, which can vary depending '''
             '''on total available memory on the GPU that is used.'''				)
         with self.args.strategy.scope():
            try:
               if self.args.trace_memory_line_by_line:
                  if not self.args.eager_mode:
                     raise ValueError(
                         '''`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'''
                         ''' consumption line by line.'''				)
                  __snake_case								=       start_memory_tracing('''transformers'''				)

               if self.args.is_tpu:
                  # tpu
                  raise NotImplementedError(
                      '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'''
                      ''' with `args.memory=False`'''				)
               elif self.args.is_gpu:
                  # gpu
                  if not is_pyanvml_available():
                     logger.warning(
                         '''py3nvml not installed, we won\'t log GPU memory usage. '''
                         '''Install py3nvml (pip install py3nvml) to log information about GPU.'''				)
                     __snake_case								=       'N/A'
                  else:
                     logger.info(
                         '''Measuring total GPU usage on GPU device. Make sure to not have additional processes'''
                         ''' running on the same GPU.'''				)
                     # init nvml
                     nvml.nvmlInit()
                     func()
                     __snake_case								=       nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx				)
                     __snake_case								=       nvml.nvmlDeviceGetMemoryInfo(snake_case__				)
                     __snake_case								=       meminfo.used
                     __snake_case								=       Memory(snake_case__				)
                     # shutdown nvml
                     nvml.nvmlShutdown()
               else:
                  # cpu
                  if self.args.trace_memory_line_by_line:
                     logger.info(
                         '''When enabling line by line tracing, the max peak memory for CPU is inaccurate in'''
                         ''' TensorFlow.'''				)
                     __snake_case								=       None
                  else:
                     __snake_case								=       measure_peak_memory_cpu(snake_case__				)
                     __snake_case								=       Memory(snake_case__				) if isinstance(snake_case__    ,      snake_case__				) else memory_bytes
               if self.args.trace_memory_line_by_line:
                  __snake_case								=       stop_memory_tracing(snake_case__				)
                  if memory is None:
                     __snake_case								=       summary.total
               else:
                  __snake_case								=       None

               return memory, summary
            except ResourceExhaustedError as e:
               self.print_fn(f"""Doesn\'t fit on GPU. {e}"""				)
               return "N/A", None








import re
import string

import numpy as np

import datasets


snake_case_           =      '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

snake_case_           =      '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

snake_case_           =      '\n'

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,				_KWARGS_DESCRIPTION  )
class 		SCREAMING_SNAKE_CASE__    (	datasets.Metric  ):


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         return datasets.MetricInfo(
             description=_DESCRIPTION    ,      citation=_CITATION    ,      inputs_description=_KWARGS_DESCRIPTION    ,      features=datasets.Features(
                 {
                     '''predictions''': datasets.Value('''string'''    ,      id='''sequence'''				),
                     '''references''': datasets.Value('''string'''    ,      id='''sequence'''				),
                 }				)    ,      reference_urls=[]    ,      )


      def        a			(self	:				List[Any]    ,      a__	:				int    ,      a__	:				Optional[int]    ,      a__	:				Optional[Any]=None    ,      a__	:				Any=False    ,      a__	:				Dict=False    ,      a__	:				Tuple=False    ,      ):





         """simple docstring"""



         if regexes_to_ignore is not None:
            for s in regexes_to_ignore:
               __snake_case								=       np.array([re.sub(a__    ,      ''''''    ,      a__				) for x in predictions]				)
               __snake_case								=       np.array([re.sub(a__    ,      ''''''    ,      a__				) for x in references]				)
         else:
            __snake_case								=       np.asarray(a__				)
            __snake_case								=       np.asarray(a__				)

         if ignore_case:
            __snake_case								=       np.char.lower(a__				)
            __snake_case								=       np.char.lower(a__				)

         if ignore_punctuation:
            __snake_case								=       string.punctuation.maketrans(''''''    ,      ''''''    ,      string.punctuation				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)

         if ignore_numbers:
            __snake_case								=       string.digits.maketrans(''''''    ,      ''''''    ,      string.digits				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)

         __snake_case								=       predictions == references

         return {"exact_match": np.mean(a__				) * 100}

