






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class     a_						(      A_     ,	unittest.TestCase    ):
  lowercase							=				CLIPTokenizer
  lowercase							=				CLIPTokenizerFast
  lowercase							=				True
  lowercase							=				{}
  lowercase							=				False

  def   A__						(				self				)  ->				str:



        """simple docstring"""






        super().setUp()

        # fmt: off
        UpperCamelCase      = ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
        # fmt: on
        UpperCamelCase      = dict(zip(snake_case__      ,		range(len(snake_case__				)				)				)				)
        UpperCamelCase      = ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
        UpperCamelCase      = {"unk_token": "<unk>"}

        UpperCamelCase      = os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES["""vocab_file"""]				)
        UpperCamelCase      = os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES["""merges_file"""]				)
        with open(self.vocab_file      ,		"""w"""      ,		encoding="""utf-8"""				) as fp:
              fp.write(json.dumps(snake_case__				) + """\n"""				)
        with open(self.merges_file      ,		"""w"""      ,		encoding="""utf-8"""				) as fp:
              fp.write("""\n""".join(snake_case__				)				)

  def   A__						(				self      ,		**_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:



        """simple docstring"""






        kwargs.update(self.special_tokens_map				)
        return CLIPTokenizer.from_pretrained(self.tmpdirname      ,		**snake_case__				)

  def   A__						(				self      ,		**_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        kwargs.update(self.special_tokens_map				)
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,		**snake_case__				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = "lower newer"
        UpperCamelCase      = "lower newer"
        return input_text, output_text

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = CLIPTokenizer(self.vocab_file      ,		self.merges_file      ,		**self.special_tokens_map				)
        UpperCamelCase      = "lower newer"
        UpperCamelCase      = ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
        UpperCamelCase      = tokenizer.tokenize(snake_case__				)
        self.assertListEqual(snake_case__      ,		snake_case__				)

        UpperCamelCase      = tokens + [tokenizer.unk_token]
        UpperCamelCase      = [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				)      ,		snake_case__				)

  @require_ftfy
  def   A__						(				self				)  ->				Optional[int]:



        """simple docstring"""






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                    UpperCamelCase      = self.tokenizer_class.from_pretrained(snake_case__      ,		**snake_case__				)
                    UpperCamelCase      = self.rust_tokenizer_class.from_pretrained(snake_case__      ,		**snake_case__				)

                    UpperCamelCase      = "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
                    UpperCamelCase      = tokenizer_s.tokenize(snake_case__				)
                    UpperCamelCase      = tokenizer_r.tokenize(snake_case__				)

                    self.assertListEqual(snake_case__      ,		snake_case__				)

                    # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                    # with Tilde) encoded in 2 different ways
                    UpperCamelCase      = "xa\u0303y" + " " + "x\xe3y"
                    UpperCamelCase      = tokenizer_s.tokenize(snake_case__				)
                    UpperCamelCase      = tokenizer_r.tokenize(snake_case__				)

                    self.assertListEqual(snake_case__      ,		snake_case__				)

                    # Test that the tokenization is identical on unicode of space type
                    UpperCamelCase      = [
                        "\u0009",  # (horizontal tab, '\t')
                        "\u000B",  # (vertical tab)
                        "\u000C",  # (form feed)
                        "\u0020",  # (space, ' ')
                        "\u200E",  # (left-to-right mark):w
                        "\u200F",  # (right-to-left mark)
                    ]
                    for unicode_seq in spaces_unicodes:
                          UpperCamelCase      = tokenizer_s.tokenize(snake_case__				)
                          UpperCamelCase      = tokenizer_r.tokenize(snake_case__				)

                          self.assertListEqual(snake_case__      ,		snake_case__				)

                    # Test that the tokenization is identical on unicode of line break type
                    UpperCamelCase      = [
                        "\u000A",  # (line feed, '\n')
                        "\r\n",  # (carriage return and line feed, '\r\n')
                        "\u000D",  # (carriage return, '\r')
                        "\r",  # (carriage return, '\r')
                        "\u000D",  # (carriage return, '\r')
                        "\u2028",  # (line separator)
                        "\u2029",  # (paragraph separator)
                        # "\u0085", # (next line)
                    ]

                    # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                    # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                    # space (and thus into an empty list).

                    for unicode_seq in line_break_unicodes:
                          UpperCamelCase      = tokenizer_s.tokenize(snake_case__				)
                          UpperCamelCase      = tokenizer_r.tokenize(snake_case__				)

                          self.assertListEqual(snake_case__      ,		snake_case__				)

  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
                    UpperCamelCase      = "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                    UpperCamelCase      = F"{text_of_1_token} {text_of_1_token}"

                    UpperCamelCase      = self.rust_tokenizer_class.from_pretrained(
                        snake_case__      ,		use_fast=snake_case__      ,		)
                    UpperCamelCase      = tokenizer_r(snake_case__      ,		return_offsets_mapping=snake_case__      ,		add_special_tokens=snake_case__				)
                    self.assertEqual(encoding.offset_mapping[0]      ,		(0, len(snake_case__				))				)
                    self.assertEqual(
                        encoding.offset_mapping[1]      ,		(len(snake_case__				) + 1, len(snake_case__				) + 1 + len(snake_case__				))      ,		)

                    UpperCamelCase      = F" {text}"

                    UpperCamelCase      = self.rust_tokenizer_class.from_pretrained(
                        snake_case__      ,		use_fast=snake_case__      ,		)
                    UpperCamelCase      = tokenizer_r(snake_case__      ,		return_offsets_mapping=snake_case__      ,		add_special_tokens=snake_case__				)
                    self.assertEqual(encoding.offset_mapping[0]      ,		(1, 1 + len(snake_case__				))				)
                    self.assertEqual(
                        encoding.offset_mapping[1]      ,		(1 + len(snake_case__				) + 1, 1 + len(snake_case__				) + 1 + len(snake_case__				))      ,		)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        with self.assertRaises(snake_case__				) as context:
              self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""				)

        self.assertTrue(
            context.exception.args[0].startswith(
                """The `backend_tokenizer` provided does not match the expected format."""				)				)

  @require_ftfy
  def   A__						(				self				)  ->				List[str]:



        """simple docstring"""






        super().test_tokenization_python_rust_equals()


  def   A__						(				self				)  ->				int:



        """simple docstring"""






        pass








'''simple docstring'''

import re
import string

import numpy as np

import datasets


SCREAMING_SNAKE_CASE__     =       '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

SCREAMING_SNAKE_CASE__     =       '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

SCREAMING_SNAKE_CASE__     =       '\n'






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,	_KWARGS_DESCRIPTION    )
class     a_						(      datasets.Metric    ):

  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        return datasets.MetricInfo(
            description=_DESCRIPTION      ,		citation=_CITATION      ,		inputs_description=_KWARGS_DESCRIPTION      ,		features=datasets.Features(
                {
                    """predictions""": datasets.Value("""string"""      ,		id="""sequence"""				),
                    """references""": datasets.Value("""string"""      ,		id="""sequence"""				),
                }				)      ,		reference_urls=[]      ,		)


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=False      ,		_SCREAMING_SNAKE_CASE=False      ,		_SCREAMING_SNAKE_CASE=False      ,		)  ->				List[Any]:



        """simple docstring"""






        if regexes_to_ignore is not None:
              for s in regexes_to_ignore:
                    UpperCamelCase      = np.array([re.sub(_SCREAMING_SNAKE_CASE      ,		""""""      ,		_SCREAMING_SNAKE_CASE				) for x in predictions]				)
                    UpperCamelCase      = np.array([re.sub(_SCREAMING_SNAKE_CASE      ,		""""""      ,		_SCREAMING_SNAKE_CASE				) for x in references]				)
        else:
              UpperCamelCase      = np.asarray(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = np.asarray(_SCREAMING_SNAKE_CASE				)

        if ignore_case:
              UpperCamelCase      = np.char.lower(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = np.char.lower(_SCREAMING_SNAKE_CASE				)

        if ignore_punctuation:
              UpperCamelCase      = string.punctuation.maketrans(""""""      ,		""""""      ,		string.punctuation				)
              UpperCamelCase      = np.char.translate(_SCREAMING_SNAKE_CASE      ,		table=_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = np.char.translate(_SCREAMING_SNAKE_CASE      ,		table=_SCREAMING_SNAKE_CASE				)

        if ignore_numbers:
              UpperCamelCase      = string.digits.maketrans(""""""      ,		""""""      ,		string.digits				)
              UpperCamelCase      = np.char.translate(_SCREAMING_SNAKE_CASE      ,		table=_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = np.char.translate(_SCREAMING_SNAKE_CASE      ,		table=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = predictions == references

        return {"exact_match": np.mean(_SCREAMING_SNAKE_CASE				) * 100}
