





import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
    import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_           =      logging.get_logger(__name__)

snake_case_           =      {'vocab_file': 'vocab.txt'}

snake_case_           =      {
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

snake_case_           =      {
    'openbmb/cpm-ant-10b': 1024,
}






def lowerCamelCase__   (							snake_case_     :  List[Any]	)				->    str:
   __snake_case								=       collections.OrderedDict()
   with open(__lowerCAmelCase	,				'''r'''	,				encoding='''utf-8'''	) as reader:
      __snake_case								=       reader.readlines()
   for index, token in enumerate(__lowerCAmelCase	):
      __snake_case								=       token.rstrip('''\n'''	)
      __snake_case								=       index
   return vocab

class 		SCREAMING_SNAKE_CASE__    (	A__  ):


      def __init__(self	:				Optional[int]    ,      a__	:				int    ,      a__	:				Union[str, Any]="<unk>"    ,      a__	:				Union[str, Any]=200				):





         """simple docstring"""



         __snake_case								=       vocab
         __snake_case								=       unk_token
         __snake_case								=       max_input_chars_per_word


      def        a			(self	:				Any    ,      a__	:				List[str]				):





         """simple docstring"""



         __snake_case								=       list(__snake_case				)
         if len(__snake_case				) > self.max_input_chars_per_word:
            return [self.unk_token]

         __snake_case								=       0
         __snake_case								=       []
         while start < len(__snake_case				):
            __snake_case								=       len(__snake_case				)
            __snake_case								=       None
            while start < end:
               __snake_case								=       ''''''.join(chars[start:end]				)
               if substr in self.vocab:
                  __snake_case								=       substr
                  break
               end -= 1
            if cur_substr is None:
               sub_tokens.append(self.unk_token				)
               start += 1
            else:
               sub_tokens.append(__snake_case				)
               __snake_case								=       end

         return sub_tokens






class 		SCREAMING_SNAKE_CASE__    (	A__  ):
      A_						:    int							=   VOCAB_FILES_NAMES
      A_						:    Optional[int]							=   PRETRAINED_VOCAB_FILES_MAP
      A_						:    Any							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A_						:    Optional[Any]							=   ['input_ids', 'attention_mask']
      A_						:    List[str]							=   False


      def __init__(self	:				int    ,      a__	:				Tuple    ,      a__	:				Optional[int]="<d>"    ,      a__	:				int="</d>"    ,      a__	:				List[Any]="<s>"    ,      a__	:				List[str]="</s>"    ,      a__	:				str="<pad>"    ,      a__	:				Union[str, Any]="<unk>"    ,      a__	:				str="</n>"    ,      a__	:				List[str]="</_>"    ,      a__	:				Union[str, Any]="left"    ,      **a__	:				Tuple    ,      ):





         """simple docstring"""



         requires_backends(self    ,      ['''jieba''']				)
         super().__init__(
             bod_token=__snake_case    ,      eod_token=__snake_case    ,      bos_token=__snake_case    ,      eos_token=__snake_case    ,      pad_token=__snake_case    ,      unk_token=__snake_case    ,      line_token=__snake_case    ,      space_token=__snake_case    ,      padding_side=__snake_case    ,      **__snake_case    ,      )
         __snake_case								=       bod_token
         __snake_case								=       eod_token
         __snake_case								=       load_vocab(__snake_case				)
         __snake_case								=       self.encoder[space_token]
         __snake_case								=       self.encoder[line_token]

         del self.encoder[space_token]
         del self.encoder[line_token]

         __snake_case								=       collections.OrderedDict(sorted(self.encoder.items()    ,      key=lambda a__				: x[1]				)				)
         __snake_case								=       {v: k for k, v in self.encoder.items()}

         __snake_case								=       WordpieceTokenizer(vocab=self.encoder    ,      unk_token=self.unk_token				)


      @property
      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         return self.encoder[self.bod_token]


      @property
      def        a			(self	:				str				):





         """simple docstring"""



         return self.encoder[self.eod_token]


      @property
      def        a			(self	:				str				):





         """simple docstring"""



         return self.encoder["\n"]


      @property
      def        a			(self	:				List[Any]				):





         """simple docstring"""



         return len(self.encoder				)


      def        a			(self	:				Any				):





         """simple docstring"""



         return dict(self.encoder    ,      **self.added_tokens_encoder				)


      def        a			(self	:				Tuple    ,      a__	:				Any				):





         """simple docstring"""



         __snake_case								=       []
         for x in jieba.cut(__snake_case    ,      cut_all=__snake_case				):
            output_tokens.extend(self.wordpiece_tokenizer.tokenize(__snake_case				)				)
         return output_tokens


      def        a			(self	:				str    ,      a__	:				Tuple    ,      **a__	:				Dict				):





         """simple docstring"""



         __snake_case								=       [i for i in token_ids if i >= 0]
         __snake_case								=       [
             x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
         ]
         return super()._decode(__snake_case    ,      **__snake_case				)


      def        a			(self	:				Union[str, Any]    ,      a__	:				Dict				):





         """simple docstring"""



         return token in self.encoder


      def        a			(self	:				Optional[Any]    ,      a__	:				List[str]				):





         """simple docstring"""



         return "".join(__snake_case				)


      def        a			(self	:				Tuple    ,      a__	:				int				):





         """simple docstring"""



         return self.encoder.get(__snake_case    ,      self.encoder.get(self.unk_token				)				)


      def        a			(self	:				str    ,      a__	:				List[Any]				):





         """simple docstring"""



         return self.decoder.get(__snake_case    ,      self.unk_token				)


      def        a			(self	:				int    ,      a__	:				str    ,      a__	:				Optional[str] = None				):





         """simple docstring"""



         if os.path.isdir(__snake_case				):
            __snake_case								=       os.path.join(
                __snake_case    ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
         else:
            __snake_case								=       (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
         __snake_case								=       0
         if " " in self.encoder:
            __snake_case								=       self.encoder[''' ''']
            del self.encoder[" "]
         if "\n" in self.encoder:
            __snake_case								=       self.encoder['''\n''']
            del self.encoder["\n"]
         __snake_case								=       collections.OrderedDict(sorted(self.encoder.items()    ,      key=lambda a__				: x[1]				)				)
         with open(__snake_case    ,      '''w'''    ,      encoding='''utf-8'''				) as writer:
            for token, token_index in self.encoder.items():
               if index != token_index:
                  logger.warning(
                      f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                      ''' Please check that the vocabulary is not corrupted!'''				)
                  __snake_case								=       token_index
               writer.write(token + '''\n'''				)
               index += 1
         return (vocab_file,)


      def        a			(self	:				Dict    ,      a__	:				List[int]    ,      a__	:				List[int] = None				):





         """simple docstring"""



         if token_ids_a is None:
            return [self.bos_token_id] + token_ids_a
         return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a


      def        a			(self	:				str    ,      a__	:				List[int]    ,      a__	:				Optional[List[int]] = None    ,      a__	:				bool = False				):





         """simple docstring"""



         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__snake_case    ,      token_ids_a=__snake_case    ,      already_has_special_tokens=__snake_case				)

         if token_ids_a is not None:
            return [1] + ([0] * len(__snake_case				)) + [1] + ([0] * len(__snake_case				))
         return [1] + ([0] * len(__snake_case				))








import re
import string

import numpy as np

import datasets


snake_case_           =      '\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

snake_case_           =      '\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

snake_case_           =      '\n'

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,				_KWARGS_DESCRIPTION  )
class 		SCREAMING_SNAKE_CASE__    (	datasets.Metric  ):


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         return datasets.MetricInfo(
             description=_DESCRIPTION    ,      citation=_CITATION    ,      inputs_description=_KWARGS_DESCRIPTION    ,      features=datasets.Features(
                 {
                     '''predictions''': datasets.Value('''string'''    ,      id='''sequence'''				),
                     '''references''': datasets.Value('''string'''    ,      id='''sequence'''				),
                 }				)    ,      reference_urls=[]    ,      )


      def        a			(self	:				List[Any]    ,      a__	:				int    ,      a__	:				Optional[int]    ,      a__	:				Optional[Any]=None    ,      a__	:				Any=False    ,      a__	:				Dict=False    ,      a__	:				Tuple=False    ,      ):





         """simple docstring"""



         if regexes_to_ignore is not None:
            for s in regexes_to_ignore:
               __snake_case								=       np.array([re.sub(a__    ,      ''''''    ,      a__				) for x in predictions]				)
               __snake_case								=       np.array([re.sub(a__    ,      ''''''    ,      a__				) for x in references]				)
         else:
            __snake_case								=       np.asarray(a__				)
            __snake_case								=       np.asarray(a__				)

         if ignore_case:
            __snake_case								=       np.char.lower(a__				)
            __snake_case								=       np.char.lower(a__				)

         if ignore_punctuation:
            __snake_case								=       string.punctuation.maketrans(''''''    ,      ''''''    ,      string.punctuation				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)

         if ignore_numbers:
            __snake_case								=       string.digits.maketrans(''''''    ,      ''''''    ,      string.digits				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)
            __snake_case								=       np.char.translate(a__    ,      table=a__				)

         __snake_case								=       predictions == references

         return {"exact_match": np.mean(a__				) * 100}

