


'''simple docstring'''






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


lowercase							=logging.get_logger(__name__)

lowercase							={'tokenizer_file': 'tokenizer.json'}

lowercase							={
    'tokenizer_file': {
        'bigscience/tokenizer': 'https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json',
        'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json',
        'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json',
        'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json',
        'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json',
        'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json',
        'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json',
    },
}






class  __magic_name__							(		lowerCAmelCase    ):
       UpperCAmelCase										=VOCAB_FILES_NAMES
       UpperCAmelCase										=PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase										=["input_ids", "attention_mask"]
       UpperCAmelCase										=None






       def __init__(      self  ,							snake_case=None  ,							snake_case=None  ,							snake_case=None  ,							snake_case="<unk>"  ,							snake_case="<s>"  ,							snake_case="</s>"  ,							snake_case="<pad>"  ,							snake_case=False  ,							snake_case=False  ,							**snake_case  ,							)						->  Tuple:




          '''simple docstring'''
          super().__init__(
              snake_case  ,							snake_case  ,							tokenizer_file=snake_case  ,							unk_token=snake_case  ,							bos_token=snake_case  ,							eos_token=snake_case  ,							pad_token=snake_case  ,							add_prefix_space=snake_case  ,							clean_up_tokenization_spaces=snake_case  ,							**snake_case  ,							)
          _UpperCAmelCase       :       int							   =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('add_prefix_space'  ,							snake_case) != add_prefix_space:
             _UpperCAmelCase       :       Union[str, Any]							   =getattr(snake_case  ,							pre_tok_state.pop('type'))
             _UpperCAmelCase       :       List[str]							   =add_prefix_space
             _UpperCAmelCase       :       Optional[int]							   =pre_tok_class(**snake_case)

          _UpperCAmelCase       :       Tuple							   =add_prefix_space






       def       lowerCAmelCase		(      self  ,							*snake_case  ,							**snake_case)						->  BatchEncoding:




          '''simple docstring'''
          _UpperCAmelCase       :       Union[str, Any]							   =kwargs.get('is_split_into_words'  ,							snake_case)
          if not (self.add_prefix_space or not is_split_into_words):
             raise Exception(
                 f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                 ' pretokenized inputs.')

          return super()._batch_encode_plus(*snake_case  ,							**snake_case)






       def       lowerCAmelCase		(      self  ,							*snake_case  ,							**snake_case)						->  BatchEncoding:




          '''simple docstring'''
          _UpperCAmelCase       :       Union[str, Any]							   =kwargs.get('is_split_into_words'  ,							snake_case)

          if not (self.add_prefix_space or not is_split_into_words):
             raise Exception(
                 f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                 ' pretokenized inputs.')

          return super()._encode_plus(*snake_case  ,							**snake_case)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case = None)						->  Tuple[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[Any]							   =self._tokenizer.model.save(snake_case  ,							name=snake_case)
          return tuple(snake_case)






       def       lowerCAmelCase		(      self  ,							snake_case)						->  List[int]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =[]
          for is_user, text in conversation.iter_texts():
             input_ids.extend(self.encode(snake_case  ,							add_special_tokens=snake_case) + [self.eos_token_id])

          if len(snake_case) > self.model_max_length:
             _UpperCAmelCase       :       Optional[int]							   =input_ids[-self.model_max_length :]
          return input_ids





'''simple docstring'''






import datasets

from .evaluate import evaluate


lowercase							='\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n'

lowercase							='\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n'

lowercase							='\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the SQuAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> squad_metric = datasets.load_metric("squad")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0}\n'






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,_KWARGS_DESCRIPTION    )
class  __magic_name__							(		datasets.Metric    ):






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          return datasets.MetricInfo(
              description=_DESCRIPTION  ,							citation=_CITATION  ,							inputs_description=_KWARGS_DESCRIPTION  ,							features=datasets.Features(
                  {
                      'predictions': {'id': datasets.Value('string'), 'prediction_text': datasets.Value('string')},
                      'references': {
                          'id': datasets.Value('string'),
                          'answers': datasets.features.Sequence(
                              {
                                  'text': datasets.Value('string'),
                                  'answer_start': datasets.Value('int32'),
                              }),
                      },
                  })  ,							codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/']  ,							reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/']  ,							)






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case)						->  int:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[int]							   ={prediction['id']: prediction['prediction_text'] for prediction in predictions}
          _UpperCAmelCase       :       Optional[int]							   =[
              {
                  'paragraphs': [
                      {
                          'qas': [
                              {
                                  'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                                  'id': ref['id'],
                              }
                              for ref in references
                          ]
                      }
                  ]
              }
          ]
          _UpperCAmelCase       :       List[Any]							   =evaluate(dataset=snake_case  ,							predictions=snake_case)
          return score

