






'''simple docstring'''

import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
SCREAMING_SNAKE_CASE__     =       {'''LayoutLMv2Config''', '''LayoutLMv3Config'''}






@is_pipeline_test
class     a_						(      unittest.TestCase    ):
  lowercase							=				MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
  lowercase							=				TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

  if model_mapping is not None:
    lowercase							=				{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
  if tf_model_mapping is not None:
    lowercase							=				{
        config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
    }

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Any:



        """simple docstring"""






        UpperCamelCase      = ZeroShotClassificationPipeline(
            model=__lowerCamelCase      ,		tokenizer=__lowerCamelCase      ,		candidate_labels=["""polics""", """health"""]				)
        return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = classifier("""Who are you voting for in 2020?"""      ,		candidate_labels="""politics"""				)
        self.assertEqual(__lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				)]}				)

        # No kwarg
        UpperCamelCase      = classifier("""Who are you voting for in 2020?"""      ,		["""politics"""]				)
        self.assertEqual(__lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				)]}				)

        UpperCamelCase      = classifier("""Who are you voting for in 2020?"""      ,		candidate_labels=["""politics"""]				)
        self.assertEqual(__lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				)]}				)

        UpperCamelCase      = classifier("""Who are you voting for in 2020?"""      ,		candidate_labels="""politics, public health"""				)
        self.assertEqual(
            __lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)]}				)
        self.assertAlmostEqual(sum(nested_simplify(outputs["""scores"""]				)				)      ,		1.0				)

        UpperCamelCase      = classifier("""Who are you voting for in 2020?"""      ,		candidate_labels=["""politics""", """public health"""]				)
        self.assertEqual(
            __lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)]}				)
        self.assertAlmostEqual(sum(nested_simplify(outputs["""scores"""]				)				)      ,		1.0				)

        UpperCamelCase      = classifier(
            """Who are you voting for in 2020?"""      ,		candidate_labels="""politics"""      ,		hypothesis_template="""This text is about {}"""				)
        self.assertEqual(__lowerCamelCase      ,		{"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				)]}				)

        # https://github.com/huggingface/transformers/issues/13846
        UpperCamelCase      = classifier(["""I am happy"""]      ,		["""positive""", """negative"""]				)
        self.assertEqual(
            __lowerCamelCase      ,		[
                {"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)]}
                for i in range(1				)
            ]      ,		)
        UpperCamelCase      = classifier(["""I am happy""", """I am sad"""]      ,		["""positive""", """negative"""]				)
        self.assertEqual(
            __lowerCamelCase      ,		[
                {"""sequence""": ANY(__lowerCamelCase				), """labels""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)], """scores""": [ANY(__lowerCamelCase				), ANY(__lowerCamelCase				)]}
                for i in range(2				)
            ]      ,		)

        with self.assertRaises(__lowerCamelCase				):
              classifier(""""""      ,		candidate_labels="""politics"""				)

        with self.assertRaises(__lowerCamelCase				):
              classifier(__lowerCamelCase      ,		candidate_labels="""politics"""				)

        with self.assertRaises(__lowerCamelCase				):
              classifier("""Who are you voting for in 2020?"""      ,		candidate_labels=""""""				)

        with self.assertRaises(__lowerCamelCase				):
              classifier("""Who are you voting for in 2020?"""      ,		candidate_labels=__lowerCamelCase				)

        with self.assertRaises(__lowerCamelCase				):
              classifier(
                  """Who are you voting for in 2020?"""      ,		candidate_labels="""politics"""      ,		hypothesis_template="""Not formatting template"""      ,		)

        with self.assertRaises(__lowerCamelCase				):
              classifier(
                  """Who are you voting for in 2020?"""      ,		candidate_labels="""politics"""      ,		hypothesis_template=__lowerCamelCase      ,		)

        self.run_entailment_id(__lowerCamelCase				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = zero_shot_classifier.model.config
        UpperCamelCase      = config.labelaid
        UpperCamelCase      = zero_shot_classifier.entailment_id

        UpperCamelCase      = {"""LABEL_0""": 0, """LABEL_1""": 1, """LABEL_2""": 2}
        self.assertEqual(zero_shot_classifier.entailment_id      ,		-1				)

        UpperCamelCase      = {"""entailment""": 0, """neutral""": 1, """contradiction""": 2}
        self.assertEqual(zero_shot_classifier.entailment_id      ,		0				)

        UpperCamelCase      = {"""ENTAIL""": 0, """NON-ENTAIL""": 1}
        self.assertEqual(zero_shot_classifier.entailment_id      ,		0				)

        UpperCamelCase      = {"""ENTAIL""": 2, """NEUTRAL""": 1, """CONTR""": 0}
        self.assertEqual(zero_shot_classifier.entailment_id      ,		2				)

        UpperCamelCase      = original_labelaid
        self.assertEqual(__lowerCamelCase      ,		zero_shot_classifier.entailment_id				)

  @require_torch
  def   A__						(				self				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = pipeline(
            """zero-shot-classification"""      ,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""      ,		framework="""pt"""      ,		)
        # There was a regression in 4.10 for this
        # Adding a test so we don't make the mistake again.
        # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
        zero_shot_classifier(
            """Who are you voting for in 2020?""" * 100      ,		candidate_labels=["""politics""", """public health""", """science"""]				)

  @require_torch
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = pipeline(
            """zero-shot-classification"""      ,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""      ,		framework="""pt"""      ,		)
        UpperCamelCase      = zero_shot_classifier(
            """Who are you voting for in 2020?"""      ,		candidate_labels=["""politics""", """public health""", """science"""]				)

        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": """Who are you voting for in 2020?""",
                """labels""": ["""science""", """public health""", """politics"""],
                """scores""": [0.3_3_3, 0.3_3_3, 0.3_3_3],
            }      ,		)

  @require_tf
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = pipeline(
            """zero-shot-classification"""      ,		model="""sshleifer/tiny-distilbert-base-cased-distilled-squad"""      ,		framework="""tf"""      ,		)
        UpperCamelCase      = zero_shot_classifier(
            """Who are you voting for in 2020?"""      ,		candidate_labels=["""politics""", """public health""", """science"""]				)

        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": """Who are you voting for in 2020?""",
                """labels""": ["""science""", """public health""", """politics"""],
                """scores""": [0.3_3_3, 0.3_3_3, 0.3_3_3],
            }      ,		)

  @slow
  @require_torch
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = pipeline("""zero-shot-classification"""      ,		model="""roberta-large-mnli"""      ,		framework="""pt"""				)
        UpperCamelCase      = zero_shot_classifier(
            """Who are you voting for in 2020?"""      ,		candidate_labels=["""politics""", """public health""", """science"""]				)

        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": """Who are you voting for in 2020?""",
                """labels""": ["""politics""", """public health""", """science"""],
                """scores""": [0.9_7_6, 0.0_1_5, 0.0_0_9],
            }      ,		)
        UpperCamelCase      = zero_shot_classifier(
            """The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"""
            """ in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"""
            """ through an attention mechanism. We propose a new simple network architecture, the Transformer, based"""
            """ solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"""
            """ machine translation tasks show these models to be superior in quality while being more parallelizable"""
            """ and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"""
            """ English-to-German translation task, improving over the existing best results, including ensembles by"""
            """ over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"""
            """ single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"""
            """ fraction of the training costs of the best models from the literature. We show that the Transformer"""
            """ generalizes well to other tasks by applying it successfully to English constituency parsing both with"""
            """ large and limited training data."""      ,		candidate_labels=["""machine learning""", """statistics""", """translation""", """vision"""]      ,		multi_label=__lowerCamelCase      ,		)
        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": (
                    """The dominant sequence transduction models are based on complex recurrent or convolutional neural"""
                    """ networks in an encoder-decoder configuration. The best performing models also connect the"""
                    """ encoder and decoder through an attention mechanism. We propose a new simple network"""
                    """ architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"""
                    """ and convolutions entirely. Experiments on two machine translation tasks show these models to be"""
                    """ superior in quality while being more parallelizable and requiring significantly less time to"""
                    """ train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"""
                    """ improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"""
                    """ English-to-French translation task, our model establishes a new single-model state-of-the-art"""
                    """ BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"""
                    """ costs of the best models from the literature. We show that the Transformer generalizes well to"""
                    """ other tasks by applying it successfully to English constituency parsing both with large and"""
                    """ limited training data."""
                ),
                """labels""": ["""translation""", """machine learning""", """vision""", """statistics"""],
                """scores""": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
            }      ,		)


  @slow
  @require_tf
  def   A__						(				self				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = pipeline("""zero-shot-classification"""      ,		model="""roberta-large-mnli"""      ,		framework="""tf"""				)
        UpperCamelCase      = zero_shot_classifier(
            """Who are you voting for in 2020?"""      ,		candidate_labels=["""politics""", """public health""", """science"""]				)

        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": """Who are you voting for in 2020?""",
                """labels""": ["""politics""", """public health""", """science"""],
                """scores""": [0.9_7_6, 0.0_1_5, 0.0_0_9],
            }      ,		)
        UpperCamelCase      = zero_shot_classifier(
            """The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"""
            """ in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"""
            """ through an attention mechanism. We propose a new simple network architecture, the Transformer, based"""
            """ solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"""
            """ machine translation tasks show these models to be superior in quality while being more parallelizable"""
            """ and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"""
            """ English-to-German translation task, improving over the existing best results, including ensembles by"""
            """ over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"""
            """ single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"""
            """ fraction of the training costs of the best models from the literature. We show that the Transformer"""
            """ generalizes well to other tasks by applying it successfully to English constituency parsing both with"""
            """ large and limited training data."""      ,		candidate_labels=["""machine learning""", """statistics""", """translation""", """vision"""]      ,		multi_label=__lowerCamelCase      ,		)
        self.assertEqual(
            nested_simplify(__lowerCamelCase				)      ,		{
                """sequence""": (
                    """The dominant sequence transduction models are based on complex recurrent or convolutional neural"""
                    """ networks in an encoder-decoder configuration. The best performing models also connect the"""
                    """ encoder and decoder through an attention mechanism. We propose a new simple network"""
                    """ architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"""
                    """ and convolutions entirely. Experiments on two machine translation tasks show these models to be"""
                    """ superior in quality while being more parallelizable and requiring significantly less time to"""
                    """ train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"""
                    """ improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"""
                    """ English-to-French translation task, our model establishes a new single-model state-of-the-art"""
                    """ BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"""
                    """ costs of the best models from the literature. We show that the Transformer generalizes well to"""
                    """ other tasks by applying it successfully to English constituency parsing both with large and"""
                    """ limited training data."""
                ),
                """labels""": ["""translation""", """machine learning""", """vision""", """statistics"""],
                """scores""": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
            }      ,		)








'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'spm_char.model'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

SCREAMING_SNAKE_CASE__     =       {
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}






class     a_						(      lowerCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				None:



        """simple docstring"""






        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=_SCREAMING_SNAKE_CASE      ,		eos_token=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**_SCREAMING_SNAKE_CASE      ,		)

        UpperCamelCase      = vocab_file

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.get_piece_size()

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def __getstate__(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        return self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,		out_type=_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.piece_to_id(_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE				)
        return token

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				) + token
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE				)
        out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				)
        return out_string.strip()

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None				)  ->				List[int]:



        """simple docstring"""






        if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_a + token_ids_a + [self.eos_token_id]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_SCREAMING_SNAKE_CASE      ,		token_ids_a=_SCREAMING_SNAKE_CASE      ,		already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = [1]
        if token_ids_a is None:
              return ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones
        return ([0] * len(_SCREAMING_SNAKE_CASE				)) + ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            _SCREAMING_SNAKE_CASE      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(_SCREAMING_SNAKE_CASE				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		_SCREAMING_SNAKE_CASE				)
        elif not os.path.isfile(self.vocab_file				):
              with open(_SCREAMING_SNAKE_CASE      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE				)

        return (out_vocab_file,)
