


'''simple docstring'''






from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


__UpperCAmelCase       =							logging.get_logger(__name__)




class 		a__    (							__UpperCamelCase  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							Union[str, Any]:
														if isinstance(lowerCamelCase_ ,   lowerCamelCase_       ):
																					lowerCAmelCase__	       =     [label.strip() for label in labels.split(''','''       ) if label.strip()]
														return labels







							def __call__(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_       )	->							Optional[Any]:
														if len(lowerCamelCase_       ) == 0 or len(lowerCamelCase_       ) == 0:
																					raise ValueError('''You must include at least one label and at least one sequence.'''       )
														if hypothesis_template.format(labels[0]       ) == hypothesis_template:
																					raise ValueError(
																					    (
																					        '''The provided hypothesis_template "{}" was not able to be formatted with the target labels. '''
																					        '''Make sure the passed template includes formatting syntax such as {{}} where the label should go.'''
																					    ).format(lowerCamelCase_       )       )

														if isinstance(lowerCamelCase_ ,   lowerCamelCase_       ):
																					lowerCAmelCase__	       =     [sequences]

														lowerCAmelCase__	       =     []
														for sequence in sequences:
																					sequence_pairs.extend([[sequence, hypothesis_template.format(lowerCamelCase_       )] for label in labels]       )

														return sequence_pairs, sequences




@add_end_docstrings(__UpperCamelCase  )
class 		a__    (							__UpperCamelCase  ):





							'''simple docstring'''






							def __init__(		self ,   lowerCamelCase_=ZeroShotClassificationArgumentHandler() ,   *lowerCamelCase_ ,   **lowerCamelCase_       )	->							List[Any]:
														lowerCAmelCase__	       =     args_parser
														super().__init__(*lowerCamelCase_ ,   **lowerCamelCase_       )
														if self.entailment_id == -1:
																					logger.warning(
																					    '''Failed to determine \'entailment\' label id from the label2id mapping in the model config. Setting to '''
																					    '''-1. Define a descriptive label2id mapping in the model config to ensure correct outputs.'''       )







							@property
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														for label, ind in self.model.config.labelaid.items():
																					if label.lower().startswith('''entail'''       ):
																												return ind
														return -1







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_=True ,   lowerCamelCase_=True ,   lowerCamelCase_=TruncationStrategy.ONLY_FIRST ,   **lowerCamelCase_       )	->							List[Any]:
														lowerCAmelCase__	       =     self.framework
														if self.tokenizer.pad_token is None:
																					# Override for tokenizers not supporting padding
																					logger.error(
																					    '''Tokenizer was not supporting padding necessary for zero-shot, attempting to use '''
																					    ''' `pad_token=eos_token`'''       )
																					lowerCAmelCase__	       =     self.tokenizer.eos_token
														try:
																					lowerCAmelCase__	       =     self.tokenizer(
																					    lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   return_tensors=lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncation=lowerCamelCase_ ,   )
														except Exception as e:
																					if "too short" in str(lowerCamelCase_       ):
																												# tokenizers might yell that we want to truncate
																												# to a value that is not even reached by the input.
																												# In that case we don't want to truncate.
																												# It seems there's not a really better way to catch that
																												# exception.

																												lowerCAmelCase__	       =     self.tokenizer(
																												    lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   return_tensors=lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncation=TruncationStrategy.DO_NOT_TRUNCATE ,   )
																					else:
																												raise e

														return inputs







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							Dict:
														if kwargs.get('''multi_class''' ,   lowerCamelCase_       ) is not None:
																					lowerCAmelCase__	       =     kwargs['multi_class']
																					logger.warning(
																					    '''The `multi_class` argument has been deprecated and renamed to `multi_label`. '''
																					    '''`multi_class` will be removed in a future version of Transformers.'''       )
														lowerCAmelCase__	       =     {}
														if "candidate_labels" in kwargs:
																					lowerCAmelCase__	       =     self._args_parser._parse_labels(kwargs['''candidate_labels''']       )
														if "hypothesis_template" in kwargs:
																					lowerCAmelCase__	       =     kwargs['hypothesis_template']

														lowerCAmelCase__	       =     {}
														if "multi_label" in kwargs:
																					lowerCAmelCase__	       =     kwargs['multi_label']
														return preprocess_params, {}, postprocess_params







							def __call__(		self ,   lowerCamelCase_ ,   *lowerCamelCase_ ,   **lowerCamelCase_ ,   )	->							List[str]:
														if len(lowerCamelCase_       ) == 0:
																					pass
														elif len(lowerCamelCase_       ) == 1 and "candidate_labels" not in kwargs:
																					lowerCAmelCase__	       =     args[0]
														else:
																					raise ValueError(F"""Unable to understand extra arguments {args}"""       )

														return super().__call__(lowerCamelCase_ ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_=None ,   lowerCamelCase_="This example is {}."       )	->							Optional[int]:
														lowerCAmelCase__	       =     self._args_parser(lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_       )

														for i, (candidate_label, sequence_pair) in enumerate(zip(lowerCamelCase_ ,   lowerCamelCase_       )       ):
																					lowerCAmelCase__	       =     self._parse_and_tokenize([sequence_pair]       )

																					yield {
																					    "candidate_label": candidate_label,
																					    "sequence": sequences[0],
																					    "is_last": i == len(lowerCamelCase_       ) - 1,
																					    **model_input,
																					}







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							List[str]:
														lowerCAmelCase__	       =     inputs['candidate_label']
														lowerCAmelCase__	       =     inputs['sequence']
														lowerCAmelCase__	       =     {k: inputs[k] for k in self.tokenizer.model_input_names}
														lowerCAmelCase__	       =     self.model(**lowerCamelCase_       )

														lowerCAmelCase__	       =     {
														    'candidate_label': candidate_label,
														    'sequence': sequence,
														    'is_last': inputs['is_last'],
														    **outputs,
														}
														return model_outputs







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_=False       )	->							List[Any]:
														lowerCAmelCase__	       =     [outputs['candidate_label'] for outputs in model_outputs]
														lowerCAmelCase__	       =     [outputs['sequence'] for outputs in model_outputs]
														lowerCAmelCase__	       =     np.concatenate([output['''logits'''].numpy() for output in model_outputs]       )
														lowerCAmelCase__	       =     logits.shape[0]
														lowerCAmelCase__	       =     len(lowerCamelCase_       )
														lowerCAmelCase__	       =     N // n
														lowerCAmelCase__	       =     logits.reshape((num_sequences, n, -1)       )

														if multi_label or len(lowerCamelCase_       ) == 1:
																					# softmax over the entailment vs. contradiction dim for each label independently
																					lowerCAmelCase__	       =     self.entailment_id
																					lowerCAmelCase__	       =     -1 if entailment_id == 0 else 0
																					lowerCAmelCase__	       =     reshaped_outputs[..., [contradiction_id, entailment_id]]
																					lowerCAmelCase__	       =     np.exp(lowerCamelCase_       ) / np.exp(lowerCamelCase_       ).sum(-1 ,   keepdims=lowerCamelCase_       )
																					lowerCAmelCase__	       =     scores[..., 1]
														else:
																					# softmax the "entailment" logits over all candidate labels
																					lowerCAmelCase__	       =     reshaped_outputs[..., self.entailment_id]
																					lowerCAmelCase__	       =     np.exp(lowerCamelCase_       ) / np.exp(lowerCamelCase_       ).sum(-1 ,   keepdims=lowerCamelCase_       )

														lowerCAmelCase__	       =     list(reversed(scores[0].argsort()       )       )
														return {
														    "sequence": sequences[0],
														    "labels": [candidate_labels[i] for i in top_inds],
														    "scores": scores[0, top_inds].tolist(),
														}





'''simple docstring'''






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       List[Any]     						=				CLIPTokenizer
							lowercase__      :       List[str]     						=				CLIPTokenizerFast
							lowercase__      :       Dict     						=				True
							lowercase__      :       Any     						=				{}
							lowercase__      :       Optional[int]     						=				False







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														super().setUp()

														# fmt: off
														lowerCAmelCase__	       =     ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
														# fmt: on
														lowerCAmelCase__	       =     dict(zip(lowerCamelCase_ ,   range(len(lowerCamelCase_       )       )       )       )
														lowerCAmelCase__	       =     ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
														lowerCAmelCase__	       =     {'''unk_token''': '''<unk>'''}

														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''merges_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write(json.dumps(lowerCamelCase_       ) + '''\n'''       )
														with open(self.merges_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							Any:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							int:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							Any:
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     '''lower newer'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     CLIPTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map       )
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokens + [tokenizer.unk_token]
														lowerCAmelCase__	       =     [10, 2, 16, 9, 3, 2, 16, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																												# with Tilde) encoded in 2 different ways
																												lowerCAmelCase__	       =     '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of space type
																												lowerCAmelCase__	       =     [
																												    '''\u0009''',  # (horizontal tab, '\t')
																												    '''\u000B''',  # (vertical tab)
																												    '''\u000C''',  # (form feed)
																												    '''\u0020''',  # (space, ' ')
																												    '''\u200E''',  # (left-to-right mark):w
																												    '''\u200F''',  # (right-to-left mark)
																												]
																												for unicode_seq in spaces_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of line break type
																												lowerCAmelCase__	       =     [
																												    '''\u000A''',  # (line feed, '\n')
																												    '''\r\n''',  # (carriage return and line feed, '\r\n')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\r''',  # (carriage return, '\r')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\u2028''',  # (line separator)
																												    '''\u2029''',  # (paragraph separator)
																												    # "\u0085", # (next line)
																												]

																												# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																												# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																												# space (and thus into an empty list).

																												for unicode_seq in line_break_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																												lowerCAmelCase__	       =     F"""{text_of_1_token} {text_of_1_token}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (0, len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (len(lowerCamelCase_       ) + 1, len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )

																												lowerCAmelCase__	       =     F""" {text}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (1, 1 + len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (1 + len(lowerCamelCase_       ) + 1, 1 + len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														# Test related to the breaking change introduced in transformers v4.17.0
														# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
														with self.assertRaises(lowerCamelCase_       ) as context:
																					self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''       )

														self.assertTrue(
														    context.exception.args[0].startswith(
														        '''The `backend_tokenizer` provided does not match the expected format.'''       )       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														super().test_tokenization_python_rust_equals()







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														# CLIP always lower cases letters
														pass