


'''simple docstring'''






from typing import List, Optional, Union

import numpy as np

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


__UpperCAmelCase       =							logging.get_logger(__name__)




class 		a__    (							a__  ):





							'''simple docstring'''






							lowercase__      :       Tuple     						=				["input_values", "padding_mask"]







							def __init__(		self ,   lowerCamelCase_ = 1 ,   lowerCamelCase_ = 2_40_00 ,   lowerCamelCase_ = 0.0 ,   lowerCamelCase_ = None ,   lowerCamelCase_ = None ,   **lowerCamelCase_ ,   )	->							Any:
														super().__init__(feature_size=lowerCamelCase_ ,   sampling_rate=lowerCamelCase_ ,   padding_value=lowerCamelCase_ ,   **lowerCamelCase_       )
														lowerCAmelCase__	       =     chunk_length_s
														lowerCAmelCase__	       =     overlap







							@property
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														if self.chunk_length_s is None:
																					return None
														else:
																					return int(self.chunk_length_s * self.sampling_rate       )







							@property
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														if self.chunk_length_s is None or self.overlap is None:
																					return None
														else:
																					return max(1 ,   int((1.0 - self.overlap) * self.chunk_length       )       )







							def __call__(		self ,   lowerCamelCase_ ,   lowerCamelCase_ = None ,   lowerCamelCase_ = False ,   lowerCamelCase_ = None ,   lowerCamelCase_ = None ,   lowerCamelCase_ = None ,   )	->							BatchFeature:
														if sampling_rate is not None:
																					if sampling_rate != self.sampling_rate:
																												raise ValueError(
																												    F"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
																												    F""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
																												    F""" {self.sampling_rate} and not {sampling_rate}."""       )
														else:
																					logger.warning(
																					    '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
																					    '''Failing to do so can result in silent errors that might be hard to debug.'''       )

														if padding and truncation:
																					raise ValueError('''Both padding and truncation were set. Make sure you only set one.'''       )
														elif padding is None:
																					# by default let's pad the inputs
																					lowerCAmelCase__	       =     True

														lowerCAmelCase__	       =     bool(
														    isinstance(lowerCamelCase_ ,   (list, tuple)       ) and (isinstance(raw_audio[0] ,   (np.ndarray, tuple, list)       ))       )

														if is_batched:
																					lowerCAmelCase__	       =     [np.asarray(lowerCamelCase_ ,   dtype=np.floataa       ).T for audio in raw_audio]
														elif not is_batched and not isinstance(lowerCamelCase_ ,   np.ndarray       ):
																					lowerCAmelCase__	       =     np.asarray(lowerCamelCase_ ,   dtype=np.floataa       )
														elif isinstance(lowerCamelCase_ ,   np.ndarray       ) and raw_audio.dtype is np.dtype(np.floataa       ):
																					lowerCAmelCase__	       =     raw_audio.astype(np.floataa       )

														# always return batch
														if not is_batched:
																					lowerCAmelCase__	       =     [np.asarray(lowerCamelCase_       ).T]

														# verify inputs are valid
														for idx, example in enumerate(lowerCamelCase_       ):
																					if example.ndim > 2:
																												raise ValueError(F"""Expected input shape (channels, length) but got shape {example.shape}"""       )
																					if self.feature_size == 1 and example.ndim != 1:
																												raise ValueError(F"""Expected mono audio but example has {example.shape[-1]} channels"""       )
																					if self.feature_size == 2 and example.shape[-1] != 2:
																												raise ValueError(F"""Expected stereo audio but example has {example.shape[-1]} channels"""       )

														lowerCAmelCase__	       =     None
														lowerCAmelCase__	       =     BatchFeature({'''input_values''': raw_audio}       )
														if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
																					if truncation:
																												lowerCAmelCase__	       =     min(array.shape[0] for array in raw_audio       )
																												lowerCAmelCase__	       =     int(np.floor(max_length / self.chunk_stride       )       )
																												lowerCAmelCase__	       =     (nb_step - 1) * self.chunk_stride + self.chunk_length
																					elif padding:
																												lowerCAmelCase__	       =     max(array.shape[0] for array in raw_audio       )
																												lowerCAmelCase__	       =     int(np.ceil(max_length / self.chunk_stride       )       )
																												lowerCAmelCase__	       =     (nb_step - 1) * self.chunk_stride + self.chunk_length
																												lowerCAmelCase__	       =     '''max_length'''
																					else:
																												lowerCAmelCase__	       =     input_values

        # normal padding on batch
														if padded_inputs is None:
																					lowerCAmelCase__	       =     self.pad(
																					    lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   truncation=lowerCamelCase_ ,   padding=lowerCamelCase_ ,   return_attention_mask=lowerCamelCase_ ,   )
																					if padding:
																												lowerCAmelCase__	       =     padded_inputs.pop('''attention_mask'''       )

														lowerCAmelCase__	       =     []
														for example in padded_inputs.pop('''input_values'''       ):
																					if self.feature_size == 1:
																												lowerCAmelCase__	       =     example[..., None]
																					input_values.append(example.T       )

														lowerCAmelCase__	       =     input_values
														if return_tensors is not None:
																					lowerCAmelCase__	       =     padded_inputs.convert_to_tensors(lowerCamelCase_       )

														return padded_inputs



'''simple docstring'''






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       List[Any]     						=				CLIPTokenizer
							lowercase__      :       List[str]     						=				CLIPTokenizerFast
							lowercase__      :       Dict     						=				True
							lowercase__      :       Any     						=				{}
							lowercase__      :       Optional[int]     						=				False







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														super().setUp()

														# fmt: off
														lowerCAmelCase__	       =     ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
														# fmt: on
														lowerCAmelCase__	       =     dict(zip(lowerCamelCase_ ,   range(len(lowerCamelCase_       )       )       )       )
														lowerCAmelCase__	       =     ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
														lowerCAmelCase__	       =     {'''unk_token''': '''<unk>'''}

														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''merges_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write(json.dumps(lowerCamelCase_       ) + '''\n'''       )
														with open(self.merges_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							Any:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							int:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							Any:
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     '''lower newer'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     CLIPTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map       )
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokens + [tokenizer.unk_token]
														lowerCAmelCase__	       =     [10, 2, 16, 9, 3, 2, 16, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																												# with Tilde) encoded in 2 different ways
																												lowerCAmelCase__	       =     '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of space type
																												lowerCAmelCase__	       =     [
																												    '''\u0009''',  # (horizontal tab, '\t')
																												    '''\u000B''',  # (vertical tab)
																												    '''\u000C''',  # (form feed)
																												    '''\u0020''',  # (space, ' ')
																												    '''\u200E''',  # (left-to-right mark):w
																												    '''\u200F''',  # (right-to-left mark)
																												]
																												for unicode_seq in spaces_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of line break type
																												lowerCAmelCase__	       =     [
																												    '''\u000A''',  # (line feed, '\n')
																												    '''\r\n''',  # (carriage return and line feed, '\r\n')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\r''',  # (carriage return, '\r')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\u2028''',  # (line separator)
																												    '''\u2029''',  # (paragraph separator)
																												    # "\u0085", # (next line)
																												]

																												# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																												# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																												# space (and thus into an empty list).

																												for unicode_seq in line_break_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																												lowerCAmelCase__	       =     F"""{text_of_1_token} {text_of_1_token}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (0, len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (len(lowerCamelCase_       ) + 1, len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )

																												lowerCAmelCase__	       =     F""" {text}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (1, 1 + len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (1 + len(lowerCamelCase_       ) + 1, 1 + len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														# Test related to the breaking change introduced in transformers v4.17.0
														# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
														with self.assertRaises(lowerCamelCase_       ) as context:
																					self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''       )

														self.assertTrue(
														    context.exception.args[0].startswith(
														        '''The `backend_tokenizer` provided does not match the expected format.'''       )       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														super().test_tokenization_python_rust_equals()







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														# CLIP always lower cases letters
														pass