



"""simple docstring"""




import itertools
import random
import unittest

import numpy as np

from transformers import WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST, WavaVecaConfig, WavaVecaFeatureExtractor
from transformers.testing_utils import require_torch, slow

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_lowerCAmelCase    : Any							=							random.Random()

def 						__snake_case						(				SCREAMING_SNAKE_CASE__     :						str						,  SCREAMING_SNAKE_CASE__     :						Optional[Any]=1.0						,  SCREAMING_SNAKE_CASE__     :						Optional[Any]=None						,  SCREAMING_SNAKE_CASE__     :						List[str]=None    )  ->       Optional[Any]:





     '''simple docstring'''

     if rng is None:
          _UpperCAmelCase      :     Tuple        =   global_rng

     _UpperCAmelCase      :     Union[str, Any]        =   []
     for batch_idx in range(shape[0]    ):
          values.append([]    )
          for _ in range(shape[1]    ):
               values[-1].append(rng.random() * scale    )

     return values






class      UpperCAmelCase_  (    unittest.TestCase						):



      def __init__(					self					:     Optional[Any]					,     A					:     int					,     A					:     Union[str, Any]=7					,     A					:     Dict=4_0_0					,     A					:     List[Any]=2_0_0_0					,     A					:     List[Any]=1					,     A					:     Dict=0.0					,     A					:     str=1_6_0_0_0					,     A					:     List[Any]=True					,     A					:     Optional[Any]=True					,     ):
           _UpperCAmelCase      :     str        =   parent
           _UpperCAmelCase      :     int        =   batch_size
           _UpperCAmelCase      :     List[str]        =   min_seq_length
           _UpperCAmelCase      :     Optional[int]        =   max_seq_length
           _UpperCAmelCase      :     Dict        =   (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
           _UpperCAmelCase      :     int        =   feature_size
           _UpperCAmelCase      :     List[Any]        =   padding_value
           _UpperCAmelCase      :     Tuple        =   sampling_rate
           _UpperCAmelCase      :     Optional[int]        =   return_attention_mask
           _UpperCAmelCase      :     List[str]        =   do_normalize



      def 						snake_case_ (					self					:     Tuple						):
           return {
               "feature_size": self.feature_size,
               "padding_value": self.padding_value,
               "sampling_rate": self.sampling_rate,
               "return_attention_mask": self.return_attention_mask,
               "do_normalize": self.do_normalize,
           }



      def 						snake_case_ (					self					:     Tuple					,     A					:     str=False					,     A					:     List[str]=False						):







           def _flatten(A					:     Any						):
                return list(itertools.chain(*_lowerCamelCase						)						)

           if equal_length:
                _UpperCAmelCase      :     Dict        =   floats_list((self.batch_size, self.max_seq_length)						)
           else:
                # make sure that inputs increase in size
                _UpperCAmelCase      :     Any        =   [
                    _flatten(floats_list((x, self.feature_size)						)						)
                    for x in range(self.min_seq_length					,     self.max_seq_length					,     self.seq_length_diff						)
                ]

           if numpify:
                _UpperCAmelCase      :     Dict        =   [np.asarray(_lowerCamelCase						) for x in speech_inputs]

           return speech_inputs







class      UpperCAmelCase_  (    a__	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       Any								=			WavaVecaFeatureExtractor



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     str        =   WavaVecaFeatureExtractionTester(self						)



      def 						snake_case_ (					self					:     Any					,     A					:     Union[str, Any]						):
           self.assertTrue(np.all(np.mean(_lowerCamelCase					,     axis=0						) < 1e-3						)						)
           self.assertTrue(np.all(np.abs(np.var(_lowerCamelCase					,     axis=0						) - 1						) < 1e-3						)						)



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     List[str]        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           # create three inputs of length 800, 1000, and 1200
           _UpperCAmelCase      :     Optional[Any]        =   [floats_list((1, x)						)[0] for x in range(8_0_0					,     1_4_0_0					,     2_0_0						)]
           _UpperCAmelCase      :     List[Any]        =   [np.asarray(_lowerCamelCase						) for speech_input in speech_inputs]

           # Test not batched input
           _UpperCAmelCase      :     Union[str, Any]        =   feat_extract(speech_inputs[0]					,     return_tensors="np"						).input_values
           _UpperCAmelCase      :     int        =   feat_extract(np_speech_inputs[0]					,     return_tensors="np"						).input_values
           self.assertTrue(np.allclose(_lowerCamelCase					,     _lowerCamelCase					,     atol=1e-3						)						)

           # Test batched
           _UpperCAmelCase      :     Dict        =   feat_extract(_lowerCamelCase					,     return_tensors="np"						).input_values
           _UpperCAmelCase      :     Dict        =   feat_extract(_lowerCamelCase					,     return_tensors="np"						).input_values
           for enc_seq_a, enc_seq_a in zip(_lowerCamelCase					,     _lowerCamelCase						):
                self.assertTrue(np.allclose(_lowerCamelCase					,     _lowerCamelCase					,     atol=1e-3						)						)

           # Test 2-D numpy arrays are batched.
           _UpperCAmelCase      :     Optional[Any]        =   [floats_list((1, x)						)[0] for x in (8_0_0, 8_0_0, 8_0_0)]
           _UpperCAmelCase      :     List[str]        =   np.asarray(_lowerCamelCase						)
           _UpperCAmelCase      :     Dict        =   feat_extract(_lowerCamelCase					,     return_tensors="np"						).input_values
           _UpperCAmelCase      :     Optional[int]        =   feat_extract(_lowerCamelCase					,     return_tensors="np"						).input_values
           for enc_seq_a, enc_seq_a in zip(_lowerCamelCase					,     _lowerCamelCase						):
                self.assertTrue(np.allclose(_lowerCamelCase					,     _lowerCamelCase					,     atol=1e-3						)						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     str        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           _UpperCAmelCase      :     int        =   [floats_list((1, x)						)[0] for x in range(8_0_0					,     1_4_0_0					,     2_0_0						)]

           _UpperCAmelCase      :     Dict        =   ['''longest''', '''max_length''', '''do_not_pad''']
           _UpperCAmelCase      :     Dict        =   [None, 1_6_0_0, None]
           for max_length, padding in zip(_lowerCamelCase					,     _lowerCamelCase						):
                _UpperCAmelCase      :     Dict        =   feat_extract(_lowerCamelCase					,     padding=_lowerCamelCase					,     max_length=_lowerCamelCase					,     return_tensors="np"						)
                _UpperCAmelCase      :     Tuple        =   processed.input_values

                self._check_zero_mean_unit_variance(input_values[0][:8_0_0]						)
                self.assertTrue(input_values[0][8_0_0:].sum() < 1e-6						)
                self._check_zero_mean_unit_variance(input_values[1][:1_0_0_0]						)
                self.assertTrue(input_values[0][1_0_0_0:].sum() < 1e-6						)
                self._check_zero_mean_unit_variance(input_values[2][:1_2_0_0]						)



      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     Optional[int]        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           _UpperCAmelCase      :     Optional[int]        =   range(8_0_0					,     1_4_0_0					,     2_0_0						)
           _UpperCAmelCase      :     Optional[Any]        =   [floats_list((1, x)						)[0] for x in lengths]

           _UpperCAmelCase      :     int        =   ['''longest''', '''max_length''', '''do_not_pad''']
           _UpperCAmelCase      :     Dict        =   [None, 1_6_0_0, None]

           for max_length, padding in zip(_lowerCamelCase					,     _lowerCamelCase						):
                _UpperCAmelCase      :     List[str]        =   feat_extract(_lowerCamelCase					,     max_length=_lowerCamelCase					,     padding=_lowerCamelCase						)
                _UpperCAmelCase      :     str        =   processed.input_values

                self._check_zero_mean_unit_variance(input_values[0][:8_0_0]						)
                self._check_zero_mean_unit_variance(input_values[1][:1_0_0_0]						)
                self._check_zero_mean_unit_variance(input_values[2][:1_2_0_0]						)



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     Optional[int]        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           _UpperCAmelCase      :     Optional[int]        =   [floats_list((1, x)						)[0] for x in range(8_0_0					,     1_4_0_0					,     2_0_0						)]
           _UpperCAmelCase      :     Optional[Any]        =   feat_extract(
               _lowerCamelCase					,     truncation=_lowerCamelCase					,     max_length=1_0_0_0					,     padding="max_length"					,     return_tensors="np"						)
           _UpperCAmelCase      :     Tuple        =   processed.input_values

           self._check_zero_mean_unit_variance(input_values[0, :8_0_0]						)
           self._check_zero_mean_unit_variance(input_values[1]						)
           self._check_zero_mean_unit_variance(input_values[2]						)



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     Optional[int]        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           _UpperCAmelCase      :     List[Any]        =   [floats_list((1, x)						)[0] for x in range(8_0_0					,     1_4_0_0					,     2_0_0						)]
           _UpperCAmelCase      :     Dict        =   feat_extract(
               _lowerCamelCase					,     truncation=_lowerCamelCase					,     max_length=1_0_0_0					,     padding="longest"					,     return_tensors="np"						)
           _UpperCAmelCase      :     List[str]        =   processed.input_values

           self._check_zero_mean_unit_variance(input_values[0, :8_0_0]						)
           self._check_zero_mean_unit_variance(input_values[1, :1_0_0_0]						)
           self._check_zero_mean_unit_variance(input_values[2]						)

           # make sure that if max_length < longest -> then pad to max_length
           self.assertTrue(input_values.shape == (3, 1_0_0_0)						)

           _UpperCAmelCase      :     Optional[Any]        =   [floats_list((1, x)						)[0] for x in range(8_0_0					,     1_4_0_0					,     2_0_0						)]
           _UpperCAmelCase      :     List[str]        =   feat_extract(
               _lowerCamelCase					,     truncation=_lowerCamelCase					,     max_length=2_0_0_0					,     padding="longest"					,     return_tensors="np"						)
           _UpperCAmelCase      :     Optional[int]        =   processed.input_values

           self._check_zero_mean_unit_variance(input_values[0, :8_0_0]						)
           self._check_zero_mean_unit_variance(input_values[1, :1_0_0_0]						)
           self._check_zero_mean_unit_variance(input_values[2]						)

           # make sure that if max_length > longest -> then pad to longest
           self.assertTrue(input_values.shape == (3, 1_2_0_0)						)



      @require_torch
      def 						snake_case_ (					self					:     int						):
           import torch

           _UpperCAmelCase      :     Union[str, Any]        =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
           _UpperCAmelCase      :     List[str]        =   np.random.rand(1_0_0						).astype(np.floataa						)
           _UpperCAmelCase      :     Optional[Any]        =   np_speech_inputs.tolist()

           for inputs in [py_speech_inputs, np_speech_inputs]:
                _UpperCAmelCase      :     Dict        =   feature_extractor.pad([{"input_values": inputs}]					,     return_tensors="np"						)
                self.assertTrue(np_processed.input_values.dtype == np.floataa						)
                _UpperCAmelCase      :     List[str]        =   feature_extractor.pad([{"input_values": inputs}]					,     return_tensors="pt"						)
                self.assertTrue(pt_processed.input_values.dtype == torch.floataa						)



      @slow
      @require_torch
      def 						snake_case_ (					self					:     List[str]						):







           for model_id in WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST:
                _UpperCAmelCase      :     Optional[int]        =   WavaVecaConfig.from_pretrained(_lowerCamelCase						)
                _UpperCAmelCase      :     Any        =   WavaVecaFeatureExtractor.from_pretrained(_lowerCamelCase						)

                # only "layer" feature extraction norm should make use of
                # attention_mask
                self.assertEqual(feat_extract.return_attention_mask					,     config.feat_extract_norm == "layer"						)








"""simple docstring"""




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import CLIPImageProcessor, CLIPProcessor






@require_vision
class      UpperCAmelCase_  (    unittest.TestCase						):



      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     List[str]        =   tempfile.mkdtemp()

           # fmt: off
           _UpperCAmelCase      :     Union[str, Any]        =   ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
           # fmt: on
           _UpperCAmelCase      :     List[Any]        =   dict(zip(A					,     range(len(A						)						)						)						)
           _UpperCAmelCase      :     Union[str, Any]        =   ["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
           _UpperCAmelCase      :     Optional[int]        =   {"unk_token": "<unk>"}

           _UpperCAmelCase      :     Optional[Any]        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["vocab_file"]						)
           _UpperCAmelCase      :     Optional[Any]        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["merges_file"]						)
           with open(self.vocab_file					,     "w"					,     encoding="utf-8"						) as fp:
                fp.write(json.dumps(A						) + "\n"						)
           with open(self.merges_file					,     "w"					,     encoding="utf-8"						) as fp:
                fp.write("\n".join(A						)						)

           _UpperCAmelCase      :     List[str]        =   {
               "do_resize": True,
               "size": 2_0,
               "do_center_crop": True,
               "crop_size": 1_8,
               "do_normalize": True,
               "image_mean": [0.48_145_466, 0.4_578_275, 0.40_821_073],
               "image_std": [0.26_862_954, 0.26_130_258, 0.27_577_711],
           }
           _UpperCAmelCase      :     Any        =   os.path.join(self.tmpdirname					,     A						)
           with open(self.image_processor_file					,     "w"					,     encoding="utf-8"						) as fp:
                json.dump(A					,     A						)



      def 						snake_case_ (					self					:     List[Any]					,     **A					:     Union[str, Any]						):
           return CLIPTokenizer.from_pretrained(self.tmpdirname					,     **A						)



      def 						snake_case_ (					self					:     int					,     **A					:     Any						):
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,     **A						)



      def 						snake_case_ (					self					:     List[str]					,     **A					:     Optional[Any]						):
           return CLIPImageProcessor.from_pretrained(self.tmpdirname					,     **A						)



      def 						snake_case_ (					self					:     Optional[int]						):
           shutil.rmtree(self.tmpdirname						)



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     int        =   [np.random.randint(2_5_5					,     size=(3, 3_0, 4_0_0)					,     dtype=np.uinta						)]

           _UpperCAmelCase      :     Dict        =   [Image.fromarray(np.moveaxis(A					,     0					,     -1						)						) for x in image_inputs]

           return image_inputs



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     int        =   self.get_tokenizer()
           _UpperCAmelCase      :     Dict        =   self.get_rust_tokenizer()
           _UpperCAmelCase      :     int        =   self.get_image_processor()

           _UpperCAmelCase      :     List[Any]        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)
           processor_slow.save_pretrained(self.tmpdirname						)
           _UpperCAmelCase      :     Optional[Any]        =   CLIPProcessor.from_pretrained(self.tmpdirname					,     use_fast=A						)

           _UpperCAmelCase      :     Optional[Any]        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)
           processor_fast.save_pretrained(self.tmpdirname						)
           _UpperCAmelCase      :     Optional[int]        =   CLIPProcessor.from_pretrained(self.tmpdirname						)

           self.assertEqual(processor_slow.tokenizer.get_vocab()					,     tokenizer_slow.get_vocab()						)
           self.assertEqual(processor_fast.tokenizer.get_vocab()					,     tokenizer_fast.get_vocab()						)
           self.assertEqual(tokenizer_slow.get_vocab()					,     tokenizer_fast.get_vocab()						)
           self.assertIsInstance(processor_slow.tokenizer					,     A						)
           self.assertIsInstance(processor_fast.tokenizer					,     A						)

           self.assertEqual(processor_slow.image_processor.to_json_string()					,     image_processor.to_json_string()						)
           self.assertEqual(processor_fast.image_processor.to_json_string()					,     image_processor.to_json_string()						)
           self.assertIsInstance(processor_slow.image_processor					,     A						)
           self.assertIsInstance(processor_fast.image_processor					,     A						)



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     List[str]        =   CLIPProcessor(tokenizer=self.get_tokenizer()					,     image_processor=self.get_image_processor()						)
           processor.save_pretrained(self.tmpdirname						)

           _UpperCAmelCase      :     Any        =   self.get_tokenizer(bos_token="(BOS)"					,     eos_token="(EOS)"						)
           _UpperCAmelCase      :     Any        =   self.get_image_processor(do_normalize=A					,     padding_value=1.0						)

           _UpperCAmelCase      :     Any        =   CLIPProcessor.from_pretrained(
               self.tmpdirname					,     bos_token="(BOS)"					,     eos_token="(EOS)"					,     do_normalize=A					,     padding_value=1.0						)

           self.assertEqual(processor.tokenizer.get_vocab()					,     tokenizer_add_kwargs.get_vocab()						)
           self.assertIsInstance(processor.tokenizer					,     A						)

           self.assertEqual(processor.image_processor.to_json_string()					,     image_processor_add_kwargs.to_json_string()						)
           self.assertIsInstance(processor.image_processor					,     A						)



      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     str        =   self.get_image_processor()
           _UpperCAmelCase      :     List[str]        =   self.get_tokenizer()

           _UpperCAmelCase      :     Any        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)

           _UpperCAmelCase      :     Dict        =   self.prepare_image_inputs()

           _UpperCAmelCase      :     Optional[int]        =   image_processor(A					,     return_tensors="np"						)
           _UpperCAmelCase      :     Any        =   processor(images=A					,     return_tensors="np"						)

           for key in input_image_proc.keys():
                self.assertAlmostEqual(input_image_proc[key].sum()					,     input_processor[key].sum()					,     delta=1e-2						)



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     Tuple        =   self.get_image_processor()
           _UpperCAmelCase      :     int        =   self.get_tokenizer()

           _UpperCAmelCase      :     List[str]        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)

           _UpperCAmelCase      :     Optional[int]        =   "lower newer"

           _UpperCAmelCase      :     Union[str, Any]        =   processor(text=A						)

           _UpperCAmelCase      :     Optional[int]        =   tokenizer(A						)

           for key in encoded_tok.keys():
                self.assertListEqual(encoded_tok[key]					,     encoded_processor[key]						)



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     Union[str, Any]        =   self.get_image_processor()
           _UpperCAmelCase      :     Tuple        =   self.get_tokenizer()

           _UpperCAmelCase      :     str        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)

           _UpperCAmelCase      :     Tuple        =   "lower newer"
           _UpperCAmelCase      :     Union[str, Any]        =   self.prepare_image_inputs()

           _UpperCAmelCase      :     str        =   processor(text=A					,     images=A						)

           self.assertListEqual(list(inputs.keys()						)					,     ["input_ids", "attention_mask", "pixel_values"]						)

           # test if it raises when no input is passed
           with pytest.raises(A						):
                processor()



      def 						snake_case_ (					self					:     int						):
           _UpperCAmelCase      :     List[str]        =   self.get_image_processor()
           _UpperCAmelCase      :     Dict        =   self.get_tokenizer()

           _UpperCAmelCase      :     List[Any]        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)

           _UpperCAmelCase      :     str        =   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           _UpperCAmelCase      :     List[str]        =   processor.batch_decode(A						)
           _UpperCAmelCase      :     int        =   tokenizer.batch_decode(A						)

           self.assertListEqual(A					,     A						)



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     Optional[Any]        =   self.get_image_processor()
           _UpperCAmelCase      :     int        =   self.get_tokenizer()

           _UpperCAmelCase      :     int        =   CLIPProcessor(tokenizer=A					,     image_processor=A						)

           _UpperCAmelCase      :     str        =   "lower newer"
           _UpperCAmelCase      :     int        =   self.prepare_image_inputs()

           _UpperCAmelCase      :     Optional[Any]        =   processor(text=A					,     images=A						)

           self.assertListEqual(list(inputs.keys()						)					,     processor.model_input_names						)
