






"""simple docstring"""


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase    :   str   			=						get_tests_dir('fixtures/spiece.model')






@require_sentencepiece
@require_tokenizers
class 						snake_case  (  UpperCAmelCase       ,			unittest.TestCase	):
    __magic_name__   					=					AlbertTokenizer
    __magic_name__   					=					AlbertTokenizerFast
    __magic_name__   					=					True
    __magic_name__   					=					True
    __magic_name__   					=					True





    def     lowerCamelCase__				(							self     : Any      ):


      '''simple docstring'''




      super().setUp()

      # We have a SentencePiece fixture for testing
      a					:      Dict    =       AlbertTokenizer(A      )
      tokenizer.save_pretrained(self.tmpdirname      )





    def     lowerCamelCase__				(							self     : str    ,		A     : Dict      ):


      '''simple docstring'''




      a					:      int    =       'this is a test'
      a					:      Dict    =       'this is a test'
      return input_text, output_text





    def     lowerCamelCase__				(							self     : Union[str, Any]      ):


      '''simple docstring'''




      a					:      Any    =       '<pad>'
      a					:      Any    =       0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(A      )    ,		A      )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(A      )    ,		A      )





    def     lowerCamelCase__				(							self     : str      ):


      '''simple docstring'''




      a					:      Dict    =       list(self.get_tokenizer().get_vocab().keys()      )

      self.assertEqual(vocab_keys[0]    ,		'<pad>'      )
      self.assertEqual(vocab_keys[1]    ,		'<unk>'      )
      self.assertEqual(vocab_keys[-1]    ,		'▁eloquent'      )
      self.assertEqual(len(A      )    ,		3_0_0_0_0      )





    def     lowerCamelCase__				(							self     : int      ):


      '''simple docstring'''




      self.assertEqual(self.get_tokenizer().vocab_size    ,		3_0_0_0_0      )





    def     lowerCamelCase__				(							self     : List[str]      ):


      '''simple docstring'''




      if not self.test_rust_tokenizer:
        return

      a					:      Union[str, Any]    =       self.get_tokenizer()
      a					:      Union[str, Any]    =       self.get_rust_tokenizer()

      a					:      str    =       'I was born in 92000, and this is falsé.'

      a					:      Tuple    =       tokenizer.tokenize(A      )
      a					:      str    =       rust_tokenizer.tokenize(A      )
      self.assertListEqual(A    ,		A      )

      a					:      Dict    =       tokenizer.encode(A    ,		add_special_tokens=A      )
      a					:      Dict    =       rust_tokenizer.encode(A    ,		add_special_tokens=A      )
      self.assertListEqual(A    ,		A      )

      a					:      str    =       self.get_rust_tokenizer()
      a					:      Dict    =       tokenizer.encode(A      )
      a					:      Any    =       rust_tokenizer.encode(A      )
      self.assertListEqual(A    ,		A      )





    def     lowerCamelCase__				(							self     : str      ):


      '''simple docstring'''




      a					:      str    =       AlbertTokenizer(A    ,		keep_accents=A      )

      a					:      Tuple    =       tokenizer.tokenize('This is a test'      )
      self.assertListEqual(A    ,		['▁this', '▁is', '▁a', '▁test']      )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(A      )    ,		[4_8, 2_5, 2_1, 1_2_8_9]      )

      a					:      int    =       tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
      self.assertListEqual(
          A    ,		['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']      )
      a					:      Union[str, Any]    =       tokenizer.convert_tokens_to_ids(A      )
      self.assertListEqual(A    ,		[3_1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]      )

      a					:      Tuple    =       tokenizer.convert_ids_to_tokens(A      )
      self.assertListEqual(
          A    ,		['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']    ,		)





    def     lowerCamelCase__				(							self     : str      ):


      '''simple docstring'''




      a					:      Union[str, Any]    =       AlbertTokenizer(A      )

      a					:      List[Any]    =       tokenizer.encode('sequence builders'      )
      a					:      Dict    =       tokenizer.encode('multi-sequence build'      )

      a					:      Union[str, Any]    =       tokenizer.build_inputs_with_special_tokens(A      )
      a					:      List[Any]    =       tokenizer.build_inputs_with_special_tokens(A    ,		A      )

      assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
      assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
          tokenizer.sep_token_id
      ]





    @slow
    def     lowerCamelCase__				(							self     : Tuple      ):


      '''simple docstring'''




      a					:      Tuple    =       {'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1_9_7_0, 1_3, 5, 6_0_9_2, 1_6_7, 2_8, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 7_0_2_8, 1_2_0_5_1, 1_8, 1_7, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 3_5_1_5, 1_8_6_8_4, 8, 4_4_6_1, 6, 1_9_2_7, 2_9_7, 8, 1_2_0_6_0, 2_6_0_7, 1_8, 1_3, 5, 4_4_6_1, 1_5, 1_0_5_3_8, 3_8, 8, 1_3_5, 1_5, 8_2_2, 5_8, 1_5, 9_9_3, 1_0_3_6_3, 1_5, 1_4_6_0, 8_0_0_5, 4_4_6_1, 1_5, 9_9_3, 2_5_5, 2_3_2_8, 9, 9, 9, 6, 2_6, 1_1_1_2, 8_1_6, 3_2_6_0, 1_3, 5, 1_0_3, 2_3_7_7, 6, 1_7, 1_1_1_2, 8_1_6, 2_7_8_2, 1_3, 5, 1_0_3, 1_0_6_4_1, 6, 2_9, 8_4, 2_5_1_2, 2_4_3_0, 7_8_2, 1_8_6_8_4, 2_7_6_1, 1_9, 8_0_8, 2_4_3_0, 2_5_5_6, 1_7, 8_5_5, 1_4_8_0, 9_4_7_7, 4_0_9_1, 1_2_8, 1_1_7_1_2, 1_5, 7_1_0_3, 2_1_5_3, 6_7_3, 1_7, 2_4_8_8_3, 9_9_9_0, 9, 3], [2, 1_1_5_0_2, 2_5, 1_0_0_6, 2_0, 7_8_2, 8, 1_1_8_0_9, 8_5_5, 1_7_3_2, 1_9_3_9_3, 1_8_6_6_7, 3_7, 3_6_7, 2_1_0_1_8, 6_9, 1_8_5_4, 3_4, 1_1_8_6_0, 1_9_1_2_4, 2_7, 1_5_6, 2_2_5, 1_7, 1_9_3, 4_1_4_1, 1_9, 6_5, 9_1_2_4, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1_4, 2_2_3_1, 8_8_6, 2_3_8_5, 1_7_6_5_9, 8_4, 1_4, 1_6_7_9_2, 1_9_5_2, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=A    ,		model_name='albert-base-v2'    ,		revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'    ,		)












"""simple docstring"""


from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


_UpperCamelCase    :   int   			=						logging.get_logger(__name__)






class 						snake_case  (  UpperCAmelCase	):
    __magic_name__   					=					['''input_features''', '''attention_mask''']





    def __init__(							self     : Optional[int]    ,		A     : Optional[Any]=8_0    ,		A     : str=1_6_0_0_0    ,		A     : List[str]=0.0    ,		A     : Any=1_0    ,		A     : Union[str, Any]=2_5    ,		A     : str="hamming_window"    ,		A     : str=3_27_68.0    ,		A     : Union[str, Any]=0.97    ,		A     : Dict=1.0    ,		A     : Any=True    ,		A     : Union[str, Any]=True    ,		A     : List[Any]=False    ,		**A     : Tuple    ,		):


      '''simple docstring'''




      super().__init__(feature_size=A    ,		sampling_rate=A    ,		padding_value=A    ,		**A      )

      a					:      Any    =       feature_size
      a					:      List[Any]    =       sampling_rate
      a					:      Any    =       padding_value
      a					:      str    =       hop_length
      a					:      Any    =       win_length
      a					:      List[Any]    =       frame_signal_scale
      a					:      Tuple    =       preemphasis_coeff
      a					:      Dict    =       mel_floor
      a					:      Optional[int]    =       normalize_means
      a					:      List[str]    =       normalize_vars
      a					:      Dict    =       win_function
      a					:      Union[str, Any]    =       return_attention_mask

      a					:      List[Any]    =       win_length * sampling_rate // 1_0_0_0
      a					:      Tuple    =       hop_length * sampling_rate // 1_0_0_0

      a					:      List[Any]    =       optimal_fft_length(self.sample_size      )
      a					:      Any    =       (self.n_fft // 2) + 1





    def     lowerCamelCase__				(							self     : List[Any]    ,		A     : np.array      ):


      '''simple docstring'''




      if self.win_function == "hamming_window":
        a					:      List[str]    =       window_function(window_length=self.sample_size    ,		name=self.win_function    ,		periodic=A      )
      else:
        a					:      Dict    =       window_function(window_length=self.sample_size    ,		name=self.win_function      )

      a					:      str    =       mel_filter_bank(
          num_frequency_bins=self.n_freqs    ,		num_mel_filters=self.feature_size    ,		min_frequency=0.0    ,		max_frequency=self.sampling_rate / 2.0    ,		sampling_rate=self.sampling_rate    ,		)

      a					:      List[Any]    =       spectrogram(
          one_waveform * self.frame_signal_scale    ,		window=A    ,		frame_length=self.sample_size    ,		hop_length=self.sample_stride    ,		fft_length=self.n_fft    ,		center=A    ,		preemphasis=self.preemphasis_coeff    ,		mel_filters=A    ,		mel_floor=self.mel_floor    ,		log_mel='log'    ,		)
      return msfc_features.T





    def     lowerCamelCase__				(							self     : int    ,		A     : Tuple    ,		A     : int    ,		A     : Optional[int]      ):


      '''simple docstring'''




      if self.normalize_means:
        a					:      Any    =       x[:input_length].mean(axis=0      )
        a					:      Dict    =       np.subtract(A    ,		A      )
      if self.normalize_vars:
        a					:      Dict    =       x[:input_length].std(axis=0      )
        a					:      Dict    =       np.divide(A    ,		A      )

      if input_length < x.shape[0]:
        a					:      Dict    =       padding_value

      # make sure array is in float32
      a					:      Optional[int]    =       x.astype(np.floataa      )

      return x





    def     lowerCamelCase__				(							self     : str    ,		A     : List[np.ndarray]    ,		A     : Optional[np.ndarray] = None      ):


      '''simple docstring'''




      a					:      str    =       attention_mask.sum(-1      ) if attention_mask is not None else [x.shape[0] for x in input_features]
      return [self._normalize_one(A    ,		A    ,		self.padding_value      ) for x, n in zip(A    ,		A      )]





    def __call__(							self     : Dict    ,		A     : Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]    ,		A     : Union[bool, str, PaddingStrategy] = False    ,		A     : Optional[int] = None    ,		A     : bool = False    ,		A     : Optional[int] = None    ,		A     : Optional[bool] = None    ,		A     : Optional[Union[str, TensorType]] = None    ,		A     : Optional[int] = None    ,		**A     : Union[str, Any]    ,		):


      '''simple docstring'''




      if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
          raise ValueError(
              F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
              F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
              F''' {self.sampling_rate} and not {sampling_rate}.'''      )
      else:
        logger.warning(
            'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
            'Failing to do so can result in silent errors that might be hard to debug.'      )

      a					:      Optional[int]    =       isinstance(A    ,		np.ndarray      ) and len(raw_speech.shape      ) > 1
      if is_batched_numpy and len(raw_speech.shape      ) > 2:
        raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''      )
      a					:      Dict    =       is_batched_numpy or (
          isinstance(A    ,		(list, tuple)      ) and (isinstance(raw_speech[0]    ,		(np.ndarray, tuple, list)      ))
      )

      if is_batched:
        a					:      str    =       [np.asarray(A    ,		dtype=np.floataa      ) for speech in raw_speech]
      elif not is_batched and not isinstance(A    ,		np.ndarray      ):
        a					:      List[str]    =       np.asarray(A    ,		dtype=np.floataa      )
      elif isinstance(A    ,		np.ndarray      ) and raw_speech.dtype is np.dtype(np.floataa      ):
        a					:      Optional[int]    =       raw_speech.astype(np.floataa      )

      # always return batch
      if not is_batched:
        a					:      Any    =       [raw_speech]

      # extract fbank features
      a					:      str    =       [self._extract_mfsc_features(A      ) for one_waveform in raw_speech]

      # convert into correct format for padding
      a					:      int    =       BatchFeature({'input_features': features}      )

      a					:      Union[str, Any]    =       self.pad(
          A    ,		padding=A    ,		max_length=A    ,		truncation=A    ,		pad_to_multiple_of=A    ,		return_attention_mask=A    ,		**A    ,		)
      # make sure list is in array format
      a					:      Optional[Any]    =       padded_inputs.get('input_features'      )
      if isinstance(input_features[0]    ,		A      ):
        a					:      List[str]    =       [np.asarray(A    ,		dtype=np.floataa      ) for feature in input_features]

      a					:      List[Any]    =       padded_inputs.get('attention_mask'      )
      if attention_mask is not None:
        a					:      int    =       [np.asarray(A    ,		dtype=np.intaa      ) for array in attention_mask]

      if self.normalize_means or self.normalize_vars:
        a					:      Any    =       (
            np.array(A    ,		dtype=np.intaa      )
            if self._get_padding_strategies(A    ,		max_length=A      ) is not PaddingStrategy.DO_NOT_PAD
            and padding
            else None
        )
        a					:      List[str]    =       self.normalize(
            padded_inputs['input_features']    ,		attention_mask=A      )

      if return_tensors is not None:
        a					:      Optional[int]    =       padded_inputs.convert_to_tensors(A      )

      return padded_inputs




