






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


_UpperCamelCase    :   List[Any]   			=						logging.getLogger(__name__)






@dataclass(frozen=UpperCAmelCase	)
class 						snake_case  :
    __magic_name__   					=					42
    __magic_name__   					=					42
    __magic_name__   					=					None
    __magic_name__   					=					None
    __magic_name__   					=					None






@dataclass(frozen=UpperCAmelCase	)
class 						snake_case  :
    __magic_name__   					=					42
    __magic_name__   					=					None
    __magic_name__   					=					None
    __magic_name__   					=					None
    __magic_name__   					=					None


if is_torch_available():
   import torch
   from torch.utils.data import Dataset






   class 						snake_case  (  UpperCAmelCase	):
       __magic_name__   					=					42





       def __init__(							self     : str    ,		A     : str    ,		A     : PreTrainedTokenizer    ,		A     : str    ,		A     : Optional[int] = None    ,		A     : str=False    ,		A     : bool = False    ,		):


         '''simple docstring'''




         a					:      str    =       hans_processors[task]()

         a					:      Any    =       os.path.join(
             _a    ,		'cached_{}_{}_{}_{}'.format(
                 'dev' if evaluate else 'train'    ,		tokenizer.__class__.__name__    ,		str(_a      )    ,		_a    ,		)    ,		)
         a					:      int    =       processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
           # HACK(label indices are swapped in RoBERTa pretrained model)
           a,					a					:      List[Any]    =       label_list[2], label_list[1]
         a					:      Tuple    =       label_list

         # Make sure only the first process in distributed training processes the dataset,
         # and the others will use the cache.
         a					:      List[str]    =       cached_features_file + '.lock'
         with FileLock(_a      ):
           if os.path.exists(_a      ) and not overwrite_cache:
             logger.info(F'''Loading features from cached file {cached_features_file}'''      )
             a					:      str    =       torch.load(_a      )
           else:
             logger.info(F'''Creating features from dataset file at {data_dir}'''      )

             a					:      Optional[int]    =       (
                 processor.get_dev_examples(_a      ) if evaluate else processor.get_train_examples(_a      )
             )

             logger.info('Training examples: %s'    ,		len(_a      )      )
             a					:      List[str]    =       hans_convert_examples_to_features(_a    ,		_a    ,		_a    ,		_a      )
             logger.info('Saving features into cached file %s'    ,		_a      )
             torch.save(self.features    ,		_a      )





       def __len__(							self     : Union[str, Any]      ):


         '''simple docstring'''




         return len(self.features      )





       def __getitem__(							self     : int    ,		A     : int      ):


         '''simple docstring'''




         return self.features[i]





       def     lowerCamelCase__				(							self     : str      ):


         '''simple docstring'''




         return self.label_list


if is_tf_available():
   import tensorflow as tf






   class 						snake_case  :
       __magic_name__   					=					42





       def __init__(							self     : List[str]    ,		A     : str    ,		A     : PreTrainedTokenizer    ,		A     : str    ,		A     : Optional[int] = 1_2_8    ,		A     : Union[str, Any]=False    ,		A     : bool = False    ,		):


         '''simple docstring'''




         a					:      List[str]    =       hans_processors[task]()
         a					:      str    =       processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
           # HACK(label indices are swapped in RoBERTa pretrained model)
           a,					a					:      Dict    =       label_list[2], label_list[1]
         a					:      Union[str, Any]    =       label_list

         a					:      int    =       processor.get_dev_examples(_a      ) if evaluate else processor.get_train_examples(_a      )
         a					:      Tuple    =       hans_convert_examples_to_features(_a    ,		_a    ,		_a    ,		_a      )

         def gen():
           for ex_index, ex in tqdm.tqdm(enumerate(self.features      )    ,		desc='convert examples to features'      ):
             if ex_index % 1_0_0_0_0 == 0:
               logger.info('Writing example %d of %d' % (ex_index, len(_a      ))      )

             yield (
                 {
                     "example_id": 0,
                     "input_ids": ex.input_ids,
                     "attention_mask": ex.attention_mask,
                     "token_type_ids": ex.token_type_ids,
                 },
                 ex.label,
             )

         a					:      Any    =       tf.data.Dataset.from_generator(
             _a    ,		(
                 {
                     'example_id': tf.intaa,
                     'input_ids': tf.intaa,
                     'attention_mask': tf.intaa,
                     'token_type_ids': tf.intaa,
                 },
                 tf.intaa,
             )    ,		(
                 {
                     'example_id': tf.TensorShape([]      ),
                     'input_ids': tf.TensorShape([None, None]      ),
                     'attention_mask': tf.TensorShape([None, None]      ),
                     'token_type_ids': tf.TensorShape([None, None]      ),
                 },
                 tf.TensorShape([]      ),
             )    ,		)





       def     lowerCamelCase__				(							self     : List[str]      ):


         '''simple docstring'''




         return self.dataset





       def __len__(							self     : int      ):


         '''simple docstring'''




         return len(self.features      )





       def __getitem__(							self     : List[str]    ,		A     : Tuple      ):


         '''simple docstring'''




         return self.features[i]





       def     lowerCamelCase__				(							self     : Optional[int]      ):


         '''simple docstring'''




         return self.label_list






class 						snake_case  (  UpperCAmelCase	):





    def     lowerCamelCase__				(							self     : str    ,		A     : List[str]      ):


      '''simple docstring'''




      return self._create_examples(self._read_tsv(os.path.join(_a    ,		'heuristics_train_set.txt'      )      )    ,		'train'      )





    def     lowerCamelCase__				(							self     : Optional[Any]    ,		A     : Optional[Any]      ):


      '''simple docstring'''




      return self._create_examples(self._read_tsv(os.path.join(_a    ,		'heuristics_evaluation_set.txt'      )      )    ,		'dev'      )





    def     lowerCamelCase__				(							self     : Optional[Any]      ):


      '''simple docstring'''




      return ["contradiction", "entailment", "neutral"]





    def     lowerCamelCase__				(							self     : List[Any]    ,		A     : Tuple    ,		A     : Optional[Any]      ):


      '''simple docstring'''




      a					:      Tuple    =       []
      for i, line in enumerate(_a      ):
        if i == 0:
          continue
        a					:      str    =       '%s-%s' % (set_type, line[0])
        a					:      Union[str, Any]    =       line[5]
        a					:      int    =       line[6]
        a					:      int    =       line[7][2:] if line[7].startswith('ex'      ) else line[7]
        a					:      List[Any]    =       line[0]
        examples.append(InputExample(guid=_a    ,		text_a=_a    ,		text_b=_a    ,		label=_a    ,		pairID=_a      )      )
      return examples




def       snake_case  (A_				:List[InputExample]     ,      A_				:List[str]     ,      A_				:int     ,      A_				:PreTrainedTokenizer     ,      ):







  '''simple docstring'''

  a					:      Optional[int]    =       {label: i for i, label in enumerate(UpperCamelCase__				)}

  a					:      Optional[int]    =       []
  for ex_index, example in tqdm.tqdm(enumerate(UpperCamelCase__				)     ,      desc='convert examples to features'				):
    if ex_index % 1_0_0_0_0 == 0:
      logger.info('Writing example %d' % (ex_index)				)

    a					:      Any    =       tokenizer(
        example.text_a     ,      example.text_b     ,      add_special_tokens=UpperCamelCase__     ,      max_length=UpperCamelCase__     ,      padding='max_length'     ,      truncation=UpperCamelCase__     ,      return_overflowing_tokens=UpperCamelCase__     ,      )

    a					:      Optional[int]    =       label_map[example.label] if example.label in label_map else 0

    a					:      Optional[int]    =       int(example.pairID				)

    features.append(InputFeatures(**UpperCamelCase__     ,      label=UpperCamelCase__     ,      pairID=UpperCamelCase__				)				)

  for i, example in enumerate(examples[:5]				):
    logger.info('*** Example ***'				)
    logger.info(f'''guid: {example}'''				)
    logger.info(f'''features: {features[i]}'''				)

  return features


_UpperCamelCase    :   Union[str, Any]   			=						{
    'hans': 3,
}






_UpperCamelCase    :   str   			=						{
    'hans': HansProcessor,
}












"""simple docstring"""


from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


_UpperCamelCase    :   int   			=						logging.get_logger(__name__)






class 						snake_case  (  UpperCAmelCase	):
    __magic_name__   					=					['''input_features''', '''attention_mask''']





    def __init__(							self     : Optional[int]    ,		A     : Optional[Any]=8_0    ,		A     : str=1_6_0_0_0    ,		A     : List[str]=0.0    ,		A     : Any=1_0    ,		A     : Union[str, Any]=2_5    ,		A     : str="hamming_window"    ,		A     : str=3_27_68.0    ,		A     : Union[str, Any]=0.97    ,		A     : Dict=1.0    ,		A     : Any=True    ,		A     : Union[str, Any]=True    ,		A     : List[Any]=False    ,		**A     : Tuple    ,		):


      '''simple docstring'''




      super().__init__(feature_size=A    ,		sampling_rate=A    ,		padding_value=A    ,		**A      )

      a					:      Any    =       feature_size
      a					:      List[Any]    =       sampling_rate
      a					:      Any    =       padding_value
      a					:      str    =       hop_length
      a					:      Any    =       win_length
      a					:      List[Any]    =       frame_signal_scale
      a					:      Tuple    =       preemphasis_coeff
      a					:      Dict    =       mel_floor
      a					:      Optional[int]    =       normalize_means
      a					:      List[str]    =       normalize_vars
      a					:      Dict    =       win_function
      a					:      Union[str, Any]    =       return_attention_mask

      a					:      List[Any]    =       win_length * sampling_rate // 1_0_0_0
      a					:      Tuple    =       hop_length * sampling_rate // 1_0_0_0

      a					:      List[Any]    =       optimal_fft_length(self.sample_size      )
      a					:      Any    =       (self.n_fft // 2) + 1





    def     lowerCamelCase__				(							self     : List[Any]    ,		A     : np.array      ):


      '''simple docstring'''




      if self.win_function == "hamming_window":
        a					:      List[str]    =       window_function(window_length=self.sample_size    ,		name=self.win_function    ,		periodic=A      )
      else:
        a					:      Dict    =       window_function(window_length=self.sample_size    ,		name=self.win_function      )

      a					:      str    =       mel_filter_bank(
          num_frequency_bins=self.n_freqs    ,		num_mel_filters=self.feature_size    ,		min_frequency=0.0    ,		max_frequency=self.sampling_rate / 2.0    ,		sampling_rate=self.sampling_rate    ,		)

      a					:      List[Any]    =       spectrogram(
          one_waveform * self.frame_signal_scale    ,		window=A    ,		frame_length=self.sample_size    ,		hop_length=self.sample_stride    ,		fft_length=self.n_fft    ,		center=A    ,		preemphasis=self.preemphasis_coeff    ,		mel_filters=A    ,		mel_floor=self.mel_floor    ,		log_mel='log'    ,		)
      return msfc_features.T





    def     lowerCamelCase__				(							self     : int    ,		A     : Tuple    ,		A     : int    ,		A     : Optional[int]      ):


      '''simple docstring'''




      if self.normalize_means:
        a					:      Any    =       x[:input_length].mean(axis=0      )
        a					:      Dict    =       np.subtract(A    ,		A      )
      if self.normalize_vars:
        a					:      Dict    =       x[:input_length].std(axis=0      )
        a					:      Dict    =       np.divide(A    ,		A      )

      if input_length < x.shape[0]:
        a					:      Dict    =       padding_value

      # make sure array is in float32
      a					:      Optional[int]    =       x.astype(np.floataa      )

      return x





    def     lowerCamelCase__				(							self     : str    ,		A     : List[np.ndarray]    ,		A     : Optional[np.ndarray] = None      ):


      '''simple docstring'''




      a					:      str    =       attention_mask.sum(-1      ) if attention_mask is not None else [x.shape[0] for x in input_features]
      return [self._normalize_one(A    ,		A    ,		self.padding_value      ) for x, n in zip(A    ,		A      )]





    def __call__(							self     : Dict    ,		A     : Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]    ,		A     : Union[bool, str, PaddingStrategy] = False    ,		A     : Optional[int] = None    ,		A     : bool = False    ,		A     : Optional[int] = None    ,		A     : Optional[bool] = None    ,		A     : Optional[Union[str, TensorType]] = None    ,		A     : Optional[int] = None    ,		**A     : Union[str, Any]    ,		):


      '''simple docstring'''




      if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
          raise ValueError(
              F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
              F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
              F''' {self.sampling_rate} and not {sampling_rate}.'''      )
      else:
        logger.warning(
            'It is strongly recommended to pass the ``sampling_rate`` argument to this function. '
            'Failing to do so can result in silent errors that might be hard to debug.'      )

      a					:      Optional[int]    =       isinstance(A    ,		np.ndarray      ) and len(raw_speech.shape      ) > 1
      if is_batched_numpy and len(raw_speech.shape      ) > 2:
        raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''      )
      a					:      Dict    =       is_batched_numpy or (
          isinstance(A    ,		(list, tuple)      ) and (isinstance(raw_speech[0]    ,		(np.ndarray, tuple, list)      ))
      )

      if is_batched:
        a					:      str    =       [np.asarray(A    ,		dtype=np.floataa      ) for speech in raw_speech]
      elif not is_batched and not isinstance(A    ,		np.ndarray      ):
        a					:      List[str]    =       np.asarray(A    ,		dtype=np.floataa      )
      elif isinstance(A    ,		np.ndarray      ) and raw_speech.dtype is np.dtype(np.floataa      ):
        a					:      Optional[int]    =       raw_speech.astype(np.floataa      )

      # always return batch
      if not is_batched:
        a					:      Any    =       [raw_speech]

      # extract fbank features
      a					:      str    =       [self._extract_mfsc_features(A      ) for one_waveform in raw_speech]

      # convert into correct format for padding
      a					:      int    =       BatchFeature({'input_features': features}      )

      a					:      Union[str, Any]    =       self.pad(
          A    ,		padding=A    ,		max_length=A    ,		truncation=A    ,		pad_to_multiple_of=A    ,		return_attention_mask=A    ,		**A    ,		)
      # make sure list is in array format
      a					:      Optional[Any]    =       padded_inputs.get('input_features'      )
      if isinstance(input_features[0]    ,		A      ):
        a					:      List[str]    =       [np.asarray(A    ,		dtype=np.floataa      ) for feature in input_features]

      a					:      List[Any]    =       padded_inputs.get('attention_mask'      )
      if attention_mask is not None:
        a					:      int    =       [np.asarray(A    ,		dtype=np.intaa      ) for array in attention_mask]

      if self.normalize_means or self.normalize_vars:
        a					:      Any    =       (
            np.array(A    ,		dtype=np.intaa      )
            if self._get_padding_strategies(A    ,		max_length=A      ) is not PaddingStrategy.DO_NOT_PAD
            and padding
            else None
        )
        a					:      List[str]    =       self.normalize(
            padded_inputs['input_features']    ,		attention_mask=A      )

      if return_tensors is not None:
        a					:      Optional[int]    =       padded_inputs.convert_to_tensors(A      )

      return padded_inputs




