




'''simple docstring'''


from typing import List, Optional, Union

import numpy as np

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


_snake_case				:				List[Any]       							=		logging.get_logger(__name__)

class       A   (		_a   ):
							lowercase_          =    ['input_values', 'padding_mask']







							def __init__(  self	:  Optional[int]							,      lowerCAmelCase_	:  int = 1							,      lowerCAmelCase_	:  int = 2_40_00							,      lowerCAmelCase_	:  float = 0.0							,      lowerCAmelCase_	:  float = None							,      lowerCAmelCase_	:  float = None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							List[str]:


														"""simple docstring"""


														super().__init__(feature_size=lowerCAmelCase_							,      sampling_rate=lowerCAmelCase_							,      padding_value=lowerCAmelCase_							,      **lowerCAmelCase_ )
														_a     					=    chunk_length_s
														_a     					=    overlap







							@property
							def        __lowerCAmelCase      (  self	:  int )      ->							Optional[int]:


														"""simple docstring"""


														if self.chunk_length_s is None:
																					return None
														else:
																					return int(self.chunk_length_s * self.sampling_rate )







							@property
							def        __lowerCAmelCase      (  self	:  str )      ->							Optional[int]:


														"""simple docstring"""


														if self.chunk_length_s is None or self.overlap is None:
																					return None
														else:
																					return max(1							,      int((1.0 - self.overlap) * self.chunk_length ) )







							def __call__(  self	:  Any							,      lowerCAmelCase_	:  Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]							,      lowerCAmelCase_	:  Optional[Union[bool, str, PaddingStrategy]] = None							,      lowerCAmelCase_	:  Optional[bool] = False							,      lowerCAmelCase_	:  Optional[int] = None							,      lowerCAmelCase_	:  Optional[Union[str, TensorType]] = None							,      lowerCAmelCase_	:  Optional[int] = None							,      )      ->							BatchFeature:


														"""simple docstring"""


														if sampling_rate is not None:
																					if sampling_rate != self.sampling_rate:
																												raise ValueError(
																												    F'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
																												    F' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'
																												    F' {self.sampling_rate} and not {sampling_rate}.' )
														else:
																					logger.warning(
																					    '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
																					    '''Failing to do so can result in silent errors that might be hard to debug.''' )

														if padding and truncation:
																					raise ValueError('''Both padding and truncation were set. Make sure you only set one.''' )
														elif padding is None:
																					# by default let's pad the inputs
																					_a     					=    True

														_a     					=    bool(
														    isinstance(lowerCAmelCase_							,      (list, tuple) ) and (isinstance(raw_audio[0]							,      (np.ndarray, tuple, list) )) )

														if is_batched:
																					_a     					=    [np.asarray(lowerCAmelCase_							,      dtype=np.floataa ).T for audio in raw_audio]
														elif not is_batched and not isinstance(lowerCAmelCase_							,      np.ndarray ):
																					_a     					=    np.asarray(lowerCAmelCase_							,      dtype=np.floataa )
														elif isinstance(lowerCAmelCase_							,      np.ndarray ) and raw_audio.dtype is np.dtype(np.floataa ):
																					_a     					=    raw_audio.astype(np.floataa )

														# always return batch
														if not is_batched:
																					_a     					=    [np.asarray(lowerCAmelCase_ ).T]

														# verify inputs are valid
														for idx, example in enumerate(lowerCAmelCase_ ):
																					if example.ndim > 2:
																												raise ValueError(F'Expected input shape (channels, length) but got shape {example.shape}' )
																					if self.feature_size == 1 and example.ndim != 1:
																												raise ValueError(F'Expected mono audio but example has {example.shape[-1]} channels' )
																					if self.feature_size == 2 and example.shape[-1] != 2:
																												raise ValueError(F'Expected stereo audio but example has {example.shape[-1]} channels' )

														_a     					=    None
														_a     					=    BatchFeature({'''input_values''': raw_audio} )
														if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
																					if truncation:
																												_a     					=    min(array.shape[0] for array in raw_audio )
																												_a     					=    int(np.floor(max_length / self.chunk_stride ) )
																												_a     					=    (nb_step - 1) * self.chunk_stride + self.chunk_length
																					elif padding:
																												_a     					=    max(array.shape[0] for array in raw_audio )
																												_a     					=    int(np.ceil(max_length / self.chunk_stride ) )
																												_a     					=    (nb_step - 1) * self.chunk_stride + self.chunk_length
																												_a     					=    '''max_length'''
																					else:
																												_a     					=    input_values

        # normal padding on batch
														if padded_inputs is None:
																					_a     					=    self.pad(
																					    lowerCAmelCase_							,      max_length=lowerCAmelCase_							,      truncation=lowerCAmelCase_							,      padding=lowerCAmelCase_							,      return_attention_mask=lowerCAmelCase_							,      )
																					if padding:
																												_a     					=    padded_inputs.pop('''attention_mask''' )

														_a     					=    []
														for example in padded_inputs.pop('''input_values''' ):
																					if self.feature_size == 1:
																												_a     					=    example[..., None]
																					input_values.append(example.T )

														_a     					=    input_values
														if return_tensors is not None:
																					_a     					=    padded_inputs.convert_to_tensors(lowerCAmelCase_ )

														return padded_inputs







'''simple docstring'''


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				str       							=		logging.get_logger(__name__)

_snake_case				:				Tuple       							=		{'vocab_file': 'spiece.model'}

_snake_case				:				Optional[int]       							=		{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_snake_case				:				Tuple       							=		{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    ['input_ids', 'attention_mask']
							lowercase_          =    []







							def __init__(  self	:  Optional[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[int]="<unk>"							,      lowerCAmelCase_	:  Tuple="<s>"							,      lowerCAmelCase_	:  List[str]="</s>"							,      lowerCAmelCase_	:  int="<pad>"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Dict="[MASK]"							,      lowerCAmelCase_	:  Optional[int]="[CLS]"							,      lowerCAmelCase_	:  Optional[Dict[str, Any]] = None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							None:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else bos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else eos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else unk_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else pad_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else cls_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else sep_token

														# Mask token behave like a normal word, i.e. include the space before it
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														_a     					=    {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      sp_model_kwargs=self.sp_model_kwargs							,      **lowerCAmelCase_							,      )

														_a     					=    vocab_file

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(lowerCAmelCase_ )







							@property
							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							int:


														"""simple docstring"""


														return self.sp_model.get_piece_size()







							def        __lowerCAmelCase      (  self	:  str )      ->							List[str]:


														"""simple docstring"""


														_a     					=    {self.convert_ids_to_tokens(lowerCAmelCase_ ): i for i in range(self.vocab_size )}
														vocab.update(self.added_tokens_encoder )
														return vocab







							def __getstate__(  self	:  str )      ->							Tuple:


														"""simple docstring"""


														_a     					=    self.__dict__.copy()
														_a     					=    None
														return state







							def __setstate__(  self	:  List[str]							,      lowerCAmelCase_	:  Any )      ->							Dict:


														"""simple docstring"""


														_a     					=    d

														# for backward compatibility
														if not hasattr(self							,      '''sp_model_kwargs''' ):
																					_a     					=    {}

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(self.vocab_file )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return self.sp_model.encode(lowerCAmelCase_							,      out_type=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  List[str] )      ->							int:


														"""simple docstring"""


														return self.sp_model.piece_to_id(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Tuple )      ->							str:


														"""simple docstring"""


														_a     					=    self.sp_model.IdToPiece(lowerCAmelCase_ )
														return token







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    []
														_a     					=    ''''''
														_a     					=    False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(lowerCAmelCase_ ) + token
																												_a     					=    True
																												_a     					=    []
																					else:
																												current_sub_tokens.append(lowerCAmelCase_ )
																												_a     					=    False
														out_string += self.sp_model.decode(lowerCAmelCase_ )
														return out_string.strip()







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  bool = False							,      lowerCAmelCase_	:  bool = None							,      lowerCAmelCase_	:  bool = True							,      **lowerCAmelCase_	:  Tuple							,      )      ->							str:


														"""simple docstring"""


														_a     					=    kwargs.pop('''use_source_tokenizer'''							,      lowerCAmelCase_ )

														_a     					=    self.convert_ids_to_tokens(lowerCAmelCase_							,      skip_special_tokens=lowerCAmelCase_ )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														_a     					=    []
														_a     					=    []
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )
																																			_a     					=    []
																												sub_texts.append(lowerCAmelCase_ )
																					else:
																												current_sub_text.append(lowerCAmelCase_ )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )

														# Mimic the behavior of the Rust tokenizer:
														# No space before [MASK] and [SEP]
														if spaces_between_special_tokens:
																					_a     					=    re.sub(R''' (\[(MASK|SEP)\])'''							,      R'''\1'''							,      ''' '''.join(lowerCAmelCase_ ) )
														else:
																					_a     					=    ''''''.join(lowerCAmelCase_ )

														_a     					=    (
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					_a     					=    self.clean_up_tokenization(lowerCAmelCase_ )
																					return clean_text
														else:
																					return text







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														if not os.path.isdir(lowerCAmelCase_ ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
																					return
														_a     					=    os.path.join(
														    lowerCAmelCase_							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

														if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase_ ) and os.path.isfile(self.vocab_file ):
																					copyfile(self.vocab_file							,      lowerCAmelCase_ )
														elif not os.path.isfile(self.vocab_file ):
																					with open(lowerCAmelCase_							,      '''wb''' ) as fi:
																												_a     					=    self.sp_model.serialized_model_proto()
																												fi.write(lowerCAmelCase_ )

														return (out_vocab_file,)







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														_a     					=    [self.sep_token_id]
														return cls + token_ids_a + sep + token_ids_a + sep







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														if token_ids_a is None:
																					return [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														return [1] + ([0] * len(lowerCAmelCase_ )) + [1] + ([0] * len(lowerCAmelCase_ )) + [1]







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]

