

'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case         =  logging.get_logger(__name__)

_snake_case         =  '▁'

_snake_case         =  {'vocab_file': 'sentencepiece.bpe.model'}

_snake_case         =  {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

_snake_case         =  {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Any     	=							['input_ids', 'attention_mask']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<s>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="<mask>"      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				int      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else mask_token

      _lowercase      :				int      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=_UpperCamelCase      ,    eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    sep_token=_UpperCamelCase      ,    cls_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )

      _lowercase      :				str      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(str(_UpperCamelCase  )  )
      _lowercase      :				Optional[int]      					=       vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      _lowercase      :				Tuple      					=       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      _lowercase      :				str      					=       1

      _lowercase      :				int      					=       len(self.sp_model  ) + self.fairseq_offset
      _lowercase      :				Tuple      					=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       self.__dict__.copy()
      _lowercase      :				int      					=       None
      _lowercase      :				int      					=       self.sp_model.serialized_model_proto()
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				str      					=       {}

      _lowercase      :				Optional[int]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowercase      :				Tuple      					=       [self.cls_token_id]
      _lowercase      :				int      					=       [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=_UpperCamelCase      ,    token_ids_a=_UpperCamelCase      ,    already_has_special_tokens=_UpperCamelCase  )

      if token_ids_a is None:
         return [1] + ([0] * len(_UpperCamelCase  )) + [1]
      return [1] + ([0] * len(_UpperCamelCase  )) + [1, 1] + ([0] * len(_UpperCamelCase  )) + [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [self.sep_token_id]
      _lowercase      :				List[Any]      					=       [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return len(self.sp_model  ) + self.fairseq_offset + 1  # Add the <mask> token


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
      _lowercase      :				Dict      					=       self.sp_model.PieceToId(_UpperCamelCase  )

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if index in self.fairseq_ids_to_tokens:
         return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Any      					=       "".join(_UpperCamelCase  ).replace(_UpperCamelCase      ,    " "  ).strip()
      return out_string






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				Any      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				Optional[Any]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)









'''simple docstring'''
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					int     	=							'Wav2Vec2FeatureExtractor'
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							'AutoTokenizer'


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase  ):





      """simple docstring"""





      super().__init__(_UpperCamelCase      ,    _UpperCamelCase  )
      _lowercase      :				List[Any]      					=       self.feature_extractor
      _lowercase      :				Optional[Any]      					=       False


   @classmethod
   def 						_lowerCamelCase       (			cls      ,    _UpperCamelCase      ,    **_UpperCamelCase  ):





      """simple docstring"""





      try:
         return super().from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )
      except OSError:
         warnings.warn(
             f'''Loading a tokenizer inside {cls.__name__} from a config that does not'''
             " include a `tokenizer_class` attribute is deprecated and will be "
             "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"
             " attribute to either your `config.json` or `tokenizer_config.json` "
             "file to suppress this warning: "      ,    _UpperCamelCase      ,    )

         _lowercase      :				Dict      					=       WavaVecaFeatureExtractor.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )
         _lowercase      :				str      					=       WavaVecaCTCTokenizer.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )

         return cls(feature_extractor=_UpperCamelCase      ,    tokenizer=_UpperCamelCase  )


   def __call__(			self      ,    *_UpperCamelCase      ,    **_UpperCamelCase  ):





      """simple docstring"""





      if self._in_target_context_manager:
         return self.current_processor(*_UpperCamelCase      ,    **_UpperCamelCase  )

      if "raw_speech" in kwargs:
         warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."  )
         _lowercase      :				int      					=       kwargs.pop("raw_speech"  )
      else:
         _lowercase      :				List[Any]      					=       kwargs.pop("audio"      ,    _UpperCamelCase  )
      _lowercase      :				List[Any]      					=       kwargs.pop("sampling_rate"      ,    _UpperCamelCase  )
      _lowercase      :				Union[str, Any]      					=       kwargs.pop("text"      ,    _UpperCamelCase  )
      if len(_UpperCamelCase  ) > 0:
         _lowercase      :				int      					=       args[0]
         _lowercase      :				Any      					=       args[1:]

      if audio is None and text is None:
         raise ValueError("You need to specify either an `audio` or `text` input to process."  )

      if audio is not None:
         _lowercase      :				Dict      					=       self.feature_extractor(_UpperCamelCase      ,    *_UpperCamelCase      ,    sampling_rate=_UpperCamelCase      ,    **_UpperCamelCase  )
      if text is not None:
         _lowercase      :				Union[str, Any]      					=       self.tokenizer(_UpperCamelCase      ,    **_UpperCamelCase  )

      if text is None:
         return inputs
      elif audio is None:
         return encodings
      else:
         _lowercase      :				int      					=       encodings["input_ids"]
         return inputs


   def 						_lowerCamelCase       (			self      ,    *_UpperCamelCase      ,    **_UpperCamelCase  ):





      """simple docstring"""





      if self._in_target_context_manager:
         return self.current_processor.pad(*_UpperCamelCase      ,    **_UpperCamelCase  )

      _lowercase      :				List[Any]      					=       kwargs.pop("input_features"      ,    _UpperCamelCase  )
      _lowercase      :				Any      					=       kwargs.pop("labels"      ,    _UpperCamelCase  )
      if len(_UpperCamelCase  ) > 0:
         _lowercase      :				Any      					=       args[0]
         _lowercase      :				Any      					=       args[1:]

      if input_features is not None:
         _lowercase      :				Any      					=       self.feature_extractor.pad(_UpperCamelCase      ,    *_UpperCamelCase      ,    **_UpperCamelCase  )
      if labels is not None:
         _lowercase      :				int      					=       self.tokenizer.pad(_UpperCamelCase      ,    **_UpperCamelCase  )

      if labels is None:
         return input_features
      elif input_features is None:
         return labels
      else:
         _lowercase      :				Optional[Any]      					=       labels["input_ids"]
         return input_features


   def 						_lowerCamelCase       (			self      ,    *_UpperCamelCase      ,    **_UpperCamelCase  ):





      """simple docstring"""





      return self.tokenizer.batch_decode(*_UpperCamelCase      ,    **_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    *_UpperCamelCase      ,    **_UpperCamelCase  ):





      """simple docstring"""





      return self.tokenizer.decode(*_UpperCamelCase      ,    **_UpperCamelCase  )






   @contextmanager
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      warnings.warn(
          "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
          "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
          "your audio inputs, or in a separate call."  )
      _lowercase      :				Optional[Any]      					=       True
      _lowercase      :				Dict      					=       self.tokenizer
      yield
      _lowercase      :				List[str]      					=       self.feature_extractor
      _lowercase      :				List[str]      					=       False






