

'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case         =  logging.get_logger(__name__)

_snake_case         =  {"vocab_file": "spm_char.model"}

_snake_case         =  {
    "vocab_file": {
        "microsoft/speecht5_asr": "https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model",
        "microsoft/speecht5_tts": "https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model",
        "microsoft/speecht5_vc": "https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model",
    }
}

_snake_case         =  {
    "microsoft/speecht5_asr": 1_024,
    "microsoft/speecht5_tts": 1_024,
    "microsoft/speecht5_vc": 1_024,
}


class        a__  (							SCREAMING_SNAKE_CASE__  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							['''input_ids''', '''attention_mask''']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<s>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Dict      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=A__      ,    eos_token=A__      ,    unk_token=A__      ,    pad_token=A__      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **A__      ,    )

      _lowercase      :				Any      					=       vocab_file

      _lowercase      :				List[str]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(A__  )


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return self.sp_model.get_piece_size()


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       {self.convert_ids_to_tokens(A__  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       self.__dict__.copy()
      _lowercase      :				List[str]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				str      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				Any      					=       {}

      _lowercase      :				List[str]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(A__      ,    out_type=A__  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.piece_to_id(A__  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       self.sp_model.IdToPiece(A__  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       []
      _lowercase      :				int      					=       ""
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(A__  ) + token
            _lowercase      :				Dict      					=       []
         else:
            current_sub_tokens.append(A__  )
      out_string += self.sp_model.decode(A__  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=A__      ,    token_ids_a=A__      ,    already_has_special_tokens=A__  )

      _lowercase      :				Optional[Any]      					=       [1]
      if token_ids_a is None:
         return ([0] * len(A__  )) + suffix_ones
      return ([0] * len(A__  )) + ([0] * len(A__  )) + suffix_ones






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(A__  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				Optional[Any]      					=       os.path.join(
          A__      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(A__  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    A__  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(A__      ,    "wb"  ) as fi:
            _lowercase      :				List[Any]      					=       self.sp_model.serialized_model_proto()
            fi.write(A__  )

      return (out_vocab_file,)









'''simple docstring'''
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


_snake_case         =  logging.getLogger()


@unittest.skip('Temporarily disable the doc tests.'  )
@require_torch
@require_tf
@slow
class        a__  (							unittest.TestCase  ):


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = True      ,    ):





      """simple docstring"""





      _lowercase      :				str      					=       [file for file in os.listdir(_UpperCamelCase  ) if os.path.isfile(os.path.join(_UpperCamelCase      ,    _UpperCamelCase  )  )]

      if identifier is not None:
         _lowercase      :				str      					=       [file for file in files if identifier in file]

      if n_identifier is not None:
         if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            for n_ in n_identifier:
               _lowercase      :				Dict      					=       [file for file in files if n_ not in file]
         else:
            _lowercase      :				Optional[Any]      					=       [file for file in files if n_identifier not in file]

      _lowercase      :				Dict      					=       ignore_files or []
      ignore_files.append("__init__.py"  )
      _lowercase      :				List[str]      					=       [file for file in files if file not in ignore_files]

      for file in files:
         # Open all files
         print("Testing"      ,    _UpperCamelCase  )

         if only_modules:
            _lowercase      :				Optional[Any]      					=       file.split("."  )[0]
            try:
               _lowercase      :				Union[str, Any]      					=       getattr(_UpperCamelCase      ,    _UpperCamelCase  )
               _lowercase      :				Optional[int]      					=       doctest.DocTestSuite(_UpperCamelCase  )
               _lowercase      :				Tuple      					=       unittest.TextTestRunner().run(_UpperCamelCase  )
               self.assertIs(len(result.failures  )      ,    0  )
            except AttributeError:
               logger.info(f'''{module_identifier} is not a module.'''  )
         else:
            _lowercase      :				Optional[Any]      					=       doctest.testfile(str(".." / directory / file  )      ,    optionflags=doctest.ELLIPSIS  )
            self.assertIs(result.failed      ,    0  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				str      					=       "modeling"
      _lowercase      :				Tuple      					=       [
          "modeling_ctrl.py",
          "modeling_tf_ctrl.py",
      ]
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase      ,    ignore_files=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       Path("src/transformers"  )
      _lowercase      :				Any      					=       "tokenization"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				Optional[Any]      					=       "configuration"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       Path("src/transformers"  )
      _lowercase      :				List[Any]      					=       ["configuration", "modeling", "tokenization"]
      self.analyze_directory(_UpperCamelCase      ,    n_identifier=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       Path("docs/source"  )
      _lowercase      :				int      					=       ["favicon.ico"]
      self.analyze_directory(_UpperCamelCase      ,    ignore_files=_UpperCamelCase      ,    only_modules=_UpperCamelCase  )






