






'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE__     =       {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

SCREAMING_SNAKE_CASE__     =       {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}






class     a_						(      _UpperCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_INIT_CONFIGURATION
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				RealmTokenizer

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=True      ,		_SCREAMING_SNAKE_CASE="[UNK]"      ,		_SCREAMING_SNAKE_CASE="[SEP]"      ,		_SCREAMING_SNAKE_CASE="[PAD]"      ,		_SCREAMING_SNAKE_CASE="[CLS]"      ,		_SCREAMING_SNAKE_CASE="[MASK]"      ,		_SCREAMING_SNAKE_CASE=True      ,		_SCREAMING_SNAKE_CASE=None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				List[Any]:



        """simple docstring"""






        super().__init__(
            _SCREAMING_SNAKE_CASE      ,		tokenizer_file=_SCREAMING_SNAKE_CASE      ,		do_lower_case=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		sep_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		cls_token=_SCREAMING_SNAKE_CASE      ,		mask_token=_SCREAMING_SNAKE_CASE      ,		tokenize_chinese_chars=_SCREAMING_SNAKE_CASE      ,		strip_accents=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE      ,		)

        UpperCamelCase      = json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
        if (
            normalizer_state.get("""lowercase"""      ,		_SCREAMING_SNAKE_CASE				) != do_lower_case
            or normalizer_state.get("""strip_accents"""      ,		_SCREAMING_SNAKE_CASE				) != strip_accents
            or normalizer_state.get("""handle_chinese_chars"""      ,		_SCREAMING_SNAKE_CASE				) != tokenize_chinese_chars
        ):
              UpperCamelCase      = getattr(_SCREAMING_SNAKE_CASE      ,		normalizer_state.pop("""type"""				)				)
              UpperCamelCase      = do_lower_case
              UpperCamelCase      = strip_accents
              UpperCamelCase      = tokenize_chinese_chars
              UpperCamelCase      = normalizer_class(**_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = do_lower_case

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        UpperCamelCase      = PaddingStrategy.MAX_LENGTH

        UpperCamelCase      = text
        UpperCamelCase      = kwargs.pop("""text_pair"""      ,		_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = kwargs.pop("""return_tensors"""      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = {
            """input_ids""": [],
            """attention_mask""": [],
            """token_type_ids""": [],
        }

        for idx, candidate_text in enumerate(_SCREAMING_SNAKE_CASE				):
              if batch_text_pair is not None:
                    UpperCamelCase      = batch_text_pair[idx]
              else:
                    UpperCamelCase      = None

              UpperCamelCase      = super().__call__(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		return_tensors=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

              UpperCamelCase      = encoded_candidates.get("""input_ids"""				)
              UpperCamelCase      = encoded_candidates.get("""attention_mask"""				)
              UpperCamelCase      = encoded_candidates.get("""token_type_ids"""				)

              if encoded_input_ids is not None:
                    output_data["input_ids"].append(_SCREAMING_SNAKE_CASE				)
              if encoded_attention_mask is not None:
                    output_data["attention_mask"].append(_SCREAMING_SNAKE_CASE				)
              if encoded_token_type_ids is not None:
                    output_data["token_type_ids"].append(_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = {key: item for key, item in output_data.items() if len(_SCREAMING_SNAKE_CASE				) != 0}

        return BatchEncoding(_SCREAMING_SNAKE_CASE      ,		tensor_type=_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[int]:



        """simple docstring"""






        UpperCamelCase      = [self.sep_token_id]
        UpperCamelCase      = [self.cls_token_id]
        if token_ids_a is None:
              return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        UpperCamelCase      = self._tokenizer.model.save(_SCREAMING_SNAKE_CASE      ,		name=_SCREAMING_SNAKE_CASE				)
        return tuple(_SCREAMING_SNAKE_CASE				)








'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'spm_char.model'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

SCREAMING_SNAKE_CASE__     =       {
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}






class     a_						(      lowerCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				None:



        """simple docstring"""






        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=_SCREAMING_SNAKE_CASE      ,		eos_token=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**_SCREAMING_SNAKE_CASE      ,		)

        UpperCamelCase      = vocab_file

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.get_piece_size()

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def __getstate__(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        return self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,		out_type=_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.piece_to_id(_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE				)
        return token

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				) + token
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE				)
        out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				)
        return out_string.strip()

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None				)  ->				List[int]:



        """simple docstring"""






        if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_a + token_ids_a + [self.eos_token_id]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_SCREAMING_SNAKE_CASE      ,		token_ids_a=_SCREAMING_SNAKE_CASE      ,		already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = [1]
        if token_ids_a is None:
              return ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones
        return ([0] * len(_SCREAMING_SNAKE_CASE				)) + ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            _SCREAMING_SNAKE_CASE      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(_SCREAMING_SNAKE_CASE				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		_SCREAMING_SNAKE_CASE				)
        elif not os.path.isfile(self.vocab_file				):
              with open(_SCREAMING_SNAKE_CASE      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE				)

        return (out_vocab_file,)
