

import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case         =  get_tests_dir('fixtures/spiece.model')


@require_sentencepiece
@require_tokenizers
class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							AlbertTokenizer
   _SCREAMING_SNAKE_CASE   :					str     	=							AlbertTokenizerFast
   _SCREAMING_SNAKE_CASE   :					int     	=							True
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							True
   _SCREAMING_SNAKE_CASE   :					int     	=							True


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().setUp()

      # We have a SentencePiece fixture for testing
      _lowercase      :				str      					=       AlbertTokenizer(_UpperCamelCase  )
      tokenizer.save_pretrained(self.tmpdirname  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       "this is a test"
      _lowercase      :				Optional[int]      					=       "this is a test"
      return input_text, output_text


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       "<pad>"
      _lowercase      :				Tuple      					=       0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase  )      ,    _UpperCamelCase  )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase  )      ,    _UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       list(self.get_tokenizer().get_vocab().keys()  )

      self.assertEqual(vocab_keys[0]      ,    "<pad>"  )
      self.assertEqual(vocab_keys[1]      ,    "<unk>"  )
      self.assertEqual(vocab_keys[-1]      ,    "▁eloquent"  )
      self.assertEqual(len(_UpperCamelCase  )      ,    30000  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.assertEqual(self.get_tokenizer().vocab_size      ,    30000  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if not self.test_rust_tokenizer:
         return

      _lowercase      :				Optional[int]      					=       self.get_tokenizer()
      _lowercase      :				Dict      					=       self.get_rust_tokenizer()

      _lowercase      :				List[str]      					=       "I was born in 92000, and this is falsé."

      _lowercase      :				Dict      					=       tokenizer.tokenize(_UpperCamelCase  )
      _lowercase      :				str      					=       rust_tokenizer.tokenize(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				Union[str, Any]      					=       tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      _lowercase      :				str      					=       rust_tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				Optional[int]      					=       self.get_rust_tokenizer()
      _lowercase      :				List[str]      					=       tokenizer.encode(_UpperCamelCase  )
      _lowercase      :				int      					=       rust_tokenizer.encode(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       AlbertTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )

      _lowercase      :				int      					=       tokenizer.tokenize("This is a test"  )
      self.assertListEqual(_UpperCamelCase      ,    ["▁this", "▁is", "▁a", "▁test"]  )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase  )      ,    [48, 25, 21, 1289]  )

      _lowercase      :				str      					=       tokenizer.tokenize("I was born in 92000, and this is falsé."  )
      self.assertListEqual(
          _UpperCamelCase      ,    ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]  )
      _lowercase      :				int      					=       tokenizer.convert_tokens_to_ids(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]  )

      _lowercase      :				List[str]      					=       tokenizer.convert_ids_to_tokens(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]      ,    )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       AlbertTokenizer(_UpperCamelCase  )

      _lowercase      :				Dict      					=       tokenizer.encode("sequence builders"  )
      _lowercase      :				List[Any]      					=       tokenizer.encode("multi-sequence build"  )

      _lowercase      :				List[str]      					=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase  )
      _lowercase      :				int      					=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase      ,    _UpperCamelCase  )

      assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
      assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
          tokenizer.sep_token_id
      ]






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       {"attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "input_ids": [[2, 21970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 12051, 18, 17, 7103, 2153, 673, 8, 3515, 18684, 8, 4461, 6, 1927, 297, 8, 12060, 2607, 18, 13, 5, 4461, 15, 10538, 38, 8, 135, 15, 822, 58, 15, 993, 10363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 10641, 6, 29, 84, 2512, 2430, 782, 18684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 11712, 15, 7103, 2153, 673, 17, 24883, 9990, 9, 3], [2, 11502, 25, 1006, 20, 782, 8, 11809, 855, 1732, 19393, 18667, 37, 367, 21018, 69, 1854, 34, 11860, 19124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 17659, 84, 14, 16792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_UpperCamelCase      ,    model_name="albert-base-v2"      ,    revision="6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"      ,    )









'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case         =  '▁'

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'}
}

_snake_case         =  {
    'google/pegasus-xsum': 512,
}


_snake_case         =  logging.get_logger(__name__)


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					str     	=							VOCAB_FILES_NAMES

   _SCREAMING_SNAKE_CASE   :					Tuple     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							['input_ids', 'attention_mask']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<mask_2>"      ,    _UpperCamelCase="<mask_1>"      ,    _UpperCamelCase=None      ,    _UpperCamelCase=103      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       offset
      if additional_special_tokens is not None:
         if not isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            raise TypeError(
                f'''additional_special_tokens should be of type {type(_UpperCamelCase  )}, but is'''
                f''' {type(_UpperCamelCase  )}'''  )

         _lowercase      :				Dict      					=       (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'''<unk_{i}>''' for i in range(len(_UpperCamelCase  )      ,    self.offset - 1  )
         ]

         if len(set(_UpperCamelCase  )  ) != len(_UpperCamelCase  ):
            raise ValueError(
                "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''  )
         _lowercase      :				List[str]      					=       additional_special_tokens_extended
      else:
         _lowercase      :				Dict      					=       [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'''<unk_{i}>''' for i in range(2      ,    self.offset  )]

      _lowercase      :				int      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    mask_token_sent=_UpperCamelCase      ,    offset=_UpperCamelCase      ,    additional_special_tokens=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )
      _lowercase      :				Union[str, Any]      					=       mask_token_sent
      _lowercase      :				str      					=       vocab_file
      _lowercase      :				Dict      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )

      # add special tokens to encoder dict
      _lowercase      :				Dict[int, str]      					=       {
          0: self.pad_token,
          1: self.eos_token,
      }

      if self.mask_token_sent is not None:
         self.encoder.update(
             {
                 2: self.mask_token_sent,
                 3: self.mask_token,
             }  )

      if self.offset > 0:
         # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
         # mask_token_sent is already added to list -> so start at 1
         self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,    self.offset - 1  )}  )

      _lowercase      :				Dict[str, int]      					=       {v: k for k, v in self.encoder.items()}


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return len(self.sp_model  ) + self.offset


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				List[Any]      					=       {}

      _lowercase      :				Any      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if token in self.decoder:
         return self.decoder[token]
      elif token in self.added_tokens_decoder:
         return self.added_tokens_decoder[token]
      _lowercase      :				int      					=       self.sp_model.piece_to_id(_UpperCamelCase  )
      return sp_id + self.offset


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if index in self.encoder:
         return self.encoder[index]
      elif index in self.added_tokens_encoder:
         return self.added_tokens_encoder[index]
      else:
         _lowercase      :				Any      					=       self.sp_model.IdToPiece(index - self.offset  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				Optional[Any]      					=       ""
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Tuple      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase=False  ):





      """simple docstring"""





      return 1


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       set(self.all_special_ids  )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

      return [1 if x in all_special_ids else 0 for x in seq]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return self._special_token_mask(_UpperCamelCase  )
      elif token_ids_a is None:
         return self._special_token_mask(_UpperCamelCase  ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				List[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				Optional[int]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)






