

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class        a__  (							_lowercase  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							CLIPTokenizer
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							CLIPTokenizerFast
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							True
   _SCREAMING_SNAKE_CASE   :					Dict     	=							{}
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							False


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().setUp()

      # fmt: off
      _lowercase      :				List[str]      					=       ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
      # fmt: on
      _lowercase      :				Dict      					=       dict(zip(__UpperCamelCase      ,    range(len(__UpperCamelCase  )  )  )  )
      _lowercase      :				Union[str, Any]      					=       ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
      _lowercase      :				Union[str, Any]      					=       {"unk_token": "<unk>"}

      _lowercase      :				Any      					=       os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["vocab_file"]  )
      _lowercase      :				str      					=       os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["merges_file"]  )
      with open(self.vocab_file      ,    "w"      ,    encoding="utf-8"  ) as fp:
         fp.write(json.dumps(__UpperCamelCase  ) + "\n"  )
      with open(self.merges_file      ,    "w"      ,    encoding="utf-8"  ) as fp:
         fp.write("\n".join(__UpperCamelCase  )  )


   def 						_lowerCamelCase       (			self      ,    **_UpperCamelCase  ):





      """simple docstring"""





      kwargs.update(self.special_tokens_map  )
      return CLIPTokenizer.from_pretrained(self.tmpdirname      ,    **__UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    **_UpperCamelCase  ):





      """simple docstring"""





      kwargs.update(self.special_tokens_map  )
      return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,    **__UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       "lower newer"
      _lowercase      :				Optional[Any]      					=       "lower newer"
      return input_text, output_text


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       CLIPTokenizer(self.vocab_file      ,    self.merges_file      ,    **self.special_tokens_map  )
      _lowercase      :				Tuple      					=       "lower newer"
      _lowercase      :				Any      					=       ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
      _lowercase      :				Dict      					=       tokenizer.tokenize(__UpperCamelCase  )
      self.assertListEqual(__UpperCamelCase      ,    __UpperCamelCase  )

      _lowercase      :				List[str]      					=       tokens + [tokenizer.unk_token]
      _lowercase      :				Optional[int]      					=       [10, 2, 16, 9, 3, 2, 16, 20]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase  )      ,    __UpperCamelCase  )


   @require_ftfy
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
            _lowercase      :				str      					=       self.tokenizer_class.from_pretrained(__UpperCamelCase      ,    **__UpperCamelCase  )
            _lowercase      :				Dict      					=       self.rust_tokenizer_class.from_pretrained(__UpperCamelCase      ,    **__UpperCamelCase  )

            _lowercase      :				Optional[Any]      					=       "A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d."
            _lowercase      :				Union[str, Any]      					=       tokenizer_s.tokenize(__UpperCamelCase  )
            _lowercase      :				Optional[int]      					=       tokenizer_r.tokenize(__UpperCamelCase  )

            self.assertListEqual(__UpperCamelCase      ,    __UpperCamelCase  )

            # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
            # with Tilde) encoded in 2 different ways
            _lowercase      :				Optional[int]      					=       "xa\u0303y" + " " + "x\xe3y"
            _lowercase      :				int      					=       tokenizer_s.tokenize(__UpperCamelCase  )
            _lowercase      :				Dict      					=       tokenizer_r.tokenize(__UpperCamelCase  )

            self.assertListEqual(__UpperCamelCase      ,    __UpperCamelCase  )

            # Test that the tokenization is identical on unicode of space type
            _lowercase      :				List[str]      					=       [
                "\u0009",  # (horizontal tab, '\t')
                "\u000B",  # (vertical tab)
                "\u000C",  # (form feed)
                "\u0020",  # (space, ' ')
                "\u200E",  # (left-to-right mark):w
                "\u200F",  # (right-to-left mark)
            ]
            for unicode_seq in spaces_unicodes:
               _lowercase      :				Optional[int]      					=       tokenizer_s.tokenize(__UpperCamelCase  )
               _lowercase      :				Tuple      					=       tokenizer_r.tokenize(__UpperCamelCase  )

               self.assertListEqual(__UpperCamelCase      ,    __UpperCamelCase  )

            # Test that the tokenization is identical on unicode of line break type
            _lowercase      :				List[str]      					=       [
                "\u000A",  # (line feed, '\n')
                "\r\n",  # (carriage return and line feed, '\r\n')
                "\u000D",  # (carriage return, '\r')
                "\r",  # (carriage return, '\r')
                "\u000D",  # (carriage return, '\r')
                "\u2028",  # (line separator)
                "\u2029",  # (paragraph separator)
                # "\u0085", # (next line)
            ]

            # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
            # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
            # space (and thus into an empty list).

            for unicode_seq in line_break_unicodes:
               _lowercase      :				Dict      					=       tokenizer_s.tokenize(__UpperCamelCase  )
               _lowercase      :				Union[str, Any]      					=       tokenizer_r.tokenize(__UpperCamelCase  )

               self.assertListEqual(__UpperCamelCase      ,    __UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
            _lowercase      :				int      					=       "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
            _lowercase      :				Tuple      					=       f'''{text_of_1_token} {text_of_1_token}'''

            _lowercase      :				Optional[Any]      					=       self.rust_tokenizer_class.from_pretrained(
                __UpperCamelCase      ,    use_fast=__UpperCamelCase      ,    )
            _lowercase      :				Union[str, Any]      					=       tokenizer_r(__UpperCamelCase      ,    return_offsets_mapping=__UpperCamelCase      ,    add_special_tokens=__UpperCamelCase  )
            self.assertEqual(encoding.offset_mapping[0]      ,    (0, len(__UpperCamelCase  ))  )
            self.assertEqual(
                encoding.offset_mapping[1]      ,    (len(__UpperCamelCase  ) + 1, len(__UpperCamelCase  ) + 1 + len(__UpperCamelCase  ))      ,    )

            _lowercase      :				Any      					=       f''' {text}'''

            _lowercase      :				Union[str, Any]      					=       self.rust_tokenizer_class.from_pretrained(
                __UpperCamelCase      ,    use_fast=__UpperCamelCase      ,    )
            _lowercase      :				Union[str, Any]      					=       tokenizer_r(__UpperCamelCase      ,    return_offsets_mapping=__UpperCamelCase      ,    add_special_tokens=__UpperCamelCase  )
            self.assertEqual(encoding.offset_mapping[0]      ,    (1, 1 + len(__UpperCamelCase  ))  )
            self.assertEqual(
                encoding.offset_mapping[1]      ,    (1 + len(__UpperCamelCase  ) + 1, 1 + len(__UpperCamelCase  ) + 1 + len(__UpperCamelCase  ))      ,    )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      with self.assertRaises(__UpperCamelCase  ) as context:
         self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"  )

      self.assertTrue(
          context.exception.args[0].startswith(
              "The `backend_tokenizer` provided does not match the expected format."  )  )


   @require_ftfy
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().test_tokenization_python_rust_equals()






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      pass









'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case         =  '▁'

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'}
}

_snake_case         =  {
    'google/pegasus-xsum': 512,
}


_snake_case         =  logging.get_logger(__name__)


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					str     	=							VOCAB_FILES_NAMES

   _SCREAMING_SNAKE_CASE   :					Tuple     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							['input_ids', 'attention_mask']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<mask_2>"      ,    _UpperCamelCase="<mask_1>"      ,    _UpperCamelCase=None      ,    _UpperCamelCase=103      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       offset
      if additional_special_tokens is not None:
         if not isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            raise TypeError(
                f'''additional_special_tokens should be of type {type(_UpperCamelCase  )}, but is'''
                f''' {type(_UpperCamelCase  )}'''  )

         _lowercase      :				Dict      					=       (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'''<unk_{i}>''' for i in range(len(_UpperCamelCase  )      ,    self.offset - 1  )
         ]

         if len(set(_UpperCamelCase  )  ) != len(_UpperCamelCase  ):
            raise ValueError(
                "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''  )
         _lowercase      :				List[str]      					=       additional_special_tokens_extended
      else:
         _lowercase      :				Dict      					=       [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'''<unk_{i}>''' for i in range(2      ,    self.offset  )]

      _lowercase      :				int      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    mask_token_sent=_UpperCamelCase      ,    offset=_UpperCamelCase      ,    additional_special_tokens=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )
      _lowercase      :				Union[str, Any]      					=       mask_token_sent
      _lowercase      :				str      					=       vocab_file
      _lowercase      :				Dict      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )

      # add special tokens to encoder dict
      _lowercase      :				Dict[int, str]      					=       {
          0: self.pad_token,
          1: self.eos_token,
      }

      if self.mask_token_sent is not None:
         self.encoder.update(
             {
                 2: self.mask_token_sent,
                 3: self.mask_token,
             }  )

      if self.offset > 0:
         # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
         # mask_token_sent is already added to list -> so start at 1
         self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,    self.offset - 1  )}  )

      _lowercase      :				Dict[str, int]      					=       {v: k for k, v in self.encoder.items()}


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return len(self.sp_model  ) + self.offset


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				List[Any]      					=       {}

      _lowercase      :				Any      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if token in self.decoder:
         return self.decoder[token]
      elif token in self.added_tokens_decoder:
         return self.added_tokens_decoder[token]
      _lowercase      :				int      					=       self.sp_model.piece_to_id(_UpperCamelCase  )
      return sp_id + self.offset


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if index in self.encoder:
         return self.encoder[index]
      elif index in self.added_tokens_encoder:
         return self.added_tokens_encoder[index]
      else:
         _lowercase      :				Any      					=       self.sp_model.IdToPiece(index - self.offset  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				Optional[Any]      					=       ""
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Tuple      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase=False  ):





      """simple docstring"""





      return 1


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       set(self.all_special_ids  )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

      return [1 if x in all_special_ids else 0 for x in seq]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return self._special_token_mask(_UpperCamelCase  )
      elif token_ids_a is None:
         return self._special_token_mask(_UpperCamelCase  ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				List[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				Optional[int]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)






