






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class       __a  (   SCREAMING_SNAKE_CASE__	,       unittest.TestCase	):
 _lowerCAmelCase	:	Any				=			MobileBertTokenizer
 _lowerCAmelCase	:	Optional[int]				=			MobileBertTokenizerFast
 _lowerCAmelCase	:	Any				=			True
 _lowerCAmelCase	:	Optional[int]				=			True
 _lowerCAmelCase	:	List[str]				=			filter_non_english
 _lowerCAmelCase	:	Any				=			'google/mobilebert-uncased'







 def  __lowercase (		self :       Any						):


  '''simple docstring'''






  super().setUp()

  UpperCamelCase__      :       List[Any]	    =  [
      "[UNK]",
      "[CLS]",
      "[SEP]",
      "[PAD]",
      "[MASK]",
      "want",
      "##want",
      "##ed",
      "wa",
      "un",
      "runn",
      "##ing",
      ",",
      "low",
      "lowest",
  ]
  UpperCamelCase__      :       str	    =  os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES["vocab_file"]						)
  with open(self.vocab_file							,       "w"							,       encoding="utf-8"						) as vocab_writer:
   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)

  UpperCamelCase__      :       Dict	    =  [
      (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
      for tokenizer_def in self.tokenizers_list
  ]







 def  __lowercase (		self :       int							,       SCREAMING_SNAKE_CASE :       str						):


  '''simple docstring'''






  UpperCamelCase__      :       int	    =  "UNwant\u00E9d,running"
  UpperCamelCase__      :       Dict	    =  "unwanted, running"
  return input_text, output_text







 def  __lowercase (		self :       List[str]						):


  '''simple docstring'''






  UpperCamelCase__      :       int	    =  self.tokenizer_class(self.vocab_file						)

  UpperCamelCase__      :       Tuple	    =  tokenizer.tokenize("UNwant\u00E9d,running"						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       ["un", "##want", "##ed", ",", "runn", "##ing"]						)
  self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)							,       [9, 6, 7, 12, 10, 11]						)







 def  __lowercase (		self :       List[str]						):


  '''simple docstring'''






  if not self.test_rust_tokenizer:
   return

  UpperCamelCase__      :       List[str]	    =  self.get_tokenizer()
  UpperCamelCase__      :       str	    =  self.get_rust_tokenizer()

  UpperCamelCase__      :       str	    =  "UNwant\u00E9d,running"

  UpperCamelCase__      :       int	    =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Tuple	    =  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       int	    =  tokenizer.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Union[str, Any]	    =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       Optional[Any]	    =  self.get_rust_tokenizer()
  UpperCamelCase__      :       Any	    =  tokenizer.encode(_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Dict	    =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  # With lower casing
  UpperCamelCase__      :       Optional[Any]	    =  self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Dict	    =  self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       int	    =  "UNwant\u00E9d,running"

  UpperCamelCase__      :       Optional[Any]	    =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       int	    =  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       List[str]	    =  tokenizer.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       int	    =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       Any	    =  self.get_rust_tokenizer()
  UpperCamelCase__      :       str	    =  tokenizer.encode(_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       List[Any]	    =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE						)
  self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       Tuple						):


  '''simple docstring'''






  UpperCamelCase__      :       Any	    =  BasicTokenizer()

  self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)							,       ["ah", "\u535A", "\u63A8", "zz"]						)







 def  __lowercase (		self :       Union[str, Any]						):


  '''simple docstring'''






  UpperCamelCase__      :       Optional[int]	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)							,       ["hello", "!", "how", "are", "you", "?"]						)
  self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)							,       ["hello"]						)







 def  __lowercase (		self :       str						):


  '''simple docstring'''






  UpperCamelCase__      :       Optional[Any]	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE							,       strip_accents=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)							,       ["hällo", "!", "how", "are", "you", "?"]						)
  self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)							,       ["h\u00E9llo"]						)







 def  __lowercase (		self :       Any						):


  '''simple docstring'''






  UpperCamelCase__      :       Dict	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE							,       strip_accents=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)							,       ["hallo", "!", "how", "are", "you", "?"]						)
  self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)							,       ["hello"]						)







 def  __lowercase (		self :       Dict						):


  '''simple docstring'''






  UpperCamelCase__      :       Union[str, Any]	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)							,       ["hallo", "!", "how", "are", "you", "?"]						)
  self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)							,       ["hello"]						)







 def  __lowercase (		self :       Tuple						):


  '''simple docstring'''






  UpperCamelCase__      :       Dict	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)							,       ["HeLLo", "!", "how", "Are", "yoU", "?"]						)







 def  __lowercase (		self :       List[str]						):


  '''simple docstring'''






  UpperCamelCase__      :       Any	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE							,       strip_accents=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)							,       ["HäLLo", "!", "how", "Are", "yoU", "?"]						)







 def  __lowercase (		self :       Optional[int]						):


  '''simple docstring'''






  UpperCamelCase__      :       Dict	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE							,       strip_accents=_SCREAMING_SNAKE_CASE						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)							,       ["HaLLo", "!", "how", "Are", "yoU", "?"]						)







 def  __lowercase (		self :       Union[str, Any]						):


  '''simple docstring'''






  UpperCamelCase__      :       Any	    =  BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE							,       never_split=["[UNK]"]						)

  self.assertListEqual(
      tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)							,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)







 def  __lowercase (		self :       int						):


  '''simple docstring'''






  UpperCamelCase__      :       Tuple	    =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

  UpperCamelCase__      :       Optional[int]	    =  {}
  for i, token in enumerate(_SCREAMING_SNAKE_CASE						):
   UpperCamelCase__      :       Optional[Any]	    =  i
  UpperCamelCase__      :       List[Any]	    =  WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE							,       unk_token="[UNK]"						)

  self.assertListEqual(tokenizer.tokenize(""						)							,       []						)

  self.assertListEqual(tokenizer.tokenize("unwanted running"						)							,       ["un", "##want", "##ed", "runn", "##ing"]						)

  self.assertListEqual(tokenizer.tokenize("unwantedX running"						)							,       ["[UNK]", "runn", "##ing"]						)







 def  __lowercase (		self :       List[str]						):


  '''simple docstring'''






  self.assertTrue(_is_whitespace(" "						)						)
  self.assertTrue(_is_whitespace("\t"						)						)
  self.assertTrue(_is_whitespace("\r"						)						)
  self.assertTrue(_is_whitespace("\n"						)						)
  self.assertTrue(_is_whitespace("\u00A0"						)						)

  self.assertFalse(_is_whitespace("A"						)						)
  self.assertFalse(_is_whitespace("-"						)						)







 def  __lowercase (		self :       Dict						):


  '''simple docstring'''






  self.assertTrue(_is_control("\u0005"						)						)

  self.assertFalse(_is_control("A"						)						)
  self.assertFalse(_is_control(" "						)						)
  self.assertFalse(_is_control("\t"						)						)
  self.assertFalse(_is_control("\r"						)						)







 def  __lowercase (		self :       List[str]						):


  '''simple docstring'''






  self.assertTrue(_is_punctuation("-"						)						)
  self.assertTrue(_is_punctuation("$"						)						)
  self.assertTrue(_is_punctuation("`"						)						)
  self.assertTrue(_is_punctuation("."						)						)

  self.assertFalse(_is_punctuation("A"						)						)
  self.assertFalse(_is_punctuation(" "						)						)







 def  __lowercase (		self :       Union[str, Any]						):


  '''simple docstring'''






  UpperCamelCase__      :       Optional[int]	    =  self.get_tokenizer()
  UpperCamelCase__      :       Any	    =  self.get_rust_tokenizer()

  # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE						) for t in ["Test", "\xad", "test"]]							,       [["[UNK]"], [], ["[UNK]"]]						)

  self.assertListEqual(
      [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE						) for t in ["Test", "\xad", "test"]]							,       [["[UNK]"], [], ["[UNK]"]]						)







 @slow
 def  __lowercase (		self :       Optional[Any]						):


  '''simple docstring'''






  UpperCamelCase__      :       Union[str, Any]	    =  self.tokenizer_class.from_pretrained("google/mobilebert-uncased"						)

  UpperCamelCase__      :       int	    =  tokenizer.encode("sequence builders"							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Tuple	    =  tokenizer.encode("multi-sequence build"							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)

  UpperCamelCase__      :       Optional[Any]	    =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE						)
  UpperCamelCase__      :       Union[str, Any]	    =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

  assert encoded_sentence == [1_01] + text + [1_02]
  assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







 def  __lowercase (		self :       str						):


  '''simple docstring'''






  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
    UpperCamelCase__      :       Dict	    =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							,       **_SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       Dict	    =  F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
    UpperCamelCase__      :       Any	    =  tokenizer_r.encode_plus(
        _SCREAMING_SNAKE_CASE							,       return_attention_mask=_SCREAMING_SNAKE_CASE							,       return_token_type_ids=_SCREAMING_SNAKE_CASE							,       return_offsets_mapping=_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE							,       )

    UpperCamelCase__      :       List[Any]	    =  tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE							,       "do_lower_case"						) else False
    UpperCamelCase__      :       Optional[int]	    =  (
        [
            ((0, 0), tokenizer_r.cls_token),
            ((0, 1), "A"),
            ((1, 2), ","),
            ((3, 5), "na"),
            ((5, 6), "##ï"),
            ((6, 8), "##ve"),
            ((9, 15), tokenizer_r.mask_token),
            ((16, 21), "Allen"),
            ((21, 23), "##NL"),
            ((23, 24), "##P"),
            ((25, 33), "sentence"),
            ((33, 34), "."),
            ((0, 0), tokenizer_r.sep_token),
        ]
        if not do_lower_case
        else [
            ((0, 0), tokenizer_r.cls_token),
            ((0, 1), "a"),
            ((1, 2), ","),
            ((3, 8), "naive"),
            ((9, 15), tokenizer_r.mask_token),
            ((16, 21), "allen"),
            ((21, 23), "##nl"),
            ((23, 24), "##p"),
            ((25, 33), "sentence"),
            ((33, 34), "."),
            ((0, 0), tokenizer_r.sep_token),
        ]
    )

    self.assertEqual(
        [e[1] for e in expected_results]							,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
    self.assertEqual([e[0] for e in expected_results]							,       tokens["offset_mapping"]						)







 def  __lowercase (		self :       Optional[Any]						):


  '''simple docstring'''






  UpperCamelCase__      :       Union[str, Any]	    =  ["的", "人", "有"]
  UpperCamelCase__      :       int	    =  "".join(_SCREAMING_SNAKE_CASE						)
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
    UpperCamelCase__      :       Tuple	    =  True
    UpperCamelCase__      :       Union[str, Any]	    =  self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							,       **_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       Optional[Any]	    =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							,       **_SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       Optional[int]	    =  tokenizer_p.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       Tuple	    =  tokenizer_r.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       Optional[Any]	    =  tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       Optional[Any]	    =  tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						)

    # it is expected that each Chinese character is not preceded by "##"
    self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)
    self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       Tuple	    =  False
    UpperCamelCase__      :       str	    =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							,       **_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       List[str]	    =  self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							,       **_SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       Optional[int]	    =  tokenizer_r.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       Tuple	    =  tokenizer_p.encode(_SCREAMING_SNAKE_CASE							,       add_special_tokens=_SCREAMING_SNAKE_CASE						)

    UpperCamelCase__      :       List[str]	    =  tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						)
    UpperCamelCase__      :       Any	    =  tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						)

    # it is expected that only the first Chinese character is not preceded by "##".
    UpperCamelCase__      :       Union[str, Any]	    =  [
        F'##{token}' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE						)
    ]
    self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)
    self.assertListEqual(_SCREAMING_SNAKE_CASE							,       _SCREAMING_SNAKE_CASE						)







import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import CLIPImageProcessor, CLIPProcessor





@require_vision
class       __a  (   unittest.TestCase	):







 def  __lowercase (		self :       Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  tempfile.mkdtemp()

      # fmt: off
      UpperCamelCase__      :       List[Any]	    =  ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
      # fmt: on
      UpperCamelCase__      :       List[Any]	    =  dict(zip(SCREAMING_SNAKE_CASE							,       range(len(SCREAMING_SNAKE_CASE						)						)						)						)
      UpperCamelCase__      :       Tuple	    =  ["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
      UpperCamelCase__      :       Tuple	    =  {"unk_token": "<unk>"}

      UpperCamelCase__      :       Tuple	    =  os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES["vocab_file"]						)
      UpperCamelCase__      :       Optional[int]	    =  os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES["merges_file"]						)
      with open(self.vocab_file							,       "w"							,       encoding="utf-8"						) as fp:
           fp.write(json.dumps(SCREAMING_SNAKE_CASE						) + "\n"						)
      with open(self.merges_file							,       "w"							,       encoding="utf-8"						) as fp:
           fp.write("\n".join(SCREAMING_SNAKE_CASE						)						)

      UpperCamelCase__      :       List[str]	    =  {
          "do_resize": True,
          "size": 20,
          "do_center_crop": True,
          "crop_size": 18,
          "do_normalize": True,
          "image_mean": [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3],
          "image_std": [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1],
      }
      UpperCamelCase__      :       int	    =  os.path.join(self.tmpdirname							,       SCREAMING_SNAKE_CASE						)
      with open(self.image_processor_file							,       "w"							,       encoding="utf-8"						) as fp:
           json.dump(SCREAMING_SNAKE_CASE							,       SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       List[str]							,       **SCREAMING_SNAKE_CASE :       List[str]						):


      '''simple docstring'''






      return CLIPTokenizer.from_pretrained(self.tmpdirname							,       **SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       List[Any]							,       **SCREAMING_SNAKE_CASE :       Union[str, Any]						):


      '''simple docstring'''






      return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,       **SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       Any							,       **SCREAMING_SNAKE_CASE :       Union[str, Any]						):


      '''simple docstring'''






      return CLIPImageProcessor.from_pretrained(self.tmpdirname							,       **SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       Optional[Any]						):


      '''simple docstring'''






      shutil.rmtree(self.tmpdirname						)







 def  __lowercase (		self :       Any						):


      '''simple docstring'''






      UpperCamelCase__      :       int	    =  [np.random.randint(2_55							,       size=(3, 30, 4_00)							,       dtype=np.uinta						)]

      UpperCamelCase__      :       int	    =  [Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE							,       0							,       -1						)						) for x in image_inputs]

      return image_inputs







 def  __lowercase (		self :       Tuple						):


      '''simple docstring'''






      UpperCamelCase__      :       Union[str, Any]	    =  self.get_tokenizer()
      UpperCamelCase__      :       List[str]	    =  self.get_rust_tokenizer()
      UpperCamelCase__      :       str	    =  self.get_image_processor()

      UpperCamelCase__      :       List[str]	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)
      processor_slow.save_pretrained(self.tmpdirname						)
      UpperCamelCase__      :       List[Any]	    =  CLIPProcessor.from_pretrained(self.tmpdirname							,       use_fast=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Tuple	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)
      processor_fast.save_pretrained(self.tmpdirname						)
      UpperCamelCase__      :       Any	    =  CLIPProcessor.from_pretrained(self.tmpdirname						)

      self.assertEqual(processor_slow.tokenizer.get_vocab()							,       tokenizer_slow.get_vocab()						)
      self.assertEqual(processor_fast.tokenizer.get_vocab()							,       tokenizer_fast.get_vocab()						)
      self.assertEqual(tokenizer_slow.get_vocab()							,       tokenizer_fast.get_vocab()						)
      self.assertIsInstance(processor_slow.tokenizer							,       SCREAMING_SNAKE_CASE						)
      self.assertIsInstance(processor_fast.tokenizer							,       SCREAMING_SNAKE_CASE						)

      self.assertEqual(processor_slow.image_processor.to_json_string()							,       image_processor.to_json_string()						)
      self.assertEqual(processor_fast.image_processor.to_json_string()							,       image_processor.to_json_string()						)
      self.assertIsInstance(processor_slow.image_processor							,       SCREAMING_SNAKE_CASE						)
      self.assertIsInstance(processor_fast.image_processor							,       SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       List[str]						):


      '''simple docstring'''






      UpperCamelCase__      :       str	    =  CLIPProcessor(tokenizer=self.get_tokenizer()							,       image_processor=self.get_image_processor()						)
      processor.save_pretrained(self.tmpdirname						)

      UpperCamelCase__      :       Union[str, Any]	    =  self.get_tokenizer(bos_token="(BOS)"							,       eos_token="(EOS)"						)
      UpperCamelCase__      :       int	    =  self.get_image_processor(do_normalize=SCREAMING_SNAKE_CASE							,       padding_value=1.0						)

      UpperCamelCase__      :       Tuple	    =  CLIPProcessor.from_pretrained(
          self.tmpdirname							,       bos_token="(BOS)"							,       eos_token="(EOS)"							,       do_normalize=SCREAMING_SNAKE_CASE							,       padding_value=1.0						)

      self.assertEqual(processor.tokenizer.get_vocab()							,       tokenizer_add_kwargs.get_vocab()						)
      self.assertIsInstance(processor.tokenizer							,       SCREAMING_SNAKE_CASE						)

      self.assertEqual(processor.image_processor.to_json_string()							,       image_processor_add_kwargs.to_json_string()						)
      self.assertIsInstance(processor.image_processor							,       SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       List[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       List[str]	    =  self.get_image_processor()
      UpperCamelCase__      :       Union[str, Any]	    =  self.get_tokenizer()

      UpperCamelCase__      :       Any	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Dict	    =  self.prepare_image_inputs()

      UpperCamelCase__      :       List[Any]	    =  image_processor(SCREAMING_SNAKE_CASE							,       return_tensors="np"						)
      UpperCamelCase__      :       Optional[Any]	    =  processor(images=SCREAMING_SNAKE_CASE							,       return_tensors="np"						)

      for key in input_image_proc.keys():
           self.assertAlmostEqual(input_image_proc[key].sum()							,       input_processor[key].sum()							,       delta=1e-2						)







 def  __lowercase (		self :       Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       str	    =  self.get_image_processor()
      UpperCamelCase__      :       int	    =  self.get_tokenizer()

      UpperCamelCase__      :       Any	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       List[Any]	    =  "lower newer"

      UpperCamelCase__      :       int	    =  processor(text=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Tuple	    =  tokenizer(SCREAMING_SNAKE_CASE						)

      for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key]							,       encoded_processor[key]						)







 def  __lowercase (		self :       int						):


      '''simple docstring'''






      UpperCamelCase__      :       List[str]	    =  self.get_image_processor()
      UpperCamelCase__      :       Dict	    =  self.get_tokenizer()

      UpperCamelCase__      :       Optional[Any]	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Dict	    =  "lower newer"
      UpperCamelCase__      :       List[Any]	    =  self.prepare_image_inputs()

      UpperCamelCase__      :       Tuple	    =  processor(text=SCREAMING_SNAKE_CASE							,       images=SCREAMING_SNAKE_CASE						)

      self.assertListEqual(list(inputs.keys()						)							,       ["input_ids", "attention_mask", "pixel_values"]						)

      # test if it raises when no input is passed
      with pytest.raises(SCREAMING_SNAKE_CASE						):
           processor()







 def  __lowercase (		self :       Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[Any]	    =  self.get_image_processor()
      UpperCamelCase__      :       Optional[int]	    =  self.get_tokenizer()

      UpperCamelCase__      :       Optional[Any]	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       List[str]	    =  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

      UpperCamelCase__      :       Optional[Any]	    =  processor.batch_decode(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       List[Any]	    =  tokenizer.batch_decode(SCREAMING_SNAKE_CASE						)

      self.assertListEqual(SCREAMING_SNAKE_CASE							,       SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       List[str]						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  self.get_image_processor()
      UpperCamelCase__      :       Tuple	    =  self.get_tokenizer()

      UpperCamelCase__      :       Dict	    =  CLIPProcessor(tokenizer=SCREAMING_SNAKE_CASE							,       image_processor=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Dict	    =  "lower newer"
      UpperCamelCase__      :       List[str]	    =  self.prepare_image_inputs()

      UpperCamelCase__      :       str	    =  processor(text=SCREAMING_SNAKE_CASE							,       images=SCREAMING_SNAKE_CASE						)

      self.assertListEqual(list(inputs.keys()						)							,       processor.model_input_names						)