



"""simple docstring"""




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class      UpperCAmelCase_  (    __lowercase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       List[Any]								=			CLIPTokenizer
      __SCREAMING_SNAKE_CASE						:       Tuple								=			CLIPTokenizerFast
      __SCREAMING_SNAKE_CASE						:       str								=			True
      __SCREAMING_SNAKE_CASE						:       Union[str, Any]								=			{}
      __SCREAMING_SNAKE_CASE						:       Union[str, Any]								=			False



      def 						snake_case_ (					self					:     int						):
           super().setUp()

           # fmt: off
           _UpperCAmelCase      :     Tuple        =   ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
           # fmt: on
           _UpperCAmelCase      :     Dict        =   dict(zip(UpperCAmelCase__					,     range(len(UpperCAmelCase__						)						)						)						)
           _UpperCAmelCase      :     Dict        =   ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
           _UpperCAmelCase      :     Optional[Any]        =   {"unk_token": "<unk>"}

           _UpperCAmelCase      :     Optional[Any]        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["vocab_file"]						)
           _UpperCAmelCase      :     Dict        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["merges_file"]						)
           with open(self.vocab_file					,     "w"					,     encoding="utf-8"						) as fp:
                fp.write(json.dumps(UpperCAmelCase__						) + "\n"						)
           with open(self.merges_file					,     "w"					,     encoding="utf-8"						) as fp:
                fp.write("\n".join(UpperCAmelCase__						)						)



      def 						snake_case_ (					self					:     Union[str, Any]					,     **A					:     List[Any]						):
           kwargs.update(self.special_tokens_map						)
           return CLIPTokenizer.from_pretrained(self.tmpdirname					,     **UpperCAmelCase__						)



      def 						snake_case_ (					self					:     Any					,     **A					:     Optional[int]						):
           kwargs.update(self.special_tokens_map						)
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,     **UpperCAmelCase__						)



      def 						snake_case_ (					self					:     Optional[Any]					,     A					:     Optional[Any]						):
           _UpperCAmelCase      :     Union[str, Any]        =   "lower newer"
           _UpperCAmelCase      :     Any        =   "lower newer"
           return input_text, output_text



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Dict        =   CLIPTokenizer(self.vocab_file					,     self.merges_file					,     **self.special_tokens_map						)
           _UpperCAmelCase      :     Optional[int]        =   "lower newer"
           _UpperCAmelCase      :     Union[str, Any]        =   ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
           _UpperCAmelCase      :     List[str]        =   tokenizer.tokenize(UpperCAmelCase__						)
           self.assertListEqual(UpperCAmelCase__					,     UpperCAmelCase__						)

           _UpperCAmelCase      :     List[Any]        =   tokens + [tokenizer.unk_token]
           _UpperCAmelCase      :     str        =   [1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__						)					,     UpperCAmelCase__						)



      @require_ftfy
      def 						snake_case_ (					self					:     List[Any]						):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
                     _UpperCAmelCase      :     int        =   self.tokenizer_class.from_pretrained(UpperCAmelCase__					,     **UpperCAmelCase__						)
                     _UpperCAmelCase      :     List[str]        =   self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__					,     **UpperCAmelCase__						)

                     _UpperCAmelCase      :     Any        =   "A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d."
                     _UpperCAmelCase      :     Tuple        =   tokenizer_s.tokenize(UpperCAmelCase__						)
                     _UpperCAmelCase      :     List[Any]        =   tokenizer_r.tokenize(UpperCAmelCase__						)

                     self.assertListEqual(UpperCAmelCase__					,     UpperCAmelCase__						)

                     # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                     # with Tilde) encoded in 2 different ways
                     _UpperCAmelCase      :     Dict        =   "xa\u0303y" + " " + "x\xe3y"
                     _UpperCAmelCase      :     Optional[Any]        =   tokenizer_s.tokenize(UpperCAmelCase__						)
                     _UpperCAmelCase      :     List[Any]        =   tokenizer_r.tokenize(UpperCAmelCase__						)

                     self.assertListEqual(UpperCAmelCase__					,     UpperCAmelCase__						)

                     # Test that the tokenization is identical on unicode of space type
                     _UpperCAmelCase      :     str        =   [
                         "\u0009",  # (horizontal tab, '\t')
                         "\u000B",  # (vertical tab)
                         "\u000C",  # (form feed)
                         "\u0020",  # (space, ' ')
                         "\u200E",  # (left-to-right mark):w
                         "\u200F",  # (right-to-left mark)
                     ]
                     for unicode_seq in spaces_unicodes:
                          _UpperCAmelCase      :     Any        =   tokenizer_s.tokenize(UpperCAmelCase__						)
                          _UpperCAmelCase      :     Optional[int]        =   tokenizer_r.tokenize(UpperCAmelCase__						)

                          self.assertListEqual(UpperCAmelCase__					,     UpperCAmelCase__						)

                     # Test that the tokenization is identical on unicode of line break type
                     _UpperCAmelCase      :     Union[str, Any]        =   [
                         "\u000A",  # (line feed, '\n')
                         "\r\n",  # (carriage return and line feed, '\r\n')
                         "\u000D",  # (carriage return, '\r')
                         "\r",  # (carriage return, '\r')
                         "\u000D",  # (carriage return, '\r')
                         "\u2028",  # (line separator)
                         "\u2029",  # (paragraph separator)
                         # "\u0085", # (next line)
                     ]

                     # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                     # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                     # space (and thus into an empty list).

                     for unicode_seq in line_break_unicodes:
                          _UpperCAmelCase      :     List[Any]        =   tokenizer_s.tokenize(UpperCAmelCase__						)
                          _UpperCAmelCase      :     Optional[Any]        =   tokenizer_r.tokenize(UpperCAmelCase__						)

                          self.assertListEqual(UpperCAmelCase__					,     UpperCAmelCase__						)



      def 						snake_case_ (					self					:     Dict						):
           # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
                     _UpperCAmelCase      :     Union[str, Any]        =   "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                     _UpperCAmelCase      :     str        =   f'{text_of_1_token} {text_of_1_token}'

                     _UpperCAmelCase      :     Any        =   self.rust_tokenizer_class.from_pretrained(
                         UpperCAmelCase__					,     use_fast=UpperCAmelCase__					,     )
                     _UpperCAmelCase      :     str        =   tokenizer_r(UpperCAmelCase__					,     return_offsets_mapping=UpperCAmelCase__					,     add_special_tokens=UpperCAmelCase__						)
                     self.assertEqual(encoding.offset_mapping[0]					,     (0, len(UpperCAmelCase__						))						)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,     (len(UpperCAmelCase__						) + 1, len(UpperCAmelCase__						) + 1 + len(UpperCAmelCase__						))					,     )

                     _UpperCAmelCase      :     List[Any]        =   f' {text}'

                     _UpperCAmelCase      :     Union[str, Any]        =   self.rust_tokenizer_class.from_pretrained(
                         UpperCAmelCase__					,     use_fast=UpperCAmelCase__					,     )
                     _UpperCAmelCase      :     int        =   tokenizer_r(UpperCAmelCase__					,     return_offsets_mapping=UpperCAmelCase__					,     add_special_tokens=UpperCAmelCase__						)
                     self.assertEqual(encoding.offset_mapping[0]					,     (1, 1 + len(UpperCAmelCase__						))						)
                     self.assertEqual(
                         encoding.offset_mapping[1]					,     (1 + len(UpperCAmelCase__						) + 1, 1 + len(UpperCAmelCase__						) + 1 + len(UpperCAmelCase__						))					,     )



      def 						snake_case_ (					self					:     Any						):
           # Test related to the breaking change introduced in transformers v4.17.0
           # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
           with self.assertRaises(UpperCAmelCase__						) as context:
                self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"						)

           self.assertTrue(
               context.exception.args[0].startswith(
                   "The `backend_tokenizer` provided does not match the expected format."						)						)



      @require_ftfy
      def 						snake_case_ (					self					:     Union[str, Any]						):
           super().test_tokenization_python_rust_equals()



      def 						snake_case_ (					self					:     List[Any]						):
           # CLIP always lower cases letters
           pass










"""simple docstring"""




from typing import Dict, Iterable, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_lowerCAmelCase    : Tuple							=							logging.get_logger(__name__)






class      UpperCAmelCase_  (    _UpperCamelCase						):
      __SCREAMING_SNAKE_CASE						:       str								=			['pixel_values']



      def __init__(					self					:     List[Any]					,     A					:     bool = True					,     A					:     Dict[str, int] = None					,     A					:     PILImageResampling = PILImageResampling.BICUBIC					,     A					:     bool = True					,     A					:     Dict[str, int] = None					,     A					:     bool = True					,     A					:     Union[int, float] = 1 / 2_5_5					,     A					:     bool = True					,     A					:     Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_MEAN					,     A					:     Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_STD					,     **A					:     List[Any]					,     ):
           super().__init__(**A						)
           _UpperCAmelCase      :     int        =   size if size is not None else {"shortest_edge": 2_2_4}
           _UpperCAmelCase      :     Tuple        =   get_size_dict(A					,     default_to_square=A						)
           _UpperCAmelCase      :     int        =   crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
           _UpperCAmelCase      :     Union[str, Any]        =   get_size_dict(A					,     param_name="crop_size"						)

           _UpperCAmelCase      :     str        =   do_resize
           _UpperCAmelCase      :     Union[str, Any]        =   size
           _UpperCAmelCase      :     Tuple        =   resample
           _UpperCAmelCase      :     List[Any]        =   do_center_crop
           _UpperCAmelCase      :     List[Any]        =   crop_size
           _UpperCAmelCase      :     Tuple        =   do_rescale
           _UpperCAmelCase      :     List[Any]        =   rescale_factor
           _UpperCAmelCase      :     Dict        =   do_normalize
           _UpperCAmelCase      :     Optional[Any]        =   image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
           _UpperCAmelCase      :     Any        =   image_std if image_std is not None else IMAGENET_DEFAULT_STD



      def 						snake_case_ (					self					:     Optional[int]					,     A					:     np.ndarray					,     A					:     Dict[str, int]					,     A					:     PILImageResampling = PILImageResampling.BICUBIC					,     A					:     Optional[Union[str, ChannelDimension]] = None					,     **A					:     Union[str, Any]					,     ):
           _UpperCAmelCase      :     Tuple        =   get_size_dict(A					,     default_to_square=A						)
           # size_dict is a dict with either keys "height" and "width" or "shortest_edge"
           if "shortest_edge" in size:
                _UpperCAmelCase      :     Any        =   int((2_5_6 / 2_2_4) * size["shortest_edge"]						)
                _UpperCAmelCase      :     Union[str, Any]        =   get_resize_output_image_size(A					,     size=A					,     default_to_square=A						)
                _UpperCAmelCase      :     str        =   {"height": output_size[0], "width": output_size[1]}
           if "height" not in size_dict or "width" not in size_dict:
                raise ValueError(
                    f'Size dict must have keys \'height\' and \'width\' or \'shortest_edge\'. Got {size_dict.keys()}'						)
           return resize(
               A					,     size=(size_dict["height"], size_dict["width"])					,     resample=A					,     data_format=A					,     **A						)



      def 						snake_case_ (					self					:     int					,     A					:     np.ndarray					,     A					:     Dict[str, int]					,     A					:     Optional[Union[str, ChannelDimension]] = None					,     **A					:     Optional[Any]					,     ):
           _UpperCAmelCase      :     Dict        =   get_size_dict(A						)
           if "height" not in size or "width" not in size:
                raise ValueError(f'Size dict must have keys \'height\' and \'width\'. Got {size.keys()}'						)
           return center_crop(A					,     size=(size["height"], size["width"])					,     data_format=A					,     **A						)



      def 						snake_case_ (					self					:     List[str]					,     A					:     np.ndarray					,     A					:     Union[int, float]					,     A					:     Optional[Union[str, ChannelDimension]] = None					,     **A					:     Dict					,     ):
           return rescale(A					,     scale=A					,     data_format=A					,     **A						)



      def 						snake_case_ (					self					:     List[str]					,     A					:     np.ndarray					,     A					:     Union[float, List[float]]					,     A					:     Union[float, List[float]]					,     A					:     Optional[Union[str, ChannelDimension]] = None					,     **A					:     str					,     ):
           return normalize(A					,     mean=A					,     std=A					,     data_format=A					,     **A						)



      def 						snake_case_ (					self					:     Tuple					,     A					:     ImageInput					,     A					:     Optional[bool] = None					,     A					:     Optional[Dict[str, int]] = None					,     A					:     PILImageResampling = None					,     A					:     Optional[bool] = None					,     A					:     Optional[Dict[str, int]] = None					,     A					:     Optional[bool] = None					,     A					:     Optional[float] = None					,     A					:     Optional[bool] = None					,     A					:     Optional[Union[float, Iterable[float]]] = None					,     A					:     Optional[Union[float, Iterable[float]]] = None					,     A					:     Optional[TensorType] = None					,     A					:     ChannelDimension = ChannelDimension.FIRST					,     **A					:     Dict					,     ):
           _UpperCAmelCase      :     Dict        =   do_resize if do_resize is not None else self.do_resize
           _UpperCAmelCase      :     List[str]        =   resample if resample is not None else self.resample
           _UpperCAmelCase      :     int        =   do_center_crop if do_center_crop is not None else self.do_center_crop
           _UpperCAmelCase      :     str        =   do_rescale if do_rescale is not None else self.do_rescale
           _UpperCAmelCase      :     Tuple        =   rescale_factor if rescale_factor is not None else self.rescale_factor
           _UpperCAmelCase      :     Optional[int]        =   do_normalize if do_normalize is not None else self.do_normalize
           _UpperCAmelCase      :     List[str]        =   image_mean if image_mean is not None else self.image_mean
           _UpperCAmelCase      :     Tuple        =   image_std if image_std is not None else self.image_std

           _UpperCAmelCase      :     Any        =   size if size is not None else self.size
           _UpperCAmelCase      :     Optional[int]        =   get_size_dict(A					,     default_to_square=A						)
           _UpperCAmelCase      :     List[str]        =   crop_size if crop_size is not None else self.crop_size
           _UpperCAmelCase      :     List[str]        =   get_size_dict(A					,     param_name="crop_size"						)

           _UpperCAmelCase      :     List[Any]        =   make_list_of_images(A						)

           if not valid_images(A						):
                raise ValueError(
                    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                    "torch.Tensor, tf.Tensor or jax.ndarray."						)

           if do_resize and size is None:
                raise ValueError("Size must be specified if do_resize is True."						)

           if do_center_crop and crop_size is None:
                raise ValueError("Crop size must be specified if do_center_crop is True."						)

           if do_rescale and rescale_factor is None:
                raise ValueError("Rescale factor must be specified if do_rescale is True."						)

           if do_normalize and (image_mean is None or image_std is None):
                raise ValueError("Image mean and std must be specified if do_normalize is True."						)

           # All transformations expect numpy arrays.
           _UpperCAmelCase      :     Any        =   [to_numpy_array(A						) for image in images]

           if do_resize:
                _UpperCAmelCase      :     str        =   [self.resize(A					,     A					,     A						) for image in images]

           if do_center_crop:
                _UpperCAmelCase      :     Dict        =   [self.center_crop(A					,     A						) for image in images]

           if do_rescale:
                _UpperCAmelCase      :     Dict        =   [self.rescale(A					,     A						) for image in images]

           if do_normalize:
                _UpperCAmelCase      :     Optional[int]        =   [self.normalize(A					,     A					,     A						) for image in images]

           _UpperCAmelCase      :     Union[str, Any]        =   [to_channel_dimension_format(A					,     A						) for image in images]

           _UpperCAmelCase      :     List[Any]        =   {"pixel_values": images}
           return BatchFeature(data=A					,     tensor_type=A						)
