





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import VivitImageProcessor

class 		SCREAMING_SNAKE_CASE__    (	unittest.TestCase  ):






      """simple docstring"""




      def __init__(self	:				Tuple    ,      a__	:				Dict    ,      a__	:				str=7    ,      a__	:				Dict=3    ,      a__	:				Union[str, Any]=10    ,      a__	:				List[Any]=18    ,      a__	:				List[str]=30    ,      a__	:				Optional[Any]=400    ,      a__	:				int=True    ,      a__	:				List[Any]=None    ,      a__	:				Dict=True    ,      a__	:				List[str]=[0.5, 0.5, 0.5]    ,      a__	:				int=[0.5, 0.5, 0.5]    ,      a__	:				Any=None    ,      ):





         """simple docstring"""



         __snake_case								=       size if size is not None else {'''shortest_edge''': 18}
         __snake_case								=       crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}

         __snake_case								=       parent
         __snake_case								=       batch_size
         __snake_case								=       num_channels
         __snake_case								=       num_frames
         __snake_case								=       image_size
         __snake_case								=       min_resolution
         __snake_case								=       max_resolution
         __snake_case								=       do_resize
         __snake_case								=       size
         __snake_case								=       do_normalize
         __snake_case								=       image_mean
         __snake_case								=       image_std
         __snake_case								=       crop_size


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         return {
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_normalize": self.do_normalize,
             "do_resize": self.do_resize,
             "size": self.size,
             "crop_size": self.crop_size,
         }






@require_torch
@require_vision
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase						,				unittest.TestCase  ):






      """simple docstring"""




      A_						:    Tuple							=   VivitImageProcessor if is_vision_available() else None


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       VivitImageProcessingTester(self				)


      @property
      def        a			(self	:				Tuple				):





         """simple docstring"""



         return self.image_processor_tester.prepare_image_processor_dict()


      def        a			(self	:				List[str]				):





         """simple docstring"""



         __snake_case								=       self.image_processing_class(**self.image_processor_dict				)
         self.assertTrue(hasattr(a__    ,      '''image_mean'''				)				)
         self.assertTrue(hasattr(a__    ,      '''image_std'''				)				)
         self.assertTrue(hasattr(a__    ,      '''do_normalize'''				)				)
         self.assertTrue(hasattr(a__    ,      '''do_resize'''				)				)
         self.assertTrue(hasattr(a__    ,      '''do_center_crop'''				)				)
         self.assertTrue(hasattr(a__    ,      '''size'''				)				)


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       self.image_processing_class.from_dict(self.image_processor_dict				)
         self.assertEqual(image_processor.size    ,      {'''shortest_edge''': 18}				)
         self.assertEqual(image_processor.crop_size    ,      {'''height''': 18, '''width''': 18}				)

         __snake_case								=       self.image_processing_class.from_dict(self.image_processor_dict    ,      size=42    ,      crop_size=84				)
         self.assertEqual(image_processor.size    ,      {'''shortest_edge''': 42}				)
         self.assertEqual(image_processor.crop_size    ,      {'''height''': 84, '''width''': 84}				)


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       self.image_processing_class(**self.image_processor_dict				)
         # create random PIL videos
         __snake_case								=       prepare_video_inputs(self.image_processor_tester    ,      equal_resolution=a__				)
         for video in video_inputs:
            self.assertIsInstance(a__    ,      a__				)
            self.assertIsInstance(video[0]    ,      Image.Image				)

         # Test not batched input
         __snake_case								=       image_processing(video_inputs[0]    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )

         # Test batched
         __snake_case								=       image_processing(a__    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       self.image_processing_class(**self.image_processor_dict				)
         # create random numpy tensors
         __snake_case								=       prepare_video_inputs(self.image_processor_tester    ,      equal_resolution=a__    ,      numpify=a__				)
         for video in video_inputs:
            self.assertIsInstance(a__    ,      a__				)
            self.assertIsInstance(video[0]    ,      np.ndarray				)

         # Test not batched input
         __snake_case								=       image_processing(video_inputs[0]    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )

         # Test batched
         __snake_case								=       image_processing(a__    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )


      def        a			(self	:				Any				):





         """simple docstring"""



         __snake_case								=       self.image_processing_class(**self.image_processor_dict				)
         # create random PyTorch tensors
         __snake_case								=       prepare_video_inputs(self.image_processor_tester    ,      equal_resolution=a__    ,      torchify=a__				)
         for video in video_inputs:
            self.assertIsInstance(a__    ,      a__				)
            self.assertIsInstance(video[0]    ,      torch.Tensor				)

         # Test not batched input
         __snake_case								=       image_processing(video_inputs[0]    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )

         # Test batched
         __snake_case								=       image_processing(a__    ,      return_tensors='''pt'''				).pixel_values
         self.assertEqual(
             encoded_videos.shape    ,      (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )    ,      )








from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


snake_case_           =      logging.get_logger(__name__)

snake_case_           =      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

snake_case_           =      {
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

snake_case_           =      {'allegro/herbert-base-cased': 514}
snake_case_           =      {}

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    Dict							=   VOCAB_FILES_NAMES
      A_						:    Optional[Any]							=   PRETRAINED_VOCAB_FILES_MAP
      A_						:    Optional[int]							=   PRETRAINED_INIT_CONFIGURATION
      A_						:    int							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A_						:    Any							=   HerbertTokenizer


      def __init__(self	:				Dict    ,      a__	:				Tuple=None    ,      a__	:				Optional[int]=None    ,      a__	:				List[str]=None    ,      a__	:				Optional[int]="<s>"    ,      a__	:				Optional[Any]="<unk>"    ,      a__	:				Any="<pad>"    ,      a__	:				List[Any]="<mask>"    ,      a__	:				Any="</s>"    ,      **a__	:				Tuple    ,      ):





         """simple docstring"""



         super().__init__(
             a__    ,      a__    ,      tokenizer_file=a__    ,      cls_token=a__    ,      unk_token=a__    ,      pad_token=a__    ,      mask_token=a__    ,      sep_token=a__    ,      **a__    ,      )


      def        a			(self	:				List[str]    ,      a__	:				List[int]    ,      a__	:				Optional[List[int]] = None				):





         """simple docstring"""



         __snake_case								=       [self.cls_token_id]
         __snake_case								=       [self.sep_token_id]
         if token_ids_a is None:
            return cls + token_ids_a + sep

         return cls + token_ids_a + sep + token_ids_a + sep


      def        a			(self	:				List[str]    ,      a__	:				List[int]    ,      a__	:				Optional[List[int]] = None    ,      a__	:				bool = False				):





         """simple docstring"""



         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=a__    ,      token_ids_a=a__    ,      already_has_special_tokens=a__				)

         if token_ids_a is None:
            return [1] + ([0] * len(a__				)) + [1]
         return [1] + ([0] * len(a__				)) + [1] + ([0] * len(a__				)) + [1]


      def        a			(self	:				Optional[int]    ,      a__	:				List[int]    ,      a__	:				Optional[List[int]] = None				):





         """simple docstring"""



         __snake_case								=       [self.sep_token_id]
         __snake_case								=       [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


      def        a			(self	:				int    ,      a__	:				str    ,      a__	:				Optional[str] = None				):





         """simple docstring"""



         __snake_case								=       self._tokenizer.model.save(a__    ,      name=a__				)
         return tuple(a__				)

