import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
   import torch

if is_vision_available():
   from PIL import Image

   from transformers import VivitImageProcessor







class __snake_case    (						unittest.TestCase		):




   def __init__(   self      :    int	,     _lowercase      :    Any	,     _lowercase      :    List[str]=7	,     _lowercase      :    Dict=3	,     _lowercase      :    Dict=10	,     _lowercase      :    Union[str, Any]=18	,     _lowercase      :    Dict=30	,     _lowercase      :    List[str]=4_00	,     _lowercase      :    Tuple=True	,     _lowercase      :    int=None	,     _lowercase      :    Union[str, Any]=True	,     _lowercase      :    Union[str, Any]=[0.5, 0.5, 0.5]	,     _lowercase      :    List[Any]=[0.5, 0.5, 0.5]	,     _lowercase      :    List[Any]=None	,     ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				size if size is not None else {"""shortest_edge""": 18}
     SCREAMING_SNAKE_CASE__      							=				crop_size if crop_size is not None else {"""height""": 18, """width""": 18}

     SCREAMING_SNAKE_CASE__      							=				parent
     SCREAMING_SNAKE_CASE__      							=				batch_size
     SCREAMING_SNAKE_CASE__      							=				num_channels
     SCREAMING_SNAKE_CASE__      							=				num_frames
     SCREAMING_SNAKE_CASE__      							=				image_size
     SCREAMING_SNAKE_CASE__      							=				min_resolution
     SCREAMING_SNAKE_CASE__      							=				max_resolution
     SCREAMING_SNAKE_CASE__      							=				do_resize
     SCREAMING_SNAKE_CASE__      							=				size
     SCREAMING_SNAKE_CASE__      							=				do_normalize
     SCREAMING_SNAKE_CASE__      							=				image_mean
     SCREAMING_SNAKE_CASE__      							=				image_std
     SCREAMING_SNAKE_CASE__      							=				crop_size




   def      __a						(   self      :    Optional[Any]				):



     """simple docstring"""






     return {
         "image_mean": self.image_mean,
         "image_std": self.image_std,
         "do_normalize": self.do_normalize,
         "do_resize": self.do_resize,
         "size": self.size,
         "crop_size": self.crop_size,
     }







@require_torch
@require_vision
class __snake_case    (						lowerCamelCase_       ,		unittest.TestCase		):
   lowerCAmelCase_														=							VivitImageProcessor if is_vision_available() else None




   def      __a						(   self      :    Union[str, Any]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				VivitImageProcessingTester(self				)




   @property
   def      __a						(   self      :    Optional[Any]				):



     """simple docstring"""






     return self.image_processor_tester.prepare_image_processor_dict()




   def      __a						(   self      :    List[Any]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class(**self.image_processor_dict				)
     self.assertTrue(hasattr(_lowercase	,     """image_mean"""				)				)
     self.assertTrue(hasattr(_lowercase	,     """image_std"""				)				)
     self.assertTrue(hasattr(_lowercase	,     """do_normalize"""				)				)
     self.assertTrue(hasattr(_lowercase	,     """do_resize"""				)				)
     self.assertTrue(hasattr(_lowercase	,     """do_center_crop"""				)				)
     self.assertTrue(hasattr(_lowercase	,     """size"""				)				)




   def      __a						(   self      :    int				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class.from_dict(self.image_processor_dict				)
     self.assertEqual(image_processor.size	,     {"""shortest_edge""": 18}				)
     self.assertEqual(image_processor.crop_size	,     {"""height""": 18, """width""": 18}				)

     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class.from_dict(self.image_processor_dict	,     size=42	,     crop_size=84				)
     self.assertEqual(image_processor.size	,     {"""shortest_edge""": 42}				)
     self.assertEqual(image_processor.crop_size	,     {"""height""": 84, """width""": 84}				)




   def      __a						(   self      :    Any				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class(**self.image_processor_dict				)
     # create random PIL videos
     SCREAMING_SNAKE_CASE__      							=				prepare_video_inputs(self.image_processor_tester	,     equal_resolution=_lowercase				)
     for video in video_inputs:
       self.assertIsInstance(_lowercase	,     _lowercase				)
       self.assertIsInstance(video[0]	,     Image.Image				)

     # Test not batched input
     SCREAMING_SNAKE_CASE__      							=				image_processing(video_inputs[0]	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )

     # Test batched
     SCREAMING_SNAKE_CASE__      							=				image_processing(_lowercase	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )




   def      __a						(   self      :    Dict				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class(**self.image_processor_dict				)
     # create random numpy tensors
     SCREAMING_SNAKE_CASE__      							=				prepare_video_inputs(self.image_processor_tester	,     equal_resolution=_lowercase	,     numpify=_lowercase				)
     for video in video_inputs:
       self.assertIsInstance(_lowercase	,     _lowercase				)
       self.assertIsInstance(video[0]	,     np.ndarray				)

     # Test not batched input
     SCREAMING_SNAKE_CASE__      							=				image_processing(video_inputs[0]	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )

     # Test batched
     SCREAMING_SNAKE_CASE__      							=				image_processing(_lowercase	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )




   def      __a						(   self      :    int				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.image_processing_class(**self.image_processor_dict				)
     # create random PyTorch tensors
     SCREAMING_SNAKE_CASE__      							=				prepare_video_inputs(self.image_processor_tester	,     equal_resolution=_lowercase	,     torchify=_lowercase				)
     for video in video_inputs:
       self.assertIsInstance(_lowercase	,     _lowercase				)
       self.assertIsInstance(video[0]	,     torch.Tensor				)

     # Test not batched input
     SCREAMING_SNAKE_CASE__      							=				image_processing(video_inputs[0]	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )

     # Test batched
     SCREAMING_SNAKE_CASE__      							=				image_processing(_lowercase	,     return_tensors="""pt"""				).pixel_values
     self.assertEqual(
         encoded_videos.shape	,     (
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size["""height"""],
             self.image_processor_tester.crop_size["""width"""],
         )	,     )

import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__lowerCamelCase      :			int							       =		logging.get_logger(__name__)


__lowerCamelCase      :			Tuple							       =		{name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}



def      __SCREAMING_SNAKE_CASE  ( __UpperCamelCase   :	int			,					__UpperCamelCase   :	Optional[int]			,					__UpperCamelCase   :	str			,					__UpperCamelCase   :	Optional[Any]     )							-> Union[str, Any]:







  """simple docstring"""





  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()     )}."""     )

  if tokenizer_name is None:
    SCREAMING_SNAKE_CASE__      							=				TOKENIZER_CLASSES
  else:
    SCREAMING_SNAKE_CASE__      							=				{tokenizer_name: getattr(__UpperCamelCase			,					tokenizer_name + """Fast"""     )}

  logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""     )

  for tokenizer_name in tokenizer_names:
    SCREAMING_SNAKE_CASE__      							=				TOKENIZER_CLASSES[tokenizer_name]

    SCREAMING_SNAKE_CASE__      							=				True
    if checkpoint_name is None:
      SCREAMING_SNAKE_CASE__      							=				list(tokenizer_class.max_model_input_sizes.keys()     )
    else:
      SCREAMING_SNAKE_CASE__      							=				[checkpoint_name]

    logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""     )

    for checkpoint in checkpoint_names:
      logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""     )

      # Load tokenizer
      SCREAMING_SNAKE_CASE__      							=				tokenizer_class.from_pretrained(__UpperCamelCase			,					force_download=__UpperCamelCase     )

      # Save fast tokenizer
      logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""     )

      # For organization names we create sub-directories
      if "/" in checkpoint:
        SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__      							=				checkpoint.split("""/"""     )
        SCREAMING_SNAKE_CASE__      							=				os.path.join(__UpperCamelCase			,					__UpperCamelCase     )
      elif add_prefix:
        SCREAMING_SNAKE_CASE__      							=				checkpoint
        SCREAMING_SNAKE_CASE__      							=				dump_path
      else:
        SCREAMING_SNAKE_CASE__      							=				None
        SCREAMING_SNAKE_CASE__      							=				dump_path

      logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""     )

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()     )[0]:
        SCREAMING_SNAKE_CASE__      							=				list(tokenizer.pretrained_vocab_files_map.values()     )[0][checkpoint]
        SCREAMING_SNAKE_CASE__      							=				file_path.split(__UpperCamelCase     )[-1][0]
        if next_char == "/":
          SCREAMING_SNAKE_CASE__      							=				os.path.join(__UpperCamelCase			,					__UpperCamelCase     )
          SCREAMING_SNAKE_CASE__      							=				None

        logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""     )

      SCREAMING_SNAKE_CASE__      							=				tokenizer.save_pretrained(
          __UpperCamelCase			,					legacy_format=__UpperCamelCase			,					filename_prefix=__UpperCamelCase     )
      logger.info(f"""=> File names {file_names}"""     )

      for file_name in file_names:
        if not file_name.endswith("""tokenizer.json"""     ):
          os.remove(__UpperCamelCase     )
          logger.info(f"""=> removing {file_name}"""     )


if __name__ == "__main__":
   __lowerCamelCase      :			str							       =		argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
   )
   parser.add_argument(
       '''--tokenizer_name''',
       default=None,
       type=str,
       help=(
           F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
           '''download and convert all the checkpoints from AWS.'''
       ),
   )
   parser.add_argument(
       '''--checkpoint_name''',
       default=None,
       type=str,
       help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
   )
   parser.add_argument(
       '''--force_download''',
       action='''store_true''',
       help='''Re-download checkpoints.''',
   )
   __lowerCamelCase      :			Any							       =		parser.parse_args()

   convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)
