




import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCamelCase    :		List[str]			   =       data_utils.TransfoXLTokenizer
lowerCamelCase    :		int			   =       data_utils.TransfoXLCorpus
lowerCamelCase    :		List[str]			   =       data_utils
lowerCamelCase    :		str			   =       data_utils

def       _SCREAMING_SNAKE_CASE      (							lowercase     :	Union[str, Any]     ,							lowercase     :	List[Any]     ,							lowercase     :	str     ,							lowercase     :	Dict					):


      '''simple docstring'''



      if transfo_xl_dataset_file:
            # Convert a pre-processed corpus (see original TensorFlow repo)
            with open(lowercase     ,							'rb'					) as fp:
                  lowerCamelCase_							=   pickle.load(lowercase     ,							encoding='latin1'					)
            # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
            lowerCamelCase_							=   pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
            print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""					)
            lowerCamelCase_							=   corpus.vocab.__dict__
            torch.save(lowercase     ,							lowercase					)

            lowerCamelCase_							=   corpus.__dict__
            corpus_dict_no_vocab.pop('vocab'     ,							lowercase					)
            lowerCamelCase_							=   pytorch_dump_folder_path + '/' + CORPUS_NAME
            print(f"""Save dataset to {pytorch_dataset_dump_path}"""					)
            torch.save(lowercase     ,							lowercase					)

      if tf_checkpoint_path:
            # Convert a pre-trained TensorFlow model
            lowerCamelCase_							=   os.path.abspath(lowercase					)
            lowerCamelCase_							=   os.path.abspath(lowercase					)

            print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""					)
            # Initialise PyTorch model
            if transfo_xl_config_file == "":
                  lowerCamelCase_							=   TransfoXLConfig()
            else:
                  lowerCamelCase_							=   TransfoXLConfig.from_json_file(lowercase					)
            print(f"""Building PyTorch model from configuration: {config}"""					)
            lowerCamelCase_							=   TransfoXLLMHeadModel(lowercase					)

            lowerCamelCase_							=   load_tf_weights_in_transfo_xl(lowercase     ,							lowercase     ,							lowercase					)
            # Save pytorch-model
            lowerCamelCase_							=   os.path.join(lowercase     ,							lowercase					)
            lowerCamelCase_							=   os.path.join(lowercase     ,							lowercase					)
            print(f"""Save PyTorch model to {os.path.abspath(lowercase					)}"""					)
            torch.save(model.state_dict()     ,							lowercase					)
            print(f"""Save configuration file to {os.path.abspath(lowercase					)}"""					)
            with open(lowercase     ,							'w'     ,							encoding='utf-8'					) as f:
                  f.write(config.to_json_string()					)


if __name__ == "__main__":
 lowerCamelCase    :		Optional[Any]			   =       argparse.ArgumentParser()
 parser.add_argument(
     "--pytorch_dump_folder_path",
     default=None,
     type=str,
     required=True,
     help="Path to the folder to store the PyTorch model or dataset/vocab.",
 )
 parser.add_argument(
     "--tf_checkpoint_path",
     default="",
     type=str,
     help="An optional path to a TensorFlow checkpoint path to be converted.",
 )
 parser.add_argument(
     "--transfo_xl_config_file",
     default="",
     type=str,
     help=(
         "An optional config json file corresponding to the pre-trained BERT model. \n"
         "This specifies the model architecture."
     ),
 )
 parser.add_argument(
     "--transfo_xl_dataset_file",
     default="",
     type=str,
     help="An optional dataset file to be converted in a vocabulary.",
 )
 lowerCamelCase    :		Union[str, Any]			   =       parser.parse_args()
 convert_transfo_xl_checkpoint_to_pytorch(
     args.tf_checkpoint_path,
     args.transfo_xl_config_file,
     args.pytorch_dump_folder_path,
     args.transfo_xl_dataset_file,
 )






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import ChineseCLIPImageProcessor







class   A(						unittest.TestCase       ):



 '''simple docstring'''


 def __init__(    self :  str						,							A_ :  int						,							A_ :  Any=7						,							A_ :  Tuple=3						,							A_ :  Union[str, Any]=18						,							A_ :  Tuple=30						,							A_ :  Union[str, Any]=400						,							A_ :  Optional[int]=True						,							A_ :  List[Any]=None						,							A_ :  Dict=True						,							A_ :  Union[str, Any]=None						,							A_ :  Optional[int]=True						,							A_ :  str=[0.48145466, 0.4578275, 0.40821073]						,							A_ :  Tuple=[0.26862954, 0.26130258, 0.27577711]						,							A_ :  Any=True						,							)       ->     str:

       """simple docstring"""






       lowerCamelCase_							=   size if size is not None else {'height': 224, 'width': 224}
       lowerCamelCase_							=   crop_size if crop_size is not None else {'height': 18, 'width': 18}
       lowerCamelCase_							=   parent
       lowerCamelCase_							=   batch_size
       lowerCamelCase_							=   num_channels
       lowerCamelCase_							=   image_size
       lowerCamelCase_							=   min_resolution
       lowerCamelCase_							=   max_resolution
       lowerCamelCase_							=   do_resize
       lowerCamelCase_							=   size
       lowerCamelCase_							=   do_center_crop
       lowerCamelCase_							=   crop_size
       lowerCamelCase_							=   do_normalize
       lowerCamelCase_							=   image_mean
       lowerCamelCase_							=   image_std
       lowerCamelCase_							=   do_convert_rgb

 def     a__			(    self :  Optional[int]  )       ->     Tuple:

       """simple docstring"""






       return {
           "do_resize": self.do_resize,
           "size": self.size,
           "do_center_crop": self.do_center_crop,
           "crop_size": self.crop_size,
           "do_normalize": self.do_normalize,
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_convert_rgb": self.do_convert_rgb,
       }





 def     a__			(    self :  Any						,							A_ :  Any=False						,							A_ :  Dict=False						,							A_ :  str=False  )       ->     Union[str, Any]:

       """simple docstring"""






       assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

       if equal_resolution:
             lowerCamelCase_							=   []
             for i in range(self.batch_size  ):
                   image_inputs.append(
                       np.random.randint(
                           255						,							size=(self.num_channels, self.max_resolution, self.max_resolution)						,							dtype=np.uinta  )  )
       else:
             lowerCamelCase_							=   []
             for i in range(self.batch_size  ):
                   lowerCamelCase_					,    lowerCamelCase_							=   np.random.choice(np.arange(self.min_resolution						,							self.max_resolution  )						,							2  )
                   image_inputs.append(np.random.randint(255						,							size=(self.num_channels, width, height)						,							dtype=np.uinta  )  )

       if not numpify and not torchify:
             # PIL expects the channel dimension as last dimension
             lowerCamelCase_							=   [Image.fromarray(np.moveaxis(A_						,							0						,							-1  )  ) for x in image_inputs]

       if torchify:
             lowerCamelCase_							=   [torch.from_numpy(A_  ) for x in image_inputs]

       return image_inputs







@require_torch
@require_vision
class   A(						UpperCamelCase							,				unittest.TestCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						ChineseCLIPImageProcessor if is_vision_available() else None

 def     a__			(    self :  int  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   ChineseCLIPImageProcessingTester(self						,							do_center_crop=A_  )

 @property
 def     a__			(    self :  str  )       ->     List[Any]:

       """simple docstring"""






       return self.image_processor_tester.prepare_image_processor_dict()

 def     a__			(    self :  int  )       ->     Optional[Any]:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       self.assertTrue(hasattr(A_						,							'do_resize'  )  )
       self.assertTrue(hasattr(A_						,							'size'  )  )
       self.assertTrue(hasattr(A_						,							'do_center_crop'  )  )
       self.assertTrue(hasattr(A_						,							'center_crop'  )  )
       self.assertTrue(hasattr(A_						,							'do_normalize'  )  )
       self.assertTrue(hasattr(A_						,							'image_mean'  )  )
       self.assertTrue(hasattr(A_						,							'image_std'  )  )
       self.assertTrue(hasattr(A_						,							'do_convert_rgb'  )  )

 def     a__			(    self :  Any  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class.from_dict(self.image_processor_dict  )
       self.assertEqual(image_processor.size						,							{'height': 224, 'width': 224}  )
       self.assertEqual(image_processor.crop_size						,							{'height': 18, 'width': 18}  )

       lowerCamelCase_							=   self.image_processing_class.from_dict(self.image_processor_dict						,							size=42						,							crop_size=84  )
       self.assertEqual(image_processor.size						,							{'shortest_edge': 42}  )
       self.assertEqual(image_processor.crop_size						,							{'height': 84, 'width': 84}  )

 def     a__			(    self :  List[Any]  )       ->     List[Any]:

       """simple docstring"""






       pass

 def     a__			(    self :  str  )       ->     str:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       # create random PIL images
       lowerCamelCase_							=   self.image_processor_tester.prepare_inputs(equal_resolution=A_  )
       for image in image_inputs:
             self.assertIsInstance(A_						,							Image.Image  )

       # Test not batched input
       lowerCamelCase_							=   image_processing(image_inputs[0]						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)

       # Test batched
       lowerCamelCase_							=   image_processing(A_						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)

 def     a__			(    self :  List[str]  )       ->     Optional[Any]:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       # create random numpy tensors
       lowerCamelCase_							=   self.image_processor_tester.prepare_inputs(equal_resolution=A_						,							numpify=A_  )
       for image in image_inputs:
             self.assertIsInstance(A_						,							np.ndarray  )

       # Test not batched input
       lowerCamelCase_							=   image_processing(image_inputs[0]						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)

       # Test batched
       lowerCamelCase_							=   image_processing(A_						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)





 def     a__			(    self :  str  )       ->     Dict:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       # create random PyTorch tensors
       lowerCamelCase_							=   self.image_processor_tester.prepare_inputs(equal_resolution=A_						,							torchify=A_  )
       for image in image_inputs:
             self.assertIsInstance(A_						,							torch.Tensor  )

       # Test not batched input
       lowerCamelCase_							=   image_processing(image_inputs[0]						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               1,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)

       # Test batched
       lowerCamelCase_							=   image_processing(A_						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)







@require_torch
@require_vision
class   A(						UpperCamelCase							,				unittest.TestCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						ChineseCLIPImageProcessor if is_vision_available() else None

 def     a__			(    self :  Dict  )       ->     int:

       """simple docstring"""






       lowerCamelCase_							=   ChineseCLIPImageProcessingTester(self						,							num_channels=4						,							do_center_crop=A_  )
       lowerCamelCase_							=   3

 @property
 def     a__			(    self :  Any  )       ->     int:

       """simple docstring"""






       return self.image_processor_tester.prepare_image_processor_dict()

 def     a__			(    self :  Dict  )       ->     Tuple:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       self.assertTrue(hasattr(A_						,							'do_resize'  )  )
       self.assertTrue(hasattr(A_						,							'size'  )  )
       self.assertTrue(hasattr(A_						,							'do_center_crop'  )  )
       self.assertTrue(hasattr(A_						,							'center_crop'  )  )
       self.assertTrue(hasattr(A_						,							'do_normalize'  )  )
       self.assertTrue(hasattr(A_						,							'image_mean'  )  )
       self.assertTrue(hasattr(A_						,							'image_std'  )  )
       self.assertTrue(hasattr(A_						,							'do_convert_rgb'  )  )

 def     a__			(    self :  Tuple  )       ->     List[str]:

       """simple docstring"""






       pass





 def     a__			(    self :  Union[str, Any]  )       ->     int:

       """simple docstring"""






       lowerCamelCase_							=   self.image_processing_class(**self.image_processor_dict  )
       # create random PIL images
       lowerCamelCase_							=   self.image_processor_tester.prepare_inputs(equal_resolution=A_  )
       for image in image_inputs:
             self.assertIsInstance(A_						,							Image.Image  )

       # Test not batched input
       lowerCamelCase_							=   image_processing(image_inputs[0]						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               1,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)

       # Test batched
       lowerCamelCase_							=   image_processing(A_						,							return_tensors='pt'  ).pixel_values
       self.assertEqual(
           encoded_images.shape						,							(
               self.image_processor_tester.batch_size,
               self.expected_encoded_image_num_channels,
               self.image_processor_tester.crop_size['height'],
               self.image_processor_tester.crop_size['width'],
           )						,							)



