






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase     :							int          =logging.getLogger(__name__)







def 		SCREAMING_SNAKE_CASE							( )			->     Tuple:
     UpperCamelCase__      :       int	    =  argparse.ArgumentParser(
         description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."      )
     parser.add_argument(
         "--dataset_name" ,   type=__lowerCAmelCase ,   default="wikitext" ,   help="Name of the training. Explore datasets at: hf.co/datasets." ,   )
     parser.add_argument(
         "--dataset_config" ,   type=__lowerCAmelCase ,   default="wikitext-103-raw-v1" ,   help="Configuration name of the dataset."      )
     parser.add_argument(
         "--tokenizer_name_or_path" ,   type=__lowerCAmelCase ,   default="sayakpaul/unigram-tokenizer-wikitext" ,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier." ,   )
     parser.add_argument(
         "--shard_size" ,   type=__lowerCAmelCase ,   default=1000 ,   help="Number of entries to go in a single shard." ,   )
     parser.add_argument("--split" ,   type=__lowerCAmelCase ,   default="train" ,   choices=["train", "test", "validation"]      )
     parser.add_argument(
         "--limit" ,   default=__lowerCAmelCase ,   type=__lowerCAmelCase ,   help="Limit the number of shards (used for debugging)." ,   )
     parser.add_argument(
         "--max_length" ,   type=__lowerCAmelCase ,   default=512 ,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
         " sequence length that is a multiple of 8." ,   )
     parser.add_argument(
         "--output_dir" ,   default="tf-tpu" ,   type=__lowerCAmelCase ,   help="Output directory where the TFRecord shards will be saved. If the"
         " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
         " shards will be directly saved to a Google Cloud Storage bucket." ,   )

     UpperCamelCase__      :       Any	    =  parser.parse_args()
     return args







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     str:
     def fn(__lowerCAmelCase      ):
          return tokenizer(examples["text"]      )

     return fn







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     Tuple:
     UpperCamelCase__      :       Any	    =  []
     for i in range(len(tokenized_data["input_ids"]      )      ):
          UpperCamelCase__      :       Dict	    =  {
              "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]      )      ),
              "attention_mask": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]      )      ),
          }
          UpperCamelCase__      :       Dict	    =  tf.train.Features(feature=__lowerCAmelCase      )
          UpperCamelCase__      :       List[Any]	    =  tf.train.Example(features=__lowerCAmelCase      )
          UpperCamelCase__      :       Union[str, Any]	    =  example.SerializeToString()
          records.append(__lowerCAmelCase      )
     return records







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase      )			->     str:
     UpperCamelCase__      :       Any	    =  datasets.load_dataset(args.dataset_name ,   args.dataset_config ,   split=args.split      )

     if args.limit is not None:
          UpperCamelCase__      :       Any	    =  min(len(__lowerCAmelCase      ) ,   args.limit      )
          UpperCamelCase__      :       Tuple	    =  dataset.select(range(__lowerCAmelCase      )      )
          print(f'Limiting the dataset to {args.limit} entries.'      )

     UpperCamelCase__      :       Dict	    =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir      ):
               os.makedirs(args.output_dir      )
          UpperCamelCase__      :       Any	    =  os.path.join(args.output_dir ,   args.split      )
          if not os.path.exists(__lowerCAmelCase      ):
               os.makedirs(__lowerCAmelCase      )
     else:
          UpperCamelCase__      :       List[str]	    =  os.path.join(args.output_dir ,   args.split      )

     # Tokenize the whole dataset at once.
     UpperCamelCase__      :       List[str]	    =  tokenize_function(__lowerCAmelCase      )
     UpperCamelCase__      :       Optional[int]	    =  dataset.map(__lowerCAmelCase ,   batched=__lowerCAmelCase ,   num_proc=4 ,   remove_columns=["text"]      )

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(__lowerCAmelCase      ):
          # Concatenate all texts.
          UpperCamelCase__      :       Optional[int]	    =  {k: sum(examples[k] ,   []      ) for k in examples.keys()}
          UpperCamelCase__      :       Union[str, Any]	    =  len(concatenated_examples[list(examples.keys()      )[0]]      )
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          UpperCamelCase__      :       Any	    =  (total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          UpperCamelCase__      :       Tuple	    =  {
              k: [t[i : i + args.max_length] for i in range(0 ,   __lowerCAmelCase ,   args.max_length      )]
              for k, t in concatenated_examples.items()
          }
          return result

     UpperCamelCase__      :       List[Any]	    =  dataset_tokenized.map(__lowerCAmelCase ,   batched=__lowerCAmelCase ,   batch_size=1000 ,   num_proc=4      )

     UpperCamelCase__      :       Tuple	    =  0
     UpperCamelCase__      :       Optional[Any]	    =  0
     for shard in range(0 ,   len(__lowerCAmelCase      ) ,   args.shard_size      ):
          UpperCamelCase__      :       Optional[Any]	    =  grouped_dataset[shard : shard + args.shard_size]
          UpperCamelCase__      :       Tuple	    =  len(dataset_snapshot["input_ids"]      )
          UpperCamelCase__      :       str	    =  os.path.join(__lowerCAmelCase ,   f'dataset-{shard_count}-{records_containing}.tfrecord'      )
          UpperCamelCase__      :       Optional[Any]	    =  get_serialized_examples(__lowerCAmelCase      )

          with tf.io.TFRecordWriter(__lowerCAmelCase      ) as out_file:
               for i in range(len(__lowerCAmelCase      )      ):
                    UpperCamelCase__      :       Union[str, Any]	    =  serialized_examples[i]
                    out_file.write(__lowerCAmelCase      )
               print("Wrote file {} containing {} records".format(__lowerCAmelCase ,   __lowerCAmelCase      )      )

          shard_count += 1
          total_records += records_containing

     with open(f'split-{args.split}-records-count.txt' ,   "w"      ) as f:
          print(f'Total {args.split} records: {total_records}' ,   file=__lowerCAmelCase      )


if __name__ == "__main__":
 lowerCamelCase     :							Any          =parse_args()
 main(args)







import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


lowerCamelCase     :							Optional[Any]    =False





class       __a  (   unittest.TestCase	):
 pass







@nightly
@require_torch_gpu
class       __a  (   unittest.TestCase	):







 def  __lowercase (		self :       Any						):


      '''simple docstring'''






      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()







 def  __lowercase (		self :       Dict						):


      '''simple docstring'''






      UpperCamelCase__      :       int	    =  VersatileDiffusionPipeline.from_pretrained("shi-labs/versatile-diffusion"							,       torch_dtype=torch.floataa						)
      pipe.to(SCREAMING_SNAKE_CASE						)
      pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Optional[int]	    =  load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg"						)

      UpperCamelCase__      :       Dict	    =  torch.manual_seed(0						)
      UpperCamelCase__      :       str	    =  pipe.dual_guided(
          prompt="first prompt"							,       image=SCREAMING_SNAKE_CASE							,       text_to_image_strength=0.7_5							,       generator=SCREAMING_SNAKE_CASE							,       guidance_scale=7.5							,       num_inference_steps=2							,       output_type="numpy"							,       ).images

      with tempfile.TemporaryDirectory() as tmpdirname:
           pipe.save_pretrained(SCREAMING_SNAKE_CASE						)
           UpperCamelCase__      :       Tuple	    =  VersatileDiffusionPipeline.from_pretrained(SCREAMING_SNAKE_CASE							,       torch_dtype=torch.floataa						)
      pipe.to(SCREAMING_SNAKE_CASE						)
      pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       int	    =  generator.manual_seed(0						)
      UpperCamelCase__      :       Dict	    =  pipe.dual_guided(
          prompt="first prompt"							,       image=SCREAMING_SNAKE_CASE							,       text_to_image_strength=0.7_5							,       generator=SCREAMING_SNAKE_CASE							,       guidance_scale=7.5							,       num_inference_steps=2							,       output_type="numpy"							,       ).images

      assert np.abs(image - new_image						).sum() < 1e-5, "Models don't have the same forward pass"







 def  __lowercase (		self :       Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  VersatileDiffusionPipeline.from_pretrained("shi-labs/versatile-diffusion"							,       torch_dtype=torch.floataa						)
      pipe.to(SCREAMING_SNAKE_CASE						)
      pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Tuple	    =  "cyberpunk 2077"
      UpperCamelCase__      :       Any	    =  load_image(
          "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg"						)
      UpperCamelCase__      :       int	    =  torch.manual_seed(0						)
      UpperCamelCase__      :       Any	    =  pipe.dual_guided(
          prompt=SCREAMING_SNAKE_CASE							,       image=SCREAMING_SNAKE_CASE							,       text_to_image_strength=0.7_5							,       generator=SCREAMING_SNAKE_CASE							,       guidance_scale=7.5							,       num_inference_steps=50							,       output_type="numpy"							,       ).images

      UpperCamelCase__      :       Tuple	    =  image[0, 2_53:2_56, 2_53:2_56, -1]

      assert image.shape == (1, 5_12, 5_12, 3)
      UpperCamelCase__      :       Optional[int]	    =  np.array([0.1_4_4_8, 0.1_6_1_9, 0.1_7_4_1, 0.1_0_8_6, 0.1_1_4_7, 0.1_1_2_8, 0.1_1_9_9, 0.1_1_6_5, 0.1_0_0_1]						)

      assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-1

      UpperCamelCase__      :       List[str]	    =  "A painting of a squirrel eating a burger "
      UpperCamelCase__      :       List[Any]	    =  torch.manual_seed(0						)
      UpperCamelCase__      :       Dict	    =  pipe.text_to_image(
          prompt=SCREAMING_SNAKE_CASE							,       generator=SCREAMING_SNAKE_CASE							,       guidance_scale=7.5							,       num_inference_steps=50							,       output_type="numpy"						).images

      UpperCamelCase__      :       Union[str, Any]	    =  image[0, 2_53:2_56, 2_53:2_56, -1]

      assert image.shape == (1, 5_12, 5_12, 3)
      UpperCamelCase__      :       Dict	    =  np.array([0.3_3_6_7, 0.3_1_6_9, 0.2_6_5_6, 0.3_8_7_0, 0.4_7_9_0, 0.3_7_9_6, 0.4_0_0_9, 0.4_8_7_8, 0.4_7_7_8]						)

      assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-1

      UpperCamelCase__      :       Any	    =  pipe.image_variation(SCREAMING_SNAKE_CASE							,       generator=SCREAMING_SNAKE_CASE							,       output_type="numpy"						).images

      UpperCamelCase__      :       Optional[Any]	    =  image[0, 2_53:2_56, 2_53:2_56, -1]

      assert image.shape == (1, 5_12, 5_12, 3)
      UpperCamelCase__      :       int	    =  np.array([0.3_0_7_6, 0.3_1_2_3, 0.3_2_8_4, 0.3_7_8_2, 0.3_7_7_0, 0.3_8_9_4, 0.4_2_9_7, 0.4_3_3_1, 0.4_4_5_6]						)

      assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-1