





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


_snake_case           =			["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :int   ,							_lowercase  :Optional[Any]   ,							_lowercase  :List[str]   ,							_lowercase  :int=None   ,							_lowercase  :Any=1 ):






     '''simple docstring'''





     lowercase__			       =  tokenizer
     lowercase__			       =  dataset
     lowercase__			       =  len(_lowercase ) if n_tasks is None else n_tasks
     lowercase__			       =  n_copies







  def __iter__(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  []
     for task in range(self.n_tasks ):
        # without strip, the model generate commented codes ...
        prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip() )
     lowercase__			       =  self.tokenizer(_lowercase   ,							padding=_lowercase   ,							return_tensors="pt" )
     for task in range(self.n_tasks ):
        for _ in range(self.n_copies ):
           yield {
               "ids": outputs.input_ids[task],
               "task_id": task,
               "input_len": outputs.attention_mask[task].sum(),
           }





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :Optional[Any]   ,							_lowercase  :int   ,							_lowercase  :List[str]   ,							_lowercase  :Any ):






     '''simple docstring'''





     lowercase__			       =  start_length
     lowercase__			       =  eof_strings
     lowercase__			       =  tokenizer







  def __call__(      self  :str   ,							_lowercase  :Dict   ,							_lowercase  :int   ,							**_lowercase  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  self.tokenizer.batch_decode(input_ids[:, self.start_length :] )
     lowercase__			       =  []
     for decoded_generation in decoded_generations:
        done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings ) )
     return all(_lowercase )


def _A       (   __magic_name__	):
   lowercase__			       =  re.split("(%s)" % "|".join(__magic_name__	)  , __magic_name__	)
   # last string should be ""
   return "".join(string_list[:-2]	)


def _A       (   __magic_name__  , __magic_name__  , __magic_name__  , __magic_name__  , __magic_name__  , __magic_name__=20  , **__magic_name__	):
   lowercase__			       =  defaultdict(__magic_name__	)  # dict of list of generated tokens
   for step, batch in tqdm(enumerate(__magic_name__	)	):
      with torch.no_grad():
         lowercase__			       =  batch["ids"].shape[-1]
         lowercase__			       =  accelerator.unwrap_model(__magic_name__	).generate(
             input_ids=batch["ids"][:, : batch["input_len"]]  , num_return_sequences=__magic_name__  , **__magic_name__	)
         # each task is generated batch_size times
         lowercase__			       =  batch["task_id"].repeat(__magic_name__	)
         lowercase__			       =  accelerator.pad_across_processes(
             __magic_name__  , dim=1  , pad_index=tokenizer.pad_token_id	)

         lowercase__       ,							lowercase__			       =  accelerator.gather((generated_tokens, generated_tasks)	)
         lowercase__			       =  generated_tokens.cpu().numpy()
         lowercase__			       =  generated_tasks.cpu().numpy()

         for task, generated_tokens in zip(__magic_name__  , __magic_name__	):
            gen_token_dict[task].append(__magic_name__	)

   lowercase__			       =  [[] for _ in range(__magic_name__	)]
   for task, generated_tokens in gen_token_dict.items():
      for s in generated_tokens:
         lowercase__			       =  tokenizer.decode(__magic_name__  , skip_special_tokens=__magic_name__  , clean_up_tokenization_spaces=__magic_name__	)
         code_gens[task].append(remove_last_block(__magic_name__	)	)
   return code_gens


def _A       (   ):
   # Setup configuration
   lowercase__			       =  HfArgumentParser(__magic_name__	)
   lowercase__			       =  parser.parse_args()

   transformers.logging.set_verbosity_error()
   # enables code execution in code_eval metric
   lowercase__			       =  args.HF_ALLOW_CODE_EVAL
   # make sure tokenizer plays nice with multiprocessing
   lowercase__			       =  "false"

   if args.num_workers is None:
      lowercase__			       =  multiprocessing.cpu_count()

   # Use dataset load to feed to accelerate
   lowercase__			       =  Accelerator()
   set_seed(args.seed  , device_specific=__magic_name__	)

   # Load model and tokenizer
   lowercase__			       =  AutoTokenizer.from_pretrained(args.model_ckpt	)
   lowercase__			       =  tokenizer.eos_token
   lowercase__			       =  AutoModelForCausalLM.from_pretrained(args.model_ckpt	)

   # Generation settings
   lowercase__			       =  {
       "do_sample": args.do_sample,
       "temperature": args.temperature,
       "max_new_tokens": args.max_new_tokens,
       "top_p": args.top_p,
       "top_k": args.top_k,
       "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0  , __magic_name__  , __magic_name__	)]	),
   }

   # Load evaluation dataset and metric
   lowercase__			       =  load_dataset("openai_humaneval"	)
   lowercase__			       =  load_metric("code_eval"	)

   lowercase__			       =  args.num_tasks if args.num_tasks is not None else len(human_eval["test"]	)
   lowercase__			       =  args.n_samples // args.batch_size

   lowercase__			       =  TokenizedDataset(__magic_name__  , human_eval["test"]  , n_copies=__magic_name__  , n_tasks=__magic_name__	)
   # do not confuse args.batch_size, which is actually the num_return_sequences
   lowercase__			       =  DataLoader(__magic_name__  , batch_size=1	)

   # Run a quick test to see if code evaluation is enabled
   try:
      lowercase__			       =  code_eval_metric.compute(references=[""]  , predictions=[[""]]	)
   except ValueError as exception:
      print(
          "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
          " flag to enable code evaluation."	)
      raise exception

   lowercase__       ,							lowercase__			       =  accelerator.prepare(__magic_name__  , __magic_name__	)

   lowercase__			       =  complete_code(
       __magic_name__  , __magic_name__  , __magic_name__  , __magic_name__  , n_tasks=__magic_name__  , batch_size=args.batch_size  , **__magic_name__  , )

   if accelerator.is_main_process:
      lowercase__			       =  []

      for task in tqdm(range(__magic_name__	)	):
         lowercase__			       =  human_eval["test"][task]["test"]
         lowercase__			       =  f'''check({human_eval['test'][task]['entry_point']})'''
         references.append("\n" + test_func + "\n" + entry_point	)

      # Evaluate completions with "code_eval" metric
      lowercase__       ,							lowercase__			       =  code_eval_metric.compute(
          references=__magic_name__  , predictions=__magic_name__  , num_workers=args.num_workers	)
      print(f'''Results: {pass_at_k}'''	)

      # Save results to json file
      with open(args.output_file  , "w"	) as fp:
         json.dump(__magic_name__  , __magic_name__	)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
  main()













import argparse
import os
import re

import torch
from flax.traverse_util import flatten_dict
from tax import checkpoints

from transformers import (
    AutoTokenizer,
    PixaStructConfig,
    PixaStructForConditionalGeneration,
    PixaStructImageProcessor,
    PixaStructProcessor,
    PixaStructTextConfig,
    PixaStructVisionConfig,
)


def _A       (   __magic_name__	):
   lowercase__			       =  checkpoints.load_tax_checkpoint(__magic_name__	)
   lowercase__			       =  flatten_dict(__magic_name__	)
   return flax_params


def _A       (   __magic_name__	):
   lowercase__			       =  {}

   lowercase__			       =  {
       "token_embedder": "embeddings",
       "encoder_norm": "layernorm",
       "kernel": "weight",
       ".out": ".output",
       "scale": "weight",
       "embedders_0.pos_embedding": "row_embedder.weight",
       "embedders_1.pos_embedding": "column_embedder.weight",
   }

   lowercase__			       =  {
       "query": "attention.query",
       "key": "attention.key",
       "value": "attention.value",
       "output.dense": "output",
       "encoder_decoder_attention.o": "encoder_decoder_attention.attention.o",
       "pre_self_attention_layer_norm": "self_attention.layer_norm",
       "pre_cross_attention_layer_norm": "encoder_decoder_attention.layer_norm",
       "mlp.": "mlp.DenseReluDense.",
       "pre_mlp_layer_norm": "mlp.layer_norm",
       "self_attention.o": "self_attention.attention.o",
       "decoder.embeddings.embedding": "decoder.embed_tokens.weight",
       "decoder.relpos_bias.rel_embedding": "decoder.layer.0.self_attention.attention.relative_attention_bias.weight",
       "decoder.decoder_norm.weight": "decoder.final_layer_norm.weight",
       "decoder.logits_dense.weight": "decoder.lm_head.weight",
   }

   for key in flax_dict.keys():
      if "target" in key:
         # remove the first prefix from the key
         lowercase__			       =  ".".join(key[1:]	)

         # rename the key
         for old, new in CONVERSION_MAPPING.items():
            lowercase__			       =  new_key.replace(__magic_name__  , __magic_name__	)

         if "decoder" in new_key:
            for old, new in DECODER_CONVERSION_MAPPING.items():
               lowercase__			       =  new_key.replace(__magic_name__  , __magic_name__	)

         if "layers" in new_key and "decoder" not in new_key:
            # use regex to replace the layer number
            lowercase__			       =  re.sub(R"layers_(\d+)"  , R"layer.\1"  , __magic_name__	)
            lowercase__			       =  new_key.replace("encoder"  , "encoder.encoder"	)

         elif "layers" in new_key and "decoder" in new_key:
            # use regex to replace the layer number
            lowercase__			       =  re.sub(R"layers_(\d+)"  , R"layer.\1"  , __magic_name__	)

         lowercase__			       =  flax_dict[key]

   lowercase__			       =  {}
   # convert converted_dict into torch format
   for key in converted_dict.keys():
      if ("embed_tokens" not in key) and ("embedder" not in key):
         lowercase__			       =  torch.from_numpy(converted_dict[key].T	)
      else:
         lowercase__			       =  torch.from_numpy(converted_dict[key]	)

   return converted_torch_dict


def _A       (   __magic_name__  , __magic_name__  , __magic_name__=False  , __magic_name__=False	):
   lowercase__			       =  get_flax_param(__magic_name__	)

   if not use_large:
      lowercase__			       =  PixaStructVisionConfig()
      lowercase__			       =  PixaStructTextConfig()
   else:
      lowercase__			       =  PixaStructVisionConfig(
          hidden_size=1536  , d_ff=3968  , num_attention_heads=24  , num_hidden_layers=18	)
      lowercase__			       =  PixaStructTextConfig(hidden_size=1536  , d_ff=3968  , num_heads=24  , num_layers=18	)
   lowercase__			       =  PixaStructConfig(
       vision_config=encoder_config.to_dict()  , text_config=decoder_config.to_dict()  , is_vqa=__magic_name__	)

   lowercase__			       =  PixaStructForConditionalGeneration(__magic_name__	)

   lowercase__			       =  rename_and_convert_flax_params(__magic_name__	)
   model.load_state_dict(__magic_name__	)

   lowercase__			       =  AutoTokenizer.from_pretrained("ybelkada/test-pix2struct-tokenizer"	)
   lowercase__			       =  PixaStructImageProcessor()
   lowercase__			       =  PixaStructProcessor(image_processor=__magic_name__  , tokenizer=__magic_name__	)

   if use_large:
      lowercase__			       =  4096

   lowercase__			       =  True

   # mkdir if needed
   os.makedirs(__magic_name__  , exist_ok=__magic_name__	)

   model.save_pretrained(__magic_name__	)
   processor.save_pretrained(__magic_name__	)

   print("Model saved in {}".format(__magic_name__	)	)


if __name__ == "__main__":
  _snake_case           =			argparse.ArgumentParser()
  parser.add_argument("""--t5x_checkpoint_path""", default=None, type=str, help="""Path to the original T5x checkpoint.""")
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  parser.add_argument("""--use_large""", action="""store_true""", help="""Use large model.""")
  parser.add_argument("""--is_vqa""", action="""store_true""", help="""Use large model.""")
  _snake_case           =			parser.parse_args()

  convert_pixastruct_original_pytorch_checkpoint_to_hf(
      args.tax_checkpoint_path, args.pytorch_dump_folder_path, args.use_large
  )






