


"""simple docstring"""







import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


__lowercase    =				logging.getLogger(__name__)

__lowercase    =				tf.data.AUTOTUNE


def 					lowercase      (    )->				Union[str, Any]:
  '''simple docstring'''







  a						:		Optional[Any]											=      argparse.ArgumentParser(description="Train a masked language model on TPU."	)
  parser.add_argument(
      "--pretrained_model_config"  ,       type=A_  ,       default="roberta-base"  ,       help="The model config to use. Note that we don't copy the model's weights, only the config!"  ,       )
  parser.add_argument(
      "--tokenizer"  ,       type=A_  ,       default="unigram-tokenizer-wikitext"  ,       help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."  ,       )

  parser.add_argument(
      "--per_replica_batch_size"  ,       type=A_  ,       default=8  ,       help="Batch size per TPU core."  ,       )

  parser.add_argument(
      "--no_tpu"  ,       action="store_true"  ,       help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."  ,       )

  parser.add_argument(
      "--tpu_name"  ,       type=A_  ,       help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."  ,       default="local"  ,       )

  parser.add_argument(
      "--tpu_zone"  ,       type=A_  ,       help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."  ,       )

  parser.add_argument(
      "--gcp_project"  ,       type=A_  ,       help="Google cloud project name. Only used for non-Colab TPU nodes."	)

  parser.add_argument(
      "--bfloat16"  ,       action="store_true"  ,       help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."  ,       )

  parser.add_argument(
      "--train_dataset"  ,       type=A_  ,       help="Path to training dataset to load. If the path begins with `gs://`"
      " then the dataset will be loaded from a Google Cloud Storage bucket."  ,       )

  parser.add_argument(
      "--shuffle_buffer_size"  ,       type=A_  ,       default=2**18  ,       help="Size of the shuffle buffer (in samples)"  ,       )

  parser.add_argument(
      "--eval_dataset"  ,       type=A_  ,       help="Path to evaluation dataset to load. If the path begins with `gs://`"
      " then the dataset will be loaded from a Google Cloud Storage bucket."  ,       )

  parser.add_argument(
      "--num_epochs"  ,       type=A_  ,       default=1  ,       help="Number of epochs to train for."  ,       )

  parser.add_argument(
      "--learning_rate"  ,       type=A_  ,       default=1e-4  ,       help="Learning rate to use for training."  ,       )

  parser.add_argument(
      "--weight_decay_rate"  ,       type=A_  ,       default=1e-3  ,       help="Weight decay rate to use for training."  ,       )

  parser.add_argument(
      "--max_length"  ,       type=A_  ,       default=512  ,       help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"  ,       )

  parser.add_argument(
      "--mlm_probability"  ,       type=A_  ,       default=0.1_5  ,       help="Fraction of tokens to mask during training."  ,       )

  parser.add_argument("--output_dir"  ,       type=A_  ,       required=A_  ,       help="Path to save model checkpoints to."	)
  parser.add_argument("--hub_model_id"  ,       type=A_  ,       help="Model ID to upload to on the Hugging Face Hub."	)

  a						:		str											=      parser.parse_args()
  return args


def 					lowercase      (    A_	)->				str:
  '''simple docstring'''







  try:
    if args.tpu_name:
      a						:		Dict											=      tf.distribute.cluster_resolver.TPUClusterResolver(
          args.tpu_name  ,       zone=args.tpu_zone  ,       project=args.gcp_project	)
    else:
      a						:		Any											=      tf.distribute.cluster_resolver.TPUClusterResolver()
  except ValueError:
    raise RuntimeError(
        "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
        "--gcp_project. When running on a TPU VM, use --tpu_name local."	)

  tf.config.experimental_connect_to_cluster(A_	)
  tf.tpu.experimental.initialize_tpu_system(A_	)

  return tpu


def 					lowercase      (    A_	)->				Union[str, Any]:
  '''simple docstring'''







  a						:		Union[str, Any]											=      0
  for file in file_list:
    a						:		str											=      file.split("/"	)[-1]
    a						:		List[Any]											=      re.search(R"-\d+-(\d+)\.tfrecord"  ,       A_	).group(1	)
    a						:		Dict											=      int(A_	)
    num_samples += sample_count

  return num_samples


def 					lowercase      (    A_  ,       A_  ,       A_  ,       A_  ,       A_  ,       A_=None	)->				List[Any]:
  '''simple docstring'''







  a						:		Any											=      count_samples(A_	)
  a						:		List[Any]											=      tf.data.Dataset.from_tensor_slices(A_	)
  if shuffle:
    a						:		int											=      dataset.shuffle(len(A_	)	)
  a						:		Optional[int]											=      tf.data.TFRecordDataset(A_  ,       num_parallel_reads=A_	)
  # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
  a						:		Dict											=      dataset.apply(tf.data.experimental.assert_cardinality(A_	)	)
  a						:		Any											=      dataset.map(A_  ,       num_parallel_calls=A_	)
  if shuffle:
    assert shuffle_buffer_size is not None
    a						:		Dict											=      dataset.shuffle(args.shuffle_buffer_size	)
  a						:		Tuple											=      dataset.batch(A_  ,       drop_remainder=A_	)
  a						:		List[Any]											=      dataset.map(A_  ,       num_parallel_calls=A_	)
  a						:		int											=      dataset.prefetch(A_	)
  return dataset


def 					lowercase      (    A_	)->				Any:
  '''simple docstring'''







  if not args.no_tpu:
    a						:		Any											=      initialize_tpu(A_	)
    a						:		Any											=      tf.distribute.TPUStrategy(A_	)
  else:
    a						:		str											=      tf.distribute.OneDeviceStrategy(device="/gpu:0"	)

  if args.bfloataa:
    tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"	)

  a						:		List[Any]											=      AutoTokenizer.from_pretrained(args.tokenizer	)
  a						:		List[Any]											=      AutoConfig.from_pretrained(args.pretrained_model_config	)
  a						:		int											=      tokenizer.vocab_size

  a						:		Union[str, Any]											=      tf.io.gfile.glob(os.path.join(args.train_dataset  ,       "*.tfrecord"	)	)
  if not training_records:
    raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.'''	)
  a						:		List[str]											=      tf.io.gfile.glob(os.path.join(args.eval_dataset  ,       "*.tfrecord"	)	)
  if not eval_records:
    raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.'''	)

  a						:		int											=      count_samples(A_	)

  a						:		Union[str, Any]											=      num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
  a						:		int											=      steps_per_epoch * args.num_epochs

  with strategy.scope():
    a						:		Tuple											=      TFAutoModelForMaskedLM.from_config(A_	)
    model(model.dummy_inputs	)  # Pass some dummy inputs through the model to ensure all the weights are built
    a						:		int											=      create_optimizer(
        num_train_steps=A_  ,       num_warmup_steps=total_train_steps // 20  ,       init_lr=args.learning_rate  ,       weight_decay_rate=args.weight_decay_rate  ,       )

    # Transformers models compute the right loss for their task by default when labels are passed, and will
    # use this for training unless you specify your own loss function in compile().
    model.compile(optimizer=A_  ,       metrics=["accuracy"]	)

  def decode_fn(A_	):
    a						:		List[Any]											=      {
        "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa  ,       shape=(args.max_length,)	),
        "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa  ,       shape=(args.max_length,)	),
    }
    return tf.io.parse_single_example(A_  ,       A_	)

  # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
  # use their methods in our data pipeline.
  a						:		List[Any]											=      DataCollatorForLanguageModeling(
      tokenizer=A_  ,       mlm_probability=args.mlm_probability  ,       mlm=A_  ,       return_tensors="tf"	)

  def mask_with_collator(A_	):
    # TF really needs an isin() function
    a						:		str											=      (
        ~tf.cast(batch["attention_mask"]  ,       tf.bool	)
        | (batch["input_ids"] == tokenizer.cls_token_id)
        | (batch["input_ids"] == tokenizer.sep_token_id)
    )
    a						:		str											=      data_collator.tf_mask_tokens(
        batch["input_ids"]  ,       vocab_size=len(A_	)  ,       mask_token_id=tokenizer.mask_token_id  ,       special_tokens_mask=A_  ,       )
    return batch

  a						:		List[Any]											=      args.per_replica_batch_size * strategy.num_replicas_in_sync

  a						:		Tuple											=      prepare_dataset(
      A_  ,       decode_fn=A_  ,       mask_fn=A_  ,       batch_size=A_  ,       shuffle=A_  ,       shuffle_buffer_size=args.shuffle_buffer_size  ,       )

  a						:		Union[str, Any]											=      prepare_dataset(
      A_  ,       decode_fn=A_  ,       mask_fn=A_  ,       batch_size=A_  ,       shuffle=A_  ,       )

  a						:		Any											=      []
  if args.hub_model_id:
    callbacks.append(
        PushToHubCallback(output_dir=args.output_dir  ,       hub_model_id=args.hub_model_id  ,       tokenizer=A_	)	)

  model.fit(
      A_  ,       validation_data=A_  ,       epochs=args.num_epochs  ,       callbacks=A_  ,       )

  model.save_pretrained(args.output_dir	)


if __name__ == "__main__":
     __lowercase    =				parse_args()
     main(args)










"""simple docstring"""







import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from timm.data import resolve_data_config
from timm.data.transforms_factory import create_transform

from transformers import (
    BitConfig,
    ViTHybridConfig,
    ViTHybridForImageClassification,
    ViTHybridImageProcessor,
    ViTHybridModel,
)
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
__lowercase    =				logging.get_logger(__name__)


def 					lowercase      (    A_  ,       A_=False	)->				int:
  '''simple docstring'''







  a						:		List[str]											=      []

  # fmt: off
  # stem:
  rename_keys.append(("cls_token", "vit.embeddings.cls_token")	)
  rename_keys.append(("pos_embed", "vit.embeddings.position_embeddings")	)

  rename_keys.append(("patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight")	)
  rename_keys.append(("patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias")	)

  # backbone
  rename_keys.append(("patch_embed.backbone.stem.conv.weight", "vit.embeddings.patch_embeddings.backbone.bit.embedder.convolution.weight")	)
  rename_keys.append(("patch_embed.backbone.stem.norm.weight", "vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.weight")	)
  rename_keys.append(("patch_embed.backbone.stem.norm.bias", "vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.bias")	)

  for stage_idx in range(len(config.backbone_config.depths	)	):
    for layer_idx in range(config.backbone_config.depths[stage_idx]	):
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv1.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.bias''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv2.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.bias''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv3.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.weight''')	)
      rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.bias''')	)

    rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.conv.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.conv.weight''')	)
    rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.weight''')	)
    rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.bias''')	)

  # transformer encoder
  for i in range(config.num_hidden_layers	):
    # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
    rename_keys.append((F'''blocks.{i}.norm1.weight''', F'''vit.encoder.layer.{i}.layernorm_before.weight''')	)
    rename_keys.append((F'''blocks.{i}.norm1.bias''', F'''vit.encoder.layer.{i}.layernorm_before.bias''')	)
    rename_keys.append((F'''blocks.{i}.attn.proj.weight''', F'''vit.encoder.layer.{i}.attention.output.dense.weight''')	)
    rename_keys.append((F'''blocks.{i}.attn.proj.bias''', F'''vit.encoder.layer.{i}.attention.output.dense.bias''')	)
    rename_keys.append((F'''blocks.{i}.norm2.weight''', F'''vit.encoder.layer.{i}.layernorm_after.weight''')	)
    rename_keys.append((F'''blocks.{i}.norm2.bias''', F'''vit.encoder.layer.{i}.layernorm_after.bias''')	)
    rename_keys.append((F'''blocks.{i}.mlp.fc1.weight''', F'''vit.encoder.layer.{i}.intermediate.dense.weight''')	)
    rename_keys.append((F'''blocks.{i}.mlp.fc1.bias''', F'''vit.encoder.layer.{i}.intermediate.dense.bias''')	)
    rename_keys.append((F'''blocks.{i}.mlp.fc2.weight''', F'''vit.encoder.layer.{i}.output.dense.weight''')	)
    rename_keys.append((F'''blocks.{i}.mlp.fc2.bias''', F'''vit.encoder.layer.{i}.output.dense.bias''')	)

  if base_model:
    # layernorm + pooler
    rename_keys.extend(
        [
            ("norm.weight", "layernorm.weight"),
            ("norm.bias", "layernorm.bias"),
            ("pre_logits.fc.weight", "pooler.dense.weight"),
            ("pre_logits.fc.bias", "pooler.dense.bias"),
        ]	)

    # if just the base model, we should remove "vit" from all keys that start with "vit"
    a						:		Tuple											=      [(pair[0], pair[1][4:]) if pair[1].startswith("vit"	) else pair for pair in rename_keys]
  else:
    # layernorm + classification head
    rename_keys.extend(
        [
            ("norm.weight", "vit.layernorm.weight"),
            ("norm.bias", "vit.layernorm.bias"),
            ("head.weight", "classifier.weight"),
            ("head.bias", "classifier.bias"),
        ]	)
  # fmt: on

  return rename_keys


def 					lowercase      (    A_  ,       A_  ,       A_=False	)->				Optional[Any]:
  '''simple docstring'''







  for i in range(config.num_hidden_layers	):
    if base_model:
      a						:		Tuple											=      ""
    else:
      a						:		Dict											=      "vit."
    # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
    a						:		str											=      state_dict.pop(F'''blocks.{i}.attn.qkv.weight'''	)
    a						:		List[Any]											=      state_dict.pop(F'''blocks.{i}.attn.qkv.bias'''	)
    # next, add query, keys and values (in that order) to the state dict
    a						:		Optional[int]											=      in_proj_weight[
        : config.hidden_size, :
    ]
    a						:		Tuple											=      in_proj_bias[: config.hidden_size]
    a						:		List[Any]											=      in_proj_weight[
        config.hidden_size : config.hidden_size * 2, :
    ]
    a						:		Optional[int]											=      in_proj_bias[
        config.hidden_size : config.hidden_size * 2
    ]
    a						:		int											=      in_proj_weight[
        -config.hidden_size :, :
    ]
    a						:		int											=      in_proj_bias[-config.hidden_size :]


def 					lowercase      (    A_	)->				Dict:
  '''simple docstring'''







  a						:		Dict											=      ["head.weight", "head.bias"]
  for k in ignore_keys:
    state_dict.pop(A_  ,       A_	)


def 					lowercase      (    A_  ,       A_  ,       A_	)->				List[Any]:
  '''simple docstring'''







  a						:		List[Any]											=      dct.pop(A_	)
  a						:		str											=      val


def 					lowercase      (    )->				List[Any]:
  '''simple docstring'''







  a						:		Any											=      "http://images.cocodataset.org/val2017/000000039769.jpg"
  a						:		Tuple											=      Image.open(requests.get(A_  ,       stream=A_	).raw	)
  return im


@torch.no_grad()
def 					lowercase      (    A_  ,       A_  ,       A_=False	)->				Union[str, Any]:
  '''simple docstring'''







  a						:		Optional[Any]											=      BitConfig(
      global_padding="same"  ,       layer_type="bottleneck"  ,       depths=(3, 4, 9)  ,       out_features=["stage3"]  ,       embedding_dynamic_padding=A_  ,       )
  a						:		Union[str, Any]											=      ViTHybridConfig(backbone_config=A_  ,       image_size=384  ,       num_labels=1_000	)
  a						:		Optional[Any]											=      False

  # load original model from timm
  a						:		Any											=      timm.create_model(A_  ,       pretrained=A_	)
  timm_model.eval()

  # load state_dict of original model, remove and rename some keys
  a						:		Optional[Any]											=      timm_model.state_dict()
  if base_model:
    remove_classification_head_(A_	)
  a						:		int											=      create_rename_keys(A_  ,       A_	)
  for src, dest in rename_keys:
    rename_key(A_  ,       A_  ,       A_	)
  read_in_q_k_v(A_  ,       A_  ,       A_	)

  a						:		Union[str, Any]											=      "huggingface/label-files"
  a						:		Optional[int]											=      "imagenet-1k-id2label.json"
  a						:		str											=      json.load(open(hf_hub_download(A_  ,       A_  ,       repo_type="dataset"	)  ,       "r"	)	)
  a						:		Optional[Any]											=      {int(A_	): v for k, v in idalabel.items()}
  a						:		str											=      idalabel
  a						:		Any											=      {v: k for k, v in idalabel.items()}

  # load HuggingFace model
  if vit_name[-5:] == "in21k":
    a						:		List[Any]											=      ViTHybridModel(A_	).eval()
  else:
    a						:		Optional[int]											=      ViTHybridForImageClassification(A_	).eval()
  model.load_state_dict(A_	)

  # create image processor
  a						:		Tuple											=      create_transform(**resolve_data_config({}  ,       model=A_	)	)
  a						:		List[Any]											=      transform.transforms

  a						:		int											=      {
      "bilinear": PILImageResampling.BILINEAR,
      "bicubic": PILImageResampling.BICUBIC,
      "nearest": PILImageResampling.NEAREST,
  }

  a						:		str											=      ViTHybridImageProcessor(
      do_resize=A_  ,       size={"shortest_edge": timm_transforms[0].size}  ,       resample=pillow_resamplings[timm_transforms[0].interpolation.value]  ,       do_center_crop=A_  ,       crop_size={"height": timm_transforms[1].size[0], "width": timm_transforms[1].size[1]}  ,       do_normalize=A_  ,       image_mean=timm_transforms[-1].mean.tolist()  ,       image_std=timm_transforms[-1].std.tolist()  ,       )

  a						:		List[Any]											=      prepare_img()
  a						:		Optional[Any]											=      transform(A_	).unsqueeze(0	)
  a						:		str											=      processor(A_  ,       return_tensors="pt"	).pixel_values

  # verify pixel values
  assert torch.allclose(A_  ,       A_	)

  # verify logits
  with torch.no_grad():
    a						:		Dict											=      model(A_	)
    a						:		Tuple											=      outputs.logits

  print("Predicted class:"  ,       logits.argmax(-1	).item()	)
  if base_model:
    a						:		str											=      timm_model.forward_features(A_	)
    assert timm_pooled_output.shape == outputs.pooler_output.shape
    assert torch.allclose(A_  ,       outputs.pooler_output  ,       atol=1e-3	)
  else:
    a						:		int											=      timm_model(A_	)
    assert timm_logits.shape == outputs.logits.shape
    assert torch.allclose(A_  ,       outputs.logits  ,       atol=1e-3	)
  print("Looks ok!"	)

  if pytorch_dump_folder_path is not None:
    Path(A_	).mkdir(exist_ok=A_	)
    print(F'''Saving model {vit_name} to {pytorch_dump_folder_path}'''	)
    model.save_pretrained(A_	)
    print(F'''Saving processor to {pytorch_dump_folder_path}'''	)
    processor.save_pretrained(A_	)

  if push_to_hub:
    print(F'''Pushing model and processor to the hub {vit_name}'''	)
    model.push_to_hub(F'''ybelkada/{vit_name}'''	)
    processor.push_to_hub(F'''ybelkada/{vit_name}'''	)


if __name__ == "__main__":
 __lowercase    =				argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--vit_name""",
     default="""vit_base_r50_s16_384""",
     type=str,
     help="""Name of the hybrid ViT timm model you'd like to convert.""",
 )
 parser.add_argument(
     """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
 )
 parser.add_argument(
     """--push_to_hub""", action="""store_true""", help="""Whether to upload the model to the HuggingFace hub."""
 )

 __lowercase    =				parser.parse_args()
 convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path, args.push_to_hub)






