




'''simple docstring'''


import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
_snake_case				:				str       							=		logging.get_logger(__name__)
def 		snake_case_   (UpperCamelCase	:  Optional[int]			,						UpperCamelCase	:  List[Any]			,						UpperCamelCase	:  Dict				):





							'''simple docstring'''


							_a     					=    os.path.abspath(UpperCamelCase				)
							logger.info(f'Converting TensorFlow checkpoint from {tf_path}'				)
							# Load weights from TF model
							_a     					=    tf.train.list_variables(UpperCamelCase				)
							_a     					=    []
							_a     					=    []
							_a     					=    []
							for full_name, shape in init_vars:
														# logger.info(f"Loading TF weight {name} with shape {shape}")
														_a     					=    full_name.split('''/'''				)
														if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
																					logger.info(f'Skipping non-model layer {full_name}'				)
																					continue
														if "optimizer" in full_name:
																					logger.info(f'Skipping optimization layer {full_name}'				)
																					continue
														if name[0] == "model":
																					# ignore initial 'model'
																					_a     					=    name[1:]
														# figure out how many levels deep the name is
														_a     					=    0
														for _name in name:
																					if _name.startswith('''layer_with_weights'''				):
																												depth += 1
																					else:
																												break
														layer_depth.append(UpperCamelCase				)
														# read data
														_a     					=    tf.train.load_variable(UpperCamelCase			,						UpperCamelCase				)
														names.append('''/'''.join(UpperCamelCase				)				)
														arrays.append(UpperCamelCase				)
							logger.info(f'Read a total of {len(UpperCamelCase				):,} layers'				)

							# Sanity check
							if len(set(UpperCamelCase				)				) != 1:
														raise ValueError(f'Found layer names with different depths (layer depth {list(set(UpperCamelCase				)				)})'				)
							_a     					=    list(set(UpperCamelCase				)				)[0]
							if layer_depth != 1:
														raise ValueError(
														    '''The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP'''
														    ''' heads.'''				)

							# convert layers
							logger.info('''Converting weights...'''				)
							for full_name, array in zip(UpperCamelCase			,						UpperCamelCase				):
														_a     					=    full_name.split('''/'''				)
														_a     					=    model
														_a     					=    []
														for i, m_name in enumerate(UpperCamelCase				):
																					if m_name == ".ATTRIBUTES":
																												# variable names end with .ATTRIBUTES/VARIABLE_VALUE
																												break
																					if m_name.startswith('''layer_with_weights'''				):
																												_a     					=    int(m_name.split('''-'''				)[-1]				)
																												if layer_num <= 2:
																																			# embedding layers
																																			# layer_num 0: word_embeddings
																																			# layer_num 1: position_embeddings
																																			# layer_num 2: token_type_embeddings
																																			continue
																												elif layer_num == 3:
																																			# embedding LayerNorm
																																			trace.extend(['''embeddings''', '''LayerNorm''']				)
																																			_a     					=    getattr(UpperCamelCase			,						'''embeddings'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''LayerNorm'''				)
																												elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
																																			# encoder layers
																																			trace.extend(['''encoder''', '''layer''', str(layer_num - 4				)]				)
																																			_a     					=    getattr(UpperCamelCase			,						'''encoder'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''layer'''				)
																																			_a     					=    pointer[layer_num - 4]
																												elif layer_num == config.num_hidden_layers + 4:
																																			# pooler layer
																																			trace.extend(['''pooler''', '''dense''']				)
																																			_a     					=    getattr(UpperCamelCase			,						'''pooler'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''dense'''				)
																					elif m_name == "embeddings":
																												trace.append('''embeddings'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''embeddings'''				)
																												if layer_num == 0:
																																			trace.append('''word_embeddings'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''word_embeddings'''				)
																												elif layer_num == 1:
																																			trace.append('''position_embeddings'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''position_embeddings'''				)
																												elif layer_num == 2:
																																			trace.append('''token_type_embeddings'''				)
																																			_a     					=    getattr(UpperCamelCase			,						'''token_type_embeddings'''				)
																												else:
																																			raise ValueError(f'Unknown embedding layer with name {full_name}'				)
																												trace.append('''weight'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''weight'''				)
																					elif m_name == "_attention_layer":
																												# self-attention layer
																												trace.extend(['''attention''', '''self''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''attention'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''self'''				)
																					elif m_name == "_attention_layer_norm":
																												# output attention norm
																												trace.extend(['''attention''', '''output''', '''LayerNorm''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''attention'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''output'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''LayerNorm'''				)
																					elif m_name == "_attention_output_dense":
																												# output attention dense
																												trace.extend(['''attention''', '''output''', '''dense''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''attention'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''output'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''dense'''				)
																					elif m_name == "_output_dense":
																												# output dense
																												trace.extend(['''output''', '''dense''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''output'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''dense'''				)
																					elif m_name == "_output_layer_norm":
																												# output dense
																												trace.extend(['''output''', '''LayerNorm''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''output'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''LayerNorm'''				)
																					elif m_name == "_key_dense":
																												# attention key
																												trace.append('''key'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''key'''				)
																					elif m_name == "_query_dense":
																												# attention query
																												trace.append('''query'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''query'''				)
																					elif m_name == "_value_dense":
																												# attention value
																												trace.append('''value'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''value'''				)
																					elif m_name == "_intermediate_dense":
																												# attention intermediate dense
																												trace.extend(['''intermediate''', '''dense''']				)
																												_a     					=    getattr(UpperCamelCase			,						'''intermediate'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''dense'''				)
																					elif m_name == "_output_layer_norm":
																												# output layer norm
																												trace.append('''output'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''output'''				)
																					# weights & biases
																					elif m_name in ["bias", "beta"]:
																												trace.append('''bias'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''bias'''				)
																					elif m_name in ["kernel", "gamma"]:
																												trace.append('''weight'''				)
																												_a     					=    getattr(UpperCamelCase			,						'''weight'''				)
																					else:
																												logger.warning(f'Ignored {m_name}'				)
        # for certain layers reshape is necessary
														_a     					=    '''.'''.join(UpperCamelCase				)
														if re.match(R'''(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)'''			,						UpperCamelCase				) or re.match(
														    R'''(\S+)\.attention\.output\.dense\.weight'''			,						UpperCamelCase				):
																					_a     					=    array.reshape(pointer.data.shape				)
														if "kernel" in full_name:
																					_a     					=    array.transpose()
														if pointer.shape == array.shape:
																					_a     					=    torch.from_numpy(UpperCamelCase				)
														else:
																					raise ValueError(
																					    f'Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'
																					    f' {array.shape}'				)
														logger.info(f'Successfully set variable {full_name} to PyTorch layer {trace}'				)
							return model
def 		snake_case_   (UpperCamelCase	:  Optional[int]			,						UpperCamelCase	:  Union[str, Any]			,						UpperCamelCase	:  Tuple				):





							'''simple docstring'''


							logger.info(f'Loading model based on config from {config_path}...'				)
							_a     					=    BertConfig.from_json_file(UpperCamelCase				)
							_a     					=    BertModel(UpperCamelCase				)

							# Load weights from checkpoint
							logger.info(f'Loading weights from checkpoint {tf_checkpoint_path}...'				)
							load_tfa_weights_in_bert(UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase				)

							# Save pytorch-model
							logger.info(f'Saving PyTorch model to {pytorch_dump_path}...'				)
							torch.save(model.state_dict()			,						UpperCamelCase				)


if __name__ == "__main__":
				_snake_case				:				Union[str, Any]       							=		argparse.ArgumentParser()
				parser.add_argument(
				    '--tf_checkpoint_path', type=str, required=True, help='Path to the TensorFlow 2.x checkpoint path.'
				)
				parser.add_argument(
				    '--bert_config_file',
				    type=str,
				    required=True,
				    help='The config json file corresponding to the BERT model. This specifies the model architecture.',
				)
				parser.add_argument(
				    '--pytorch_dump_path',
				    type=str,
				    required=True,
				    help='Path to the output PyTorch model (must include filename).',
				)
				_snake_case				:				Optional[int]       							=		parser.parse_args()
				convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)







'''simple docstring'''


import copy
import random

from transformers import CLIPTokenizer

class       A   (		_a   ):







							def __init__(  self	:  str							,      *lowerCAmelCase_	:  int							,      **lowerCAmelCase_	:  List[str] )      ->							List[Any]:


														"""simple docstring"""


														super().__init__(*lowerCAmelCase_							,      **lowerCAmelCase_ )
														_a     					=    {}







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  Tuple							,      *lowerCAmelCase_	:  Any							,      **lowerCAmelCase_	:  Tuple )      ->							str:


														"""simple docstring"""


														_a     					=    super().add_tokens(lowerCAmelCase_							,      *lowerCAmelCase_							,      **lowerCAmelCase_ )
														if num_added_tokens == 0:
																					raise ValueError(
																					    F'The tokenizer already contains the token {placeholder_token}. Please pass a different'
																					    ''' `placeholder_token` that is not already in the tokenizer.''' )







							def        __lowerCAmelCase      (  self	:  Optional[Any]							,      lowerCAmelCase_	:  Dict							,      *lowerCAmelCase_	:  Tuple							,      lowerCAmelCase_	:  Tuple=1							,      **lowerCAmelCase_	:  int )      ->							Any:


														"""simple docstring"""


														_a     					=    []
														if num_vec_per_token == 1:
																					self.try_adding_tokens(lowerCAmelCase_							,      *lowerCAmelCase_							,      **lowerCAmelCase_ )
																					output.append(lowerCAmelCase_ )
														else:
																					_a     					=    []
																					for i in range(lowerCAmelCase_ ):
																												_a     					=    placeholder_token + F'_{i}'
																												self.try_adding_tokens(lowerCAmelCase_							,      *lowerCAmelCase_							,      **lowerCAmelCase_ )
																												output.append(lowerCAmelCase_ )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
														for token in self.token_map:
																					if token in placeholder_token:
																												raise ValueError(
																												    F'The tokenizer already has placeholder token {token} that can get confused with'
																												    F' {placeholder_token}keep placeholder tokens independent' )
														_a     					=    output







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  List[Any]							,      lowerCAmelCase_	:  Tuple=False							,      lowerCAmelCase_	:  Optional[int]=1.0 )      ->							Tuple:


														"""simple docstring"""


														if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ):
																					_a     					=    []
																					for i in range(len(lowerCAmelCase_ ) ):
																												output.append(self.replace_placeholder_tokens_in_text(text[i]							,      vector_shuffle=lowerCAmelCase_ ) )
																					return output
														for placeholder_token in self.token_map:
																					if placeholder_token in text:
																												_a     					=    self.token_map[placeholder_token]
																												_a     					=    tokens[: 1 + int(len(lowerCAmelCase_ ) * prop_tokens_to_load )]
																												if vector_shuffle:
																																			_a     					=    copy.copy(lowerCAmelCase_ )
																																			random.shuffle(lowerCAmelCase_ )
																												_a     					=    text.replace(lowerCAmelCase_							,      ''' '''.join(lowerCAmelCase_ ) )
														return text







							def __call__(  self	:  List[str]							,      lowerCAmelCase_	:  str							,      *lowerCAmelCase_	:  int							,      lowerCAmelCase_	:  Optional[int]=False							,      lowerCAmelCase_	:  Union[str, Any]=1.0							,      **lowerCAmelCase_	:  Union[str, Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														return super().__call__(
														    self.replace_placeholder_tokens_in_text(
														        lowerCAmelCase_							,      vector_shuffle=lowerCAmelCase_							,      prop_tokens_to_load=lowerCAmelCase_ )							,      *lowerCAmelCase_							,      **lowerCAmelCase_							,      )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  int							,      *lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Any=False							,      lowerCAmelCase_	:  List[Any]=1.0							,      **lowerCAmelCase_	:  Union[str, Any] )      ->							Dict:


														"""simple docstring"""


														return super().encode(
														    self.replace_placeholder_tokens_in_text(
														        lowerCAmelCase_							,      vector_shuffle=lowerCAmelCase_							,      prop_tokens_to_load=lowerCAmelCase_ )							,      *lowerCAmelCase_							,      **lowerCAmelCase_							,      )

