


import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowerCamelCase						=		'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
lowerCamelCase						=		direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowerCamelCase						=		re.compile(R'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
lowerCamelCase						=		re.compile(R'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCamelCase						=		re.compile(R'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowerCamelCase						=		[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]






def 					lowerCamelCase_   (	_a	):




		"""simple docstring"""






		lowerCAmelCase__       :      Union[str, Any]   					= re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''     ,       _a	)
		return [m.group(0	) for m in matches]






def 					lowerCamelCase_   (	):




		"""simple docstring"""






		lowerCAmelCase__       :      str   					= transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
		lowerCAmelCase__       :      str   					= {
		    config.replace('''Config'''     ,       ''''''	): model_type for model_type, config in config_maping_names.items()
		}

		# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
		lowerCAmelCase__       :      str   					= collections.defaultdict(_a	)
		lowerCAmelCase__       :      List[Any]   					= collections.defaultdict(_a	)
		lowerCAmelCase__       :      int   					= collections.defaultdict(_a	)

		# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
		for attr_name in dir(_a	):
				lowerCAmelCase__       :      Dict   					= None
				if _re_tf_models.match(_a	) is not None:
						lowerCAmelCase__       :      List[str]   					= tf_models
						lowerCAmelCase__       :      Optional[int]   					= _re_tf_models.match(_a	).groups()[0]
				elif _re_flax_models.match(_a	) is not None:
						lowerCAmelCase__       :      List[Any]   					= flax_models
						lowerCAmelCase__       :      Optional[int]   					= _re_flax_models.match(_a	).groups()[0]
				elif _re_pt_models.match(_a	) is not None:
						lowerCAmelCase__       :      Any   					= pt_models
						lowerCAmelCase__       :      Optional[int]   					= _re_pt_models.match(_a	).groups()[0]

				if lookup_dict is not None:
						while len(_a	) > 0:
								if attr_name in model_prefix_to_model_type:
										lowerCAmelCase__       :      List[Any]   					= True
										break
								# Try again after removing the last word in the name
								lowerCAmelCase__       :      Tuple   					= ''''''.join(camel_case_split(_a	)[:-1]	)

		lowerCAmelCase__       :      Tuple   					= set(list(pt_models.keys()	) + list(tf_models.keys()	) + list(flax_models.keys()	)	)
		lowerCAmelCase__       :      int   					= list(_a	)
		all_models.sort()

		lowerCAmelCase__       :      Union[str, Any]   					= {'''model_type''': all_models}
		lowerCAmelCase__       :      List[Any]   					= [pt_models[t] for t in all_models]
		lowerCAmelCase__       :      List[Any]   					= [tf_models[t] for t in all_models]
		lowerCAmelCase__       :      Optional[int]   					= [flax_models[t] for t in all_models]

		# Now let's use the auto-mapping names to make sure
		lowerCAmelCase__       :      List[str]   					= {}
		for t in all_models:
				if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
						lowerCAmelCase__       :      Dict   					= '''AutoProcessor'''
				elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
						lowerCAmelCase__       :      List[Any]   					= '''AutoTokenizer'''
				elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
						lowerCAmelCase__       :      Tuple   					= '''AutoFeatureExtractor'''
				else:
						# Default to AutoTokenizer if a model has nothing, for backward compatibility.
						lowerCAmelCase__       :      str   					= '''AutoTokenizer'''

		lowerCAmelCase__       :      str   					= [processors[t] for t in all_models]

		return pd.DataFrame(_a	)






def 					lowerCamelCase_   (	_a	):




		"""simple docstring"""






		lowerCAmelCase__       :      Optional[int]   					= [
		    transformers_module.models.auto.modeling_auto,
		    transformers_module.models.auto.modeling_tf_auto,
		    transformers_module.models.auto.modeling_flax_auto,
		]
		for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
				lowerCAmelCase__       :      str   					= [model_mapping, f'TF_{model_mapping}', f'FLAX_{model_mapping}']
				lowerCAmelCase__       :      str   					= [auto_class, f'TF_{auto_class}', f'Flax_{auto_class}']
				# Loop through all three frameworks
				for module, cls, mapping in zip(_a     ,       _a     ,       _a	):
						# The type of pipeline may not exist in this framework
						if not hasattr(_a     ,       _a	):
								continue
						# First extract all model_names
						lowerCAmelCase__       :      Any   					= []
						for name in getattr(_a     ,       _a	).values():
								if isinstance(_a     ,       _a	):
										model_names.append(_a	)
								else:
										model_names.extend(list(_a	)	)

            # Add pipeline tag and auto model class for those models
						table.update({model_name: (pipeline_tag, cls) for model_name in model_names}	)

		return table






def 					lowerCamelCase_   (	_a     ,       _a	):




		"""simple docstring"""






		lowerCAmelCase__       :      Union[str, Any]   					= get_frameworks_table()
		lowerCAmelCase__       :      str   					= Dataset.from_pandas(_a	)

		lowerCAmelCase__       :      Dict   					= hf_hub_download(
		    '''huggingface/transformers-metadata'''     ,       '''pipeline_tags.json'''     ,       repo_type='''dataset'''     ,       token=_a	)
		lowerCAmelCase__       :      Union[str, Any]   					= Dataset.from_json(_a	)
		lowerCAmelCase__       :      Optional[Any]   					= {
		    tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
		    for i in range(len(_a	)	)
		}
		lowerCAmelCase__       :      Union[str, Any]   					= update_pipeline_and_auto_class_table(_a	)

		# Sort the model classes to avoid some nondeterministic updates to create false update commits.
		lowerCAmelCase__       :      Optional[Any]   					= sorted(table.keys()	)
		lowerCAmelCase__       :      List[str]   					= pd.DataFrame(
		    {
		        '''model_class''': model_classes,
		        '''pipeline_tag''': [table[m][0] for m in model_classes],
		        '''auto_class''': [table[m][1] for m in model_classes],
		    }	)
		lowerCAmelCase__       :      Dict   					= Dataset.from_pandas(_a	)

		with tempfile.TemporaryDirectory() as tmp_dir:
				frameworks_dataset.to_json(os.path.join(_a     ,       '''frameworks.json'''	)	)
				tags_dataset.to_json(os.path.join(_a     ,       '''pipeline_tags.json'''	)	)

				if commit_sha is not None:
						lowerCAmelCase__       :      List[Any]   					= (
						    f'Update with commit {commit_sha}\n\nSee: '
						    f'https://github.com/huggingface/transformers/commit/{commit_sha}'
						)
				else:
						lowerCAmelCase__       :      Union[str, Any]   					= '''Update'''

				upload_folder(
				    repo_id='''huggingface/transformers-metadata'''     ,       folder_path=_a     ,       repo_type='''dataset'''     ,       token=_a     ,       commit_message=_a     ,       )






def 					lowerCamelCase_   (	):




		"""simple docstring"""






		lowerCAmelCase__       :      Union[str, Any]   					= {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
		lowerCAmelCase__       :      Tuple   					= transformers_module.pipelines.SUPPORTED_TASKS
		lowerCAmelCase__       :      Any   					= []
		for key in pipeline_tasks:
				if key not in in_table:
						lowerCAmelCase__       :      Optional[int]   					= pipeline_tasks[key]['''pt''']
						if isinstance(_a     ,       (list, tuple)	):
								lowerCAmelCase__       :      List[Any]   					= model[0]
						lowerCAmelCase__       :      Tuple   					= model.__name__
						if model not in in_table.values():
								missing.append(_a	)

		if len(_a	) > 0:
				lowerCAmelCase__       :      List[Any]   					= ''', '''.join(_a	)
				raise ValueError(
				    '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
				    f'`utils/update_metadata.py`: {msg}. Please add them!'	)


if __name__ == "__main__":
		lowerCamelCase						=		argparse.ArgumentParser()
		parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
		parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
		parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
		lowerCamelCase						=		parser.parse_args()

		if args.check_only:
				check_pipeline_tags()
		else:
				update_metadata(args.token, args.commit_sha)








from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{
    '''facebook/dpr-ctx_encoder-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-question_encoder-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-reader-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-ctx_encoder-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-question_encoder-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-reader-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json'''
    ),
}





class  _a							(    _lowercase):
	_a  :  List[Any]				   =							'''dpr'''




	def __init__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		List[str]=3_0522		, _SCREAMING_SNAKE_CASE						:		Optional[int]=768		, _SCREAMING_SNAKE_CASE						:		List[str]=12		, _SCREAMING_SNAKE_CASE						:		Tuple=12		, _SCREAMING_SNAKE_CASE						:		str=3072		, _SCREAMING_SNAKE_CASE						:		Optional[Any]="gelu"		, _SCREAMING_SNAKE_CASE						:		Dict=0.1		, _SCREAMING_SNAKE_CASE						:		Optional[int]=0.1		, _SCREAMING_SNAKE_CASE						:		List[str]=512		, _SCREAMING_SNAKE_CASE						:		int=2		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=0.02		, _SCREAMING_SNAKE_CASE						:		Tuple=1E-12		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=0		, _SCREAMING_SNAKE_CASE						:		List[str]="absolute"		, _SCREAMING_SNAKE_CASE						:		int = 0		, **_SCREAMING_SNAKE_CASE						:		Optional[Any]		, )->	Optional[int]:
			super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      Union[str, Any]   					= vocab_size
			lowerCAmelCase__       :      Tuple   					= hidden_size
			lowerCAmelCase__       :      Union[str, Any]   					= num_hidden_layers
			lowerCAmelCase__       :      List[Any]   					= num_attention_heads
			lowerCAmelCase__       :      Optional[int]   					= hidden_act
			lowerCAmelCase__       :      Optional[Any]   					= intermediate_size
			lowerCAmelCase__       :      Optional[Any]   					= hidden_dropout_prob
			lowerCAmelCase__       :      int   					= attention_probs_dropout_prob
			lowerCAmelCase__       :      str   					= max_position_embeddings
			lowerCAmelCase__       :      List[Any]   					= type_vocab_size
			lowerCAmelCase__       :      Optional[int]   					= initializer_range
			lowerCAmelCase__       :      List[str]   					= layer_norm_eps
			lowerCAmelCase__       :      Dict   					= projection_dim
			lowerCAmelCase__       :      int   					= position_embedding_type




