



import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowerCAmelCase__      =    '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__      =    direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowerCAmelCase__      =    re.compile(R'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
lowerCAmelCase__      =    re.compile(R'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCAmelCase__      =    re.compile(R'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowerCAmelCase__      =    [
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]






def 	snake_case_  (				A_ :		Optional[int]						):


       '''simple docstring'''





       _lowerCamelCase						:    Union[str, Any]         =			re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)''',    __SCREAMING_SNAKE_CASE						)
       return [m.group(0						) for m in matches]






def 	snake_case_  (				):


       '''simple docstring'''





       _lowerCamelCase						:    Optional[int]         =			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
       _lowerCamelCase						:    Tuple         =			{
           config.replace('''Config''',    ''''''						): model_type for model_type, config in config_maping_names.items()
       }

       # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
       _lowerCamelCase						:    str         =			collections.defaultdict(__SCREAMING_SNAKE_CASE						)
       _lowerCamelCase						:    Tuple         =			collections.defaultdict(__SCREAMING_SNAKE_CASE						)
       _lowerCamelCase						:    Union[str, Any]         =			collections.defaultdict(__SCREAMING_SNAKE_CASE						)

       # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
       for attr_name in dir(__SCREAMING_SNAKE_CASE						):
              _lowerCamelCase						:    str         =			None
              if _re_tf_models.match(__SCREAMING_SNAKE_CASE						) is not None:
                     _lowerCamelCase						:    List[Any]         =			tf_models
                     _lowerCamelCase						:    Dict         =			_re_tf_models.match(__SCREAMING_SNAKE_CASE						).groups()[0]
              elif _re_flax_models.match(__SCREAMING_SNAKE_CASE						) is not None:
                     _lowerCamelCase						:    List[str]         =			flax_models
                     _lowerCamelCase						:    Any         =			_re_flax_models.match(__SCREAMING_SNAKE_CASE						).groups()[0]
              elif _re_pt_models.match(__SCREAMING_SNAKE_CASE						) is not None:
                     _lowerCamelCase						:    Union[str, Any]         =			pt_models
                     _lowerCamelCase						:    Union[str, Any]         =			_re_pt_models.match(__SCREAMING_SNAKE_CASE						).groups()[0]

              if lookup_dict is not None:
                     while len(__SCREAMING_SNAKE_CASE						) > 0:
                            if attr_name in model_prefix_to_model_type:
                                   _lowerCamelCase						:    Any         =			True
                                   break
                            # Try again after removing the last word in the name
                            _lowerCamelCase						:    int         =			"""""".join(camel_case_split(__SCREAMING_SNAKE_CASE						)[:-1]						)

       _lowerCamelCase						:    Union[str, Any]         =			set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
       _lowerCamelCase						:    List[Any]         =			list(__SCREAMING_SNAKE_CASE						)
       all_models.sort()

       _lowerCamelCase						:    Dict         =			{"""model_type""": all_models}
       _lowerCamelCase						:    Optional[Any]         =			[pt_models[t] for t in all_models]
       _lowerCamelCase						:    List[Any]         =			[tf_models[t] for t in all_models]
       _lowerCamelCase						:    Tuple         =			[flax_models[t] for t in all_models]

       # Now let's use the auto-mapping names to make sure
       _lowerCamelCase						:    Any         =			{}
       for t in all_models:
              if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
                     _lowerCamelCase						:    Union[str, Any]         =			"""AutoProcessor"""
              elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
                     _lowerCamelCase						:    Dict         =			"""AutoTokenizer"""
              elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
                     _lowerCamelCase						:    Tuple         =			"""AutoFeatureExtractor"""
              else:
                     # Default to AutoTokenizer if a model has nothing, for backward compatibility.
                     _lowerCamelCase						:    Tuple         =			"""AutoTokenizer"""

       _lowerCamelCase						:    Optional[int]         =			[processors[t] for t in all_models]

       return pd.DataFrame(__SCREAMING_SNAKE_CASE						)






def 	snake_case_  (				A_ :		List[Any]						):


       '''simple docstring'''





       _lowerCamelCase						:    Union[str, Any]         =			[
           transformers_module.models.auto.modeling_auto,
           transformers_module.models.auto.modeling_tf_auto,
           transformers_module.models.auto.modeling_flax_auto,
       ]
       for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
              _lowerCamelCase						:    str         =			[model_mapping, F'''TF_{model_mapping}''', F'''FLAX_{model_mapping}''']
              _lowerCamelCase						:    int         =			[auto_class, F'''TF_{auto_class}''', F'''Flax_{auto_class}''']
              # Loop through all three frameworks
              for module, cls, mapping in zip(__SCREAMING_SNAKE_CASE,    __SCREAMING_SNAKE_CASE,    __SCREAMING_SNAKE_CASE						):
                     # The type of pipeline may not exist in this framework
                     if not hasattr(__SCREAMING_SNAKE_CASE,    __SCREAMING_SNAKE_CASE						):
                            continue
                     # First extract all model_names
                     _lowerCamelCase						:    List[str]         =			[]
                     for name in getattr(__SCREAMING_SNAKE_CASE,    __SCREAMING_SNAKE_CASE						).values():
                            if isinstance(__SCREAMING_SNAKE_CASE,    __SCREAMING_SNAKE_CASE						):
                                   model_names.append(__SCREAMING_SNAKE_CASE						)
                            else:
                                   model_names.extend(list(__SCREAMING_SNAKE_CASE						)						)

            # Add pipeline tag and auto model class for those models
                     table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

       return table






def 	snake_case_  (				A_ :		str,    A_ :		List[Any]						):


       '''simple docstring'''





       _lowerCamelCase						:    Dict         =			get_frameworks_table()
       _lowerCamelCase						:    Tuple         =			Dataset.from_pandas(__SCREAMING_SNAKE_CASE						)

       _lowerCamelCase						:    int         =			hf_hub_download(
           '''huggingface/transformers-metadata''',    '''pipeline_tags.json''',    repo_type='''dataset''',    token=__SCREAMING_SNAKE_CASE						)
       _lowerCamelCase						:    Dict         =			Dataset.from_json(__SCREAMING_SNAKE_CASE						)
       _lowerCamelCase						:    Any         =			{
           tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
           for i in range(len(__SCREAMING_SNAKE_CASE						)						)
       }
       _lowerCamelCase						:    Optional[int]         =			update_pipeline_and_auto_class_table(__SCREAMING_SNAKE_CASE						)

       # Sort the model classes to avoid some nondeterministic updates to create false update commits.
       _lowerCamelCase						:    Optional[int]         =			sorted(table.keys()						)
       _lowerCamelCase						:    Union[str, Any]         =			pd.DataFrame(
           {
               '''model_class''': model_classes,
               '''pipeline_tag''': [table[m][0] for m in model_classes],
               '''auto_class''': [table[m][1] for m in model_classes],
           }						)
       _lowerCamelCase						:    Any         =			Dataset.from_pandas(__SCREAMING_SNAKE_CASE						)

       with tempfile.TemporaryDirectory() as tmp_dir:
              frameworks_dataset.to_json(os.path.join(__SCREAMING_SNAKE_CASE,    '''frameworks.json'''						)						)
              tags_dataset.to_json(os.path.join(__SCREAMING_SNAKE_CASE,    '''pipeline_tags.json'''						)						)

              if commit_sha is not None:
                     _lowerCamelCase						:    Dict         =			(
                         F'''Update with commit {commit_sha}\n\nSee: '''
                         F'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
                     )
              else:
                     _lowerCamelCase						:    List[Any]         =			"""Update"""

              upload_folder(
                  repo_id='''huggingface/transformers-metadata''',    folder_path=__SCREAMING_SNAKE_CASE,    repo_type='''dataset''',    token=__SCREAMING_SNAKE_CASE,    commit_message=__SCREAMING_SNAKE_CASE,    )






def 	snake_case_  (				):


       '''simple docstring'''





       _lowerCamelCase						:    List[Any]         =			{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
       _lowerCamelCase						:    Dict         =			transformers_module.pipelines.SUPPORTED_TASKS
       _lowerCamelCase						:    Tuple         =			[]
       for key in pipeline_tasks:
              if key not in in_table:
                     _lowerCamelCase						:    Dict         =			pipeline_tasks[key]["""pt"""]
                     if isinstance(__SCREAMING_SNAKE_CASE,    (list, tuple)						):
                            _lowerCamelCase						:    Tuple         =			model[0]
                     _lowerCamelCase						:    Optional[Any]         =			model.__name__
                     if model not in in_table.values():
                            missing.append(__SCREAMING_SNAKE_CASE						)

       if len(__SCREAMING_SNAKE_CASE						) > 0:
              _lowerCamelCase						:    Union[str, Any]         =			""", """.join(__SCREAMING_SNAKE_CASE						)
              raise ValueError(
                  '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
                  F'''`utils/update_metadata.py`: {msg}. Please add them!'''						)


if __name__ == "__main__":
   lowerCAmelCase__      =    argparse.ArgumentParser()
   parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
   parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
   parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
   lowerCAmelCase__      =    parser.parse_args()

   if args.check_only:
      check_pipeline_tags()
   else:
      update_metadata(args.token, args.commit_sha)







"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__      =    logging.get_logger(__name__)

lowerCAmelCase__      =    {
    '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/config.json''',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}






class 	__snake_case	(     _lowercase):
  snake_case__    :       List[Any]        =						"xglm"
  snake_case__    :       Dict        =						["past_key_values"]

  snake_case__    :       str        =						{
      "num_attention_heads": "attention_heads",
      "hidden_size": "d_model",
      "num_hidden_layers": "num_layers",
  }




  def __init__(			self					:						List[str]						, __lowerCAmelCase					:						List[Any]=2_5_6_0_0_8						, __lowerCAmelCase					:						int=2_0_4_8						, __lowerCAmelCase					:						Dict=1_0_2_4						, __lowerCAmelCase					:						List[str]=4_0_9_6						, __lowerCAmelCase					:						Tuple=2_4						, __lowerCAmelCase					:						Dict=1_6						, __lowerCAmelCase					:						Tuple="gelu"						, __lowerCAmelCase					:						Tuple=0.1						, __lowerCAmelCase					:						Tuple=0.1						, __lowerCAmelCase					:						Optional[Any]=0.0						, __lowerCAmelCase					:						List[Any]=0.0						, __lowerCAmelCase					:						int=0.02						, __lowerCAmelCase					:						Any=True						, __lowerCAmelCase					:						Tuple=True						, __lowerCAmelCase					:						str=2						, __lowerCAmelCase					:						Dict=1						, __lowerCAmelCase					:						Dict=0						, __lowerCAmelCase					:						List[Any]=2						, **__lowerCAmelCase					:						Optional[Any]						, ):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			vocab_size
         _lowerCamelCase						:    List[Any]         =			max_position_embeddings
         _lowerCamelCase						:    int         =			d_model
         _lowerCamelCase						:    Optional[Any]         =			ffn_dim
         _lowerCamelCase						:    Any         =			num_layers
         _lowerCamelCase						:    Union[str, Any]         =			attention_heads
         _lowerCamelCase						:    List[str]         =			activation_function
         _lowerCamelCase						:    Union[str, Any]         =			dropout
         _lowerCamelCase						:    int         =			attention_dropout
         _lowerCamelCase						:    Optional[int]         =			activation_dropout
         _lowerCamelCase						:    Any         =			layerdrop
         _lowerCamelCase						:    List[str]         =			init_std
         _lowerCamelCase						:    Union[str, Any]         =			scale_embedding  # scale factor will be sqrt(d_model) if True
         _lowerCamelCase						:    str         =			use_cache

         super().__init__(
             pad_token_id=__lowerCAmelCase						, bos_token_id=__lowerCAmelCase						, eos_token_id=__lowerCAmelCase						, decoder_start_token_id=__lowerCAmelCase						, **__lowerCAmelCase						, )


