


"""simple docstring"""




import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowercase_	   =     'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
lowercase_	   =     direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowercase_	   =     re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
lowercase_	   =     re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowercase_	   =     re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowercase_	   =     [
    ('pretraining', 'MODEL_FOR_PRETRAINING_MAPPING_NAMES', 'AutoModelForPreTraining'),
    ('feature-extraction', 'MODEL_MAPPING_NAMES', 'AutoModel'),
    ('audio-classification', 'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForAudioClassification'),
    ('text-generation', 'MODEL_FOR_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForCausalLM'),
    ('automatic-speech-recognition', 'MODEL_FOR_CTC_MAPPING_NAMES', 'AutoModelForCTC'),
    ('image-classification', 'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForImageClassification'),
    ('image-segmentation', 'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES', 'AutoModelForImageSegmentation'),
    ('fill-mask', 'MODEL_FOR_MASKED_LM_MAPPING_NAMES', 'AutoModelForMaskedLM'),
    ('object-detection', 'MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES', 'AutoModelForObjectDetection'),
    (
        'zero-shot-object-detection',
        'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES',
        'AutoModelForZeroShotObjectDetection',
    ),
    ('question-answering', 'MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES', 'AutoModelForQuestionAnswering'),
    ('text2text-generation', 'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForSeq2SeqLM'),
    ('text-classification', 'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForSequenceClassification'),
    ('automatic-speech-recognition', 'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES', 'AutoModelForSpeechSeq2Seq'),
    (
        'table-question-answering',
        'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForTableQuestionAnswering',
    ),
    ('token-classification', 'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForTokenClassification'),
    ('multiple-choice', 'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES', 'AutoModelForMultipleChoice'),
    (
        'next-sentence-prediction',
        'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES',
        'AutoModelForNextSentencePrediction',
    ),
    (
        'audio-frame-classification',
        'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForAudioFrameClassification',
    ),
    ('audio-xvector', 'MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES', 'AutoModelForAudioXVector'),
    (
        'document-question-answering',
        'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForDocumentQuestionAnswering',
    ),
    (
        'visual-question-answering',
        'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForVisualQuestionAnswering',
    ),
    ('image-to-text', 'MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES', 'AutoModelForVision2Seq'),
    (
        'zero-shot-image-classification',
        'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForZeroShotImageClassification',
    ),
    ('depth-estimation', 'MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES', 'AutoModelForDepthEstimation'),
    ('video-classification', 'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForVideoClassification'),
    ('mask-generation', 'MODEL_FOR_MASK_GENERATION_MAPPING_NAMES', 'AutoModelForMaskGeneration'),
]
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							Any		 =						re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'      ,	UpperCAmelCase_						)
 return [m.group(0						) for m in matches]
def 				UpperCamelCase__					(  ):
 __lowerCamelCase		:							List[Any]		 =						transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
 __lowerCamelCase		:							Tuple		 =						{
     config.replace('Config'      ,	''						): model_type for model_type, config in config_maping_names.items()
 }

 # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
 __lowerCamelCase		:							Dict		 =						collections.defaultdict(UpperCAmelCase_						)
 __lowerCamelCase		:							Optional[Any]		 =						collections.defaultdict(UpperCAmelCase_						)
 __lowerCamelCase		:							Tuple		 =						collections.defaultdict(UpperCAmelCase_						)

 # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
 for attr_name in dir(UpperCAmelCase_						):
  __lowerCamelCase		:							List[str]		 =						None
  if _re_tf_models.match(UpperCAmelCase_						) is not None:
   __lowerCamelCase		:							Union[str, Any]		 =						tf_models
   __lowerCamelCase		:							Dict		 =						_re_tf_models.match(UpperCAmelCase_						).groups()[0]
  elif _re_flax_models.match(UpperCAmelCase_						) is not None:
   __lowerCamelCase		:							Tuple		 =						flax_models
   __lowerCamelCase		:							Union[str, Any]		 =						_re_flax_models.match(UpperCAmelCase_						).groups()[0]
  elif _re_pt_models.match(UpperCAmelCase_						) is not None:
   __lowerCamelCase		:							Optional[int]		 =						pt_models
   __lowerCamelCase		:							Union[str, Any]		 =						_re_pt_models.match(UpperCAmelCase_						).groups()[0]

  if lookup_dict is not None:
   while len(UpperCAmelCase_						) > 0:
    if attr_name in model_prefix_to_model_type:
     __lowerCamelCase		:							int		 =						True
     break
    # Try again after removing the last word in the name
    __lowerCamelCase		:							List[Any]		 =						''.join(camel_case_split(UpperCAmelCase_						)[:-1]						)

 __lowerCamelCase		:							Union[str, Any]		 =						set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
 __lowerCamelCase		:							List[Any]		 =						list(UpperCAmelCase_						)
 all_models.sort()

 __lowerCamelCase		:							Dict		 =						{'model_type': all_models}
 __lowerCamelCase		:							List[str]		 =						[pt_models[t] for t in all_models]
 __lowerCamelCase		:							Optional[Any]		 =						[tf_models[t] for t in all_models]
 __lowerCamelCase		:							List[str]		 =						[flax_models[t] for t in all_models]

 # Now let's use the auto-mapping names to make sure
 __lowerCamelCase		:							Union[str, Any]		 =						{}
 for t in all_models:
  if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
   __lowerCamelCase		:							Any		 =						'AutoProcessor'
  elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
   __lowerCamelCase		:							List[Any]		 =						'AutoTokenizer'
  elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
   __lowerCamelCase		:							Union[str, Any]		 =						'AutoFeatureExtractor'
  else:
   # Default to AutoTokenizer if a model has nothing, for backward compatibility.
   __lowerCamelCase		:							str		 =						'AutoTokenizer'

 __lowerCamelCase		:							Tuple		 =						[processors[t] for t in all_models]

 return pd.DataFrame(UpperCAmelCase_						)
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							List[str]		 =						[
     transformers_module.models.auto.modeling_auto,
     transformers_module.models.auto.modeling_tf_auto,
     transformers_module.models.auto.modeling_flax_auto,
 ]
 for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
  __lowerCamelCase		:							List[str]		 =						[model_mapping, f'TF_{model_mapping}', f'FLAX_{model_mapping}']
  __lowerCamelCase		:							Optional[int]		 =						[auto_class, f'TF_{auto_class}', f'Flax_{auto_class}']
  # Loop through all three frameworks
  for module, cls, mapping in zip(UpperCAmelCase_      ,	UpperCAmelCase_      ,	UpperCAmelCase_						):
   # The type of pipeline may not exist in this framework
   if not hasattr(UpperCAmelCase_      ,	UpperCAmelCase_						):
    continue
   # First extract all model_names
   __lowerCamelCase		:							int		 =						[]
   for name in getattr(UpperCAmelCase_      ,	UpperCAmelCase_						).values():
    if isinstance(UpperCAmelCase_      ,	UpperCAmelCase_						):
     model_names.append(UpperCAmelCase_						)
    else:
     model_names.extend(list(UpperCAmelCase_						)						)

            # Add pipeline tag and auto model class for those models
   table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

 return table
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 __lowerCamelCase		:							List[Any]		 =						get_frameworks_table()
 __lowerCamelCase		:							Optional[int]		 =						Dataset.from_pandas(UpperCAmelCase_						)

 __lowerCamelCase		:							Union[str, Any]		 =						hf_hub_download(
     'huggingface/transformers-metadata'      ,	'pipeline_tags.json'      ,	repo_type='dataset'      ,	token=UpperCAmelCase_						)
 __lowerCamelCase		:							List[str]		 =						Dataset.from_json(UpperCAmelCase_						)
 __lowerCamelCase		:							Dict		 =						{
     tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
     for i in range(len(UpperCAmelCase_						)						)
 }
 __lowerCamelCase		:							List[Any]		 =						update_pipeline_and_auto_class_table(UpperCAmelCase_						)

 # Sort the model classes to avoid some nondeterministic updates to create false update commits.
 __lowerCamelCase		:							Union[str, Any]		 =						sorted(table.keys()						)
 __lowerCamelCase		:							List[str]		 =						pd.DataFrame(
     {
         'model_class': model_classes,
         'pipeline_tag': [table[m][0] for m in model_classes],
         'auto_class': [table[m][1] for m in model_classes],
     }						)
 __lowerCamelCase		:							Tuple		 =						Dataset.from_pandas(UpperCAmelCase_						)

 with tempfile.TemporaryDirectory() as tmp_dir:
  frameworks_dataset.to_json(os.path.join(UpperCAmelCase_      ,	'frameworks.json'						)						)
  tags_dataset.to_json(os.path.join(UpperCAmelCase_      ,	'pipeline_tags.json'						)						)

  if commit_sha is not None:
   __lowerCamelCase		:							Union[str, Any]		 =						(
       f'Update with commit {commit_sha}\n\nSee: '
       f'https://github.com/huggingface/transformers/commit/{commit_sha}'
   )
  else:
   __lowerCamelCase		:							Optional[Any]		 =						'Update'

  upload_folder(
      repo_id='huggingface/transformers-metadata'      ,	folder_path=UpperCAmelCase_      ,	repo_type='dataset'      ,	token=UpperCAmelCase_      ,	commit_message=UpperCAmelCase_      ,	)
def 				UpperCamelCase__					(  ):
 __lowerCamelCase		:							str		 =						{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
 __lowerCamelCase		:							Tuple		 =						transformers_module.pipelines.SUPPORTED_TASKS
 __lowerCamelCase		:							int		 =						[]
 for key in pipeline_tasks:
  if key not in in_table:
   __lowerCamelCase		:							List[str]		 =						pipeline_tasks[key]['pt']
   if isinstance(UpperCAmelCase_      ,	(list, tuple)						):
    __lowerCamelCase		:							str		 =						model[0]
   __lowerCamelCase		:							str		 =						model.__name__
   if model not in in_table.values():
    missing.append(UpperCAmelCase_						)

 if len(UpperCAmelCase_						) > 0:
  __lowerCamelCase		:							Any		 =						', '.join(UpperCAmelCase_						)
  raise ValueError(
      'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
      f'`utils/update_metadata.py`: {msg}. Please add them!'						)


if __name__ == "__main__":
       lowercase_	   =     argparse.ArgumentParser()
       parser.add_argument('--token', type=str, help='The token to use to push to the transformers-metadata dataset.')
       parser.add_argument('--commit_sha', type=str, help='The sha of the commit going with this update.')
       parser.add_argument('--check-only', action='store_true', help='Activate to just check all pipelines are present.')
       lowercase_	   =     parser.parse_args()

       if args.check_only:
              check_pipeline_tags()
       else:
              update_metadata(args.token, args.commit_sha)





import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
       import tensorflow as tf
       from tensorflow.python.eager import context
       from tensorflow.python.framework import ops

       from transformers import GradientAccumulator, create_optimizer


@require_tf
class 	A_ (	unittest.TestCase   ):




      '''simple docstring'''


      def 					_snake_case     (    self:	Optional[int]						,				a:	int						,				a:	List[Any]						,				a:	Any    ):
       self.assertEqual(len(a    )						,				len(a    )    )
       for a, b in zip(a						,				a    ):
        self.assertAlmostEqual(a						,				a						,				delta=a    )




      def 					_snake_case     (    self:	Union[str, Any]    ):
       __lowerCamelCase		:							Union[str, Any]		 =						GradientAccumulator()
       accumulator([tf.constant([1.0, 2.0]    )]    )
       accumulator([tf.constant([-2.0, 1.0]    )]    )
       accumulator([tf.constant([-1.0, 2.0]    )]    )
       with self.assertRaises(a    ):
        accumulator([tf.constant([1.0, 1.0]    ), tf.constant([2.0, 2.0]    )]    )
       self.assertEqual(accumulator.step						,				3    )
       self.assertEqual(len(accumulator.gradients    )						,				1    )
       self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()						,				[-2.0, 5.0]						,				tol=1e-2    )
       accumulator.reset()
       self.assertEqual(accumulator.step						,				0    )
       self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()						,				[0.0, 0.0]						,				tol=1e-2    )




      def 					_snake_case     (    self:	str    ):
       __lowerCamelCase		:							Any		 =						None
       ops.enable_eager_execution_internal()
       __lowerCamelCase		:							List[str]		 =						tf.config.list_physical_devices('CPU'    )
       if len(a    ) == 1:
        tf.config.set_logical_device_configuration(
            physical_devices[0]						,				[tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]    )
       __lowerCamelCase		:							Dict		 =						tf.config.list_logical_devices(device_type='CPU'    )
       __lowerCamelCase		:							List[Any]		 =						tf.distribute.MirroredStrategy(devices=devices[:2]    )

       with strategy.scope():
        __lowerCamelCase		:							Optional[Any]		 =						GradientAccumulator()
        __lowerCamelCase		:							int		 =						tf.Variable([4.0, 3.0]    )
        __lowerCamelCase ,					__lowerCamelCase		:							Any		 =						create_optimizer(5e-5						,				10						,				5    )
        __lowerCamelCase		:							Any		 =						tf.Variable([0.0, 0.0]						,				trainable=a    )

       def accumulate_on_replica(a:	str    ):
        accumulator([gradient]    )

       def apply_on_replica():
        optimizer.apply_gradients(list(zip(accumulator.gradients						,				[variable]    )    )    )

       @tf.function
       def accumulate(a:	List[Any]						,				a:	List[str]    ):
        with strategy.scope():
         __lowerCamelCase		:							Optional[int]		 =						strategy.experimental_local_results(a    )
         local_variables[0].assign(a    )
         local_variables[1].assign(a    )
         strategy.run(a						,				args=(gradient_placeholder,)    )

       @tf.function
       def apply_grad():
        with strategy.scope():
         strategy.run(a    )

       def _check_local_values(a:	str						,				a:	Union[str, Any]    ):
        __lowerCamelCase		:							Optional[Any]		 =						strategy.experimental_local_results(accumulator._gradients[0]    )
        self.assertListAlmostEqual(values[0].value()						,				a						,				tol=1e-2    )
        self.assertListAlmostEqual(values[1].value()						,				a						,				tol=1e-2    )

       accumulate([1.0, 2.0]						,				[-1.0, 1.0]    )
       accumulate([3.0, -1.0]						,				[-1.0, -1.0]    )
       accumulate([-2.0, 2.0]						,				[3.0, -2.0]    )
       self.assertEqual(accumulator.step						,				3    )
       _check_local_values([2.0, 3.0]						,				[1.0, -2.0]    )
       apply_grad()
       self.assertListAlmostEqual(variable.value()						,				[4.0, 3.0]						,				tol=1e-2    )
       accumulator.reset()
       self.assertEqual(accumulator.step						,				0    )
       _check_local_values([0.0, 0.0]						,				[0.0, 0.0]    )

