






"""simple docstring"""







import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


_UpperCamelCase					      =  logging.getLogger(__name__)

_UpperCamelCase					      =  """pytorch_model.bin"""







@dataclasses.dataclass
class  lowerCamelCase__ :
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co.'''}   ,					)







@dataclasses.dataclass
class  lowerCamelCase__ :
    SCREAMING_SNAKE_CASE		=      dataclasses.field(metadata={'''help''': '''A csv or a json file containing the training data.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(metadata={'''help''': '''A csv or a json file containing the data to predict on.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''A csv or a json file containing the validation data.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''The name of the task to train on.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''The list of labels for the task.'''}			)







@dataclasses.dataclass
class  lowerCamelCase__ :
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        metadata={'''help''': '''The output directory where the model predictions and checkpoints will be written.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default='''accuracy'''   ,					metadata={'''help''': '''The evaluation metric used for the task.'''}			)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default='''no'''   ,					metadata={
            '''help''': '''The evaluation strategy to adopt during training. Possible values are: ["no", "step", "epoch]'''
        }   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=10   ,					metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=0.0   ,					metadata={
            '''help''': '''How much the specified evaluation metric must improve to satisfy early stopping conditions.'''
        }   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the confidence score.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the validation performance.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''Whether to fine-tune on labeled data after pseudo training.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=0.0   ,					metadata={'''help''': '''Confidence threshold for pseudo-labeled data filtering.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=100   ,					metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}   ,					)
    SCREAMING_SNAKE_CASE		=      dataclasses.field(
        default=__SCREAMING_SNAKE_CASE   ,					metadata={'''help''': '''Random seed for initialization.'''}   ,					)







def _a			(		_snake_case				,	_snake_case				,	_snake_case				,	_snake_case				,	_snake_case				,	_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	datasets.concatenate_datasets([infer_input, infer_output]				,	axis=1							)

      if args.do_filter_by_confidence:
            UpperCAmelCase          =	dataset.filter(lambda _snake_case							: example["probability"] > args.confidence_threshold							)

      if args.do_filter_by_val_performance:
            assert eval_result >= 0.0 and eval_result <= 1.0
            UpperCAmelCase          =	int(eval_result * len(lowerCAmelCase__							)							)
            print(lowerCAmelCase__							)
            UpperCAmelCase          =	dataset.sort("""probability"""				,	reverse=lowerCAmelCase__							)
            UpperCAmelCase          =	dataset.select(range(lowerCAmelCase__							)							)

      UpperCAmelCase          =	dataset.remove_columns(["""label""", """probability"""]							)
      UpperCAmelCase          =	dataset.rename_column("""prediction"""				,	"""label"""							)
      UpperCAmelCase          =	dataset.map(lambda _snake_case							: {"label": idalabel[example["label"]]}							)
      UpperCAmelCase          =	dataset.shuffle(seed=args.seed							)

      UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	F'''train_pseudo.{args.data_file_extension}'''							)
      if args.data_file_extension == "csv":
            dataset.to_csv(lowerCAmelCase__				,	index=lowerCAmelCase__							)
      else:
            dataset.to_json(lowerCAmelCase__							)






def _a			(		_snake_case				,	_snake_case				,	_snake_case				,	_snake_case				,	**_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	Accelerator()
      # Make one log on every process with the configuration for debugging.
      logging.basicConfig(
          format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""				,	datefmt="""%m/%d/%Y %H:%M:%S"""				,	level=logging.INFO				,	)
      logger.info(accelerator.state							)

      # Setup logging, we only want one process per machine to log things on the
      # screen. accelerator.is_local_main_process is only True for one process per
      # machine.
      logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR							)

      if accelerator.is_local_main_process:
            datasets.utils.logging.set_verbosity_warning()
            transformers.utils.logging.set_verbosity_info()
      else:
            datasets.utils.logging.set_verbosity_error()
            transformers.utils.logging.set_verbosity_error()

      UpperCAmelCase          =	STModelArguments(model_name_or_path=lowerCAmelCase__							)
      UpperCAmelCase          =	STDataArguments(train_file=lowerCAmelCase__				,	infer_file=lowerCAmelCase__							)
      UpperCAmelCase          =	STTrainingArguments(output_dir=lowerCAmelCase__							)
      UpperCAmelCase          =	argparse.Namespace()

      for arg_class in (model_args, data_args, training_args):
            for key, value in vars(lowerCAmelCase__							).items():
                  setattr(lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__							)

      for key, value in kwargs.items():
            if hasattr(lowerCAmelCase__				,	lowerCAmelCase__							):
                  setattr(lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__							)

    # Sanity checks
      UpperCAmelCase          =	{}
      UpperCAmelCase          =	None

      # You need to provide the training data and the data to predict on
      assert args.train_file is not None
      assert args.infer_file is not None
      UpperCAmelCase          =	args.train_file
      UpperCAmelCase          =	args.infer_file

      if args.evaluation_strategy != IntervalStrategy.NO.value:
            assert args.eval_file is not None
            UpperCAmelCase          =	args.eval_file

      for key in data_files:
            UpperCAmelCase          =	data_files[key].split("""."""							)[-1]
            assert extension in ["csv", "json"], F'''`{key}_file` should be a csv or a json file.'''
            if args.data_file_extension is None:
                  UpperCAmelCase          =	extension
            else:
                  assert extension == args.data_file_extension, F'''`{key}_file` should be a {args.data_file_extension} file`.'''

      assert (
          args.eval_metric in datasets.list_metrics()
      ), F'''{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}.'''

      # If passed along, set the training seed now.
      if args.seed is not None:
            set_seed(args.seed							)

      logger.info("""Creating the initial data directory for self-training..."""							)
      UpperCAmelCase          =	F'''{args.output_dir}/self-train_iter-{{}}'''.format
      UpperCAmelCase          =	data_dir_format(0							)

      if accelerator.is_main_process:
            if args.output_dir is not None:
                  os.makedirs(args.output_dir				,	exist_ok=lowerCAmelCase__							)
                  os.makedirs(lowerCAmelCase__				,	exist_ok=lowerCAmelCase__							)
      accelerator.wait_for_everyone()

      UpperCAmelCase          =	None
      UpperCAmelCase          =	None
      UpperCAmelCase          =	0
      UpperCAmelCase          =	False
      # Show the progress bar
      UpperCAmelCase          =	tqdm(range(args.max_selftrain_iterations							)				,	disable=not accelerator.is_local_main_process							)

      # Self-train
      for iteration in range(0				,	int(args.max_selftrain_iterations							)							):
            UpperCAmelCase          =	data_dir_format(lowerCAmelCase__							)
            assert os.path.exists(lowerCAmelCase__							)

            # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
            # iteration > 0
            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""stage-1"""							)
            UpperCAmelCase          =	{
                """accelerator""": accelerator,
                """model_name_or_path""": args.model_name_or_path,
                """cache_dir""": args.cache_dir,
                """do_train""": True,
                """train_file""": data_files["""train"""] if iteration == 0 else data_files["""train_pseudo"""],
                """do_eval""": True if args.eval_file is not None else False,
                """eval_file""": data_files["""eval"""],
                """do_predict""": True,
                """infer_file""": data_files["""infer"""],
                """task_name""": args.task_name,
                """label_list""": args.label_list,
                """output_dir""": current_output_dir,
                """eval_metric""": args.eval_metric,
                """evaluation_strategy""": args.evaluation_strategy,
                """early_stopping_patience""": args.early_stopping_patience,
                """early_stopping_threshold""": args.early_stopping_threshold,
                """seed""": args.seed,
            }
            # Add additional training arguments
            for key, value in kwargs.items():
                  if key not in arguments_dict and not hasattr(lowerCAmelCase__				,	lowerCAmelCase__							):
                        arguments_dict.update({key: value}							)

            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""best-checkpoint"""				,	lowerCAmelCase__							)
            if os.path.exists(lowerCAmelCase__							):
                  logger.info(
                      """Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1."""				,	lowerCAmelCase__				,	lowerCAmelCase__				,	)
            else:
                  logger.info("""***** Running self-training: iteration: %d, stage: 1 *****"""				,	lowerCAmelCase__							)
                  finetune(**lowerCAmelCase__							)
                  accelerator.wait_for_everyone()
                  assert os.path.exists(lowerCAmelCase__							)
                  logger.info("""Self-training job completed: iteration: %d, stage: 1."""				,	lowerCAmelCase__							)

            if iteration > 0 and args.finetune_on_labeled_data:
                  # Stage 2 (optional): fine-tuning on the original labeled data
                  UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""best-checkpoint"""							)
                  UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""stage-2"""							)
                  # Update arguments_dict
                  UpperCAmelCase          =	model_path
                  UpperCAmelCase          =	data_files["""train"""]
                  UpperCAmelCase          =	current_output_dir

                  UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""best-checkpoint"""				,	lowerCAmelCase__							)
                  if os.path.exists(lowerCAmelCase__							):
                        logger.info(
                            """Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2."""				,	lowerCAmelCase__				,	lowerCAmelCase__				,	)
                  else:
                        logger.info("""***** Running self-training: iteration: %d, stage: 2 *****"""				,	lowerCAmelCase__							)
                        finetune(**lowerCAmelCase__							)
                        accelerator.wait_for_everyone()
                        assert os.path.exists(lowerCAmelCase__							)
                        logger.info("""Self-training job completed: iteration: %d, stage: 2."""				,	lowerCAmelCase__							)

            UpperCAmelCase          =	iteration
            UpperCAmelCase          =	data_dir_format(iteration + 1							)

            UpperCAmelCase          =	AutoConfig.from_pretrained(os.path.join(lowerCAmelCase__				,	"""best-checkpoint"""							)							)
            UpperCAmelCase          =	config.idalabel
            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""eval_results_best-checkpoint.json"""							)
            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""test_results_best-checkpoint.json"""							)
            assert os.path.exists(lowerCAmelCase__							)

            with open(lowerCAmelCase__				,	"""r"""							) as f:
                  UpperCAmelCase          =	float(json.load(lowerCAmelCase__							)[args.eval_metric]							)
            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	"""infer_output_best-checkpoint.csv"""							)
            assert os.path.exists(lowerCAmelCase__							)
            # Loading the dataset from local csv or json files.
            UpperCAmelCase          =	load_dataset(args.data_file_extension				,	data_files={"""data""": data_files["""infer"""]}							)["""data"""]
            UpperCAmelCase          =	load_dataset("""csv"""				,	data_files={"""data""": infer_output_file}							)["""data"""]

            if accelerator.is_main_process:
                  os.makedirs(lowerCAmelCase__				,	exist_ok=lowerCAmelCase__							)
                  shutil.copy(lowerCAmelCase__				,	os.path.join(lowerCAmelCase__				,	F'''eval_results_iter-{iteration}.json'''							)							)
                  if os.path.exists(lowerCAmelCase__							):
                        shutil.copy(lowerCAmelCase__				,	os.path.join(lowerCAmelCase__				,	F'''test_results_iter-{iteration}.json'''							)							)
                  create_pseudo_labeled_data(lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__				,	lowerCAmelCase__							)
            accelerator.wait_for_everyone()

            UpperCAmelCase          =	os.path.join(lowerCAmelCase__				,	F'''train_pseudo.{args.data_file_extension}'''							)

            if args.evaluation_strategy != IntervalStrategy.NO.value:
                  UpperCAmelCase          =	eval_result

                  if best_iteration is None:
                        UpperCAmelCase          =	new_iteration
                        UpperCAmelCase          =	new_eval_result
                  else:
                        if new_eval_result - best_eval_result > args.early_stopping_threshold:
                              UpperCAmelCase          =	new_iteration
                              UpperCAmelCase          =	new_eval_result
                              UpperCAmelCase          =	0
                        else:
                              if new_eval_result == best_eval_result:
                                    UpperCAmelCase          =	new_iteration
                                    UpperCAmelCase          =	new_eval_result
                              early_stopping_patience_counter += 1

                        if early_stopping_patience_counter >= args.early_stopping_patience:
                              UpperCAmelCase          =	True

            progress_bar.update(1							)

            if should_training_stop:
                  break

      if best_iteration is not None:
            # Save the best iteration
            logger.info("""Best iteration: %d"""				,	lowerCAmelCase__							)
            logger.info("""Best evaluation result: %s = %f"""				,	args.eval_metric				,	lowerCAmelCase__							)
            accelerator.wait_for_everyone()
            if accelerator.is_main_process:
                  shutil.copy(
                      os.path.join(lowerCAmelCase__				,	F'''eval_results_iter-{iteration}.json'''							)				,	os.path.join(lowerCAmelCase__				,	"""eval_results_best-iteration.json"""							)				,	)
      else:
            # Assume that the last iteration is the best
            logger.info("""Best iteration: %d"""				,	args.max_selftrain_iterations - 1							)
            logger.info("""Best evaluation result: %s = %f"""				,	args.eval_metric				,	lowerCAmelCase__							)
            accelerator.wait_for_everyone()
            if accelerator.is_main_process:
                  shutil.copy(
                      os.path.join(lowerCAmelCase__				,	F'''eval_results_iter-{args.max_selftrain_iterations - 1}.json'''							)				,	os.path.join(lowerCAmelCase__				,	"""eval_results_best-iteration.json"""							)				,	)











"""simple docstring"""







from __future__ import annotations

import json

import requests
from bsa import BeautifulSoup
from fake_useragent import UserAgent

_UpperCamelCase					      =  {"""UserAgent""": UserAgent().random}

def _a			(		_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	script.contents[0]
      UpperCAmelCase          =	json.loads(data[data.find("""{\"config\""""							) : -1]							)
      return info["entry_data"]["ProfilePage"][0]["graphql"]["user"]







class  lowerCamelCase__ :






    def __init__(			self     ,A ):
          UpperCAmelCase          =	F'''https://www.instagram.com/{username}/'''
          UpperCAmelCase          =	self.get_json()






    def     _UpperCamelCase							(			self ):
          UpperCAmelCase          =	requests.get(self.url     ,headers=A ).text
          UpperCAmelCase          =	BeautifulSoup(A     ,"""html.parser""" ).find_all("""script""" )
          try:
                return extract_user_profile(scripts[4] )
          except (json.decoder.JSONDecodeError, KeyError):
                return extract_user_profile(scripts[3] )






    def __repr__(			self ):
          return F'''{self.__class__.__name__}(\'{self.username}\')'''






    def __str__(			self ):
          return F'''{self.fullname} ({self.username}) is {self.biography}'''






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["username"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["full_name"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["biography"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["business_email"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["external_url"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["edge_followed_by"]["count"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["edge_follow"]["count"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["edge_owner_to_timeline_media"]["count"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["profile_pic_url_hd"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["is_verified"]






    @property
    def     _UpperCamelCase							(			self ):
          return self.user_data["is_private"]







def _a			(		_snake_case = "github"							):







      """simple docstring"""






      import os

      if os.environ.get("""CI"""							):
            return  # test failing on GitHub Actions
      UpperCAmelCase          =	InstagramUser(_snake_case							)
      assert instagram_user.user_data
      assert isinstance(instagram_user.user_data				,	_snake_case							)
      assert instagram_user.username == username
      if username != "github":
            return
      assert instagram_user.fullname == "GitHub"
      assert instagram_user.biography == "Built for developers."
      assert instagram_user.number_of_posts > 150
      assert instagram_user.number_of_followers > 12_0000
      assert instagram_user.number_of_followings > 15
      assert instagram_user.email == "support@github.com"
      assert instagram_user.website == "https://github.com/readme"
      assert instagram_user.profile_picture_url.startswith("""https://instagram."""							)
      assert instagram_user.is_verified is True
      assert instagram_user.is_private is False


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  _UpperCamelCase					      =  InstagramUser("""github""")
  print(instagram_user)
  print(F"""{instagram_user.number_of_posts = }""")
  print(F"""{instagram_user.number_of_followers = }""")
  print(F"""{instagram_user.number_of_followings = }""")
  print(F"""{instagram_user.email = }""")
  print(F"""{instagram_user.website = }""")
  print(F"""{instagram_user.profile_picture_url = }""")
  print(F"""{instagram_user.is_verified = }""")
  print(F"""{instagram_user.is_private = }""")



