





'''simple docstring'''






import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A__    :       List[Any]              =16
A__    :       Union[str, Any]              =32
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase = 16  ):





      """simple docstring"""


      _lowerCAmelCase												=							AutoTokenizer.from_pretrained("""bert-base-cased"""  )
      _lowerCAmelCase												=							DatasetDict(
          {
              """train""": dataset["""train"""].select(lowerCAmelCase  ),
              """validation""": dataset["""train"""].select(lowerCAmelCase  ),
              """test""": dataset["""validation"""],
          }  )

      def tokenize_function(lowerCAmelCase  ):
            # max_length=None => use the model max length (it's actually the default)
            _lowerCAmelCase												=							tokenizer(examples["""sentence1"""]							,     examples["""sentence2"""]							,     truncation=lowerCAmelCase							,     max_length=lowerCAmelCase  )
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            _lowerCAmelCase												=							datasets.map(
                lowerCAmelCase							,     batched=lowerCAmelCase							,     remove_columns=["""idx""", """sentence1""", """sentence2"""]							,     )

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      _lowerCAmelCase												=							tokenized_datasets.rename_column("""label"""							,     """labels"""  )

      def collate_fn(lowerCAmelCase  ):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            _lowerCAmelCase												=							1_28 if accelerator.distributed_type == DistributedType.TPU else None
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  _lowerCAmelCase												=							16
            elif accelerator.mixed_precision != "no":
                  _lowerCAmelCase												=							8
            else:
                  _lowerCAmelCase												=							None

            return tokenizer.pad(
                lowerCAmelCase							,     padding="""longest"""							,     max_length=lowerCAmelCase							,     pad_to_multiple_of=lowerCAmelCase							,     return_tensors="""pt"""							,     )

      # Instantiate dataloaders.
      _lowerCAmelCase												=							DataLoader(
          tokenized_datasets["""train"""]							,     shuffle=lowerCAmelCase							,     collate_fn=lowerCAmelCase							,     batch_size=lowerCAmelCase  )
      _lowerCAmelCase												=							DataLoader(
          tokenized_datasets["""validation"""]							,     shuffle=lowerCAmelCase							,     collate_fn=lowerCAmelCase							,     batch_size=lowerCAmelCase  )

      _lowerCAmelCase												=							DataLoader(
          tokenized_datasets["""test"""]							,     shuffle=lowerCAmelCase							,     collate_fn=lowerCAmelCase							,     batch_size=lowerCAmelCase  )

      return train_dataloader, eval_dataloader, test_dataloader
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      _lowerCAmelCase												=							[]
      # Download the dataset
      _lowerCAmelCase												=							load_dataset("""glue"""							,     """mrpc"""  )
      # Create our splits
      _lowerCAmelCase												=							StratifiedKFold(n_splits=int(args.num_folds  )  )
      # Initialize accelerator
      _lowerCAmelCase												=							Accelerator(cpu=args.cpu							,     mixed_precision=args.mixed_precision  )
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      _lowerCAmelCase												=							config["""lr"""]
      _lowerCAmelCase												=							int(config["""num_epochs"""]  )
      _lowerCAmelCase												=							int(config["""seed"""]  )
      _lowerCAmelCase												=							int(config["""batch_size"""]  )

      _lowerCAmelCase												=							evaluate.load("""glue"""							,     """mrpc"""  )

      # If the batch size is too big we use gradient accumulation
      _lowerCAmelCase												=							1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
            _lowerCAmelCase												=							batch_size // MAX_GPU_BATCH_SIZE
            _lowerCAmelCase												=							MAX_GPU_BATCH_SIZE

      set_seed(lowerCAmelCase  )

      # New Code #
      # Create our folds:
      _lowerCAmelCase												=							kfold.split(np.zeros(datasets["""train"""].num_rows  )							,     datasets["""train"""]["""label"""]  )
      _lowerCAmelCase												=							[]
      # Iterate over them
      for i, (train_idxs, valid_idxs) in enumerate(lowerCAmelCase  ):
            _lowerCAmelCase					,							_lowerCAmelCase					,							_lowerCAmelCase												=							get_fold_dataloaders(
                lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     )
            # Instantiate the model (we build the model here so that the seed also control new weights initialization)
            _lowerCAmelCase												=							AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""							,     return_dict=lowerCAmelCase  )

            # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
            # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
            # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
            _lowerCAmelCase												=							model.to(accelerator.device  )

            # Instantiate optimizer
            _lowerCAmelCase												=							AdamW(params=model.parameters()							,     lr=lowerCAmelCase  )

            # Instantiate scheduler
            _lowerCAmelCase												=							get_linear_schedule_with_warmup(
                optimizer=lowerCAmelCase							,     num_warmup_steps=1_00							,     num_training_steps=(len(lowerCAmelCase  ) * num_epochs) // gradient_accumulation_steps							,     )

            # Prepare everything
            # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
            # prepare method.
            _lowerCAmelCase					,							_lowerCAmelCase					,							_lowerCAmelCase					,							_lowerCAmelCase					,							_lowerCAmelCase												=							accelerator.prepare(
                lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  )

            # Now we train the model
            for epoch in range(lowerCAmelCase  ):
                  model.train()
                  for step, batch in enumerate(lowerCAmelCase  ):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device  )
                        _lowerCAmelCase												=							model(**lowerCAmelCase  )
                        _lowerCAmelCase												=							outputs.loss
                        _lowerCAmelCase												=							loss / gradient_accumulation_steps
                        accelerator.backward(lowerCAmelCase  )
                        if step % gradient_accumulation_steps == 0:
                              optimizer.step()
                              lr_scheduler.step()
                              optimizer.zero_grad()

                  model.eval()
                  for step, batch in enumerate(lowerCAmelCase  ):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device  )
                        with torch.no_grad():
                              _lowerCAmelCase												=							model(**lowerCAmelCase  )
                        _lowerCAmelCase												=							outputs.logits.argmax(dim=-1  )
                        _lowerCAmelCase					,							_lowerCAmelCase												=							accelerator.gather_for_metrics((predictions, batch["""labels"""])  )
                        metric.add_batch(
                            predictions=lowerCAmelCase							,     references=lowerCAmelCase							,     )

                  _lowerCAmelCase												=							metric.compute()
                  # Use accelerator.print to print only on the main process.
                  accelerator.print(f"epoch {epoch}:"							,     lowerCAmelCase  )

            # New Code #
            # We also run predictions on the test set at the very end
            _lowerCAmelCase												=							[]
            for step, batch in enumerate(lowerCAmelCase  ):
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device  )
                  with torch.no_grad():
                        _lowerCAmelCase												=							model(**lowerCAmelCase  )
                  _lowerCAmelCase												=							outputs.logits
                  _lowerCAmelCase					,							_lowerCAmelCase												=							accelerator.gather_for_metrics((predictions, batch["""labels"""])  )
                  fold_predictions.append(predictions.cpu()  )
                  if i == 0:
                        # We need all of the test predictions
                        test_references.append(references.cpu()  )
        # Use accelerator.print to print only on the main process.
            test_predictions.append(torch.cat(lowerCAmelCase							,     dim=0  )  )
            # We now need to release all our memory and get rid of the current model, optimizer, etc
            accelerator.free_memory()
      # New Code #
      # Finally we check the accuracy of our folded results:
      _lowerCAmelCase												=							torch.cat(lowerCAmelCase							,     dim=0  )
      _lowerCAmelCase												=							torch.stack(lowerCAmelCase							,     dim=0  ).sum(dim=0  ).div(int(args.num_folds  )  ).argmax(dim=-1  )
      _lowerCAmelCase												=							metric.compute(predictions=lowerCAmelCase							,     references=lowerCAmelCase  )
      accelerator.print("""Average test metrics from all folds:"""							,     lowerCAmelCase  )
def        UpperCamelCase__				(		):





      """simple docstring"""


      _lowerCAmelCase												=							argparse.ArgumentParser(description="""Simple example of training script."""  )
      parser.add_argument(
          """--mixed_precision"""							,     type=lowerCAmelCase							,     default=lowerCAmelCase							,     choices=["""no""", """fp16""", """bf16""", """fp8"""]							,     help="""Whether to use mixed precision. Choose"""
          """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
          """and an Nvidia Ampere GPU."""							,     )
      parser.add_argument("""--cpu"""							,     action="""store_true"""							,     help="""If passed, will train on the CPU."""  )
      # New Code #
      parser.add_argument("""--num_folds"""							,     type=lowerCAmelCase							,     default=3							,     help="""The number of splits to perform across the dataset"""  )
      _lowerCAmelCase												=							parser.parse_args()
      _lowerCAmelCase												=							{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
      training_function(lowerCAmelCase							,     lowerCAmelCase  )


if __name__ == "__main__":
      main()













'''simple docstring'''






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import CLIPSegProcessor, ViTImageProcessor




@require_vision
class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			List[str]  )    ->				Dict:
            _lowerCAmelCase												=							tempfile.mkdtemp()

            # fmt: off
            _lowerCAmelCase												=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
            # fmt: on
            _lowerCAmelCase												=							dict(zip(__snake_case      ,      range(len(__snake_case  )  )  )  )
            _lowerCAmelCase												=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
            _lowerCAmelCase												=							{"""unk_token""": """<unk>"""}

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["""vocab_file"""]  )
            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["""merges_file"""]  )
            with open(self.vocab_file      ,      """w"""      ,      encoding="""utf-8"""  ) as fp:
                  fp.write(json.dumps(__snake_case  ) + """\n"""  )
            with open(self.merges_file      ,      """w"""      ,      encoding="""utf-8"""  ) as fp:
                  fp.write("""\n""".join(__snake_case  )  )

            _lowerCAmelCase												=							{
                """do_resize""": True,
                """size""": 20,
                """do_center_crop""": True,
                """crop_size""": 18,
                """do_normalize""": True,
                """image_mean""": [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
                """image_std""": [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
            }
            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      __snake_case  )
            with open(self.image_processor_file      ,      """w"""      ,      encoding="""utf-8"""  ) as fp:
                  json.dump(__snake_case      ,      __snake_case  )




      def     lowercase__   (      self      :			Optional[int]      ,      **__snake_case      :			Dict  )    ->				List[str]:
            return CLIPTokenizer.from_pretrained(self.tmpdirname      ,      **__snake_case  )




      def     lowercase__   (      self      :			List[str]      ,      **__snake_case      :			Any  )    ->				List[Any]:
            return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,      **__snake_case  )




      def     lowercase__   (      self      :			Tuple      ,      **__snake_case      :			List[str]  )    ->				int:
            return ViTImageProcessor.from_pretrained(self.tmpdirname      ,      **__snake_case  )




      def     lowercase__   (      self      :			str  )    ->				Optional[Any]:
            shutil.rmtree(self.tmpdirname  )




      def     lowercase__   (      self      :			str  )    ->				List[Any]:
            _lowerCAmelCase												=							[np.random.randint(2_55      ,      size=(3, 30, 4_00)      ,      dtype=np.uinta  )]

            _lowerCAmelCase												=							[Image.fromarray(np.moveaxis(__snake_case      ,      0      ,      -1  )  ) for x in image_inputs]

            return image_inputs




      def     lowercase__   (      self      :			Optional[Any]  )    ->				List[str]:
            _lowerCAmelCase												=							self.get_tokenizer()
            _lowerCAmelCase												=							self.get_rust_tokenizer()
            _lowerCAmelCase												=							self.get_image_processor()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )
            processor_slow.save_pretrained(self.tmpdirname  )
            _lowerCAmelCase												=							CLIPSegProcessor.from_pretrained(self.tmpdirname      ,      use_fast=__snake_case  )

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )
            processor_fast.save_pretrained(self.tmpdirname  )
            _lowerCAmelCase												=							CLIPSegProcessor.from_pretrained(self.tmpdirname  )

            self.assertEqual(processor_slow.tokenizer.get_vocab()      ,      tokenizer_slow.get_vocab()  )
            self.assertEqual(processor_fast.tokenizer.get_vocab()      ,      tokenizer_fast.get_vocab()  )
            self.assertEqual(tokenizer_slow.get_vocab()      ,      tokenizer_fast.get_vocab()  )
            self.assertIsInstance(processor_slow.tokenizer      ,      __snake_case  )
            self.assertIsInstance(processor_fast.tokenizer      ,      __snake_case  )

            self.assertEqual(processor_slow.image_processor.to_json_string()      ,      image_processor.to_json_string()  )
            self.assertEqual(processor_fast.image_processor.to_json_string()      ,      image_processor.to_json_string()  )
            self.assertIsInstance(processor_slow.image_processor      ,      __snake_case  )
            self.assertIsInstance(processor_fast.image_processor      ,      __snake_case  )




      def     lowercase__   (      self      :			Optional[int]  )    ->				Dict:
            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=self.get_tokenizer()      ,      image_processor=self.get_image_processor()  )
            processor.save_pretrained(self.tmpdirname  )

            _lowerCAmelCase												=							self.get_tokenizer(bos_token="""(BOS)"""      ,      eos_token="""(EOS)"""  )
            _lowerCAmelCase												=							self.get_image_processor(do_normalize=__snake_case      ,      padding_value=1.0  )

            _lowerCAmelCase												=							CLIPSegProcessor.from_pretrained(
                self.tmpdirname      ,      bos_token="""(BOS)"""      ,      eos_token="""(EOS)"""      ,      do_normalize=__snake_case      ,      padding_value=1.0  )

            self.assertEqual(processor.tokenizer.get_vocab()      ,      tokenizer_add_kwargs.get_vocab()  )
            self.assertIsInstance(processor.tokenizer      ,      __snake_case  )

            self.assertEqual(processor.image_processor.to_json_string()      ,      image_processor_add_kwargs.to_json_string()  )
            self.assertIsInstance(processor.image_processor      ,      __snake_case  )




      def     lowercase__   (      self      :			Tuple  )    ->				List[Any]:
            _lowerCAmelCase												=							self.get_image_processor()
            _lowerCAmelCase												=							self.get_tokenizer()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )

            _lowerCAmelCase												=							self.prepare_image_inputs()

            _lowerCAmelCase												=							image_processor(__snake_case      ,      return_tensors="""np"""  )
            _lowerCAmelCase												=							processor(images=__snake_case      ,      return_tensors="""np"""  )

            for key in input_feat_extract.keys():
                  self.assertAlmostEqual(input_feat_extract[key].sum()      ,      input_processor[key].sum()      ,      delta=1E-2  )




      def     lowercase__   (      self      :			Any  )    ->				str:
            _lowerCAmelCase												=							self.get_image_processor()
            _lowerCAmelCase												=							self.get_tokenizer()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )

            _lowerCAmelCase												=							"""lower newer"""

            _lowerCAmelCase												=							processor(text=__snake_case  )

            _lowerCAmelCase												=							tokenizer(__snake_case  )

            for key in encoded_tok.keys():
                  self.assertListEqual(encoded_tok[key]      ,      encoded_processor[key]  )




      def     lowercase__   (      self      :			str  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							self.get_image_processor()
            _lowerCAmelCase												=							self.get_tokenizer()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )

            _lowerCAmelCase												=							"""lower newer"""
            _lowerCAmelCase												=							self.prepare_image_inputs()

            _lowerCAmelCase												=							processor(text=__snake_case      ,      images=__snake_case  )

            self.assertListEqual(list(inputs.keys()  )      ,      ["""input_ids""", """attention_mask""", """pixel_values"""]  )

            # test if it raises when no input is passed
            with pytest.raises(__snake_case  ):
                  processor()




      def     lowercase__   (      self      :			Union[str, Any]  )    ->				Dict:
            _lowerCAmelCase												=							self.get_image_processor()
            _lowerCAmelCase												=							self.get_tokenizer()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )

            _lowerCAmelCase												=							self.prepare_image_inputs()
            _lowerCAmelCase												=							self.prepare_image_inputs()

            _lowerCAmelCase												=							processor(images=__snake_case      ,      visual_prompt=__snake_case  )

            self.assertListEqual(list(inputs.keys()  )      ,      ["""pixel_values""", """conditional_pixel_values"""]  )

            # test if it raises when no input is passed
            with pytest.raises(__snake_case  ):
                  processor()




      def     lowercase__   (      self      :			int  )    ->				Tuple:
            _lowerCAmelCase												=							self.get_image_processor()
            _lowerCAmelCase												=							self.get_tokenizer()

            _lowerCAmelCase												=							CLIPSegProcessor(tokenizer=__snake_case      ,      image_processor=__snake_case  )

            _lowerCAmelCase												=							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

            _lowerCAmelCase												=							processor.batch_decode(__snake_case  )
            _lowerCAmelCase												=							tokenizer.batch_decode(__snake_case  )

            self.assertListEqual(__snake_case      ,      __snake_case  )






