



'''simple docstring'''

import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import OwlViTImageProcessor, OwlViTProcessor

@require_vision
class __snake_case(				unittest.TestCase ):


						'''simple docstring'''



						def     __snake_case   ( self				)     ->     int:
							lowerCAmelCase   =     tempfile.mkdtemp()

							# fmt: off
							lowerCAmelCase   =     ["""""", """l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
							# fmt: on
							lowerCAmelCase   =     dict(zip(A_     ,	range(len(A_				)				)				)				)
							lowerCAmelCase   =     ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
							lowerCAmelCase   =     {"""unk_token""": """<unk>"""}

							lowerCAmelCase   =     os.path.join(self.tmpdirname     ,	VOCAB_FILES_NAMES["""vocab_file"""]				)
							lowerCAmelCase   =     os.path.join(self.tmpdirname     ,	VOCAB_FILES_NAMES["""merges_file"""]				)
							with open(self.vocab_file     ,	"""w"""     ,	encoding="""utf-8"""				) as fp:
								fp.write(json.dumps(A_				) + """\n"""				)
							with open(self.merges_file     ,	"""w"""     ,	encoding="""utf-8"""				) as fp:
								fp.write("""\n""".join(A_				)				)

							lowerCAmelCase   =     {
							    """do_resize""": True,
							    """size""": 20,
							    """do_center_crop""": True,
							    """crop_size""": 18,
							    """do_normalize""": True,
							    """image_mean""": [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3],
							    """image_std""": [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1],
							}
							lowerCAmelCase   =     os.path.join(self.tmpdirname     ,	A_				)
							with open(self.image_processor_file     ,	"""w"""     ,	encoding="""utf-8"""				) as fp:
								json.dump(A_     ,	A_				)


						def     __snake_case   ( self     ,	**A_				)     ->     Tuple:
							return CLIPTokenizer.from_pretrained(self.tmpdirname     ,	pad_token="""!"""     ,	**A_				)


						def     __snake_case   ( self     ,	**A_				)     ->     List[str]:
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname     ,	pad_token="""!"""     ,	**A_				)


						def     __snake_case   ( self     ,	**A_				)     ->     List[Any]:
							return OwlViTImageProcessor.from_pretrained(self.tmpdirname     ,	**A_				)


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							shutil.rmtree(self.tmpdirname				)


						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     [np.random.randint(255     ,	size=(3, 30, 400)     ,	dtype=np.uinta				)]

							lowerCAmelCase   =     [Image.fromarray(np.moveaxis(A_     ,	0     ,	-1				)				) for x in image_inputs]

							return image_inputs


						def     __snake_case   ( self				)     ->     Any:
							lowerCAmelCase   =     self.get_tokenizer()
							lowerCAmelCase   =     self.get_rust_tokenizer()
							lowerCAmelCase   =     self.get_image_processor()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)
							processor_slow.save_pretrained(self.tmpdirname				)
							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(self.tmpdirname     ,	use_fast=A_				)

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)
							processor_fast.save_pretrained(self.tmpdirname				)
							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(self.tmpdirname				)

							self.assertEqual(processor_slow.tokenizer.get_vocab()     ,	tokenizer_slow.get_vocab()				)
							self.assertEqual(processor_fast.tokenizer.get_vocab()     ,	tokenizer_fast.get_vocab()				)
							self.assertEqual(tokenizer_slow.get_vocab()     ,	tokenizer_fast.get_vocab()				)
							self.assertIsInstance(processor_slow.tokenizer     ,	A_				)
							self.assertIsInstance(processor_fast.tokenizer     ,	A_				)

							self.assertEqual(processor_slow.image_processor.to_json_string()     ,	image_processor.to_json_string()				)
							self.assertEqual(processor_fast.image_processor.to_json_string()     ,	image_processor.to_json_string()				)
							self.assertIsInstance(processor_slow.image_processor     ,	A_				)
							self.assertIsInstance(processor_fast.image_processor     ,	A_				)


						def     __snake_case   ( self				)     ->     Dict:
							lowerCAmelCase   =     OwlViTProcessor(tokenizer=self.get_tokenizer()     ,	image_processor=self.get_image_processor()				)
							processor.save_pretrained(self.tmpdirname				)

							lowerCAmelCase   =     self.get_tokenizer(bos_token="""(BOS)"""     ,	eos_token="""(EOS)"""				)
							lowerCAmelCase   =     self.get_image_processor(do_normalize=A_				)

							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(
							    self.tmpdirname     ,	bos_token="""(BOS)"""     ,	eos_token="""(EOS)"""     ,	do_normalize=A_				)

							self.assertEqual(processor.tokenizer.get_vocab()     ,	tokenizer_add_kwargs.get_vocab()				)
							self.assertIsInstance(processor.tokenizer     ,	A_				)

							self.assertEqual(processor.image_processor.to_json_string()     ,	image_processor_add_kwargs.to_json_string()				)
							self.assertIsInstance(processor.image_processor     ,	A_				)


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.get_image_processor()
							lowerCAmelCase   =     self.get_tokenizer()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)

							lowerCAmelCase   =     self.prepare_image_inputs()

							lowerCAmelCase   =     image_processor(A_     ,	return_tensors="""np"""				)
							lowerCAmelCase   =     processor(images=A_     ,	return_tensors="""np"""				)

							for key in input_image_proc.keys():
								self.assertAlmostEqual(input_image_proc[key].sum()     ,	input_processor[key].sum()     ,	delta=1e-2				)


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     self.get_image_processor()
							lowerCAmelCase   =     self.get_tokenizer()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)

							lowerCAmelCase   =     """lower newer"""

							lowerCAmelCase   =     processor(text=A_     ,	return_tensors="""np"""				)

							lowerCAmelCase   =     tokenizer(A_     ,	return_tensors="""np"""				)

							for key in encoded_tok.keys():
								self.assertListEqual(encoded_tok[key][0].tolist()     ,	encoded_processor[key][0].tolist()				)


						def     __snake_case   ( self				)     ->     Dict:
							lowerCAmelCase   =     self.get_image_processor()
							lowerCAmelCase   =     self.get_tokenizer()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)

							lowerCAmelCase   =     """lower newer"""
							lowerCAmelCase   =     self.prepare_image_inputs()

							lowerCAmelCase   =     processor(text=A_     ,	images=A_				)

							self.assertListEqual(list(inputs.keys()				)     ,	["""input_ids""", """attention_mask""", """pixel_values"""]				)

							# test if it raises when no input is passed
							with pytest.raises(A_				):
								processor()


						def     __snake_case   ( self				)     ->     str:
							lowerCAmelCase   =     """google/owlvit-base-patch32"""
							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(A_				)

							lowerCAmelCase   =     ["""cat""", """nasa badge"""]
							lowerCAmelCase   =     processor(text=A_				)

							lowerCAmelCase   =     16
							self.assertListEqual(list(inputs.keys()				)     ,	["""input_ids""", """attention_mask"""]				)
							self.assertEqual(inputs["""input_ids"""].shape     ,	(2, seq_length)				)

							# test if it raises when no input is passed
							with pytest.raises(A_				):
								processor()


						def     __snake_case   ( self				)     ->     List[Any]:
							lowerCAmelCase   =     """google/owlvit-base-patch32"""
							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(A_				)

							lowerCAmelCase   =     [["""cat""", """nasa badge"""], ["""person"""]]
							lowerCAmelCase   =     processor(text=A_				)

							lowerCAmelCase   =     16
							lowerCAmelCase   =     len(A_				)
							lowerCAmelCase   =     max([len(A_				) for texts in input_texts]				)

							self.assertListEqual(list(inputs.keys()				)     ,	["""input_ids""", """attention_mask"""]				)
							self.assertEqual(inputs["""input_ids"""].shape     ,	(batch_size * num_max_text_queries, seq_length)				)

							# test if it raises when no input is passed
							with pytest.raises(A_				):
								processor()


						def     __snake_case   ( self				)     ->     Union[str, Any]:
							lowerCAmelCase   =     """google/owlvit-base-patch32"""
							lowerCAmelCase   =     OwlViTProcessor.from_pretrained(A_				)

							lowerCAmelCase   =     ["""cat""", """nasa badge"""]
							lowerCAmelCase   =     processor(text=A_				)

							lowerCAmelCase   =     16
							lowerCAmelCase   =     inputs["""input_ids"""]
							lowerCAmelCase   =     [
							    [4_9406, 2368, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    [4_9406, 6841, 1_1301, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							]

							self.assertListEqual(list(inputs.keys()				)     ,	["""input_ids""", """attention_mask"""]				)
							self.assertEqual(inputs["""input_ids"""].shape     ,	(2, seq_length)				)
							self.assertListEqual(list(input_ids[0]				)     ,	predicted_ids[0]				)
							self.assertListEqual(list(input_ids[1]				)     ,	predicted_ids[1]				)


						def     __snake_case   ( self				)     ->     Tuple:
							lowerCAmelCase   =     self.get_image_processor()
							lowerCAmelCase   =     self.get_tokenizer()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)

							lowerCAmelCase   =     self.prepare_image_inputs()
							lowerCAmelCase   =     self.prepare_image_inputs()

							lowerCAmelCase   =     processor(images=A_     ,	query_images=A_				)

							self.assertListEqual(list(inputs.keys()				)     ,	["""query_pixel_values""", """pixel_values"""]				)

							# test if it raises when no input is passed
							with pytest.raises(A_				):
								processor()




						def     __snake_case   ( self				)     ->     Dict:
							lowerCAmelCase   =     self.get_image_processor()
							lowerCAmelCase   =     self.get_tokenizer()

							lowerCAmelCase   =     OwlViTProcessor(tokenizer=A_     ,	image_processor=A_				)

							lowerCAmelCase   =     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

							lowerCAmelCase   =     processor.batch_decode(A_				)
							lowerCAmelCase   =     tokenizer.batch_decode(A_				)

							self.assertListEqual(A_     ,	A_				)




'''simple docstring'''

import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
				raise RuntimeError('At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training')

# TF training parameters
UpperCAmelCase     		=   False
UpperCAmelCase     		=   False






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Namespace							)      ->						Tuple:







	"""simple docstring"""
	return TrainCommand(_SCREAMING_SNAKE_CASE							)





class __snake_case(				_lowerCAmelCase ):


						'''simple docstring'''



						@staticmethod
						def     __snake_case   ( A_				)     ->     Optional[int]:
							lowerCAmelCase   =     parser.add_parser("""train"""     ,	help="""CLI tool to train a model on a task."""				)

							train_parser.add_argument(
							    """--train_data"""     ,	type=A_     ,	required=A_     ,	help="""path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences."""     ,	)
							train_parser.add_argument(
							    """--column_label"""     ,	type=A_     ,	default=0     ,	help="""Column of the dataset csv file with example labels."""				)
							train_parser.add_argument(
							    """--column_text"""     ,	type=A_     ,	default=1     ,	help="""Column of the dataset csv file with example texts."""				)
							train_parser.add_argument(
							    """--column_id"""     ,	type=A_     ,	default=2     ,	help="""Column of the dataset csv file with example ids."""				)
							train_parser.add_argument(
							    """--skip_first_row"""     ,	action="""store_true"""     ,	help="""Skip the first row of the csv file (headers)."""				)

							train_parser.add_argument("""--validation_data"""     ,	type=A_     ,	default=""""""     ,	help="""path to validation dataset."""				)
							train_parser.add_argument(
							    """--validation_split"""     ,	type=A_     ,	default=0.1     ,	help="""if validation dataset is not provided, fraction of train dataset to use as validation dataset."""     ,	)

							train_parser.add_argument("""--output"""     ,	type=A_     ,	default="""./"""     ,	help="""path to saved the trained model."""				)

							train_parser.add_argument(
							    """--task"""     ,	type=A_     ,	default="""text_classification"""     ,	help="""Task to train the model on."""				)
							train_parser.add_argument(
							    """--model"""     ,	type=A_     ,	default="""bert-base-uncased"""     ,	help="""Model's name or path to stored model."""				)
							train_parser.add_argument("""--train_batch_size"""     ,	type=A_     ,	default=32     ,	help="""Batch size for training."""				)
							train_parser.add_argument("""--valid_batch_size"""     ,	type=A_     ,	default=64     ,	help="""Batch size for validation."""				)
							train_parser.add_argument("""--learning_rate"""     ,	type=A_     ,	default=3e-5     ,	help="""Learning rate."""				)
							train_parser.add_argument("""--adam_epsilon"""     ,	type=A_     ,	default=1e-08     ,	help="""Epsilon for Adam optimizer."""				)
							train_parser.set_defaults(func=A_				)


						def __init__( self     ,	A_				)     ->     Tuple:
							lowerCAmelCase   =     logging.get_logger("""transformers-cli/training"""				)

							lowerCAmelCase   =     """tf""" if is_tf_available() else """torch"""

							os.makedirs(args.output     ,	exist_ok=A_				)
							lowerCAmelCase   =     args.output

							lowerCAmelCase   =     args.column_label
							lowerCAmelCase   =     args.column_text
							lowerCAmelCase   =     args.column_id

							self.logger.info(f'Loading {args.task} pipeline for {args.model}'				)
							if args.task == "text_classification":
								lowerCAmelCase   =     TextClassificationPipeline.from_pretrained(args.model				)
							elif args.task == "token_classification":
								raise NotImplementedError
							elif args.task == "question_answering":
								raise NotImplementedError

							self.logger.info(f'Loading dataset from {args.train_data}'				)
							lowerCAmelCase   =     Processor.create_from_csv(
							    args.train_data     ,	column_label=args.column_label     ,	column_text=args.column_text     ,	column_id=args.column_id     ,	skip_first_row=args.skip_first_row     ,	)
							lowerCAmelCase   =     None
							if args.validation_data:
								self.logger.info(f'Loading validation dataset from {args.validation_data}'				)
								lowerCAmelCase   =     Processor.create_from_csv(
								    args.validation_data     ,	column_label=args.column_label     ,	column_text=args.column_text     ,	column_id=args.column_id     ,	skip_first_row=args.skip_first_row     ,	)

							lowerCAmelCase   =     args.validation_split
							lowerCAmelCase   =     args.train_batch_size
							lowerCAmelCase   =     args.valid_batch_size
							lowerCAmelCase   =     args.learning_rate
							lowerCAmelCase   =     args.adam_epsilon


						def     __snake_case   ( self				)     ->     Optional[int]:
							if self.framework == "tf":
								return self.run_tf()
							return self.run_torch()


						def     __snake_case   ( self				)     ->     Tuple:
							raise NotImplementedError




						def     __snake_case   ( self				)     ->     Tuple:
							self.pipeline.fit(
							    self.train_dataset     ,	validation_data=self.valid_dataset     ,	validation_split=self.validation_split     ,	learning_rate=self.learning_rate     ,	adam_epsilon=self.adam_epsilon     ,	train_batch_size=self.train_batch_size     ,	valid_batch_size=self.valid_batch_size     ,	)

							# Save trained pipeline
							self.pipeline.save_pretrained(self.output				)