





from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
		from fairscale.optim import OSS


A	     =					logging.get_logger(__name__)

A	     =					{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
}




class 			__lowercase     (     a_							):







					'''simple docstring'''




					def __init__(  self		,     _UpperCAmelCase=None		,     _UpperCAmelCase=None		,     *_UpperCAmelCase		,     **_UpperCAmelCase       ):
									super().__init__(*_UpperCAmelCase		,     **_UpperCAmelCase       )

									if config is None:
													assert isinstance(self.model		,     _UpperCAmelCase       ), (
													    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
													    f""" {self.model.__class__}"""
													)
													__a	:  int         =					self.model.config
									else:
													__a	:  Union[str, Any]         =					config

									__a	:  str         =					data_args
									__a	:  Optional[Any]         =					self.config.tgt_vocab_size if isinstance(self.config		,     _UpperCAmelCase       ) else self.config.vocab_size

									if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
													assert self.config.pad_token_id is not None, (
													    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
													    " calculation or doing label smoothing."
													)

									if self.config.pad_token_id is None and self.config.eos_token_id is not None:
													logger.warning(
													    f"""The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for"""
													    ''' padding..'''       )

									if self.args.label_smoothing == 0:
													__a	:  Union[str, Any]         =					torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id       )
									else:
													# dynamically import label_smoothed_nll_loss
													from utils import label_smoothed_nll_loss

													__a	:  str         =					label_smoothed_nll_loss







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):







									if self.optimizer is None:
													__a	:  Union[str, Any]         =					['''bias''', '''LayerNorm.weight''']
													__a	:  List[str]         =					[
													    {
													        '''params''': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay       )],
													        '''weight_decay''': self.args.weight_decay,
													    },
													    {
													        '''params''': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay       )],
													        '''weight_decay''': 0.0,
													    },
													]
													__a	:  str         =					Adafactor if self.args.adafactor else AdamW
													if self.args.adafactor:
																	__a	:  Tuple         =					Adafactor
																	__a	:  int         =					{'''scale_parameter''': False, '''relative_step''': False}
													else:
																	__a	:  List[str]         =					AdamW
																	__a	:  Dict         =					{
																	    '''betas''': (self.args.adam_betaa, self.args.adam_betaa),
																	    '''eps''': self.args.adam_epsilon,
																	}
													__a	:  Optional[Any]         =					self.args.learning_rate
													if self.sharded_ddp:
																	__a	:  Any         =					OSS(
																	    params=_UpperCAmelCase		,     optim=_UpperCAmelCase		,     **_UpperCAmelCase		,     )
													else:
																	__a	:  str         =					optimizer_cls(_UpperCAmelCase		,     **_UpperCAmelCase       )

									if self.lr_scheduler is None:
													__a	:  Any         =					self._get_lr_scheduler(_UpperCAmelCase       )
									else:  # ignoring --lr_scheduler
													logger.warning('''scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'''       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									__a	:  List[Any]         =					arg_to_scheduler[self.args.lr_scheduler]
									if self.args.lr_scheduler == "constant":
													__a	:  List[str]         =					schedule_func(self.optimizer       )
									elif self.args.lr_scheduler == "constant_w_warmup":
													__a	:  List[str]         =					schedule_func(self.optimizer		,     num_warmup_steps=self.args.warmup_steps       )
									else:
													__a	:  List[str]         =					schedule_func(
													    self.optimizer		,     num_warmup_steps=self.args.warmup_steps		,     num_training_steps=_UpperCAmelCase       )
									return scheduler







					def 	_lowerCamelCase						(  self       ):







									if isinstance(self.train_dataset		,     torch.utils.data.IterableDataset       ):
													return None
									elif is_torch_tpu_available():
													return get_tpu_sampler(self.train_dataset       )
									else:
													if self.args.sortish_sampler:
																	self.train_dataset.make_sortish_sampler(
																	    self.args.per_device_train_batch_size		,     distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)		,     )

													return (
													    RandomSampler(self.train_dataset       )
													    if self.args.local_rank == -1
													    else DistributedSampler(self.train_dataset       )
													)







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):







									if self.args.label_smoothing == 0:
													if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																	# force training to ignore pad token
																	__a	:  str         =					model(**_UpperCAmelCase		,     use_cache=_UpperCAmelCase       )[0]
																	__a	:  List[str]         =					self.loss_fn(logits.view(-1		,     logits.shape[-1]       )		,     labels.view(-1       )       )
													else:
																	# compute usual loss via models
																	__a    ,  __a	:  Any         =					model(**_UpperCAmelCase		,     labels=_UpperCAmelCase		,     use_cache=_UpperCAmelCase       )[:2]
									else:
													# compute label smoothed loss
													__a	:  Optional[Any]         =					model(**_UpperCAmelCase		,     use_cache=_UpperCAmelCase       )[0]
													__a	:  List[Any]         =					torch.nn.functional.log_softmax(_UpperCAmelCase		,     dim=-1       )
													__a    ,  __a	:  Optional[int]         =					self.loss_fn(_UpperCAmelCase		,     _UpperCAmelCase		,     self.args.label_smoothing		,     ignore_index=self.config.pad_token_id       )
									return loss, logits







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  Optional[int]         =					inputs.pop('''labels'''       )
									__a    ,  __a	:  Any         =					self._compute_loss(_UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       )
									return loss







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase = None		,     ):
									__a	:  List[Any]         =					self._prepare_inputs(_UpperCAmelCase       )

									__a	:  str         =					{
									    '''max_length''': self.data_args.val_max_target_length
									    if self.data_args is not None
									    else self.config.max_length,
									    '''num_beams''': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
									}

									if self.args.predict_with_generate and not self.args.prediction_loss_only:
													__a	:  List[Any]         =					self.model.generate(
													    inputs['''input_ids''']		,     attention_mask=inputs['''attention_mask''']		,     **_UpperCAmelCase		,     )
													# in case the batch is shorter than max length, the output should be padded
													if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																	__a	:  Any         =					self._pad_tensors_to_max_len(_UpperCAmelCase		,     gen_kwargs['''max_length''']       )

									__a	:  str         =					inputs.pop('''labels'''       )
									with torch.no_grad():
													# compute loss on predict data
													__a    ,  __a	:  Optional[int]         =					self._compute_loss(_UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       )

									__a	:  Tuple         =					loss.mean().detach()
									if self.args.prediction_loss_only:
													return (loss, None, None)

									__a	:  int         =					generated_tokens if self.args.predict_with_generate else logits

									if labels.shape[-1] < gen_kwargs["max_length"]:
													__a	:  Optional[Any]         =					self._pad_tensors_to_max_len(_UpperCAmelCase		,     gen_kwargs['''max_length''']       )

									return (loss, logits, labels)







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  str         =					self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

									if pad_token_id is None:
													raise ValueError(
													    '''Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'''
													    f""" padded to `max_length`={max_length}"""       )

									__a	:  int         =					pad_token_id * torch.ones(
									    (tensor.shape[0], max_length)		,     dtype=tensor.dtype		,     device=tensor.device       )
									__a	:  List[str]         =					tensor
									return padded_tensor






"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A	     =					{
    '''configuration_instructblip''': [
        '''INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''InstructBlipConfig''',
        '''InstructBlipQFormerConfig''',
        '''InstructBlipVisionConfig''',
    ],
    '''processing_instructblip''': ['''InstructBlipProcessor'''],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A	     =					[
		    '''INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''InstructBlipQFormerModel''',
		    '''InstructBlipPreTrainedModel''',
		    '''InstructBlipForConditionalGeneration''',
		    '''InstructBlipVisionModel''',
		]

if TYPE_CHECKING:
		from .configuration_instructblip import (
		    INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    InstructBlipConfig,
		    InstructBlipQFormerConfig,
		    InstructBlipVisionConfig,
		)
		from .processing_instructblip import InstructBlipProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_instructblip import (
				    INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    InstructBlipForConditionalGeneration,
				    InstructBlipPreTrainedModel,
				    InstructBlipQFormerModel,
				    InstructBlipVisionModel,
				)

else:
		import sys

		A	     =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)