






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy






@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """     ,	_lowerCAmelCase     ,	)
class     a_						(      _lowerCAmelCase    ):
  lowercase							=				RobertaConfig
  lowercase							=				"""roberta"""

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        super().__init__(SCREAMING_SNAKE_CASE_				)

        UpperCamelCase      = RobertaEmbeddings(SCREAMING_SNAKE_CASE_				)
        self.init_weights()






@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """     ,	_lowerCAmelCase     ,	)
class     a_						(      _lowerCAmelCase    ):
  lowercase							=				RobertaConfig
  lowercase							=				"""roberta"""

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:



        """simple docstring"""






        super().__init__(SCREAMING_SNAKE_CASE_				)
        UpperCamelCase      = config.num_labels
        UpperCamelCase      = config.num_hidden_layers

        UpperCamelCase      = DeeRobertaModel(SCREAMING_SNAKE_CASE_				)
        UpperCamelCase      = nn.Dropout(config.hidden_dropout_prob				)
        UpperCamelCase      = nn.Linear(config.hidden_size      ,		self.config.num_labels				)


  @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_				)
  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=-1      ,		_SCREAMING_SNAKE_CASE=False      ,		)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = self.num_layers
        try:
              UpperCamelCase      = self.roberta(
                  SCREAMING_SNAKE_CASE_      ,		attention_mask=SCREAMING_SNAKE_CASE_      ,		token_type_ids=SCREAMING_SNAKE_CASE_      ,		position_ids=SCREAMING_SNAKE_CASE_      ,		head_mask=SCREAMING_SNAKE_CASE_      ,		inputs_embeds=SCREAMING_SNAKE_CASE_      ,		)

              UpperCamelCase      = outputs[1]

              UpperCamelCase      = self.dropout(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase      = self.classifier(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase      = (logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
              UpperCamelCase      = e.message
              UpperCamelCase      = e.exit_layer
              UpperCamelCase      = outputs[0]

        if not self.training:
              UpperCamelCase      = entropy(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase      = []
              UpperCamelCase      = []
        if labels is not None:
              if self.num_labels == 1:
                    #  We are doing regression
                    UpperCamelCase      = MSELoss()
                    UpperCamelCase      = loss_fct(logits.view(-1				)      ,		labels.view(-1				)				)
              else:
                    UpperCamelCase      = CrossEntropyLoss()
                    UpperCamelCase      = loss_fct(logits.view(-1      ,		self.num_labels				)      ,		labels.view(-1				)				)

              # work with highway exits
              UpperCamelCase      = []
              for highway_exit in outputs[-1]:
                    UpperCamelCase      = highway_exit[0]
                    if not self.training:
                          highway_logits_all.append(SCREAMING_SNAKE_CASE_				)
                          highway_entropy.append(highway_exit[2]				)
                    if self.num_labels == 1:
                          #  We are doing regression
                          UpperCamelCase      = MSELoss()
                          UpperCamelCase      = loss_fct(highway_logits.view(-1				)      ,		labels.view(-1				)				)
                    else:
                          UpperCamelCase      = CrossEntropyLoss()
                          UpperCamelCase      = loss_fct(highway_logits.view(-1      ,		self.num_labels				)      ,		labels.view(-1				)				)
                    highway_losses.append(SCREAMING_SNAKE_CASE_				)

              if train_highway:
                    UpperCamelCase      = (sum(highway_losses[:-1]				),) + outputs
                    # exclude the final highway, of course
              else:
                    UpperCamelCase      = (loss,) + outputs
        if not self.training:
              UpperCamelCase      = outputs + ((original_entropy, highway_entropy), exit_layer)
              if output_layer >= 0:
                    UpperCamelCase      = (
                        (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                    )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy








'''simple docstring'''

import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


SCREAMING_SNAKE_CASE__     =       'scheduler_config.json'






class     a_						(      lowerCamelCase    ):
  lowercase							=				1
  lowercase							=				2
  lowercase							=				3
  lowercase							=				4
  lowercase							=				5
  lowercase							=				6
  lowercase							=				7
  lowercase							=				8
  lowercase							=				9
  lowercase							=				10
  lowercase							=				11
  lowercase							=				12
  lowercase							=				13
  lowercase							=				14






@dataclass
class     a_						(      lowerCamelCase    ):
  lowercase							=				42






class     a_						:
  lowercase							=				SCHEDULER_CONFIG_NAME
  lowercase							=				[]
  lowercase							=				True

  @classmethod
  def   A__						(				cls      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE=False      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase      = cls.load_config(
            pretrained_model_name_or_path=_SCREAMING_SNAKE_CASE      ,		subfolder=_SCREAMING_SNAKE_CASE      ,		return_unused_kwargs=_SCREAMING_SNAKE_CASE      ,		return_commit_hash=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE      ,		)
        return cls.from_config(_SCREAMING_SNAKE_CASE      ,		return_unused_kwargs=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = False      ,		**_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        self.save_config(save_directory=_SCREAMING_SNAKE_CASE      ,		push_to_hub=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self._get_compatibles()


  @classmethod
  def   A__						(				cls				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = list(set([cls.__name__] + cls._compatibles				)				)
        UpperCamelCase      = importlib.import_module(__name__.split("""."""				)[0]				)
        UpperCamelCase      = [
            getattr(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				) for c in compatible_classes_str if hasattr(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        ]
        return compatible_classes
