





"""simple docstring"""




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''			,  __UpperCAmelCase			,  )
class 			A__   (							__UpperCAmelCase     ):




   """simple docstring"""



   __A       :    List[str]	  =  RobertaConfig
   __A       :    Dict	  =  '''roberta'''

   def __init__( self ,  lowercase)      ->						str:







          '''simple docstring'''

          super().__init__(lowercase)

          a__						:     Dict			=    RobertaEmbeddings(lowercase)
          self.init_weights()





@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''			,  __UpperCAmelCase			,  )
class 			A__   (							__UpperCAmelCase     ):




   """simple docstring"""



   __A       :    Optional[int]	  =  RobertaConfig
   __A       :    Tuple	  =  '''roberta'''

   def __init__( self ,  lowercase)      ->						Union[str, Any]:







          '''simple docstring'''

          super().__init__(lowercase)
          a__						:     int			=    config.num_labels
          a__						:     List[str]			=    config.num_hidden_layers

          a__						:     Any			=    DeeRobertaModel(lowercase)
          a__						:     Tuple			=    nn.Dropout(config.hidden_dropout_prob)
          a__						:     Union[str, Any]			=    nn.Linear(config.hidden_size ,  self.config.num_labels)

   @add_start_docstrings_to_model_forward(lowercase)
   def 							__lowercase		( self ,  lowercase=None ,  lowercase=None ,  lowercase=None ,  lowercase=None ,  lowercase=None ,  lowercase=None ,  lowercase=None ,  lowercase=-1 ,  lowercase=False ,  )      ->						Tuple:







          '''simple docstring'''

          a__						:     Optional[Any]			=    self.num_layers
          try:
                 a__						:     Any			=    self.roberta(
                     lowercase ,  attention_mask=lowercase ,  token_type_ids=lowercase ,  position_ids=lowercase ,  head_mask=lowercase ,  inputs_embeds=lowercase ,  )

                 a__						:     Dict			=    outputs[1]

                 a__						:     List[Any]			=    self.dropout(lowercase)
                 a__						:     List[Any]			=    self.classifier(lowercase)
                 a__						:     str			=    (logits,) + outputs[2:]  # add hidden states and attention if they are here
          except HighwayException as e:
                 a__						:     str			=    e.message
                 a__						:     str			=    e.exit_layer
                 a__						:     Tuple			=    outputs[0]

          if not self.training:
                 a__						:     Dict			=    entropy(lowercase)
                 a__						:     Tuple			=    []
                 a__						:     List[str]			=    []
          if labels is not None:
                 if self.num_labels == 1:
                        #  We are doing regression
                        a__						:     Optional[Any]			=    MSELoss()
                        a__						:     List[str]			=    loss_fct(logits.view(-1) ,  labels.view(-1))
                 else:
                        a__						:     Union[str, Any]			=    CrossEntropyLoss()
                        a__						:     Optional[int]			=    loss_fct(logits.view(-1 ,  self.num_labels) ,  labels.view(-1))

                 # work with highway exits
                 a__						:     Optional[int]			=    []
                 for highway_exit in outputs[-1]:
                        a__						:     Optional[Any]			=    highway_exit[0]
                        if not self.training:
                               highway_logits_all.append(lowercase)
                               highway_entropy.append(highway_exit[2])
                        if self.num_labels == 1:
                               #  We are doing regression
                               a__						:     Optional[int]			=    MSELoss()
                               a__						:     Tuple			=    loss_fct(highway_logits.view(-1) ,  labels.view(-1))
                        else:
                               a__						:     Any			=    CrossEntropyLoss()
                               a__						:     Tuple			=    loss_fct(highway_logits.view(-1 ,  self.num_labels) ,  labels.view(-1))
                        highway_losses.append(lowercase)

                 if train_highway:
                        a__						:     Optional[Any]			=    (sum(highway_losses[:-1]),) + outputs
                        # exclude the final highway, of course
                 else:
                        a__						:     Dict			=    (loss,) + outputs
          if not self.training:
                 a__						:     List[Any]			=    outputs + ((original_entropy, highway_entropy), exit_layer)
                 if output_layer >= 0:
                        a__						:     Dict			=    (
                            (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                        )  # use the highway of the last layer

          return outputs  # (loss), logits, (hidden_states), (attentions), entropy









import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
    from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm



def A_     (		A__    )  ->		Optional[int]:
       if is_torch_version('<'		,				'2.0.0'    ) or not hasattr(A__		,				'_dynamo'    ):
              return False
       return isinstance(A__		,				torch._dynamo.eval_frame.OptimizedModule    )



def A_     (		A__		,				A__ = True    )  ->		int:
       a__						:     Optional[Any]			=    (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

       a__						:     Union[str, Any]			=    is_compiled_module(A__    )
       if is_compiled:
              a__						:     List[str]			=    model
              a__						:     Dict			=    model._orig_mod

       if is_deepspeed_available():
              options += (DeepSpeedEngine,)

       while isinstance(A__		,				A__    ):
              a__						:     str			=    model.module

       if not keep_fpaa_wrapper:
              a__						:     Union[str, Any]			=    getattr(A__		,				'forward'    )
              a__						:     List[Any]			=    model.__dict__.pop('_original_forward'		,				A__    )
              if original_forward is not None:
                     while hasattr(A__		,				'__wrapped__'    ):
                            a__						:     int			=    forward.__wrapped__
                            if forward == original_forward:
                                   break
                     a__						:     List[Any]			=    forward
              if getattr(A__		,				'_converted_to_transformer_engine'		,				A__    ):
                     convert_model(A__		,				to_transformer_engine=A__    )

       if is_compiled:
              a__						:     List[str]			=    model
              a__						:     Any			=    compiled_model

       return model



def A_     (		)  ->		int:
       PartialState().wait_for_everyone()



def A_     (		A__		,				A__    )  ->		Dict:
       if PartialState().distributed_type == DistributedType.TPU:
              xm.save(A__		,				A__    )
       elif PartialState().local_process_index == 0:
              torch.save(A__		,				A__    )



@contextmanager
def A_     (		**A__    )  ->		Any:
       for key, value in kwargs.items():
              a__						:     Optional[int]			=    str(A__    )

       yield

       for key in kwargs:
              if key.upper() in os.environ:
                     del os.environ[key.upper()]



def A_     (		A__    )  ->		List[str]:
       if not hasattr(A__		,				'__qualname__'    ) and not hasattr(A__		,				'__name__'    ):
              a__						:     Dict			=    getattr(A__		,				'__class__'		,				A__    )
       if hasattr(A__		,				'__qualname__'    ):
              return obj.__qualname__
       if hasattr(A__		,				'__name__'    ):
              return obj.__name__
       return str(A__    )



def A_     (		A__		,				A__    )  ->		Dict:
       for key, value in source.items():
              if isinstance(A__		,				A__    ):
                     a__						:     Optional[Any]			=    destination.setdefault(A__		,				{}    )
                     merge_dicts(A__		,				A__    )
              else:
                     a__						:     Optional[int]			=    value

       return destination



def A_     (		A__ = None    )  ->		bool:
       if port is None:
              a__						:     List[Any]			=    2_9500
       with socket.socket(socket.AF_INET		,				socket.SOCK_STREAM    ) as s:
              return s.connect_ex(('localhost', port)    ) == 0


