





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


snake_case_           =      logging.getLogger(__name__)

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):


      def        a			(self	:				Tuple    ,      a__	:				Optional[int]    ,      a__	:				Dict    ,      a__	:				Optional[Any]=None    ,      a__	:				int=None				):





         """simple docstring"""



         __snake_case								=       self.layer[current_layer](a__    ,      a__    ,      head_mask[current_layer]				)

         __snake_case								=       layer_outputs[0]

         return hidden_states






@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'						,				_UpperCAmelCase						,				)
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):


      def __init__(self	:				Optional[int]    ,      a__	:				int				):





         """simple docstring"""



         super().__init__(a__				)

         __snake_case								=       BertEncoderWithPabee(a__				)

         self.init_weights()
         __snake_case								=       0
         __snake_case								=       0
         __snake_case								=       0

         __snake_case								=       0


      def        a			(self	:				List[str]    ,      a__	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       threshold


      def        a			(self	:				Tuple    ,      a__	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       patience


      def        a			(self	:				Any				):





         """simple docstring"""



         __snake_case								=       0
         __snake_case								=       0


      def        a			(self	:				Tuple				):





         """simple docstring"""



         __snake_case								=       self.inference_layers_num / self.inference_instances_num
         __snake_case								=       (
             f"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
             f""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
         )
         print(a__				)


      @add_start_docstrings_to_model_forward(a__				)
      def        a			(self	:				Optional[Any]    ,      a__	:				Dict=None    ,      a__	:				int=None    ,      a__	:				str=None    ,      a__	:				List[Any]=None    ,      a__	:				List[Any]=None    ,      a__	:				Dict=None    ,      a__	:				Tuple=None    ,      a__	:				str=None    ,      a__	:				Dict=None    ,      a__	:				Any=None    ,      a__	:				Optional[Any]=False    ,      ):





         """simple docstring"""



         if input_ids is not None and inputs_embeds is not None:
            raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''				)
         elif input_ids is not None:
            __snake_case								=       input_ids.size()
         elif inputs_embeds is not None:
            __snake_case								=       inputs_embeds.size()[:-1]
         else:
            raise ValueError('''You have to specify either input_ids or inputs_embeds'''				)

         __snake_case								=       input_ids.device if input_ids is not None else inputs_embeds.device

         if attention_mask is None:
            __snake_case								=       torch.ones(a__    ,      device=a__				)
         if token_type_ids is None:
            __snake_case								=       torch.zeros(a__    ,      dtype=torch.long    ,      device=a__				)

         # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
         # ourselves in which case we just need to make it broadcastable to all heads.
         __snake_case								=       self.get_extended_attention_mask(a__    ,      a__    ,      a__				)

         # If a 2D ou 3D attention mask is provided for the cross-attention
         # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
         if self.config.is_decoder and encoder_hidden_states is not None:
            __snake_case	,     __snake_case	,     __snake_case								=       encoder_hidden_states.size()
            __snake_case								=       (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
               __snake_case								=       torch.ones(a__    ,      device=a__				)
            __snake_case								=       self.invert_attention_mask(a__				)
         else:
            __snake_case								=       None

         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape bsz x n_heads x N x N
         # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
         # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
         __snake_case								=       self.get_head_mask(a__    ,      self.config.num_hidden_layers				)

         __snake_case								=       self.embeddings(
             input_ids=a__    ,      position_ids=a__    ,      token_type_ids=a__    ,      inputs_embeds=a__				)
         __snake_case								=       embedding_output

         if self.training:
            __snake_case								=       []
            for i in range(self.config.num_hidden_layers				):
               __snake_case								=       self.encoder.adaptive_forward(
                   a__    ,      current_layer=a__    ,      attention_mask=a__    ,      head_mask=a__				)

               __snake_case								=       self.pooler(a__				)
               __snake_case								=       output_layers[i](output_dropout(a__				)				)
               res.append(a__				)
         elif self.patience == 0:  # Use all layers for inference
            __snake_case								=       self.encoder(
                a__    ,      attention_mask=a__    ,      head_mask=a__    ,      encoder_hidden_states=a__    ,      encoder_attention_mask=a__    ,      )
            __snake_case								=       self.pooler(encoder_outputs[0]				)
            __snake_case								=       [output_layers[self.config.num_hidden_layers - 1](a__				)]
         else:
            __snake_case								=       0
            __snake_case								=       None
            __snake_case								=       0
            for i in range(self.config.num_hidden_layers				):
               calculated_layer_num += 1
               __snake_case								=       self.encoder.adaptive_forward(
                   a__    ,      current_layer=a__    ,      attention_mask=a__    ,      head_mask=a__				)

               __snake_case								=       self.pooler(a__				)
               __snake_case								=       output_layers[i](a__				)
               if regression:
                  __snake_case								=       logits.detach()
                  if patient_result is not None:
                     __snake_case								=       patient_result.detach()
                  if (patient_result is not None) and torch.abs(patient_result - labels				) < self.regression_threshold:
                     patient_counter += 1
                  else:
                     __snake_case								=       0
               else:
                  __snake_case								=       logits.detach().argmax(dim=1				)
                  if patient_result is not None:
                     __snake_case								=       patient_result.detach().argmax(dim=1				)
                  if (patient_result is not None) and torch.all(labels.eq(a__				)				):
                     patient_counter += 1
                  else:
                     __snake_case								=       0

               __snake_case								=       logits
               if patient_counter == self.patience:
                  break
            __snake_case								=       [patient_result]
            self.inference_layers_num += calculated_layer_num
            self.inference_instances_num += 1

         return res






@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '						,				_UpperCAmelCase						,				)
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):


      def __init__(self	:				Any    ,      a__	:				Union[str, Any]				):





         """simple docstring"""



         super().__init__(a__				)
         __snake_case								=       config.num_labels

         __snake_case								=       BertModelWithPabee(a__				)
         __snake_case								=       nn.Dropout(config.hidden_dropout_prob				)
         __snake_case								=       nn.ModuleList(
             [nn.Linear(config.hidden_size    ,      self.config.num_labels				) for _ in range(config.num_hidden_layers				)]				)

         self.init_weights()


      @add_start_docstrings_to_model_forward(a__				)
      def        a			(self	:				List[Any]    ,      a__	:				Dict=None    ,      a__	:				Any=None    ,      a__	:				List[str]=None    ,      a__	:				Tuple=None    ,      a__	:				Optional[Any]=None    ,      a__	:				Union[str, Any]=None    ,      a__	:				str=None    ,      ):





         """simple docstring"""



         __snake_case								=       self.bert(
             input_ids=a__    ,      attention_mask=a__    ,      token_type_ids=a__    ,      position_ids=a__    ,      head_mask=a__    ,      inputs_embeds=a__    ,      output_dropout=self.dropout    ,      output_layers=self.classifiers    ,      regression=self.num_labels == 1    ,      )

         __snake_case								=       (logits[-1],)

         if labels is not None:
            __snake_case								=       None
            __snake_case								=       0
            for ix, logits_item in enumerate(a__				):
               if self.num_labels == 1:
                  #  We are doing regression
                  __snake_case								=       MSELoss()
                  __snake_case								=       loss_fct(logits_item.view(-1				)    ,      labels.view(-1				)				)
               else:
                  __snake_case								=       CrossEntropyLoss()
                  __snake_case								=       loss_fct(logits_item.view(-1    ,      self.num_labels				)    ,      labels.view(-1				)				)
               if total_loss is None:
                  __snake_case								=       loss
               else:
                  total_loss += loss * (ix + 1)
               total_weights += ix + 1
            __snake_case								=       (total_loss / total_weights,) + outputs

         return outputs








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_           =      {}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_           =      ['NllbTokenizer']

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_           =      ['NllbTokenizerFast']


if TYPE_CHECKING:
    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_nllb import NllbTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_nllb_fast import NllbTokenizerFast

else:
    import sys

    snake_case_           =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

