



"""simple docstring"""





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


lowerCamelCase       	=						logging.getLogger(__name__)
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       def 					lowercase__    (							self      :  Optional[Any]		,    _UpperCAmelCase      :  int		,    _UpperCAmelCase      :  str		,    _UpperCAmelCase      :  str=None		,    _UpperCAmelCase      :  List[str]=None   )			->						int:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.layer[current_layer](_UpperCAmelCase		,    _UpperCAmelCase		,    head_mask[current_layer]   )

             UpperCAmelCase_      			=							layer_outputs[0]

             return hidden_states



@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''    , SCREAMING_SNAKE_CASE    , )
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       def __init__(							self      :  List[str]		,    _UpperCAmelCase      :  Union[str, Any]   )			->						Any:


             '''simple docstring'''




             super().__init__(_UpperCAmelCase   )

             UpperCAmelCase_      			=							BertEncoderWithPabee(_UpperCAmelCase   )

             self.init_weights()
             UpperCAmelCase_      			=							0
             UpperCAmelCase_      			=							0
             UpperCAmelCase_      			=							0

             UpperCAmelCase_      			=							0



       def 					lowercase__    (							self      :  Optional[int]		,    _UpperCAmelCase      :  Dict   )			->						Any:


             '''simple docstring'''




             UpperCAmelCase_      			=							threshold



       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  int   )			->						Any:


             '''simple docstring'''




             UpperCAmelCase_      			=							patience



       def 					lowercase__    (							self      :  Any   )			->						List[Any]:


             '''simple docstring'''




             UpperCAmelCase_      			=							0
             UpperCAmelCase_      			=							0



       def 					lowercase__    (							self      :  List[Any]   )			->						Tuple:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.inference_layers_num / self.inference_instances_num
             UpperCAmelCase_      			=							(
                 F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
                 F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
             )
             print(_UpperCAmelCase   )







       @add_start_docstrings_to_model_forward(_UpperCAmelCase   )
       def 					lowercase__    (							self      :  int		,    _UpperCAmelCase      :  Optional[int]=None		,    _UpperCAmelCase      :  Union[str, Any]=None		,    _UpperCAmelCase      :  Any=None		,    _UpperCAmelCase      :  str=None		,    _UpperCAmelCase      :  Optional[Any]=None		,    _UpperCAmelCase      :  Any=None		,    _UpperCAmelCase      :  Dict=None		,    _UpperCAmelCase      :  str=None		,    _UpperCAmelCase      :  List[str]=None		,    _UpperCAmelCase      :  List[Any]=None		,    _UpperCAmelCase      :  Any=False		,    )			->						Optional[int]:


             '''simple docstring'''




             if input_ids is not None and inputs_embeds is not None:
                   raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"   )
             elif input_ids is not None:
                   UpperCAmelCase_      			=							input_ids.size()
             elif inputs_embeds is not None:
                   UpperCAmelCase_      			=							inputs_embeds.size()[:-1]
             else:
                   raise ValueError("You have to specify either input_ids or inputs_embeds"   )

             UpperCAmelCase_      			=							input_ids.device if input_ids is not None else inputs_embeds.device

             if attention_mask is None:
                   UpperCAmelCase_      			=							torch.ones(_UpperCAmelCase		,    device=_UpperCAmelCase   )
             if token_type_ids is None:
                   UpperCAmelCase_      			=							torch.zeros(_UpperCAmelCase		,    dtype=torch.long		,    device=_UpperCAmelCase   )

             # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
             # ourselves in which case we just need to make it broadcastable to all heads.
             UpperCAmelCase_      			=							self.get_extended_attention_mask(_UpperCAmelCase		,    _UpperCAmelCase		,    _UpperCAmelCase   )

             # If a 2D ou 3D attention mask is provided for the cross-attention
             # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
             if self.config.is_decoder and encoder_hidden_states is not None:
                   UpperCAmelCase_			,   UpperCAmelCase_			,   UpperCAmelCase_      			=							encoder_hidden_states.size()
                   UpperCAmelCase_      			=							(encoder_batch_size, encoder_sequence_length)
                   if encoder_attention_mask is None:
                         UpperCAmelCase_      			=							torch.ones(_UpperCAmelCase		,    device=_UpperCAmelCase   )
                   UpperCAmelCase_      			=							self.invert_attention_mask(_UpperCAmelCase   )
             else:
                   UpperCAmelCase_      			=							None

             # Prepare head mask if needed
             # 1.0 in head_mask indicate we keep the head
             # attention_probs has shape bsz x n_heads x N x N
             # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
             # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
             UpperCAmelCase_      			=							self.get_head_mask(_UpperCAmelCase		,    self.config.num_hidden_layers   )

             UpperCAmelCase_      			=							self.embeddings(
                 input_ids=_UpperCAmelCase		,    position_ids=_UpperCAmelCase		,    token_type_ids=_UpperCAmelCase		,    inputs_embeds=_UpperCAmelCase   )
             UpperCAmelCase_      			=							embedding_output

             if self.training:
                   UpperCAmelCase_      			=							[]
                   for i in range(self.config.num_hidden_layers   ):
                         UpperCAmelCase_      			=							self.encoder.adaptive_forward(
                             _UpperCAmelCase		,    current_layer=_UpperCAmelCase		,    attention_mask=_UpperCAmelCase		,    head_mask=_UpperCAmelCase   )

                         UpperCAmelCase_      			=							self.pooler(_UpperCAmelCase   )
                         UpperCAmelCase_      			=							output_layers[i](output_dropout(_UpperCAmelCase   )   )
                         res.append(_UpperCAmelCase   )
             elif self.patience == 0:  # Use all layers for inference
                   UpperCAmelCase_      			=							self.encoder(
                       _UpperCAmelCase		,    attention_mask=_UpperCAmelCase		,    head_mask=_UpperCAmelCase		,    encoder_hidden_states=_UpperCAmelCase		,    encoder_attention_mask=_UpperCAmelCase		,    )
                   UpperCAmelCase_      			=							self.pooler(encoder_outputs[0]   )
                   UpperCAmelCase_      			=							[output_layers[self.config.num_hidden_layers - 1](_UpperCAmelCase   )]
             else:
                   UpperCAmelCase_      			=							0
                   UpperCAmelCase_      			=							None
                   UpperCAmelCase_      			=							0
                   for i in range(self.config.num_hidden_layers   ):
                         calculated_layer_num += 1
                         UpperCAmelCase_      			=							self.encoder.adaptive_forward(
                             _UpperCAmelCase		,    current_layer=_UpperCAmelCase		,    attention_mask=_UpperCAmelCase		,    head_mask=_UpperCAmelCase   )

                         UpperCAmelCase_      			=							self.pooler(_UpperCAmelCase   )
                         UpperCAmelCase_      			=							output_layers[i](_UpperCAmelCase   )
                         if regression:
                               UpperCAmelCase_      			=							logits.detach()
                               if patient_result is not None:
                                     UpperCAmelCase_      			=							patient_result.detach()
                               if (patient_result is not None) and torch.abs(patient_result - labels   ) < self.regression_threshold:
                                     patient_counter += 1
                               else:
                                     UpperCAmelCase_      			=							0
                         else:
                               UpperCAmelCase_      			=							logits.detach().argmax(dim=1   )
                               if patient_result is not None:
                                     UpperCAmelCase_      			=							patient_result.detach().argmax(dim=1   )
                               if (patient_result is not None) and torch.all(labels.eq(_UpperCAmelCase   )   ):
                                     patient_counter += 1
                               else:
                                     UpperCAmelCase_      			=							0

                         UpperCAmelCase_      			=							logits
                         if patient_counter == self.patience:
                               break
                   UpperCAmelCase_      			=							[patient_result]
                   self.inference_layers_num += calculated_layer_num
                   self.inference_instances_num += 1

             return res



@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''    , SCREAMING_SNAKE_CASE    , )
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       def __init__(							self      :  str		,    _UpperCAmelCase      :  List[Any]   )			->						List[str]:


             '''simple docstring'''




             super().__init__(_UpperCAmelCase   )
             UpperCAmelCase_      			=							config.num_labels

             UpperCAmelCase_      			=							BertModelWithPabee(_UpperCAmelCase   )
             UpperCAmelCase_      			=							nn.Dropout(config.hidden_dropout_prob   )
             UpperCAmelCase_      			=							nn.ModuleList(
                 [nn.Linear(config.hidden_size		,    self.config.num_labels   ) for _ in range(config.num_hidden_layers   )]   )

             self.init_weights()







       @add_start_docstrings_to_model_forward(_UpperCAmelCase   )
       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  Tuple=None		,    _UpperCAmelCase      :  Optional[Any]=None		,    _UpperCAmelCase      :  str=None		,    _UpperCAmelCase      :  Dict=None		,    _UpperCAmelCase      :  Tuple=None		,    _UpperCAmelCase      :  List[str]=None		,    _UpperCAmelCase      :  str=None		,    )			->						int:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.bert(
                 input_ids=_UpperCAmelCase		,    attention_mask=_UpperCAmelCase		,    token_type_ids=_UpperCAmelCase		,    position_ids=_UpperCAmelCase		,    head_mask=_UpperCAmelCase		,    inputs_embeds=_UpperCAmelCase		,    output_dropout=self.dropout		,    output_layers=self.classifiers		,    regression=self.num_labels == 1		,    )

             UpperCAmelCase_      			=							(logits[-1],)

             if labels is not None:
                   UpperCAmelCase_      			=							None
                   UpperCAmelCase_      			=							0
                   for ix, logits_item in enumerate(_UpperCAmelCase   ):
                         if self.num_labels == 1:
                               #  We are doing regression
                               UpperCAmelCase_      			=							MSELoss()
                               UpperCAmelCase_      			=							loss_fct(logits_item.view(-1   )		,    labels.view(-1   )   )
                         else:
                               UpperCAmelCase_      			=							CrossEntropyLoss()
                               UpperCAmelCase_      			=							loss_fct(logits_item.view(-1		,    self.num_labels   )		,    labels.view(-1   )   )
                         if total_loss is None:
                               UpperCAmelCase_      			=							loss
                         else:
                               total_loss += loss * (ix + 1)
                         total_weights += ix + 1
                   UpperCAmelCase_      			=							(total_loss / total_weights,) + outputs

             return outputs









"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase       	=						logging.get_logger(__name__)

lowerCamelCase       	=						{
    """RWKV/rwkv-4-169m-pile""": """https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-430m-pile""": """https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-1b5-pile""": """https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-3b-pile""": """https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-7b-pile""": """https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-14b-pile""": """https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json""",
    """RWKV/rwkv-raven-1b5""": """https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json""",
    """RWKV/rwkv-raven-3b""": """https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json""",
    """RWKV/rwkv-raven-7b""": """https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json""",
    """RWKV/rwkv-raven-14b""": """https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json""",
}
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       UpperCamelCase						=					'''rwkv'''
       UpperCamelCase						=					{'''max_position_embeddings''': '''context_length'''}



       def __init__(							self      :  str		,    _UpperCAmelCase      :  int=50277		,    _UpperCAmelCase      :  Optional[Any]=1024		,    _UpperCAmelCase      :  str=4096		,    _UpperCAmelCase      :  Any=32		,    _UpperCAmelCase      :  Any=None		,    _UpperCAmelCase      :  Optional[int]=None		,    _UpperCAmelCase      :  Tuple=1e-5		,    _UpperCAmelCase      :  str=0		,    _UpperCAmelCase      :  Any=0		,    _UpperCAmelCase      :  Union[str, Any]=6		,    _UpperCAmelCase      :  Dict=False		,    _UpperCAmelCase      :  Any=True		,    **_UpperCAmelCase      :  Optional[Any]		,    )			->						str:


             '''simple docstring'''




             UpperCAmelCase_      			=							vocab_size
             UpperCAmelCase_      			=							context_length
             UpperCAmelCase_      			=							hidden_size
             UpperCAmelCase_      			=							num_hidden_layers
             UpperCAmelCase_      			=							attention_hidden_size if attention_hidden_size is not None else hidden_size
             UpperCAmelCase_      			=							intermediate_size if intermediate_size is not None else 4 * hidden_size
             UpperCAmelCase_      			=							layer_norm_epsilon
             UpperCAmelCase_      			=							rescale_every
             UpperCAmelCase_      			=							use_cache

             UpperCAmelCase_      			=							bos_token_id
             UpperCAmelCase_      			=							eos_token_id

             super().__init__(
                 tie_word_embeddings=_UpperCAmelCase		,    bos_token_id=_UpperCAmelCase		,    eos_token_id=_UpperCAmelCase		,    **_UpperCAmelCase   )




