




import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class       a					:

     '''simple docstring'''





     def __init__(     self       :      str ,   __snake_case       :      Dict ,   __snake_case       :      Any=13 ,   __snake_case       :      Optional[Any]=7 ,   __snake_case       :      str=True ,   __snake_case       :      str=True ,   __snake_case       :      Dict=True ,   __snake_case       :      List[Any]=True ,   __snake_case       :      List[str]=99 ,   __snake_case       :      Union[str, Any]=16 ,   __snake_case       :      Optional[Any]=36 ,   __snake_case       :      Dict=6 ,   __snake_case       :      str=6 ,   __snake_case       :      Tuple=6 ,   __snake_case       :      Any=37 ,   __snake_case       :      Optional[Any]="gelu" ,   __snake_case       :      str=0.1 ,   __snake_case       :      Optional[Any]=0.1 ,   __snake_case       :      List[str]=5_12 ,   __snake_case       :      str=16 ,   __snake_case       :      Optional[Any]=2 ,   __snake_case       :      Optional[int]=0.02 ,   __snake_case       :      List[Any]=3 ,   __snake_case       :      Tuple=4 ,   __snake_case       :      Optional[Any]=None ,   ):
         UpperCAmelCase_										=					parent
         UpperCAmelCase_										=					batch_size
         UpperCAmelCase_										=					seq_length
         UpperCAmelCase_										=					is_training
         UpperCAmelCase_										=					use_input_mask
         UpperCAmelCase_										=					use_token_type_ids
         UpperCAmelCase_										=					use_labels
         UpperCAmelCase_										=					vocab_size
         UpperCAmelCase_										=					embedding_size
         UpperCAmelCase_										=					hidden_size
         UpperCAmelCase_										=					num_hidden_layers
         UpperCAmelCase_										=					num_hidden_groups
         UpperCAmelCase_										=					num_attention_heads
         UpperCAmelCase_										=					intermediate_size
         UpperCAmelCase_										=					hidden_act
         UpperCAmelCase_										=					hidden_dropout_prob
         UpperCAmelCase_										=					attention_probs_dropout_prob
         UpperCAmelCase_										=					max_position_embeddings
         UpperCAmelCase_										=					type_vocab_size
         UpperCAmelCase_										=					type_sequence_label_size
         UpperCAmelCase_										=					initializer_range
         UpperCAmelCase_										=					num_labels
         UpperCAmelCase_										=					num_choices
         UpperCAmelCase_										=					scope
     def   lowerCamelCase_       (     self       :      Optional[int]    ):
         UpperCAmelCase_										=					ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size    )

         UpperCAmelCase_										=					None
         if self.use_input_mask:
             UpperCAmelCase_										=					random_attention_mask([self.batch_size, self.seq_length]    )

         UpperCAmelCase_										=					None
         if self.use_token_type_ids:
             UpperCAmelCase_										=					ids_tensor([self.batch_size, self.seq_length] ,   self.type_vocab_size    )

         UpperCAmelCase_										=					None
         UpperCAmelCase_										=					None
         UpperCAmelCase_										=					None
         if self.use_labels:
             UpperCAmelCase_										=					ids_tensor([self.batch_size] ,   self.type_sequence_label_size    )
             UpperCAmelCase_										=					ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels    )
             UpperCAmelCase_										=					ids_tensor([self.batch_size] ,   self.num_choices    )

         UpperCAmelCase_										=					self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     def   lowerCamelCase_       (     self       :      List[Any]    ):
         return AlbertConfig(
             vocab_size=self.vocab_size ,   hidden_size=self.hidden_size ,   num_hidden_layers=self.num_hidden_layers ,   num_attention_heads=self.num_attention_heads ,   intermediate_size=self.intermediate_size ,   hidden_act=self.hidden_act ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   max_position_embeddings=self.max_position_embeddings ,   type_vocab_size=self.type_vocab_size ,   initializer_range=self.initializer_range ,   num_hidden_groups=self.num_hidden_groups ,   )
     def   lowerCamelCase_       (     self       :      Union[str, Any] ,   __snake_case       :      str ,   __snake_case       :      Tuple ,   __snake_case       :      Any ,   __snake_case       :      Optional[Any] ,   __snake_case       :      str ,   __snake_case       :      Any ,   __snake_case       :      List[str]    ):
         UpperCAmelCase_										=					AlbertModel(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(__snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case    )
         UpperCAmelCase_										=					model(__snake_case ,   token_type_ids=__snake_case    )
         UpperCAmelCase_										=					model(__snake_case    )
         self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size)    )
         self.parent.assertEqual(result.pooler_output.shape ,   (self.batch_size, self.hidden_size)    )
     def   lowerCamelCase_       (     self       :      Any ,   __snake_case       :      int ,   __snake_case       :      List[Any] ,   __snake_case       :      Union[str, Any] ,   __snake_case       :      Dict ,   __snake_case       :      int ,   __snake_case       :      Any ,   __snake_case       :      Optional[Any]    ):
         UpperCAmelCase_										=					AlbertForPreTraining(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(
             __snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   labels=__snake_case ,   sentence_order_label=__snake_case ,   )
         self.parent.assertEqual(result.prediction_logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size)    )
         self.parent.assertEqual(result.sop_logits.shape ,   (self.batch_size, config.num_labels)    )
     def   lowerCamelCase_       (     self       :      List[str] ,   __snake_case       :      Tuple ,   __snake_case       :      int ,   __snake_case       :      Any ,   __snake_case       :      Tuple ,   __snake_case       :      Optional[Any] ,   __snake_case       :      str ,   __snake_case       :      Optional[int]    ):
         UpperCAmelCase_										=					AlbertForMaskedLM(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(__snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   labels=__snake_case    )
         self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size)    )
     def   lowerCamelCase_       (     self       :      Optional[Any] ,   __snake_case       :      Optional[int] ,   __snake_case       :      Any ,   __snake_case       :      str ,   __snake_case       :      Any ,   __snake_case       :      str ,   __snake_case       :      Dict ,   __snake_case       :      str    ):
         UpperCAmelCase_										=					AlbertForQuestionAnswering(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(
             __snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   start_positions=__snake_case ,   end_positions=__snake_case ,   )
         self.parent.assertEqual(result.start_logits.shape ,   (self.batch_size, self.seq_length)    )
         self.parent.assertEqual(result.end_logits.shape ,   (self.batch_size, self.seq_length)    )
     def   lowerCamelCase_       (     self       :      List[str] ,   __snake_case       :      Dict ,   __snake_case       :      Dict ,   __snake_case       :      Union[str, Any] ,   __snake_case       :      Any ,   __snake_case       :      List[Any] ,   __snake_case       :      Any ,   __snake_case       :      Dict    ):
         UpperCAmelCase_										=					self.num_labels
         UpperCAmelCase_										=					AlbertForSequenceClassification(__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(__snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   labels=__snake_case    )
         self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_labels)    )
     def   lowerCamelCase_       (     self       :      Any ,   __snake_case       :      Dict ,   __snake_case       :      str ,   __snake_case       :      Optional[int] ,   __snake_case       :      Dict ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Optional[int] ,   __snake_case       :      Optional[int]    ):
         UpperCAmelCase_										=					self.num_labels
         UpperCAmelCase_										=					AlbertForTokenClassification(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					model(__snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   labels=__snake_case    )
         self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels)    )
     def   lowerCamelCase_       (     self       :      Dict ,   __snake_case       :      List[str] ,   __snake_case       :      Tuple ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Optional[int] ,   __snake_case       :      Optional[int] ,   __snake_case       :      Any ,   __snake_case       :      List[str]    ):
         UpperCAmelCase_										=					self.num_choices
         UpperCAmelCase_										=					AlbertForMultipleChoice(config=__snake_case    )
         model.to(__snake_case    )
         model.eval()
         UpperCAmelCase_										=					input_ids.unsqueeze(1    ).expand(-1 ,   self.num_choices ,   -1    ).contiguous()
         UpperCAmelCase_										=					token_type_ids.unsqueeze(1    ).expand(-1 ,   self.num_choices ,   -1    ).contiguous()
         UpperCAmelCase_										=					input_mask.unsqueeze(1    ).expand(-1 ,   self.num_choices ,   -1    ).contiguous()
         UpperCAmelCase_										=					model(
             __snake_case ,   attention_mask=__snake_case ,   token_type_ids=__snake_case ,   labels=__snake_case ,   )
         self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_choices)    )




     def   lowerCamelCase_       (     self       :      Tuple    ):
         UpperCAmelCase_										=					self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,							(
             UpperCAmelCase_							
         )			,														
         )										=					config_and_inputs
         UpperCAmelCase_										=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict



@require_torch
class       a					(       _A     ,       _A     ,       unittest.TestCase      ):

     '''simple docstring'''





     lowerCAmelCase    :	Optional[Any]       		=  (
         (
             AlbertModel,
             AlbertForPreTraining,
             AlbertForMaskedLM,
             AlbertForMultipleChoice,
             AlbertForSequenceClassification,
             AlbertForTokenClassification,
             AlbertForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     lowerCAmelCase    :	List[str]       		=  (
         {
             'feature-extraction': AlbertModel,
             'fill-mask': AlbertForMaskedLM,
             'question-answering': AlbertForQuestionAnswering,
             'text-classification': AlbertForSequenceClassification,
             'token-classification': AlbertForTokenClassification,
             'zero-shot': AlbertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     lowerCAmelCase    :	Tuple       		=  True
     def   lowerCamelCase_       (     self       :      List[str] ,   __snake_case       :      Union[str, Any] ,   __snake_case       :      Optional[Any] ,   __snake_case       :      int=False    ):
         UpperCAmelCase_										=					super()._prepare_for_class(__snake_case ,   __snake_case ,   return_labels=__snake_case    )

         if return_labels:
             if model_class in get_values(__snake_case    ):
                 UpperCAmelCase_										=					torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.seq_length) ,   dtype=torch.long ,   device=__snake_case    )
                 UpperCAmelCase_										=					torch.zeros(
                     self.model_tester.batch_size ,   dtype=torch.long ,   device=__snake_case    )
         return inputs_dict
     def   lowerCamelCase_       (     self       :      str    ):
         UpperCAmelCase_										=					AlbertModelTester(self    )
         UpperCAmelCase_										=					ConfigTester(self ,   config_class=__snake_case ,   hidden_size=37    )
     def   lowerCamelCase_       (     self       :      str    ):
         self.config_tester.run_common_tests()
     def   lowerCamelCase_       (     self       :      Union[str, Any]    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__snake_case    )
     def   lowerCamelCase_       (     self       :      List[str]    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*__snake_case    )
     def   lowerCamelCase_       (     self       :      int    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*__snake_case    )
     def   lowerCamelCase_       (     self       :      int    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*__snake_case    )
     def   lowerCamelCase_       (     self       :      str    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*__snake_case    )
     def   lowerCamelCase_       (     self       :      List[Any]    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*__snake_case    )
     def   lowerCamelCase_       (     self       :      int    ):
         UpperCAmelCase_										=					self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
             UpperCAmelCase_										=					type
             self.model_tester.create_and_check_model(*__snake_case    )




     @slow
     def   lowerCamelCase_       (     self       :      str    ):
         for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             UpperCAmelCase_										=					AlbertModel.from_pretrained(__snake_case    )
             self.assertIsNotNone(__snake_case    )



@require_torch
class       a					(       unittest.TestCase      ):

     '''simple docstring'''





     @slow
     def   lowerCamelCase_       (     self       :      Optional[int]    ):
         UpperCAmelCase_										=					AlbertModel.from_pretrained('''albert-base-v2'''    )
         UpperCAmelCase_										=					torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]    )
         UpperCAmelCase_										=					torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]    )
         with torch.no_grad():
             UpperCAmelCase_										=					model(__snake_case ,   attention_mask=__snake_case    )[0]
         UpperCAmelCase_										=					torch.Size((1, 11, 7_68)    )
         self.assertEqual(output.shape ,   __snake_case    )
         UpperCAmelCase_										=					torch.tensor(
             [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]    )

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,   __snake_case ,   atol=1E-4    )    )











import numpy as np




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						np.array				)  ->							np.array:
    return (2 / (1 + np.exp(-2 * vector				))) - 1


if __name__ == "__main__":
 import doctest

 doctest.testmod()





