
"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_						     =					logging.get_logger(__name__)

lowerCamelCase_						     =					{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}




class      __A(    __lowerCamelCase			):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__         =      """megatron-bert"""







  def __init__(self	,      SCREAMING_SNAKE_CASE_=2_90_56	,      SCREAMING_SNAKE_CASE_=10_24	,      SCREAMING_SNAKE_CASE_=24	,      SCREAMING_SNAKE_CASE_=16	,      SCREAMING_SNAKE_CASE_=40_96	,      SCREAMING_SNAKE_CASE_="gelu"	,      SCREAMING_SNAKE_CASE_=0.1	,      SCREAMING_SNAKE_CASE_=0.1	,      SCREAMING_SNAKE_CASE_=5_12	,      SCREAMING_SNAKE_CASE_=2	,      SCREAMING_SNAKE_CASE_=0.02	,      SCREAMING_SNAKE_CASE_=1E-12	,      SCREAMING_SNAKE_CASE_=0	,      SCREAMING_SNAKE_CASE_="absolute"	,      SCREAMING_SNAKE_CASE_=True	,      **SCREAMING_SNAKE_CASE_	,      ):
       super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_	,      **SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    vocab_size
       UpperCamelCase__				=    hidden_size
       UpperCamelCase__				=    num_hidden_layers
       UpperCamelCase__				=    num_attention_heads
       UpperCamelCase__				=    hidden_act
       UpperCamelCase__				=    intermediate_size
       UpperCamelCase__				=    hidden_dropout_prob
       UpperCamelCase__				=    attention_probs_dropout_prob
       UpperCamelCase__				=    max_position_embeddings
       UpperCamelCase__				=    type_vocab_size
       UpperCamelCase__				=    initializer_range
       UpperCamelCase__				=    layer_norm_eps
       UpperCamelCase__				=    position_embedding_type
       UpperCamelCase__				=    use_cache







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowerCamelCase_						     =					{
    '''configuration_canine''': ['''CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CanineConfig'''],
    '''tokenization_canine''': ['''CanineTokenizer'''],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase_						     =					[
        '''CANINE_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''CanineForMultipleChoice''',
        '''CanineForQuestionAnswering''',
        '''CanineForSequenceClassification''',
        '''CanineForTokenClassification''',
        '''CanineLayer''',
        '''CanineModel''',
        '''CaninePreTrainedModel''',
        '''load_tf_weights_in_canine''',
    ]


if TYPE_CHECKING:
    from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
    from .tokenization_canine import CanineTokenizer

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_canine import (
            CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
            CanineForMultipleChoice,
            CanineForQuestionAnswering,
            CanineForSequenceClassification,
            CanineForTokenClassification,
            CanineLayer,
            CanineModel,
            CaninePreTrainedModel,
            load_tf_weights_in_canine,
        )


else:
    import sys

    lowerCamelCase_						     =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





