

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase		:       List[str]						    =	{
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase		:       Dict						    =	[
					    """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MegatronBertForCausalLM""",
					    """MegatronBertForMaskedLM""",
					    """MegatronBertForMultipleChoice""",
					    """MegatronBertForNextSentencePrediction""",
					    """MegatronBertForPreTraining""",
					    """MegatronBertForQuestionAnswering""",
					    """MegatronBertForSequenceClassification""",
					    """MegatronBertForTokenClassification""",
					    """MegatronBertModel""",
					    """MegatronBertPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_megatron_bert import (
										    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MegatronBertForCausalLM,
										    MegatronBertForMaskedLM,
										    MegatronBertForMultipleChoice,
										    MegatronBertForNextSentencePrediction,
										    MegatronBertForPreTraining,
										    MegatronBertForQuestionAnswering,
										    MegatronBertForSequenceClassification,
										    MegatronBertForTokenClassification,
										    MegatronBertModel,
										    MegatronBertPreTrainedModel,
										)

else:
					import sys

					_UpperCAmelCase		:       List[Any]						    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase		:       str						    =	logging.get_logger(__name__)

_UpperCAmelCase		:       int						    =	{
    """google/canine-s""": """https://huggingface.co/google/canine-s/resolve/main/config.json""",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}







class 			lowercase    (	lowercase_      ):
				__SCREAMING_SNAKE_CASE :			int	      =      '''canine'''




				def __init__( self				,       snake_case=768				,       snake_case=12				,       snake_case=12				,       snake_case=3072				,       snake_case="gelu"				,       snake_case=0.1				,       snake_case=0.1				,       snake_case=1_6384				,       snake_case=16				,       snake_case=0.02				,       snake_case=1e-1_2				,       snake_case=0				,       snake_case=0xE000				,       snake_case=0xE001				,       snake_case=4				,       snake_case=4				,       snake_case=8				,       snake_case=1_6384				,       snake_case=128				,       **snake_case				,       ):
											super().__init__(pad_token_id=snake_case				,       bos_token_id=snake_case				,       eos_token_id=snake_case				,       **snake_case	)

											snake_case_          =   max_position_embeddings
											snake_case_          =   hidden_size
											snake_case_          =   num_hidden_layers
											snake_case_          =   num_attention_heads
											snake_case_          =   intermediate_size
											snake_case_          =   hidden_act
											snake_case_          =   hidden_dropout_prob
											snake_case_          =   attention_probs_dropout_prob
											snake_case_          =   initializer_range
											snake_case_          =   type_vocab_size
											snake_case_          =   layer_norm_eps

											# Character config:
											snake_case_          =   downsampling_rate
											snake_case_          =   upsampling_kernel_size
											snake_case_          =   num_hash_functions
											snake_case_          =   num_hash_buckets
											snake_case_          =   local_transformer_stride




