


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase      :int			    =       logging.get_logger(__name__)

__UpperCAmelCase      :str			    =       {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class      a    (		_a	):


     """simple docstring"""






     SCREAMING_SNAKE_CASE	:			Tuple							       =       "megatron-bert"

     def __init__(    self					:		Union[str, Any]							,			snake_case					:		List[Any]=2_9056							,			snake_case					:		List[Any]=1024							,			snake_case					:		List[Any]=24							,			snake_case					:		Tuple=16							,			snake_case					:		int=4096							,			snake_case					:		Optional[int]="gelu"							,			snake_case					:		int=0.1							,			snake_case					:		List[Any]=0.1							,			snake_case					:		Optional[int]=512							,			snake_case					:		Optional[Any]=2							,			snake_case					:		Optional[Any]=0.02							,			snake_case					:		List[Any]=1E-12							,			snake_case					:		List[str]=0							,			snake_case					:		Optional[int]="absolute"							,			snake_case					:		Dict=True							,			**snake_case					:		Optional[Any]							,			)						->							str:
          super().__init__(pad_token_id=lowerCamelCase_							,			**lowerCamelCase_      )

          __UpperCAmelCase     :			str  							=      vocab_size
          __UpperCAmelCase     :			Dict  							=      hidden_size
          __UpperCAmelCase     :			List[str]  							=      num_hidden_layers
          __UpperCAmelCase     :			Union[str, Any]  							=      num_attention_heads
          __UpperCAmelCase     :			Optional[int]  							=      hidden_act
          __UpperCAmelCase     :			Tuple  							=      intermediate_size
          __UpperCAmelCase     :			int  							=      hidden_dropout_prob
          __UpperCAmelCase     :			Union[str, Any]  							=      attention_probs_dropout_prob
          __UpperCAmelCase     :			Union[str, Any]  							=      max_position_embeddings
          __UpperCAmelCase     :			Dict  							=      type_vocab_size
          __UpperCAmelCase     :			Tuple  							=      initializer_range
          __UpperCAmelCase     :			Optional[int]  							=      layer_norm_eps
          __UpperCAmelCase     :			Optional[int]  							=      position_embedding_type
          __UpperCAmelCase     :			str  							=      use_cache



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__UpperCAmelCase      :Union[str, Any]			    =       {
    "configuration_rag": ["RagConfig"],
    "retrieval_rag": ["RagRetriever"],
    "tokenization_rag": ["RagTokenizer"],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      :Dict			    =       [
          "RagModel",
          "RagPreTrainedModel",
          "RagSequenceForGeneration",
          "RagTokenForGeneration",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      :Dict			    =       [
          "TFRagModel",
          "TFRagPreTrainedModel",
          "TFRagSequenceForGeneration",
          "TFRagTokenForGeneration",
      ]


if TYPE_CHECKING:
      from .configuration_rag import RagConfig
      from .retrieval_rag import RagRetriever
      from .tokenization_rag import RagTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_rag import (
                TFRagModel,
                TFRagPreTrainedModel,
                TFRagSequenceForGeneration,
                TFRagTokenForGeneration,
            )

else:
      import sys

      __UpperCAmelCase      :Any			    =       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)