


"""simple docstring"""







import flax.linen as nn
import jax
import jax.numpy as jnp






class 						_A  (  nn.Module  ):







    """simple docstring"""




    UpperCAmelCase	:							int
    UpperCAmelCase	:							jnp.dtype     =	jnp.floataa






    def      __snake_case							( self    :   List[Any]):
      a						:		Dict											=      nn.Conv(
          self.out_channels    ,			kernel_size=(3, 3)    ,			strides=(1, 1)    ,			padding=((1, 1), (1, 1))    ,			dtype=self.dtype    ,			)






    def __call__( self    :   List[str]    ,			__UpperCAmelCase    :   Optional[Any]):
      a							,		a							,		a							,		a						:		Union[str, Any]											=      hidden_states.shape
      a						:		List[str]											=      jax.image.resize(
          __UpperCAmelCase    ,			shape=(batch, height * 2, width * 2, channels)    ,			method="nearest"    ,			)
      a						:		List[Any]											=      self.conv(__UpperCAmelCase)
      return hidden_states






class 						_A  (  nn.Module  ):







    """simple docstring"""




    UpperCAmelCase	:							int
    UpperCAmelCase	:							jnp.dtype     =	jnp.floataa






    def      __snake_case							( self    :   str):
      a						:		Optional[Any]											=      nn.Conv(
          self.out_channels    ,			kernel_size=(3, 3)    ,			strides=(2, 2)    ,			padding=((1, 1), (1, 1))    ,			dtype=self.dtype    ,			)






    def __call__( self    :   int    ,			__UpperCAmelCase    :   str):
      # pad = ((0, 0), (0, 1), (0, 1), (0, 0))  # pad height and width dim
      # hidden_states = jnp.pad(hidden_states, pad_width=pad)
      a						:		int											=      self.conv(__UpperCAmelCase)
      return hidden_states






class 						_A  (  nn.Module  ):







    """simple docstring"""




    UpperCAmelCase	:							int
    UpperCAmelCase	:							int     =	None
    UpperCAmelCase	:							float     =	0.0
    UpperCAmelCase	:							bool     =	None
    UpperCAmelCase	:							jnp.dtype     =	jnp.floataa






    def      __snake_case							( self    :   Optional[Any]):
      a						:		Any											=      self.in_channels if self.out_channels is None else self.out_channels

      a						:		List[str]											=      nn.GroupNorm(num_groups=32    ,			epsilon=1e-5)
      a						:		str											=      nn.Conv(
          __UpperCAmelCase    ,			kernel_size=(3, 3)    ,			strides=(1, 1)    ,			padding=((1, 1), (1, 1))    ,			dtype=self.dtype    ,			)

      a						:		Union[str, Any]											=      nn.Dense(__UpperCAmelCase    ,			dtype=self.dtype)

      a						:		List[Any]											=      nn.GroupNorm(num_groups=32    ,			epsilon=1e-5)
      a						:		str											=      nn.Dropout(self.dropout_prob)
      a						:		str											=      nn.Conv(
          __UpperCAmelCase    ,			kernel_size=(3, 3)    ,			strides=(1, 1)    ,			padding=((1, 1), (1, 1))    ,			dtype=self.dtype    ,			)

      a						:		List[Any]											=      self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut

      a						:		Tuple											=      None
      if use_nin_shortcut:
        a						:		Optional[Any]											=      nn.Conv(
            __UpperCAmelCase    ,			kernel_size=(1, 1)    ,			strides=(1, 1)    ,			padding="VALID"    ,			dtype=self.dtype    ,			)






    def __call__( self    :   List[str]    ,			__UpperCAmelCase    :   Dict    ,			__UpperCAmelCase    :   Optional[int]    ,			__UpperCAmelCase    :   Any=True):
      a						:		Optional[int]											=      hidden_states
      a						:		List[Any]											=      self.norma(__UpperCAmelCase)
      a						:		Union[str, Any]											=      nn.swish(__UpperCAmelCase)
      a						:		List[Any]											=      self.conva(__UpperCAmelCase)

      a						:		Optional[int]											=      self.time_emb_proj(nn.swish(__UpperCAmelCase))
      a						:		Optional[Any]											=      jnp.expand_dims(jnp.expand_dims(__UpperCAmelCase    ,			1)    ,			1)
      a						:		Any											=      hidden_states + temb

      a						:		Optional[int]											=      self.norma(__UpperCAmelCase)
      a						:		Optional[Any]											=      nn.swish(__UpperCAmelCase)
      a						:		List[str]											=      self.dropout(__UpperCAmelCase    ,			__UpperCAmelCase)
      a						:		int											=      self.conva(__UpperCAmelCase)

      if self.conv_shortcut is not None:
        a						:		str											=      self.conv_shortcut(__UpperCAmelCase)

      return hidden_states + residual










"""simple docstring"""







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
 from ...onnx.config import PatchingSpec
 from ...tokenization_utils_base import PreTrainedTokenizerBase


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096""": """https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096-finetuned-triviaqa""": (
        """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"""
    ),
    """allenai/longformer-base-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
    """allenai/longformer-large-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Optional[Any]     =	"""longformer"""






    def __init__( self    :   List[Any]    ,			__UpperCAmelCase    :   Union[List[int], int] = 512    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   int = 1    ,			__UpperCAmelCase    :   int = 0    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   int = 30522    ,			__UpperCAmelCase    :   int = 768    ,			__UpperCAmelCase    :   int = 12    ,			__UpperCAmelCase    :   int = 12    ,			__UpperCAmelCase    :   int = 3072    ,			__UpperCAmelCase    :   str = "gelu"    ,			__UpperCAmelCase    :   float = 0.1    ,			__UpperCAmelCase    :   float = 0.1    ,			__UpperCAmelCase    :   int = 512    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   float = 0.02    ,			__UpperCAmelCase    :   float = 1e-12    ,			__UpperCAmelCase    :   bool = False    ,			**__UpperCAmelCase    :   int    ,			):
      super().__init__(pad_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase)

      a						:		int											=      attention_window
      a						:		List[str]											=      sep_token_id
      a						:		List[Any]											=      bos_token_id
      a						:		int											=      eos_token_id
      a						:		Union[str, Any]											=      vocab_size
      a						:		Dict											=      hidden_size
      a						:		int											=      num_hidden_layers
      a						:		int											=      num_attention_heads
      a						:		Optional[int]											=      hidden_act
      a						:		Union[str, Any]											=      intermediate_size
      a						:		Tuple											=      hidden_dropout_prob
      a						:		Union[str, Any]											=      attention_probs_dropout_prob
      a						:		str											=      max_position_embeddings
      a						:		Dict											=      type_vocab_size
      a						:		Optional[Any]											=      initializer_range
      a						:		List[Any]											=      layer_norm_eps
      a						:		List[str]											=      onnx_export






class 						_A  (  _a  ):







    """simple docstring"""




    def __init__( self    :   Tuple    ,			__UpperCAmelCase    :   "PretrainedConfig"    ,			__UpperCAmelCase    :   str = "default"    ,			__UpperCAmelCase    :   "List[PatchingSpec]" = None):
      super().__init__(__UpperCAmelCase    ,			__UpperCAmelCase    ,			__UpperCAmelCase)
      a						:		Optional[Any]											=      True






    @property
    def      __snake_case							( self    :   int):
      if self.task == "multiple-choice":
        a						:		List[Any]											=      {0: "batch", 1: "choice", 2: "sequence"}
      else:
        a						:		int											=      {0: "batch", 1: "sequence"}
      return OrderedDict(
          [
              ("input_ids", dynamic_axis),
              ("attention_mask", dynamic_axis),
              ("global_attention_mask", dynamic_axis),
          ])






    @property
    def      __snake_case							( self    :   Union[str, Any]):
      a						:		str											=      super().outputs
      if self.task == "default":
        a						:		Tuple											=      {0: "batch"}
      return outputs






    @property
    def      __snake_case							( self    :   Optional[int]):
      return 1e-4






    @property
    def      __snake_case							( self    :   str):
      # needs to be >= 14 to support tril operator
      return max(super().default_onnx_opset    ,			14)






    def      __snake_case							( self    :   List[str]    ,			__UpperCAmelCase    :   "PreTrainedTokenizerBase"    ,			__UpperCAmelCase    :   int = -1    ,			__UpperCAmelCase    :   int = -1    ,			__UpperCAmelCase    :   bool = False    ,			__UpperCAmelCase    :   Optional[TensorType] = None    ,			):
      a						:		str											=      super().generate_dummy_inputs(
          preprocessor=__UpperCAmelCase    ,			batch_size=__UpperCAmelCase    ,			seq_length=__UpperCAmelCase    ,			is_pair=__UpperCAmelCase    ,			framework=__UpperCAmelCase)
      import torch

      # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
      # makes the export fail randomly
      a						:		Union[str, Any]											=      torch.zeros_like(inputs["input_ids"])
      # make every second token global
      a						:		Dict											=      1

      return inputs






