


"""simple docstring"""







import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase    =				{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase    =				{
    """Salesforce/codegen-350M-mono""": 2048,
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Optional[Any]     =	VOCAB_FILES_NAMES
    UpperCAmelCase	:							List[Any]     =	PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase	:							Tuple     =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase	:							List[Any]     =	["""input_ids""", """attention_mask"""]
    UpperCAmelCase	:							int     =	CodeGenTokenizer






    def __init__( self    :   List[Any]    ,			__UpperCAmelCase    :   str=None    ,			__UpperCAmelCase    :   List[str]=None    ,			__UpperCAmelCase    :   List[str]=None    ,			__UpperCAmelCase    :   int="<|endoftext|>"    ,			__UpperCAmelCase    :   Dict="<|endoftext|>"    ,			__UpperCAmelCase    :   str="<|endoftext|>"    ,			__UpperCAmelCase    :   List[str]=False    ,			**__UpperCAmelCase    :   Union[str, Any]    ,			):
      super().__init__(
          __UpperCAmelCase    ,			__UpperCAmelCase    ,			tokenizer_file=__UpperCAmelCase    ,			unk_token=__UpperCAmelCase    ,			bos_token=__UpperCAmelCase    ,			eos_token=__UpperCAmelCase    ,			add_prefix_space=__UpperCAmelCase    ,			**__UpperCAmelCase    ,			)

      if kwargs.pop("add_bos_token"    ,			__UpperCAmelCase):
        a						:		Union[str, Any]											=      kwargs.pop("name_or_path"    ,			"")
        raise ValueError(
            "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
            "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
            f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
            f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
            "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
            " so that the fast tokenizer works correctly.")

      a						:		Dict											=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
      if pre_tok_state.get("add_prefix_space"    ,			__UpperCAmelCase) != add_prefix_space:
        a						:		str											=      getattr(__UpperCAmelCase    ,			pre_tok_state.pop("type"))
        a						:		Dict											=      add_prefix_space
        a						:		str											=      pre_tok_class(**__UpperCAmelCase)

      a						:		Optional[int]											=      add_prefix_space






    def      __snake_case							( self    :   Optional[Any]    ,			*__UpperCAmelCase    :   Any    ,			**__UpperCAmelCase    :   Any):
      a						:		List[str]											=      kwargs.get("is_split_into_words"    ,			__UpperCAmelCase)
      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__UpperCAmelCase    ,			**__UpperCAmelCase)






    def      __snake_case							( self    :   Dict    ,			*__UpperCAmelCase    :   Union[str, Any]    ,			**__UpperCAmelCase    :   List[str]):
      a						:		List[str]											=      kwargs.get("is_split_into_words"    ,			__UpperCAmelCase)

      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__UpperCAmelCase    ,			**__UpperCAmelCase)






    def      __snake_case							( self    :   List[str]    ,			__UpperCAmelCase    :   str    ,			__UpperCAmelCase    :   Optional[str] = None):
      a						:		Any											=      self._tokenizer.model.save(__UpperCAmelCase    ,			name=__UpperCAmelCase)
      return tuple(__UpperCAmelCase)






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]    ,			__UpperCAmelCase    :   bool = False    ,			__UpperCAmelCase    :   bool = None    ,			__UpperCAmelCase    :   Optional[List[str]] = None    ,			**__UpperCAmelCase    :   Union[str, Any]    ,			):
      a						:		Optional[int]											=      super().decode(
          token_ids=__UpperCAmelCase    ,			skip_special_tokens=__UpperCAmelCase    ,			clean_up_tokenization_spaces=__UpperCAmelCase    ,			**__UpperCAmelCase    ,			)

      if truncate_before_pattern is not None and len(__UpperCAmelCase) > 0:
        a						:		Any											=      self.truncate(__UpperCAmelCase    ,			__UpperCAmelCase)

      return decoded_text






    def      __snake_case							( self    :   int    ,			__UpperCAmelCase    :   Optional[Any]    ,			__UpperCAmelCase    :   Any):
      def find_re(__UpperCAmelCase    :   Union[str, Any]    ,			__UpperCAmelCase    :   Optional[Any]    ,			__UpperCAmelCase    :   Union[str, Any]):
        a						:		int											=      pattern.search(__UpperCAmelCase    ,			__UpperCAmelCase)
        return m.start() if m else -1

      a						:		List[Any]											=      [re.compile(__UpperCAmelCase    ,			re.MULTILINE) for pattern in truncate_before_pattern]

      a						:		List[Any]											=      list(re.finditer("^print"    ,			__UpperCAmelCase    ,			re.MULTILINE))

      if len(__UpperCAmelCase) > 1:
        a						:		int											=      completion[: prints[1].start()]

      a						:		Optional[int]											=      list(re.finditer("^def"    ,			__UpperCAmelCase    ,			re.MULTILINE))

      if len(__UpperCAmelCase) > 1:
        a						:		Optional[Any]											=      completion[: defs[1].start()]

      a						:		int											=      0

      a						:		List[Any]											=      [
          pos for pos in [find_re(__UpperCAmelCase    ,			__UpperCAmelCase    ,			__UpperCAmelCase) for terminal in terminals] if pos != -1
      ]

      if len(__UpperCAmelCase) > 0:
        return completion[: min(__UpperCAmelCase)]
      else:
        return completion










"""simple docstring"""







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
 from ...onnx.config import PatchingSpec
 from ...tokenization_utils_base import PreTrainedTokenizerBase


__lowercase    =				logging.get_logger(__name__)

__lowercase    =				{
    """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096""": """https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096-finetuned-triviaqa""": (
        """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"""
    ),
    """allenai/longformer-base-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
    """allenai/longformer-large-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Optional[Any]     =	"""longformer"""






    def __init__( self    :   List[Any]    ,			__UpperCAmelCase    :   Union[List[int], int] = 512    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   int = 1    ,			__UpperCAmelCase    :   int = 0    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   int = 30522    ,			__UpperCAmelCase    :   int = 768    ,			__UpperCAmelCase    :   int = 12    ,			__UpperCAmelCase    :   int = 12    ,			__UpperCAmelCase    :   int = 3072    ,			__UpperCAmelCase    :   str = "gelu"    ,			__UpperCAmelCase    :   float = 0.1    ,			__UpperCAmelCase    :   float = 0.1    ,			__UpperCAmelCase    :   int = 512    ,			__UpperCAmelCase    :   int = 2    ,			__UpperCAmelCase    :   float = 0.02    ,			__UpperCAmelCase    :   float = 1e-12    ,			__UpperCAmelCase    :   bool = False    ,			**__UpperCAmelCase    :   int    ,			):
      super().__init__(pad_token_id=__UpperCAmelCase    ,			**__UpperCAmelCase)

      a						:		int											=      attention_window
      a						:		List[str]											=      sep_token_id
      a						:		List[Any]											=      bos_token_id
      a						:		int											=      eos_token_id
      a						:		Union[str, Any]											=      vocab_size
      a						:		Dict											=      hidden_size
      a						:		int											=      num_hidden_layers
      a						:		int											=      num_attention_heads
      a						:		Optional[int]											=      hidden_act
      a						:		Union[str, Any]											=      intermediate_size
      a						:		Tuple											=      hidden_dropout_prob
      a						:		Union[str, Any]											=      attention_probs_dropout_prob
      a						:		str											=      max_position_embeddings
      a						:		Dict											=      type_vocab_size
      a						:		Optional[Any]											=      initializer_range
      a						:		List[Any]											=      layer_norm_eps
      a						:		List[str]											=      onnx_export






class 						_A  (  _a  ):







    """simple docstring"""




    def __init__( self    :   Tuple    ,			__UpperCAmelCase    :   "PretrainedConfig"    ,			__UpperCAmelCase    :   str = "default"    ,			__UpperCAmelCase    :   "List[PatchingSpec]" = None):
      super().__init__(__UpperCAmelCase    ,			__UpperCAmelCase    ,			__UpperCAmelCase)
      a						:		Optional[Any]											=      True






    @property
    def      __snake_case							( self    :   int):
      if self.task == "multiple-choice":
        a						:		List[Any]											=      {0: "batch", 1: "choice", 2: "sequence"}
      else:
        a						:		int											=      {0: "batch", 1: "sequence"}
      return OrderedDict(
          [
              ("input_ids", dynamic_axis),
              ("attention_mask", dynamic_axis),
              ("global_attention_mask", dynamic_axis),
          ])






    @property
    def      __snake_case							( self    :   Union[str, Any]):
      a						:		str											=      super().outputs
      if self.task == "default":
        a						:		Tuple											=      {0: "batch"}
      return outputs






    @property
    def      __snake_case							( self    :   Optional[int]):
      return 1e-4






    @property
    def      __snake_case							( self    :   str):
      # needs to be >= 14 to support tril operator
      return max(super().default_onnx_opset    ,			14)






    def      __snake_case							( self    :   List[str]    ,			__UpperCAmelCase    :   "PreTrainedTokenizerBase"    ,			__UpperCAmelCase    :   int = -1    ,			__UpperCAmelCase    :   int = -1    ,			__UpperCAmelCase    :   bool = False    ,			__UpperCAmelCase    :   Optional[TensorType] = None    ,			):
      a						:		str											=      super().generate_dummy_inputs(
          preprocessor=__UpperCAmelCase    ,			batch_size=__UpperCAmelCase    ,			seq_length=__UpperCAmelCase    ,			is_pair=__UpperCAmelCase    ,			framework=__UpperCAmelCase)
      import torch

      # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
      # makes the export fail randomly
      a						:		Union[str, Any]											=      torch.zeros_like(inputs["input_ids"])
      # make every second token global
      a						:		Dict											=      1

      return inputs






