



"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase       	=						logging.get_logger(__name__)


lowerCamelCase       	=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

lowerCamelCase       	=						{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

lowerCamelCase       	=						{
    """allenai/longformer-base-4096""": 4_096,
    """allenai/longformer-large-4096""": 4_096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_096,
}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def        a__     (				):
      UpperCAmelCase_      			=							(
          list(range(ord("!"				)     ,      ord("~"				) + 1				)				) + list(range(ord("¡"				)     ,      ord("¬"				) + 1				)				) + list(range(ord("®"				)     ,      ord("ÿ"				) + 1				)				)
      )
      UpperCAmelCase_      			=							bs[:]
      UpperCAmelCase_      			=							0
      for b in range(2**8				):
            if b not in bs:
                  bs.append(lowerCAmelCase__				)
                  cs.append(2**8 + n				)
                  n += 1
      UpperCAmelCase_      			=							[chr(lowerCAmelCase__				) for n in cs]
      return dict(zip(lowerCAmelCase__     ,      lowerCAmelCase__				)				)
def        a__     (				lowerCAmelCase__				):
      UpperCAmelCase_      			=							set()
      UpperCAmelCase_      			=							word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            UpperCAmelCase_      			=							char
      return pairs


class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       UpperCamelCase						=					VOCAB_FILES_NAMES
       UpperCamelCase						=					PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase						=					['''input_ids''', '''attention_mask''']



       def __init__(							self      :  Tuple		,    _UpperCAmelCase      :  List[str]		,    _UpperCAmelCase      :  Dict		,    _UpperCAmelCase      :  Optional[Any]="replace"		,    _UpperCAmelCase      :  List[Any]="<s>"		,    _UpperCAmelCase      :  List[Any]="</s>"		,    _UpperCAmelCase      :  Optional[int]="</s>"		,    _UpperCAmelCase      :  List[Any]="<s>"		,    _UpperCAmelCase      :  str="<unk>"		,    _UpperCAmelCase      :  List[Any]="<pad>"		,    _UpperCAmelCase      :  Union[str, Any]="<mask>"		,    _UpperCAmelCase      :  Optional[int]=False		,    **_UpperCAmelCase      :  List[Any]		,    )			->						Dict:


             '''simple docstring'''




             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else bos_token
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else eos_token
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else sep_token
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else cls_token
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else unk_token
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else mask_token

             super().__init__(
                 errors=_UpperCAmelCase		,    bos_token=_UpperCAmelCase		,    eos_token=_UpperCAmelCase		,    unk_token=_UpperCAmelCase		,    sep_token=_UpperCAmelCase		,    cls_token=_UpperCAmelCase		,    pad_token=_UpperCAmelCase		,    mask_token=_UpperCAmelCase		,    add_prefix_space=_UpperCAmelCase		,    **_UpperCAmelCase		,    )

             with open(_UpperCAmelCase		,    encoding="utf-8"   ) as vocab_handle:
                   UpperCAmelCase_      			=							json.load(_UpperCAmelCase   )
             UpperCAmelCase_      			=							{v: k for k, v in self.encoder.items()}
             UpperCAmelCase_      			=							errors  # how to handle errors in decoding
             UpperCAmelCase_      			=							bytes_to_unicode()
             UpperCAmelCase_      			=							{v: k for k, v in self.byte_encoder.items()}
             with open(_UpperCAmelCase		,    encoding="utf-8"   ) as merges_handle:
                   UpperCAmelCase_      			=							merges_handle.read().split("\n"   )[1:-1]
             UpperCAmelCase_      			=							[tuple(merge.split()   ) for merge in bpe_merges]
             UpperCAmelCase_      			=							dict(zip(_UpperCAmelCase		,    range(len(_UpperCAmelCase   )   )   )   )
             UpperCAmelCase_      			=							{}
             UpperCAmelCase_      			=							add_prefix_space

             # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
             UpperCAmelCase_      			=							re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"   )



       @property
       def 					lowercase__    (							self      :  Any   )			->						List[str]:


             '''simple docstring'''




             return len(self.encoder   )



       def 					lowercase__    (							self      :  Union[str, Any]   )			->						Dict:


             '''simple docstring'''




             return dict(self.encoder		,    **self.added_tokens_encoder   )



       def 					lowercase__    (							self      :  Optional[Any]		,    _UpperCAmelCase      :  int   )			->						Union[str, Any]:


             '''simple docstring'''




             if token in self.cache:
                   return self.cache[token]
             UpperCAmelCase_      			=							tuple(_UpperCAmelCase   )
             UpperCAmelCase_      			=							get_pairs(_UpperCAmelCase   )

             if not pairs:
                   return token

             while True:
                   UpperCAmelCase_      			=							min(_UpperCAmelCase		,    key=lambda _UpperCAmelCase   : self.bpe_ranks.get(_UpperCAmelCase		,    float("inf"   )   )   )
                   if bigram not in self.bpe_ranks:
                         break
                   UpperCAmelCase_			,   UpperCAmelCase_      			=							bigram
                   UpperCAmelCase_      			=							[]
                   UpperCAmelCase_      			=							0
                   while i < len(_UpperCAmelCase   ):
                         try:
                               UpperCAmelCase_      			=							word.index(_UpperCAmelCase		,    _UpperCAmelCase   )
                         except ValueError:
                               new_word.extend(word[i:]   )
                               break
                         else:
                               new_word.extend(word[i:j]   )
                               UpperCAmelCase_      			=							j

                         if word[i] == first and i < len(_UpperCAmelCase   ) - 1 and word[i + 1] == second:
                               new_word.append(first + second   )
                               i += 2
                         else:
                               new_word.append(word[i]   )
                               i += 1
                   UpperCAmelCase_      			=							tuple(_UpperCAmelCase   )
                   UpperCAmelCase_      			=							new_word
                   if len(_UpperCAmelCase   ) == 1:
                         break
                   else:
                         UpperCAmelCase_      			=							get_pairs(_UpperCAmelCase   )
             UpperCAmelCase_      			=							" ".join(_UpperCAmelCase   )
             UpperCAmelCase_      			=							word
             return word



       def 					lowercase__    (							self      :  Any		,    _UpperCAmelCase      :  Optional[Any]   )			->						Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_      			=							[]
             for token in re.findall(self.pat		,    _UpperCAmelCase   ):
                   UpperCAmelCase_      			=							"".join(
                       self.byte_encoder[b] for b in token.encode("utf-8"   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                   bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase   ).split(" "   )   )
             return bpe_tokens



       def 					lowercase__    (							self      :  List[Any]		,    _UpperCAmelCase      :  int   )			->						Union[str, Any]:


             '''simple docstring'''




             return self.encoder.get(_UpperCAmelCase		,    self.encoder.get(self.unk_token   )   )



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  int   )			->						Optional[Any]:


             '''simple docstring'''




             return self.decoder.get(_UpperCAmelCase   )



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  List[Any]   )			->						Dict:


             '''simple docstring'''




             UpperCAmelCase_      			=							"".join(_UpperCAmelCase   )
             UpperCAmelCase_      			=							bytearray([self.byte_decoder[c] for c in text]   ).decode("utf-8"		,    errors=self.errors   )
             return text



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  str		,    _UpperCAmelCase      :  Optional[str] = None   )			->						Tuple[str]:


             '''simple docstring'''




             if not os.path.isdir(_UpperCAmelCase   ):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
                   return
             UpperCAmelCase_      			=							os.path.join(
                 _UpperCAmelCase		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
             UpperCAmelCase_      			=							os.path.join(
                 _UpperCAmelCase		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

             with open(_UpperCAmelCase		,    "w"		,    encoding="utf-8"   ) as f:
                   f.write(json.dumps(self.encoder		,    indent=2		,    sort_keys=_UpperCAmelCase		,    ensure_ascii=_UpperCAmelCase   ) + "\n"   )

             UpperCAmelCase_      			=							0
             with open(_UpperCAmelCase		,    "w"		,    encoding="utf-8"   ) as writer:
                   writer.write("#version: 0.2\n"   )
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,    key=lambda _UpperCAmelCase   : kv[1]   ):
                         if index != token_index:
                               logger.warning(
                                   F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                   " Please check that the tokenizer is not corrupted!"   )
                               UpperCAmelCase_      			=							token_index
                         writer.write(" ".join(_UpperCAmelCase   ) + "\n"   )
                         index += 1

             return vocab_file, merge_file



       def 					lowercase__    (							self      :  Any		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None   )			->						List[int]:


             '''simple docstring'''




             if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             UpperCAmelCase_      			=							[self.cls_token_id]
             UpperCAmelCase_      			=							[self.sep_token_id]
             return cls + token_ids_a + sep + sep + token_ids_a + sep



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None		,    _UpperCAmelCase      :  bool = False   )			->						List[int]:


             '''simple docstring'''




             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=_UpperCAmelCase		,    token_ids_a=_UpperCAmelCase		,    already_has_special_tokens=_UpperCAmelCase   )

             if token_ids_a is None:
                   return [1] + ([0] * len(_UpperCAmelCase   )) + [1]
             return [1] + ([0] * len(_UpperCAmelCase   )) + [1, 1] + ([0] * len(_UpperCAmelCase   )) + [1]



       def 					lowercase__    (							self      :  List[Any]		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None   )			->						List[int]:


             '''simple docstring'''




             UpperCAmelCase_      			=							[self.sep_token_id]
             UpperCAmelCase_      			=							[self.cls_token_id]

             if token_ids_a is None:
                   return len(cls + token_ids_a + sep   ) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







       def 					lowercase__    (							self      :  Union[str, Any]		,    _UpperCAmelCase      :  Union[str, Any]		,    _UpperCAmelCase      :  Dict=False		,    **_UpperCAmelCase      :  Optional[int]   )			->						Dict:


             '''simple docstring'''




             UpperCAmelCase_      			=							kwargs.pop("add_prefix_space"		,    self.add_prefix_space   )
             if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase   ) > 0 and not text[0].isspace()):
                   UpperCAmelCase_      			=							" " + text
             return (text, kwargs)









"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase       	=						logging.get_logger(__name__)

lowerCamelCase       	=						{
    """RWKV/rwkv-4-169m-pile""": """https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-430m-pile""": """https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-1b5-pile""": """https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-3b-pile""": """https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-7b-pile""": """https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-14b-pile""": """https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json""",
    """RWKV/rwkv-raven-1b5""": """https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json""",
    """RWKV/rwkv-raven-3b""": """https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json""",
    """RWKV/rwkv-raven-7b""": """https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json""",
    """RWKV/rwkv-raven-14b""": """https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json""",
}
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       UpperCamelCase						=					'''rwkv'''
       UpperCamelCase						=					{'''max_position_embeddings''': '''context_length'''}



       def __init__(							self      :  str		,    _UpperCAmelCase      :  int=50277		,    _UpperCAmelCase      :  Optional[Any]=1024		,    _UpperCAmelCase      :  str=4096		,    _UpperCAmelCase      :  Any=32		,    _UpperCAmelCase      :  Any=None		,    _UpperCAmelCase      :  Optional[int]=None		,    _UpperCAmelCase      :  Tuple=1e-5		,    _UpperCAmelCase      :  str=0		,    _UpperCAmelCase      :  Any=0		,    _UpperCAmelCase      :  Union[str, Any]=6		,    _UpperCAmelCase      :  Dict=False		,    _UpperCAmelCase      :  Any=True		,    **_UpperCAmelCase      :  Optional[Any]		,    )			->						str:


             '''simple docstring'''




             UpperCAmelCase_      			=							vocab_size
             UpperCAmelCase_      			=							context_length
             UpperCAmelCase_      			=							hidden_size
             UpperCAmelCase_      			=							num_hidden_layers
             UpperCAmelCase_      			=							attention_hidden_size if attention_hidden_size is not None else hidden_size
             UpperCAmelCase_      			=							intermediate_size if intermediate_size is not None else 4 * hidden_size
             UpperCAmelCase_      			=							layer_norm_epsilon
             UpperCAmelCase_      			=							rescale_every
             UpperCAmelCase_      			=							use_cache

             UpperCAmelCase_      			=							bos_token_id
             UpperCAmelCase_      			=							eos_token_id

             super().__init__(
                 tie_word_embeddings=_UpperCAmelCase		,    bos_token_id=_UpperCAmelCase		,    eos_token_id=_UpperCAmelCase		,    **_UpperCAmelCase   )




