





import os
import random
import sys

from . import cryptomath_module as cryptomath
from . import rabin_miller

_UpperCAmelCase		       =							3






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:int   )      ->   List[Any]:
 print("""Generating primitive root of p"""   )
 while True:
  __lowerCAmelCase					:	Tuple      =       random.randrange(3   ,   A__   )
  if pow(A__   ,   2   ,   A__   ) == 1:
   continue
  if pow(A__   ,   A__   ,   A__   ) == 1:
   continue
  return g






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:int   )      ->   Optional[int]:
 print("""Generating prime p..."""   )
 __lowerCAmelCase					:	Union[str, Any]      =       rabin_miller.generate_large_prime(A__   )  # select large prime number.
 __lowerCAmelCase					:	Optional[int]      =       primitive_root(A__   )  # one primitive root on modulo p.
 __lowerCAmelCase					:	int      =       random.randrange(3   ,   A__   )  # private_key -> have to be greater than 2 for safety.
 __lowerCAmelCase					:	Tuple      =       cryptomath.find_mod_inverse(pow(A__   ,   A__   ,   A__   )   ,   A__   )

 __lowerCAmelCase					:	int      =       (key_size, e_a, e_a, p)
 __lowerCAmelCase					:	str      =       (key_size, d)

 return public_key, private_key






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:str   ,   SCREAMING_SNAKE_CASE		:int   )      ->   Tuple:





 if os.path.exists(F'''{name}_pubkey.txt'''   ) or os.path.exists(F'''{name}_privkey.txt'''   ):
  print("""\nWARNING:"""   )
  print(
      F'''"{name}_pubkey.txt" or "{name}_privkey.txt" already exists. \n'''
      """Use a different name or delete these files and re-run this program."""   )
  sys.exit()

 __lowerCAmelCase					:	Optional[Any]      =       generate_key(A__   )
 print(F'''\nWriting public key to file {name}_pubkey.txt...'''   )
 with open(F'''{name}_pubkey.txt'''   ,   """w"""   ) as fo:
  fo.write(F'''{public_key[0]},{public_key[1]},{public_key[2]},{public_key[3]}'''   )

 print(F'''Writing private key to file {name}_privkey.txt...'''   )
 with open(F'''{name}_privkey.txt'''   ,   """w"""   ) as fo:
  fo.write(F'''{private_key[0]},{private_key[1]}'''   )






def 	_SCREAMING_SNAKE_CASE		(      )      ->   Optional[Any]:
 print("""Making key files..."""   )
 make_key_files("""elgamal"""   ,   2_048   )
 print("""Key files generation successful"""   )


if __name__ == "__main__":
      main()






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


_UpperCAmelCase             =							logging.get_logger(__name__)

_UpperCAmelCase             =							{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

_UpperCAmelCase             =							{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

_UpperCAmelCase             =							{
    'abeja/gpt-neox-japanese-2.7b': 2048,
}






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:int   ,   SCREAMING_SNAKE_CASE		:Optional[int]   )      ->   Optional[Any]:
 with open(SCREAMING_SNAKE_CASE   ,   """r"""   ,   encoding="""utf-8"""   ) as f:
  __lowerCAmelCase					:	int      =       json.loads(f.read()   )

 __lowerCAmelCase					:	Dict      =       collections.OrderedDict()
 __lowerCAmelCase					:	str      =       collections.OrderedDict()
 __lowerCAmelCase					:	Union[str, Any]      =       collections.OrderedDict()
 with open(SCREAMING_SNAKE_CASE   ,   """r"""   ,   encoding="""utf-8"""   ) as f:
  __lowerCAmelCase					:	Tuple      =       f.readlines()
 __lowerCAmelCase					:	Tuple      =       [[t.rstrip("""\n"""   )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""   ).split(""","""   ) for t in token]
 for idx, b in enumerate(SCREAMING_SNAKE_CASE   ):
  __lowerCAmelCase					:	Dict      =       b
  __lowerCAmelCase					:	Dict      =       idx
  for wd in b:
   __lowerCAmelCase					:	List[str]      =       idx

 return vocab, raw_vocab, ids_to_tokens, emoji






class        snake_case_    (		__lowercase						):
  A_      							=   VOCAB_FILES_NAMES
  A_      							=   PRETRAINED_VOCAB_FILES_MAP
  A_      							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_      							=   ['input_ids', 'attention_mask']






  def __init__(							self			:					str       ,  _snake_case			:					Union[str, Any]       ,  _snake_case			:					Optional[Any]       ,  _snake_case			:					Any="<|endoftext|>"       ,  _snake_case			:					str="<|endoftext|>"       ,  _snake_case			:					str="<|startoftext|>"       ,  _snake_case			:					List[Any]="<|endoftext|>"       ,  _snake_case			:					str=False       ,  **_snake_case			:					List[Any]       ,  )->Union[str, Any]:






   '''simple docstring'''

   super().__init__(
       unk_token=_snake_case       ,  pad_token=_snake_case       ,  bos_token=_snake_case       ,  eos_token=_snake_case       ,  do_clean_text=_snake_case       ,  **_snake_case       ,  )
   if not os.path.isfile(_snake_case      ):
    raise ValueError(
        F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
        """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""      )
   if not os.path.isfile(_snake_case      ):
    raise ValueError(
        F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
        """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""      )
   __lowerCAmelCase					:	Any      =       do_clean_text
   __lowerCAmelCase ,					__lowerCAmelCase ,					__lowerCAmelCase ,					__lowerCAmelCase					:	Union[str, Any]      =       load_vocab_and_emoji(_snake_case       ,  _snake_case      )
   __lowerCAmelCase					:	int      =       SubWordJapaneseTokenizer(
       vocab=self.vocab       ,  ids_to_tokens=self.ids_to_tokens       ,  emoji=self.emoji      )






  @property
  def 				UpperCAmelCase__						(							self			:					int      )->str:






   '''simple docstring'''

   return len(self.raw_vocab      )






  def 				UpperCAmelCase__						(							self			:					Tuple      )->Any:






   '''simple docstring'''

   return dict(self.raw_vocab       ,  **self.added_tokens_encoder      )






  def 				UpperCAmelCase__						(							self			:					Any       ,  _snake_case			:					str      )->Optional[int]:






   '''simple docstring'''

   return self.subword_tokenizer.tokenize(_snake_case       ,  clean=self.do_clean_text      )






  def 				UpperCAmelCase__						(							self			:					Optional[Any]       ,  _snake_case			:					Optional[Any]      )->Any:






   '''simple docstring'''

   return self.vocab.get(_snake_case       ,  self.vocab.get(self.unk_token      )      )






  def 				UpperCAmelCase__						(							self			:					int       ,  _snake_case			:					Any      )->int:






   '''simple docstring'''

   return self.subword_tokenizer.convert_id_to_token(_snake_case      )






  def 				UpperCAmelCase__						(							self			:					Optional[int]       ,  _snake_case			:					int      )->List[Any]:






   '''simple docstring'''

   __lowerCAmelCase					:	str      =       """""".join(_snake_case      ).strip()
   return out_string






  def 				UpperCAmelCase__						(							self			:					List[str]       ,  _snake_case			:					"Conversation"      )->List[int]:






   '''simple docstring'''

   __lowerCAmelCase					:	Union[str, Any]      =       []
   for is_user, text in conversation.iter_texts():
    input_ids.extend(self.encode(_snake_case       ,  add_special_tokens=_snake_case      ) + [self.eos_token_id]      )

   if len(_snake_case      ) > self.model_max_length:
    __lowerCAmelCase					:	List[str]      =       input_ids[-self.model_max_length :]
   return input_ids






  def 				UpperCAmelCase__						(							self			:					Optional[Any]       ,  _snake_case			:					str       ,  _snake_case			:					Optional[str] = None      )->Tuple[str]:






   '''simple docstring'''

   __lowerCAmelCase					:	Optional[Any]      =       0
   if os.path.isdir(_snake_case      ):
    __lowerCAmelCase					:	Dict      =       os.path.join(
        _snake_case       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
    __lowerCAmelCase					:	List[Any]      =       os.path.join(
        _snake_case       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]      )
   else:
    __lowerCAmelCase					:	Union[str, Any]      =       (
        (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
    )
    __lowerCAmelCase					:	Dict      =       (
        (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
    )
   with open(_snake_case       ,  """w"""       ,  encoding="""utf-8"""      ) as writer:
    for token_index, token in self.ids_to_tokens.items():
     if index != token_index:
      logger.warning(
          F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
          """ Please check that the vocabulary is not corrupted!"""      )
      __lowerCAmelCase					:	List[str]      =       token_index
     writer.write(""",""".join(_snake_case      ) + """\n"""      )
     index += 1
   with open(_snake_case       ,  """w"""       ,  encoding="""utf-8"""      ) as writer:
    json.dump(self.emoji       ,  _snake_case      )
   return vocab_file, emoji_file






class        snake_case_    (		__lowercase						):






  def __init__(							self			:					Optional[Any]       ,  _snake_case			:					str       ,  _snake_case			:					Union[str, Any]       ,  _snake_case			:					Optional[int]      )->List[Any]:






   '''simple docstring'''

   __lowerCAmelCase					:	Optional[Any]      =       vocab  # same as swe
   __lowerCAmelCase					:	str      =       ids_to_tokens  # same as bpe
   __lowerCAmelCase					:	Dict      =       emoji
   __lowerCAmelCase					:	int      =       np.max([len(_snake_case      ) for w in self.vocab.keys()]      )
   __lowerCAmelCase					:	str      =       re.compile(R"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""      )
   __lowerCAmelCase					:	Optional[Any]      =       re.compile(R"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""      )
   __lowerCAmelCase					:	Tuple      =       re.compile(R"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""      )
   __lowerCAmelCase					:	Optional[Any]      =       re.compile(
       R"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""      )
   __lowerCAmelCase					:	Union[str, Any]      =       re.compile(
       R"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""      )
   __lowerCAmelCase					:	str      =       re.compile(
       R"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""      )
   __lowerCAmelCase					:	List[Any]      =       """─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
   __lowerCAmelCase					:	Union[str, Any]      =       """▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
   __lowerCAmelCase					:	str      =       str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}      )






  def __len__(							self			:					int      )->int:






   '''simple docstring'''

   return len(self.ids_to_tokens      )






  def 				UpperCAmelCase__						(							self			:					List[str]       ,  _snake_case			:					Any      )->str:






   '''simple docstring'''

   __lowerCAmelCase					:	List[str]      =       self.content_repattera.sub("""<URL>"""       ,  _snake_case      )
   __lowerCAmelCase					:	Tuple      =       self.content_repattera.sub("""<EMAIL>"""       ,  _snake_case      )
   __lowerCAmelCase					:	Optional[Any]      =       self.content_repattera.sub("""<TEL>"""       ,  _snake_case      )
   __lowerCAmelCase					:	str      =       self.content_repattera.sub("""<DATE>"""       ,  _snake_case      )
   __lowerCAmelCase					:	Tuple      =       self.content_repattera.sub("""<DATE>"""       ,  _snake_case      )
   __lowerCAmelCase					:	Tuple      =       self.content_repattera.sub("""<PRICE>"""       ,  _snake_case      )
   __lowerCAmelCase					:	List[Any]      =       content.translate(self.content_transa      )
   while "<BLOCK><BLOCK>" in content:
    __lowerCAmelCase					:	str      =       content.replace("""<BLOCK><BLOCK>"""       ,  """<BLOCK>"""      )
   return content






  def 				UpperCAmelCase__						(							self			:					str       ,  _snake_case			:					List[Any]       ,  _snake_case			:					Optional[int]=False      )->int:






   '''simple docstring'''

   __lowerCAmelCase					:	Optional[int]      =       text.replace(""" """       ,  """<SP>"""      )
   __lowerCAmelCase					:	Optional[int]      =       text.replace("""　"""       ,  """<SP>"""      )
   __lowerCAmelCase					:	Union[str, Any]      =       text.replace("""\r\n"""       ,  """<BR>"""      )
   __lowerCAmelCase					:	Tuple      =       text.replace("""\n"""       ,  """<BR>"""      )
   __lowerCAmelCase					:	List[str]      =       text.replace("""\r"""       ,  """<BR>"""      )
   __lowerCAmelCase					:	Dict      =       text.replace("""\t"""       ,  """<TAB>"""      )
   __lowerCAmelCase					:	Dict      =       text.replace("""—"""       ,  """ー"""      )
   __lowerCAmelCase					:	Tuple      =       text.replace("""−"""       ,  """ー"""      )
   for k, v in self.emoji["emoji"].items():
    if k in text:
     __lowerCAmelCase					:	Optional[Any]      =       text.replace(_snake_case       ,  _snake_case      )
   if clean:
    __lowerCAmelCase					:	List[Any]      =       self.clean_text(_snake_case      )

   def check_simbol(_snake_case			:					List[str]      ):
    __lowerCAmelCase					:	Optional[int]      =       x.encode()
    if len(_snake_case      ) == 1 and len(_snake_case      ) == 2:
     __lowerCAmelCase					:	Optional[Any]      =       (int(e[0]      ) << 8) + int(e[1]      )
     if (
         (c >= 0xc2a1 and c <= 0xc2bf)
         or (c >= 0xc780 and c <= 0xc783)
         or (c >= 0xcab9 and c <= 0xcbbf)
         or (c >= 0xcc80 and c <= 0xcda2)
     ):
      return True
    return False

   def checkuae(_snake_case			:					Union[str, Any]      ):
    __lowerCAmelCase					:	Dict      =       x.encode()
    if len(_snake_case      ) == 1 and len(_snake_case      ) == 3:
     __lowerCAmelCase					:	List[str]      =       (int(e[0]      ) << 16) + (int(e[1]      ) << 8) + int(e[2]      )
     if c >= 0xe2_8080 and c <= 0xe2_b07f:
      return True
    return False

   __lowerCAmelCase					:	Dict      =       0
   __lowerCAmelCase					:	Dict      =       []
   while pos < len(_snake_case      ):
    __lowerCAmelCase					:	str      =       min(len(_snake_case      )       ,  pos + self.maxlen + 1      ) if text[pos] == """<""" else pos + 3
    __lowerCAmelCase					:	Tuple      =       []  # (token_id, token, pos)
    for e in range(_snake_case       ,  _snake_case       ,  -1      ):
     __lowerCAmelCase					:	Optional[int]      =       text[pos:e]
     if wd in self.vocab:
      if wd[0] == "<" and len(_snake_case      ) > 2:
       __lowerCAmelCase					:	Tuple      =       [(self.vocab[wd], wd, e)]
       break
      else:
       candidates.append((self.vocab[wd], wd, e)      )
    if len(_snake_case      ) > 0:
     # the smallest token_id is adopted
     __lowerCAmelCase ,					__lowerCAmelCase ,					__lowerCAmelCase					:	int      =       sorted(_snake_case       ,  key=lambda _snake_case      : x[0]      )[0]
     result.append(_snake_case      )
     __lowerCAmelCase					:	int      =       e
    else:
     __lowerCAmelCase					:	Dict      =       pos + 1
     __lowerCAmelCase					:	Dict      =       text[pos:end]
     if check_simbol(_snake_case      ):
      result.append("""<KIGOU>"""      )
     elif checkuae(_snake_case      ):
      result.append("""<U2000U2BFF>"""      )
     else:
      for i in wd.encode("""utf-8"""      ):
       result.append("""<|byte%d|>""" % i      )
     __lowerCAmelCase					:	int      =       end
   return result






  def 				UpperCAmelCase__						(							self			:					List[str]       ,  _snake_case			:					Optional[int]       ,  _snake_case			:					List[Any]="\n"      )->List[Any]:






   '''simple docstring'''

   __lowerCAmelCase					:	List[str]      =       []
   __lowerCAmelCase					:	Union[str, Any]      =       []
   __lowerCAmelCase					:	Optional[Any]      =       self.ids_to_tokens[index][0]
   if word[:6] == "<|byte" and word[-2:] == "|>":
    byte_tokens.append(int(word[6:-2]      )      )
   else:
    if len(_snake_case      ) > 0:
     words.append(bytearray(_snake_case      ).decode("""utf-8"""       ,  errors="""replace"""      )      )
     __lowerCAmelCase					:	Optional[Any]      =       []
    if word[:7] == "<|emoji" and word[-2:] == "|>":
     words.append(self.emoji["""emoji_inv"""][word]      )
    elif word == "<SP>":
     words.append(""" """      )
    elif word == "<BR>":
     words.append(_snake_case      )
    elif word == "<TAB>":
     words.append("""\t"""      )
    elif word == "<BLOCK>":
     words.append("""▀"""      )
    elif word == "<KIGOU>":
     words.append("""ǀ"""      )
    elif word == "<U2000U2BFF>":
     words.append("""‖"""      )
    else:
     words.append(_snake_case      )
   if len(_snake_case      ) > 0:
    words.append(bytearray(_snake_case      ).decode("""utf-8"""       ,  errors="""replace"""      )      )
   __lowerCAmelCase					:	Dict      =       """""".join(_snake_case      )
   return text