





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


snake_case_           =      logging.get_logger(__name__)

snake_case_           =      {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

snake_case_           =      {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

snake_case_           =      {
    'abeja/gpt-neox-japanese-2.7b': 2048,
}






def lowerCamelCase__   (							snake_case_     :  Tuple	,				snake_case_     :  str	)				->    int:
   with open(snake_case_	,				'''r'''	,				encoding='''utf-8'''	) as f:
      __snake_case								=       json.loads(f.read()	)

   __snake_case								=       collections.OrderedDict()
   __snake_case								=       collections.OrderedDict()
   __snake_case								=       collections.OrderedDict()
   with open(snake_case_	,				'''r'''	,				encoding='''utf-8'''	) as f:
      __snake_case								=       f.readlines()
   __snake_case								=       [[t.rstrip('''\n'''	)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''	).split(''','''	) for t in token]
   for idx, b in enumerate(snake_case_	):
      __snake_case								=       b
      __snake_case								=       idx
      for wd in b:
         __snake_case								=       idx

   return vocab, raw_vocab, ids_to_tokens, emoji

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    List[Any]							=   VOCAB_FILES_NAMES
      A_						:    List[str]							=   PRETRAINED_VOCAB_FILES_MAP
      A_						:    List[Any]							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A_						:    Any							=   ['input_ids', 'attention_mask']


      def __init__(self	:				Any    ,      a__	:				Any    ,      a__	:				Optional[Any]    ,      a__	:				int="<|endoftext|>"    ,      a__	:				Union[str, Any]="<|endoftext|>"    ,      a__	:				List[Any]="<|startoftext|>"    ,      a__	:				List[Any]="<|endoftext|>"    ,      a__	:				Any=False    ,      **a__	:				Optional[int]    ,      ):





         """simple docstring"""



         super().__init__(
             unk_token=a__    ,      pad_token=a__    ,      bos_token=a__    ,      eos_token=a__    ,      do_clean_text=a__    ,      **a__    ,      )
         if not os.path.isfile(a__				):
            raise ValueError(
                f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
                ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''				)
         if not os.path.isfile(a__				):
            raise ValueError(
                f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
                ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''				)
         __snake_case								=       do_clean_text
         __snake_case	,     __snake_case	,     __snake_case	,     __snake_case								=       load_vocab_and_emoji(a__    ,      a__				)
         __snake_case								=       SubWordJapaneseTokenizer(
             vocab=self.vocab    ,      ids_to_tokens=self.ids_to_tokens    ,      emoji=self.emoji				)


      @property
      def        a			(self	:				Dict				):





         """simple docstring"""



         return len(self.raw_vocab				)


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         return dict(self.raw_vocab    ,      **self.added_tokens_encoder				)


      def        a			(self	:				List[str]    ,      a__	:				Optional[int]				):





         """simple docstring"""



         return self.subword_tokenizer.tokenize(a__    ,      clean=self.do_clean_text				)


      def        a			(self	:				Any    ,      a__	:				str				):





         """simple docstring"""



         return self.vocab.get(a__    ,      self.vocab.get(self.unk_token				)				)


      def        a			(self	:				List[Any]    ,      a__	:				List[str]				):





         """simple docstring"""



         return self.subword_tokenizer.convert_id_to_token(a__				)


      def        a			(self	:				Tuple    ,      a__	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       ''''''.join(a__				).strip()
         return out_string


      def        a			(self	:				List[Any]    ,      a__	:				"Conversation"				):





         """simple docstring"""



         __snake_case								=       []
         for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(a__    ,      add_special_tokens=a__				) + [self.eos_token_id]				)

         if len(a__				) > self.model_max_length:
            __snake_case								=       input_ids[-self.model_max_length :]
         return input_ids


      def        a			(self	:				List[str]    ,      a__	:				str    ,      a__	:				Optional[str] = None				):





         """simple docstring"""



         __snake_case								=       0
         if os.path.isdir(a__				):
            __snake_case								=       os.path.join(
                a__    ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
            __snake_case								=       os.path.join(
                a__    ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']				)
         else:
            __snake_case								=       (
                (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
            )
            __snake_case								=       (
                (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
            )
         with open(a__    ,      '''w'''    ,      encoding='''utf-8'''				) as writer:
            for token_index, token in self.ids_to_tokens.items():
               if index != token_index:
                  logger.warning(
                      f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                      ''' Please check that the vocabulary is not corrupted!'''				)
                  __snake_case								=       token_index
               writer.write(''','''.join(a__				) + '''\n'''				)
               index += 1
         with open(a__    ,      '''w'''    ,      encoding='''utf-8'''				) as writer:
            json.dump(self.emoji    ,      a__				)
         return vocab_file, emoji_file






class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):


      def __init__(self	:				List[str]    ,      a__	:				Optional[int]    ,      a__	:				Union[str, Any]    ,      a__	:				str				):





         """simple docstring"""



         __snake_case								=       vocab  # same as swe
         __snake_case								=       ids_to_tokens  # same as bpe
         __snake_case								=       emoji
         __snake_case								=       np.max([len(a__				) for w in self.vocab.keys()]				)
         __snake_case								=       re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''				)
         __snake_case								=       re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''				)
         __snake_case								=       re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''				)
         __snake_case								=       re.compile(
             R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''				)
         __snake_case								=       re.compile(
             R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''				)
         __snake_case								=       re.compile(
             R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''				)
         __snake_case								=       '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
         __snake_case								=       '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
         __snake_case								=       str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}				)


      def __len__(self	:				Optional[Any]				):





         """simple docstring"""



         return len(self.ids_to_tokens				)


      def        a			(self	:				str    ,      a__	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       self.content_repattera.sub('''<URL>'''    ,      a__				)
         __snake_case								=       self.content_repattera.sub('''<EMAIL>'''    ,      a__				)
         __snake_case								=       self.content_repattera.sub('''<TEL>'''    ,      a__				)
         __snake_case								=       self.content_repattera.sub('''<DATE>'''    ,      a__				)
         __snake_case								=       self.content_repattera.sub('''<DATE>'''    ,      a__				)
         __snake_case								=       self.content_repattera.sub('''<PRICE>'''    ,      a__				)
         __snake_case								=       content.translate(self.content_transa				)
         while "<BLOCK><BLOCK>" in content:
            __snake_case								=       content.replace('''<BLOCK><BLOCK>'''    ,      '''<BLOCK>'''				)
         return content


      def        a			(self	:				str    ,      a__	:				Optional[Any]    ,      a__	:				Dict=False				):





         """simple docstring"""



         __snake_case								=       text.replace(''' '''    ,      '''<SP>'''				)
         __snake_case								=       text.replace('''　'''    ,      '''<SP>'''				)
         __snake_case								=       text.replace('''\r\n'''    ,      '''<BR>'''				)
         __snake_case								=       text.replace('''\n'''    ,      '''<BR>'''				)
         __snake_case								=       text.replace('''\r'''    ,      '''<BR>'''				)
         __snake_case								=       text.replace('''\t'''    ,      '''<TAB>'''				)
         __snake_case								=       text.replace('''—'''    ,      '''ー'''				)
         __snake_case								=       text.replace('''−'''    ,      '''ー'''				)
         for k, v in self.emoji["emoji"].items():
            if k in text:
               __snake_case								=       text.replace(a__    ,      a__				)
         if clean:
            __snake_case								=       self.clean_text(a__				)

         def check_simbol(a__	:				Union[str, Any]				):
            __snake_case								=       x.encode()
            if len(a__				) == 1 and len(a__				) == 2:
               __snake_case								=       (int(e[0]				) << 8) + int(e[1]				)
               if (
                   (c >= 0xc_2_a_1 and c <= 0xc_2_b_f)
                   or (c >= 0xc_7_8_0 and c <= 0xc_7_8_3)
                   or (c >= 0xc_a_b_9 and c <= 0xc_b_b_f)
                   or (c >= 0xc_c_8_0 and c <= 0xc_d_a_2)
               ):
                  return True
            return False

         def checkuae(a__	:				Dict				):
            __snake_case								=       x.encode()
            if len(a__				) == 1 and len(a__				) == 3:
               __snake_case								=       (int(e[0]				) << 16) + (int(e[1]				) << 8) + int(e[2]				)
               if c >= 0xe_2_8_0_8_0 and c <= 0xe_2_b_0_7_f:
                  return True
            return False

         __snake_case								=       0
         __snake_case								=       []
         while pos < len(a__				):
            __snake_case								=       min(len(a__				)    ,      pos + self.maxlen + 1				) if text[pos] == '''<''' else pos + 3
            __snake_case								=       []  # (token_id, token, pos)
            for e in range(a__    ,      a__    ,      -1				):
               __snake_case								=       text[pos:e]
               if wd in self.vocab:
                  if wd[0] == "<" and len(a__				) > 2:
                     __snake_case								=       [(self.vocab[wd], wd, e)]
                     break
                  else:
                     candidates.append((self.vocab[wd], wd, e)				)
            if len(a__				) > 0:
               # the smallest token_id is adopted
               __snake_case	,     __snake_case	,     __snake_case								=       sorted(a__    ,      key=lambda a__				: x[0]				)[0]
               result.append(a__				)
               __snake_case								=       e
            else:
               __snake_case								=       pos + 1
               __snake_case								=       text[pos:end]
               if check_simbol(a__				):
                  result.append('''<KIGOU>'''				)
               elif checkuae(a__				):
                  result.append('''<U2000U2BFF>'''				)
               else:
                  for i in wd.encode('''utf-8'''				):
                     result.append('''<|byte%d|>''' % i				)
               __snake_case								=       end
         return result


      def        a			(self	:				Union[str, Any]    ,      a__	:				Any    ,      a__	:				Union[str, Any]="\n"				):





         """simple docstring"""



         __snake_case								=       []
         __snake_case								=       []
         __snake_case								=       self.ids_to_tokens[index][0]
         if word[:6] == "<|byte" and word[-2:] == "|>":
            byte_tokens.append(int(word[6:-2]				)				)
         else:
            if len(a__				) > 0:
               words.append(bytearray(a__				).decode('''utf-8'''    ,      errors='''replace'''				)				)
               __snake_case								=       []
            if word[:7] == "<|emoji" and word[-2:] == "|>":
               words.append(self.emoji['''emoji_inv'''][word]				)
            elif word == "<SP>":
               words.append(''' '''				)
            elif word == "<BR>":
               words.append(a__				)
            elif word == "<TAB>":
               words.append('''\t'''				)
            elif word == "<BLOCK>":
               words.append('''▀'''				)
            elif word == "<KIGOU>":
               words.append('''ǀ'''				)
            elif word == "<U2000U2BFF>":
               words.append('''‖'''				)
            else:
               words.append(a__				)
         if len(a__				) > 0:
            words.append(bytearray(a__				).decode('''utf-8'''    ,      errors='''replace'''				)				)
         __snake_case								=       ''''''.join(a__				)
         return text








class 		SCREAMING_SNAKE_CASE__    :


      def __init__(self	:				str    ,      a__	:				list				):





         """simple docstring"""



         __snake_case								=       set_counts
         __snake_case								=       max(a__				)
         __snake_case								=       len(a__				)
         __snake_case								=       [1] * num_sets
         __snake_case								=       list(range(a__				)				)


      def        a			(self	:				str    ,      a__	:				int    ,      a__	:				int				):





         """simple docstring"""



         __snake_case								=       self.get_parent(a__				)
         __snake_case								=       self.get_parent(a__				)

         if src_parent == dst_parent:
            return False

         if self.ranks[dst_parent] >= self.ranks[src_parent]:
            self.set_counts[dst_parent] += self.set_counts[src_parent]
            __snake_case								=       0
            __snake_case								=       dst_parent
            if self.ranks[dst_parent] == self.ranks[src_parent]:
               self.ranks[dst_parent] += 1
            __snake_case								=       self.set_counts[dst_parent]
         else:
            self.set_counts[src_parent] += self.set_counts[dst_parent]
            __snake_case								=       0
            __snake_case								=       src_parent
            __snake_case								=       self.set_counts[src_parent]

         __snake_case								=       max(self.max_set    ,      a__				)
         return True


      def        a			(self	:				Union[str, Any]    ,      a__	:				int				):





         """simple docstring"""



         if self.parents[disj_set] == disj_set:
            return disj_set
         __snake_case								=       self.get_parent(self.parents[disj_set]				)
         return self.parents[disj_set]

