



"""simple docstring"""






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowerCAmelCase__      =    logging.get_logger(__name__)

lowerCAmelCase__      =    {'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowerCAmelCase__      =    {
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowerCAmelCase__      =    {
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}






def 	snake_case_  (				A_ :		Union[str, Any],    A_ :		List[Any]						):


       '''simple docstring'''





       with open(A_,    '''r''',    encoding='''utf-8'''						) as f:
              _lowerCamelCase						:    Optional[Any]         =			json.loads(f.read()						)

       _lowerCamelCase						:    int         =			collections.OrderedDict()
       _lowerCamelCase						:    Dict         =			collections.OrderedDict()
       _lowerCamelCase						:    str         =			collections.OrderedDict()
       with open(A_,    '''r''',    encoding='''utf-8'''						) as f:
              _lowerCamelCase						:    Tuple         =			f.readlines()
       _lowerCamelCase						:    Any         =			[[t.rstrip('''\n'''						)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''						).split(''','''						) for t in token]
       for idx, b in enumerate(A_						):
              _lowerCamelCase						:    Optional[int]         =			b
              _lowerCamelCase						:    Optional[int]         =			idx
              for wd in b:
                     _lowerCamelCase						:    str         =			idx

       return vocab, raw_vocab, ids_to_tokens, emoji






class 	__snake_case	(     _lowercase):
  snake_case__    :       Tuple        =						VOCAB_FILES_NAMES
  snake_case__    :       List[str]        =						PRETRAINED_VOCAB_FILES_MAP
  snake_case__    :       Any        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case__    :       List[str]        =						["input_ids", "attention_mask"]




  def __init__(			self					:						List[str]						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						int="<|endoftext|>"						, __lowerCAmelCase					:						Optional[Any]="<|endoftext|>"						, __lowerCAmelCase					:						Optional[Any]="<|startoftext|>"						, __lowerCAmelCase					:						Optional[Any]="<|endoftext|>"						, __lowerCAmelCase					:						Optional[int]=False						, **__lowerCAmelCase					:						Optional[Any]						, ):






         """simple docstring"""






         super().__init__(
             unk_token=__lowerCAmelCase						, pad_token=__lowerCAmelCase						, bos_token=__lowerCAmelCase						, eos_token=__lowerCAmelCase						, do_clean_text=__lowerCAmelCase						, **__lowerCAmelCase						, )
         if not os.path.isfile(__lowerCAmelCase		):
                raise ValueError(
                    f'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
                    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''		)
         if not os.path.isfile(__lowerCAmelCase		):
                raise ValueError(
                    f'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
                    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''		)
         _lowerCamelCase						:    Any         =			do_clean_text
         _lowerCamelCase      ,   _lowerCamelCase      ,   _lowerCamelCase      ,   _lowerCamelCase						:    Optional[Any]         =			load_vocab_and_emoji(__lowerCAmelCase						, __lowerCAmelCase		)
         _lowerCamelCase						:    Optional[Any]         =			SubWordJapaneseTokenizer(
             vocab=self.vocab						, ids_to_tokens=self.ids_to_tokens						, emoji=self.emoji		)




  @property
  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]		):






         """simple docstring"""






         return len(self.raw_vocab		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						int		):






         """simple docstring"""






         return dict(self.raw_vocab						, **self.added_tokens_encoder		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[str]						, __lowerCAmelCase					:						Dict		):






         """simple docstring"""






         return self.subword_tokenizer.tokenize(__lowerCAmelCase						, clean=self.do_clean_text		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[str]						, __lowerCAmelCase					:						Optional[Any]		):






         """simple docstring"""






         return self.vocab.get(__lowerCAmelCase						, self.vocab.get(self.unk_token		)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						Any		):






         """simple docstring"""






         return self.subword_tokenizer.convert_id_to_token(__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]						, __lowerCAmelCase					:						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:    List[str]         =			''''''.join(__lowerCAmelCase		).strip()
         return out_string




  def 						SCREAMING_SNAKE_CASE    (			self					:						Dict						, __lowerCAmelCase					:						"Conversation"		):






         """simple docstring"""






         _lowerCamelCase						:    int         =			[]
         for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(__lowerCAmelCase						, add_special_tokens=__lowerCAmelCase		) + [self.eos_token_id]		)

         if len(__lowerCAmelCase		) > self.model_max_length:
                _lowerCamelCase						:    Optional[int]         =			input_ids[-self.model_max_length :]
         return input_ids




  def 						SCREAMING_SNAKE_CASE    (			self					:						Any						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[str] = None		):






         """simple docstring"""






         _lowerCamelCase						:    Any         =			0
         if os.path.isdir(__lowerCAmelCase		):
                _lowerCamelCase						:    Tuple         =			os.path.join(
                    __lowerCAmelCase						, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
                _lowerCamelCase						:    Any         =			os.path.join(
                    __lowerCAmelCase						, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']		)
         else:
                _lowerCamelCase						:    Any         =			(
                    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
                )
                _lowerCamelCase						:    Union[str, Any]         =			(
                    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
                )
         with open(__lowerCAmelCase						, '''w'''						, encoding='''utf-8'''		) as writer:
                for token_index, token in self.ids_to_tokens.items():
                       if index != token_index:
                              logger.warning(
                                  f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                                  ''' Please check that the vocabulary is not corrupted!'''		)
                              _lowerCamelCase						:    List[Any]         =			token_index
                       writer.write(''','''.join(__lowerCAmelCase		) + '''\n'''		)
                       index += 1
         with open(__lowerCAmelCase						, '''w'''						, encoding='''utf-8'''		) as writer:
                json.dump(self.emoji						, __lowerCAmelCase		)
         return vocab_file, emoji_file






class 	__snake_case	(     _lowercase):




  def __init__(			self					:						Optional[int]						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[int]						, __lowerCAmelCase					:						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			vocab  # same as swe
         _lowerCamelCase						:    Dict         =			ids_to_tokens  # same as bpe
         _lowerCamelCase						:    Any         =			emoji
         _lowerCamelCase						:    Optional[int]         =			np.max([len(__lowerCAmelCase		) for w in self.vocab.keys()]		)
         _lowerCamelCase						:    Tuple         =			re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''		)
         _lowerCamelCase						:    List[Any]         =			re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''		)
         _lowerCamelCase						:    Any         =			re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''		)
         _lowerCamelCase						:    List[Any]         =			re.compile(
             R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''		)
         _lowerCamelCase						:    List[str]         =			re.compile(
             R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''		)
         _lowerCamelCase						:    Dict         =			re.compile(
             R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''		)
         _lowerCamelCase						:    List[Any]         =			'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
         _lowerCamelCase						:    Optional[Any]         =			'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
         _lowerCamelCase						:    Optional[Any]         =			str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}		)




  def __len__(			self					:						Dict		):






         """simple docstring"""






         return len(self.ids_to_tokens		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple						, __lowerCAmelCase					:						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:    Optional[Any]         =			self.content_repattera.sub('''<URL>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    Optional[Any]         =			self.content_repattera.sub('''<EMAIL>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    Tuple         =			self.content_repattera.sub('''<TEL>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    Optional[Any]         =			self.content_repattera.sub('''<DATE>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    Any         =			self.content_repattera.sub('''<DATE>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    Optional[int]         =			self.content_repattera.sub('''<PRICE>'''						, __lowerCAmelCase		)
         _lowerCamelCase						:    List[Any]         =			content.translate(self.content_transa		)
         while "<BLOCK><BLOCK>" in content:
                _lowerCamelCase						:    Optional[int]         =			content.replace('''<BLOCK><BLOCK>'''						, '''<BLOCK>'''		)
         return content




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						Union[str, Any]						, __lowerCAmelCase					:						List[str]=False		):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			text.replace(''' '''						, '''<SP>'''		)
         _lowerCamelCase						:    int         =			text.replace('''　'''						, '''<SP>'''		)
         _lowerCamelCase						:    int         =			text.replace('''\r\n'''						, '''<BR>'''		)
         _lowerCamelCase						:    Any         =			text.replace('''\n'''						, '''<BR>'''		)
         _lowerCamelCase						:    int         =			text.replace('''\r'''						, '''<BR>'''		)
         _lowerCamelCase						:    Any         =			text.replace('''\t'''						, '''<TAB>'''		)
         _lowerCamelCase						:    str         =			text.replace('''—'''						, '''ー'''		)
         _lowerCamelCase						:    Dict         =			text.replace('''−'''						, '''ー'''		)
         for k, v in self.emoji["emoji"].items():
                if k in text:
                       _lowerCamelCase						:    Optional[Any]         =			text.replace(__lowerCAmelCase						, __lowerCAmelCase		)
         if clean:
                _lowerCamelCase						:    List[Any]         =			self.clean_text(__lowerCAmelCase		)

         def check_simbol(__lowerCAmelCase					:						Dict		):
                _lowerCamelCase						:    Optional[Any]         =			x.encode()
                if len(__lowerCAmelCase		) == 1 and len(__lowerCAmelCase		) == 2:
                       _lowerCamelCase						:    Optional[Any]         =			(int(e[0]		) << 8) + int(e[1]		)
                       if (
                           (c >= 0xc2_a1 and c <= 0xc2_bf)
                           or (c >= 0xc7_80 and c <= 0xc7_83)
                           or (c >= 0xca_b9 and c <= 0xcb_bf)
                           or (c >= 0xcc_80 and c <= 0xcd_a2)
                       ):
                              return True
                return False

         def checkuae(__lowerCAmelCase					:						List[str]		):
                _lowerCamelCase						:    List[str]         =			x.encode()
                if len(__lowerCAmelCase		) == 1 and len(__lowerCAmelCase		) == 3:
                       _lowerCamelCase						:    Any         =			(int(e[0]		) << 1_6) + (int(e[1]		) << 8) + int(e[2]		)
                       if c >= 0xe2_80_80 and c <= 0xe2_b0_7f:
                              return True
                return False

         _lowerCamelCase						:    Tuple         =			0
         _lowerCamelCase						:    Optional[int]         =			[]
         while pos < len(__lowerCAmelCase		):
                _lowerCamelCase						:    Union[str, Any]         =			min(len(__lowerCAmelCase		)						, pos + self.maxlen + 1		) if text[pos] == '''<''' else pos + 3
                _lowerCamelCase						:    Dict         =			[]  # (token_id, token, pos)
                for e in range(__lowerCAmelCase						, __lowerCAmelCase						, -1		):
                       _lowerCamelCase						:    Tuple         =			text[pos:e]
                       if wd in self.vocab:
                              if wd[0] == "<" and len(__lowerCAmelCase		) > 2:
                                     _lowerCamelCase						:    Optional[Any]         =			[(self.vocab[wd], wd, e)]
                                     break
                              else:
                                     candidates.append((self.vocab[wd], wd, e)		)
                if len(__lowerCAmelCase		) > 0:
                       # the smallest token_id is adopted
                       _lowerCamelCase      ,   _lowerCamelCase      ,   _lowerCamelCase						:    Any         =			sorted(__lowerCAmelCase						, key=lambda __lowerCAmelCase		: x[0]		)[0]
                       result.append(__lowerCAmelCase		)
                       _lowerCamelCase						:    Union[str, Any]         =			e
                else:
                       _lowerCamelCase						:    List[Any]         =			pos + 1
                       _lowerCamelCase						:    Optional[int]         =			text[pos:end]
                       if check_simbol(__lowerCAmelCase		):
                              result.append('''<KIGOU>'''		)
                       elif checkuae(__lowerCAmelCase		):
                              result.append('''<U2000U2BFF>'''		)
                       else:
                              for i in wd.encode('''utf-8'''		):
                                     result.append('''<|byte%d|>''' % i		)
                       _lowerCamelCase						:    Optional[Any]         =			end
         return result




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						Optional[Any]="\n"		):






         """simple docstring"""






         _lowerCamelCase						:    Optional[Any]         =			[]
         _lowerCamelCase						:    str         =			[]
         _lowerCamelCase						:    Any         =			self.ids_to_tokens[index][0]
         if word[:6] == "<|byte" and word[-2:] == "|>":
                byte_tokens.append(int(word[6:-2]		)		)
         else:
                if len(__lowerCAmelCase		) > 0:
                       words.append(bytearray(__lowerCAmelCase		).decode('''utf-8'''						, errors='''replace'''		)		)
                       _lowerCamelCase						:    Optional[Any]         =			[]
                if word[:7] == "<|emoji" and word[-2:] == "|>":
                       words.append(self.emoji['''emoji_inv'''][word]		)
                elif word == "<SP>":
                       words.append(''' '''		)
                elif word == "<BR>":
                       words.append(__lowerCAmelCase		)
                elif word == "<TAB>":
                       words.append('''\t'''		)
                elif word == "<BLOCK>":
                       words.append('''▀'''		)
                elif word == "<KIGOU>":
                       words.append('''ǀ'''		)
                elif word == "<U2000U2BFF>":
                       words.append('''‖'''		)
                else:
                       words.append(__lowerCAmelCase		)
         if len(__lowerCAmelCase		) > 0:
                words.append(bytearray(__lowerCAmelCase		).decode('''utf-8'''						, errors='''replace'''		)		)
         _lowerCamelCase						:    List[str]         =			''''''.join(__lowerCAmelCase		)
         return text







"""simple docstring"""






import timeit

import numpy as np

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features.features import _ArrayXD






def 	snake_case_  (				A_ :		Optional[int]						):


       '''simple docstring'''





       def wrapper(*A_ :		int,    **A_ :		List[str]						):
              _lowerCamelCase						:    Tuple         =			timeit.default_timer()
              _lowerCamelCase						:    Optional[Any]         =			func(*A_,    **A_						)
              _lowerCamelCase						:    Tuple         =			timeit.default_timer() - starttime
              return delta

       _lowerCamelCase						:    Optional[Any]         =			func.__name__

       return wrapper






def 	snake_case_  (				A_ :		dict,    A_ :		List[str]=1_00,    A_ :		Optional[Any]=None						):


       '''simple docstring'''





       _lowerCamelCase						:    Union[str, Any]         =			[]
       _lowerCamelCase						:    Optional[int]         =			seq_shapes or {}
       for i in range(A_						):
              _lowerCamelCase						:    Any         =			{}
              for col_id, (k, v) in enumerate(features.items()						):
                     if isinstance(A_,    _ArrayXD						):
                            _lowerCamelCase						:    str         =			np.random.rand(*v.shape						).astype(v.dtype						)
                     elif isinstance(A_,    datasets.Value						):
                            if v.dtype == "string":
                                   _lowerCamelCase						:    List[str]         =			'''The small grey turtle was surprisingly fast when challenged.'''
                            else:
                                   _lowerCamelCase						:    int         =			np.random.randint(10,    size=1						).astype(v.dtype						).item()
                     elif isinstance(A_,    datasets.Sequence						):
                            while isinstance(A_,    datasets.Sequence						):
                                   _lowerCamelCase						:    Union[str, Any]         =			v.feature
                            _lowerCamelCase						:    Dict         =			seq_shapes[k]
                            _lowerCamelCase						:    int         =			np.random.rand(*A_						).astype(v.dtype						)
                     _lowerCamelCase						:    Optional[Any]         =			data

              dummy_data.append((i, example)						)

       return dummy_data







def 	snake_case_  (				A_ :		List[str],    A_ :		Optional[int],    A_ :		Optional[int]=1_00,    A_ :		Optional[Any]=None						):


       '''simple docstring'''





       _lowerCamelCase						:    Optional[int]         =			generate_examples(A_,    num_examples=A_,    seq_shapes=A_						)

       with ArrowWriter(features=A_,    path=A_						) as writer:
              for key, record in dummy_data:
                     _lowerCamelCase						:    Optional[Any]         =			features.encode_example(A_						)
                     writer.write(A_						)

              _lowerCamelCase      ,   _lowerCamelCase						:    Dict         =			writer.finalize()

       if not num_final_examples == num_examples:
              raise ValueError(
                  F'''Error writing the dataset, wrote {num_final_examples} examples but should have written {num_examples}.'''						)

       _lowerCamelCase						:    List[Any]         =			datasets.Dataset.from_file(filename=A_,    info=datasets.DatasetInfo(features=A_						)						)

       return dataset


