






"""simple docstring"""



import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A__	:    Optional[Any]			 = logging.get_logger(__name__)

A__	:    str			 = {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

A__	:    Optional[int]			 = {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

A__	:    str			 = {
    'abeja/gpt-neox-japanese-2.7b': 2_048,
}

def 			_snake_case    (					lowerCamelCase__     :   int    ,						lowerCamelCase__     :   Any  )    ->     List[Any]:
							with open(lowerCamelCase__    ,						"r"    ,						encoding="utf-8"  ) as f:
														lowerCamelCase_	:							List[str]									=json.loads(f.read()  )

							lowerCamelCase_	:							Union[str, Any]									=collections.OrderedDict()
							lowerCamelCase_	:							Union[str, Any]									=collections.OrderedDict()
							lowerCamelCase_	:							int									=collections.OrderedDict()
							with open(lowerCamelCase__    ,						"r"    ,						encoding="utf-8"  ) as f:
														lowerCamelCase_	:							Optional[Any]									=f.readlines()
							lowerCamelCase_	:							List[str]									=[[t.rstrip("\n"  )] if (t == "," or "," not in t) else t.rstrip("\n"  ).split(","  ) for t in token]
							for idx, b in enumerate(lowerCamelCase__  ):
														lowerCamelCase_	:							Union[str, Any]									=b
														lowerCamelCase_	:							Optional[int]									=idx
														for wd in b:
																					lowerCamelCase_	:							int									=idx

							return vocab, raw_vocab, ids_to_tokens, emoji







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:Any             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:Tuple             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:List[Any]             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:Tuple             =			["input_ids", "attention_mask"]


			def __init__(				self					:  List[Any]       ,	snake_case__					:  Union[str, Any]       ,	snake_case__					:  List[str]       ,	snake_case__					:  List[Any]="<|endoftext|>"       ,	snake_case__					:  Tuple="<|endoftext|>"       ,	snake_case__					:  List[Any]="<|startoftext|>"       ,	snake_case__					:  List[str]="<|endoftext|>"       ,	snake_case__					:  Optional[Any]=False       ,	**snake_case__					:  int       ,	):
										super().__init__(
										    unk_token=snake_case__       ,	pad_token=snake_case__       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	do_clean_text=snake_case__       ,	**snake_case__       ,	)
										if not os.path.isfile(snake_case__ ):
																	raise ValueError(
																	    F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
																	    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" )
										if not os.path.isfile(snake_case__ ):
																	raise ValueError(
																	    F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
																	    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" )
										lowerCamelCase_	:							str									=do_clean_text
										lowerCamelCase_							,						lowerCamelCase_							,						lowerCamelCase_							,						lowerCamelCase_	:							Optional[Any]									=load_vocab_and_emoji(snake_case__       ,	snake_case__ )
										lowerCamelCase_	:							int									=SubWordJapaneseTokenizer(
										    vocab=self.vocab       ,	ids_to_tokens=self.ids_to_tokens       ,	emoji=self.emoji )


			@property
			def      UpperCAmelCase__						(				self					:  Any ):
										# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
										return len(self.raw_vocab )


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										return dict(self.raw_vocab       ,	**self.added_tokens_encoder )


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  Optional[Any] ):
										return self.subword_tokenizer.tokenize(snake_case__       ,	clean=self.do_clean_text )


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  int ):
										return self.vocab.get(snake_case__       ,	self.vocab.get(self.unk_token ) )


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  Optional[int] ):
										return self.subword_tokenizer.convert_id_to_token(snake_case__ )


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  Dict ):
										lowerCamelCase_	:							List[str]									="".join(snake_case__ ).strip()
										return out_string


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  "Conversation" ):
										lowerCamelCase_	:							Dict									=[]
										for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(snake_case__       ,	add_special_tokens=snake_case__ ) + [self.eos_token_id] )

										if len(snake_case__ ) > self.model_max_length:
																	lowerCamelCase_	:							Any									=input_ids[-self.model_max_length :]
										return input_ids


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  str       ,	snake_case__					:  Optional[str] = None ):
										lowerCamelCase_	:							int									=0
										if os.path.isdir(snake_case__ ):
																	lowerCamelCase_	:							Optional[Any]									=os.path.join(
																	    snake_case__       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )
																	lowerCamelCase_	:							Any									=os.path.join(
																	    snake_case__       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"] )
										else:
																	lowerCamelCase_	:							Tuple									=(
																	    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
																	)
																	lowerCamelCase_	:							str									=(
																	    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
																	)
										with open(snake_case__       ,	"w"       ,	encoding="utf-8" ) as writer:
																	for token_index, token in self.ids_to_tokens.items():
																								if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																															    " Please check that the vocabulary is not corrupted!" )
																															lowerCamelCase_	:							Optional[int]									=token_index
																								writer.write(",".join(snake_case__ ) + "\n" )
																								index += 1
										with open(snake_case__       ,	"w"       ,	encoding="utf-8" ) as writer:
																	json.dump(self.emoji       ,	snake_case__ )
										return vocab_file, emoji_file







class        lowercase__      (							snake_case__       ):


			def __init__(				self					:  Optional[int]       ,	snake_case__					:  Optional[int]       ,	snake_case__					:  List[Any]       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							str									=vocab  # same as swe
										lowerCamelCase_	:							Dict									=ids_to_tokens  # same as bpe
										lowerCamelCase_	:							List[Any]									=emoji
										lowerCamelCase_	:							List[str]									=np.max([len(snake_case__ ) for w in self.vocab.keys()] )
										lowerCamelCase_	:							str									=re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)" )
										lowerCamelCase_	:							Dict									=re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*" )
										lowerCamelCase_	:							Optional[int]									=re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}" )
										lowerCamelCase_	:							List[str]									=re.compile(
										    r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*" )
										lowerCamelCase_	:							int									=re.compile(
										    r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*" )
										lowerCamelCase_	:							Optional[Any]									=re.compile(
										    r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*" )
										lowerCamelCase_	:							Union[str, Any]									="─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
										lowerCamelCase_	:							Any									="▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
										lowerCamelCase_	:							int									=str.maketrans({k: "<BLOCK>" for k in keisen + blocks} )


			def __len__(				self					:  List[Any] ):
										return len(self.ids_to_tokens )


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							Union[str, Any]									=self.content_repattera.sub("<URL>"       ,	snake_case__ )
										lowerCamelCase_	:							int									=self.content_repattera.sub("<EMAIL>"       ,	snake_case__ )
										lowerCamelCase_	:							Union[str, Any]									=self.content_repattera.sub("<TEL>"       ,	snake_case__ )
										lowerCamelCase_	:							Any									=self.content_repattera.sub("<DATE>"       ,	snake_case__ )
										lowerCamelCase_	:							str									=self.content_repattera.sub("<DATE>"       ,	snake_case__ )
										lowerCamelCase_	:							Union[str, Any]									=self.content_repattera.sub("<PRICE>"       ,	snake_case__ )
										lowerCamelCase_	:							Optional[int]									=content.translate(self.content_transa )
										while "<BLOCK><BLOCK>" in content:
																	lowerCamelCase_	:							List[str]									=content.replace("<BLOCK><BLOCK>"       ,	"<BLOCK>" )
										return content


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  Optional[int]       ,	snake_case__					:  List[str]=False ):
										lowerCamelCase_	:							Dict									=text.replace(" "       ,	"<SP>" )
										lowerCamelCase_	:							List[Any]									=text.replace("　"       ,	"<SP>" )
										lowerCamelCase_	:							Union[str, Any]									=text.replace("\r\n"       ,	"<BR>" )
										lowerCamelCase_	:							int									=text.replace("\n"       ,	"<BR>" )
										lowerCamelCase_	:							Tuple									=text.replace("\r"       ,	"<BR>" )
										lowerCamelCase_	:							str									=text.replace("\t"       ,	"<TAB>" )
										lowerCamelCase_	:							List[str]									=text.replace("—"       ,	"ー" )
										lowerCamelCase_	:							List[str]									=text.replace("−"       ,	"ー" )
										for k, v in self.emoji["emoji"].items():
																	if k in text:
																								lowerCamelCase_	:							Any									=text.replace(snake_case__       ,	snake_case__ )
										if clean:
																	lowerCamelCase_	:							Optional[int]									=self.clean_text(snake_case__ )

										def check_simbol(snake_case__					:  Any ):
																	lowerCamelCase_	:							str									=x.encode()
																	if len(snake_case__ ) == 1 and len(snake_case__ ) == 2:
																								lowerCamelCase_	:							str									=(int(e[0] ) << 8) + int(e[1] )
																								if (
																								    (c >= 0xc_2a1 and c <= 0xc_2bf)
																								    or (c >= 0xc_780 and c <= 0xc_783)
																								    or (c >= 0xc_ab9 and c <= 0xc_bbf)
																								    or (c >= 0xc_c80 and c <= 0xc_da2)
																								):
																															return True
																	return False

										def checkuae(snake_case__					:  Optional[Any] ):
																	lowerCamelCase_	:							Optional[Any]									=x.encode()
																	if len(snake_case__ ) == 1 and len(snake_case__ ) == 3:
																								lowerCamelCase_	:							Union[str, Any]									=(int(e[0] ) << 16) + (int(e[1] ) << 8) + int(e[2] )
																								if c >= 0xe28_080 and c <= 0xe2b_07f:
																															return True
																	return False

										lowerCamelCase_	:							Dict									=0
										lowerCamelCase_	:							str									=[]
										while pos < len(snake_case__ ):
																	lowerCamelCase_	:							Optional[Any]									=min(len(snake_case__ )       ,	pos + self.maxlen + 1 ) if text[pos] == "<" else pos + 3
																	lowerCamelCase_	:							Union[str, Any]									=[]  # (token_id, token, pos)
																	for e in range(snake_case__       ,	snake_case__       ,	-1 ):
																								lowerCamelCase_	:							int									=text[pos:e]
																								if wd in self.vocab:
																															if wd[0] == "<" and len(snake_case__ ) > 2:
																																						lowerCamelCase_	:							List[str]									=[(self.vocab[wd], wd, e)]
																																						break
																															else:
																																						candidates.append((self.vocab[wd], wd, e) )
																	if len(snake_case__ ) > 0:
																								# the smallest token_id is adopted
																								lowerCamelCase_							,						lowerCamelCase_							,						lowerCamelCase_	:							str									=sorted(snake_case__       ,	key=lambda snake_case__ : x[0] )[0]
																								result.append(snake_case__ )
																								lowerCamelCase_	:							Any									=e
																	else:
																								lowerCamelCase_	:							int									=pos + 1
																								lowerCamelCase_	:							Union[str, Any]									=text[pos:end]
																								if check_simbol(snake_case__ ):
																															result.append("<KIGOU>" )
																								elif checkuae(snake_case__ ):
																															result.append("<U2000U2BFF>" )
																								else:
																															for i in wd.encode("utf-8" ):
																																						result.append("<|byte%d|>" % i )
																								lowerCamelCase_	:							Any									=end
										return result


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  int       ,	snake_case__					:  str="\n" ):
										lowerCamelCase_	:							Any									=[]
										lowerCamelCase_	:							Any									=[]
										lowerCamelCase_	:							Optional[Any]									=self.ids_to_tokens[index][0]
										if word[:6] == "<|byte" and word[-2:] == "|>":
																	byte_tokens.append(int(word[6:-2] ) )
										else:
																	if len(snake_case__ ) > 0:
																								words.append(bytearray(snake_case__ ).decode("utf-8"       ,	errors="replace" ) )
																								lowerCamelCase_	:							int									=[]
																	if word[:7] == "<|emoji" and word[-2:] == "|>":
																								words.append(self.emoji["emoji_inv"][word] )
																	elif word == "<SP>":
																								words.append(" " )
																	elif word == "<BR>":
																								words.append(snake_case__ )
																	elif word == "<TAB>":
																								words.append("\t" )
																	elif word == "<BLOCK>":
																								words.append("▀" )
																	elif word == "<KIGOU>":
																								words.append("ǀ" )
																	elif word == "<U2000U2BFF>":
																								words.append("‖" )
																	else:
																								words.append(snake_case__ )
										if len(snake_case__ ) > 0:
																	words.append(bytearray(snake_case__ ).decode("utf-8"       ,	errors="replace" ) )
										lowerCamelCase_	:							str									="".join(snake_case__ )
										return text











"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


A__	:    Optional[int]			 = {
    'configuration_speech_to_text': ['SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2TextConfig'],
    'processing_speech_to_text': ['Speech2TextProcessor'],
}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Dict			 = ['Speech2TextTokenizer']

try:
						if not is_speech_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Optional[Any]			 = ['Speech2TextFeatureExtractor']

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Dict			 = [
						    'TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'TFSpeech2TextForConditionalGeneration',
						    'TFSpeech2TextModel',
						    'TFSpeech2TextPreTrainedModel',
						]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Optional[int]			 = [
						    'SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'Speech2TextForConditionalGeneration',
						    'Speech2TextModel',
						    'Speech2TextPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
						from .processing_speech_to_text import SpeechaTextProcessor

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_speech_to_text import SpeechaTextTokenizer

						try:
												if not is_speech_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_speech_to_text import (
												    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFSpeechaTextForConditionalGeneration,
												    TFSpeechaTextModel,
												    TFSpeechaTextPreTrainedModel,
												)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_speech_to_text import (
												    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    SpeechaTextForConditionalGeneration,
												    SpeechaTextModel,
												    SpeechaTextPreTrainedModel,
												)

else:
						import sys

						A__	:    List[Any]			 = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



