





'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self       ,_lowerCamelCase = "▁"       ,_lowerCamelCase = True       ,_lowerCamelCase = "<unk>"       ,_lowerCamelCase = "</s>"       ,_lowerCamelCase = "<pad>"       ,)							->							Any:







				'''simple docstring'''





				__lowercase			  =							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase			  =							[None] * len(self.special_tokens	)
				for token_dict in self.special_tokens.values():
							__lowercase			  =							token_dict['''token''']

				__lowercase			  =							Tokenizer(Unigram()	)

				__lowercase			  =							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}'''	)       ,''' '''	),
				        normalizers.Lowercase(),
				    ]	)
				__lowercase			  =							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase       ,add_prefix_space=_lowerCamelCase	),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase	),
				        pre_tokenizers.Punctuation(),
				    ]	)
				__lowercase			  =							decoders.Metaspace(replacement=_lowerCamelCase       ,add_prefix_space=_lowerCamelCase	)

				__lowercase			  =							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"       ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]       ,)

				__lowercase			  =							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase       ,_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = 8000       ,_lowerCamelCase = True       ,)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase       ,special_tokens=self.special_tokens_list       ,show_progress=_lowerCamelCase       ,)

				if isinstance(_lowerCamelCase       ,_lowerCamelCase	):
							__lowercase			  =							[files]
				self._tokenizer.train(_lowerCamelCase       ,trainer=_lowerCamelCase	)

				self.add_unk_id()


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = 8000       ,_lowerCamelCase = True       ,)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase       ,special_tokens=self.special_tokens_list       ,show_progress=_lowerCamelCase       ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase       ,trainer=_lowerCamelCase	)

				self.add_unk_id()


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							json.loads(self._tokenizer.to_str()	)

				__lowercase			  =							self.special_tokens['''unk''']['''id''']

				__lowercase			  =							Tokenizer.from_str(json.dumps(_lowerCamelCase	)	)












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Union[str, Any]     =    "gpt_neox_japanese"


	def __init__(self       ,_lowerCamelCase=32000       ,_lowerCamelCase=2560       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase=4       ,_lowerCamelCase="gelu"       ,_lowerCamelCase=1.0_0       ,_lowerCamelCase=10000       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-5       ,_lowerCamelCase=True       ,_lowerCamelCase=31996       ,_lowerCamelCase=31999       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.0       ,**_lowerCamelCase       ,)							->							Optional[int]:







				'''simple docstring'''





				super().__init__(bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,**_lowerCamelCase	)
				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							intermediate_multiple_size
				__lowercase			  =							hidden_act
				__lowercase			  =							rotary_pct
				__lowercase			  =							rotary_emb_base
				__lowercase			  =							initializer_range
				__lowercase			  =							layer_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							attention_dropout
				__lowercase			  =							hidden_dropout





