





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {'vocab_file': 'spiece.model'}

_SCREAMING_SNAKE_CASE   =      {
    'vocab_file': {
        'bert_for_seq_generation': (
            'https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model'
        ),
    }
}

_SCREAMING_SNAKE_CASE   =      {'bert_for_seq_generation': 5_1_2}






class 							__lowercase    ( _lowerCamelCase  ):







	'''simple docstring'''

	a      :			str     =    VOCAB_FILES_NAMES
	a      :			Union[str, Any]     =    PRETRAINED_VOCAB_FILES_MAP
	a      :			Tuple     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a      :			List[Any]     =    []
	a      :			Optional[Any]     =    ["input_ids", "attention_mask"]


	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase="<s>"       ,_lowerCamelCase="</s>"       ,_lowerCamelCase="<unk>"       ,_lowerCamelCase="<pad>"       ,_lowerCamelCase="<::::>"       ,_lowerCamelCase = None       ,**_lowerCamelCase       ,)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							{} if sp_model_kwargs is None else sp_model_kwargs

				# Add extra_ids to the special token list
				super().__init__(
				    bos_token=_lowerCamelCase       ,eos_token=_lowerCamelCase       ,unk_token=_lowerCamelCase       ,pad_token=_lowerCamelCase       ,sep_token=_lowerCamelCase       ,sp_model_kwargs=self.sp_model_kwargs       ,**_lowerCamelCase       ,)

				__lowercase			  =							vocab_file

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(_lowerCamelCase	)


	@property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return self.sp_model.get_piece_size()


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				__lowercase			  =							{self.convert_ids_to_tokens(_lowerCamelCase	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab


	def __getstate__(self	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							self.__dict__.copy()
				__lowercase			  =							None
				return state


	def __setstate__(self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							d

				# for backward compatibility
				if not hasattr(self       ,'''sp_model_kwargs'''	):
							__lowercase			  =							{}

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(self.vocab_file	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				return self.sp_model.encode(_lowerCamelCase       ,out_type=_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				return self.sp_model.piece_to_id(_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							self.sp_model.IdToPiece(_lowerCamelCase	)
				return token


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							[]
				__lowercase			  =							''''''
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										out_string += self.sp_model.decode(_lowerCamelCase	) + token
										__lowercase			  =							[]
							else:
										current_sub_tokens.append(_lowerCamelCase	)
				out_string += self.sp_model.decode(_lowerCamelCase	)
				return out_string.strip()


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							Any:







				'''simple docstring'''





				if not os.path.isdir(_lowerCamelCase	):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
							return
				__lowercase			  =							os.path.join(
				    _lowerCamelCase       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

				if os.path.abspath(self.vocab_file	) != os.path.abspath(_lowerCamelCase	) and os.path.isfile(self.vocab_file	):
							copyfile(self.vocab_file       ,_lowerCamelCase	)
				elif not os.path.isfile(self.vocab_file	):
							with open(_lowerCamelCase       ,'''wb'''	) as fi:
										__lowercase			  =							self.sp_model.serialized_model_proto()
										fi.write(_lowerCamelCase	)

				return (out_vocab_file,)












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Dict     =    "open-llama"


	def __init__(self       ,_lowerCamelCase=100000       ,_lowerCamelCase=4096       ,_lowerCamelCase=11008       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase="silu"       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-6       ,_lowerCamelCase=True       ,_lowerCamelCase=0       ,_lowerCamelCase=1       ,_lowerCamelCase=2       ,_lowerCamelCase=False       ,_lowerCamelCase=True       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.1       ,_lowerCamelCase=True       ,_lowerCamelCase=True       ,_lowerCamelCase=None       ,**_lowerCamelCase       ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							intermediate_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							hidden_act
				__lowercase			  =							initializer_range
				__lowercase			  =							rms_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							kwargs.pop(
				    '''use_memorry_efficient_attention'''       ,_lowerCamelCase	)
				__lowercase			  =							hidden_dropout_prob
				__lowercase			  =							attention_dropout_prob
				__lowercase			  =							use_stable_embedding
				__lowercase			  =							shared_input_output_embedding
				__lowercase			  =							rope_scaling
				self._rope_scaling_validation()

				super().__init__(
				    pad_token_id=_lowerCamelCase       ,bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,tie_word_embeddings=_lowerCamelCase       ,**_lowerCamelCase       ,)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				if self.rope_scaling is None:
							return

				if not isinstance(self.rope_scaling       ,_lowerCamelCase	) or len(self.rope_scaling	) != 2:
							raise ValueError(
							    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
							    f"got {self.rope_scaling}"	)
				__lowercase			  =							self.rope_scaling.get('''type'''       ,_lowerCamelCase	)
				__lowercase			  =							self.rope_scaling.get('''factor'''       ,_lowerCamelCase	)
				if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
							raise ValueError(
							    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"	)
				if rope_scaling_factor is None or not isinstance(_lowerCamelCase       ,_lowerCamelCase	) or rope_scaling_factor <= 1.0:
							raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"	)





