






"""simple docstring"""



import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
						import torch


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A__	:    Optional[int]			 = logging.get_logger(__name__)
A__	:    Optional[int]			 = {'vocab_file': 'spiece.model'}

A__	:    Optional[Any]			 = {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

A__	:    str			 = {
    'AI-Sweden/gpt-sw3-126m': 2_048,
    'AI-Sweden/gpt-sw3-350m': 2_048,
    'AI-Sweden/gpt-sw3-1.6b': 2_048,
    'AI-Sweden/gpt-sw3-6.7b': 2_048,
    'AI-Sweden/gpt-sw3-20b': 2_048,
}







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:Optional[int]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:int             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:str             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:List[str]             =			["input_ids", "attention_mask"]


			def __init__(				self					:  Dict       ,	snake_case__					:  Optional[int]       ,	snake_case__					:  int=False       ,	snake_case__					:  List[Any]=False       ,	snake_case__					:  Optional[int]=False       ,	snake_case__					:  Tuple=None       ,	snake_case__					:  Any=None       ,	snake_case__					:  Optional[Any]=None       ,	snake_case__					:  Optional[Any]=None       ,	snake_case__					:  Optional[Dict[str, Any]] = None       ,	**snake_case__					:  Optional[int]       ,	):
										lowerCamelCase_	:							int									={} if sp_model_kwargs is None else sp_model_kwargs

										lowerCamelCase_	:							Optional[int]									=kwargs.get("name_or_path" )
										if name_or_path is None:
																	logger.warning(
																	    "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
																	    " you are testing the model, this can safely be ignored" )
																	lowerCamelCase_	:							int									="None"

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										lowerCamelCase_	:							List[Any]									="<|endoftext|>" if eos_token is None else eos_token
										lowerCamelCase_	:							Optional[Any]									="<unk>" if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
																	lowerCamelCase_	:							Union[str, Any]									=unk_token if pad_token is None else pad_token
																	lowerCamelCase_	:							List[str]									=eos_token if bos_token is None else bos_token
										else:
																	lowerCamelCase_	:							Any									="<pad>" if pad_token is None else pad_token
																	lowerCamelCase_	:							List[Any]									="<s>" if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=snake_case__       ,	remove_space=snake_case__       ,	keep_accents=snake_case__       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	unk_token=snake_case__       ,	pad_token=snake_case__       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**snake_case__       ,	)

										lowerCamelCase_	:							Tuple									=do_lower_case
										lowerCamelCase_	:							List[Any]									=remove_space
										lowerCamelCase_	:							Union[str, Any]									=keep_accents
										lowerCamelCase_	:							str									=vocab_file

										lowerCamelCase_	:							int									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(snake_case__ )

										# Used for whitespace normalization in input texts
										# fmt : off
										lowerCamelCase_	:							Dict									={" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										lowerCamelCase_	:							int									=re.compile(
										    F"""[{''.join(map(snake_case__       ,	list(range(0       ,	9 ) ) + list(range(11       ,	32 ) ) + list(range(127       ,	160 ) ) + [160, 173, 8203] ) )}]""" )


			def __getstate__(				self					:  Optional[int] ):
										lowerCamelCase_	:							int									=self.__dict__.copy()
										lowerCamelCase_	:							Tuple									=None
										return state


			def __setstate__(				self					:  Optional[Any]       ,	snake_case__					:  Optional[Any] ):
										lowerCamelCase_	:							Union[str, Any]									=d

										# for backward compatibility
										if not hasattr(self       ,	"sp_model_kwargs" ):
																	lowerCamelCase_	:							Tuple									={}

										lowerCamelCase_	:							Dict									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(self.vocab_file )


			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def      UpperCAmelCase__						(				self					:  Tuple ):
										return len(self.sp_model )


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  str ):
										lowerCamelCase_	:							List[Any]									=self.non_printing_characters_re.sub(""       ,	snake_case__ )

										# Normalize whitespaces
										lowerCamelCase_	:							Union[str, Any]									="".join([char if char not in self.whitespaces else " " for char in text] )

										# NFC Unicode normalization
										lowerCamelCase_	:							str									=unicodedata.normalize("NFC"       ,	snake_case__ )
										return text


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  str       ,	**snake_case__					:  Dict ):
										lowerCamelCase_	:							Any									=self.preprocess_text(snake_case__ )
										return self.sp_model.encode(snake_case__       ,	out_type=snake_case__ )


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  str ):
										return self.sp_model.PieceToId(snake_case__ )


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  int ):
										return self.sp_model.IdToPiece(snake_case__ )


			@staticmethod
			def      UpperCAmelCase__						(				snake_case__					:  str ):
										return out_string


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							Union[str, Any]									=[]
										lowerCamelCase_	:							Any									=""
										lowerCamelCase_	:							List[str]									=False
										for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																								# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																								if not prev_is_special:
																															out_string += " "

																								out_string += self.sp_model.decode(snake_case__ ) + token
																								lowerCamelCase_	:							List[str]									=True
																								lowerCamelCase_	:							List[Any]									=[]
																	else:
																								current_sub_tokens.append(snake_case__ )
																								lowerCamelCase_	:							Tuple									=False
										out_string += self.sp_model.decode(snake_case__ )

										return out_string


			def      UpperCAmelCase__						(				self					:  Any ):
										lowerCamelCase_	:							Tuple									={self.convert_ids_to_tokens(snake_case__ ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  str       ,	snake_case__					:  Optional[str] = None ):
										if not os.path.isdir(snake_case__ ):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
																	return
										lowerCamelCase_	:							Any									=os.path.join(
										    snake_case__       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case__ ) and os.path.isfile(self.vocab_file ):
																	copyfile(self.vocab_file       ,	snake_case__ )
										elif not os.path.isfile(self.vocab_file ):
																	with open(snake_case__       ,	"wb" ) as fi:
																								lowerCamelCase_	:							Any									=self.sp_model.serialized_model_proto()
																								fi.write(snake_case__ )

										return (out_vocab_file,)


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  Union[str, List[str]]       ,	snake_case__					:  Union[str, bool] = False ):

										if isinstance(snake_case__       ,	snake_case__ ):
																	lowerCamelCase_	:							Optional[int]									=self.preprocess_text(snake_case__ )
																	lowerCamelCase_	:							Optional[int]									=self.sp_model.encode(snake_case__ )
										else:
																	lowerCamelCase_	:							Dict									=[self.preprocess_text(snake_case__ ) for t in text]
																	lowerCamelCase_	:							List[Any]									=self.sp_model.encode(snake_case__ )

										if return_tensors is True or return_tensors == "pt":
																	lowerCamelCase_	:							Optional[int]									=torch.tensor(snake_case__ )

										return token_ids


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  Union[int, List[int]] ):
										return self.sp_model.decode(snake_case__ )


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  "Conversation" ):
										lowerCamelCase_	:							Tuple									=[F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
										lowerCamelCase_	:							int									=(
										    F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(snake_case__ ) + F"""{self.bos_token}Bot:"""
										)
										return self.encode(text=snake_case__ )












"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__	:    List[Any]			 = {
    'configuration_lilt': ['LILT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LiltConfig'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    List[str]			 = [
						    'LILT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'LiltForQuestionAnswering',
						    'LiltForSequenceClassification',
						    'LiltForTokenClassification',
						    'LiltModel',
						    'LiltPreTrainedModel',
						]

if TYPE_CHECKING:
						from .configuration_lilt import LILT_PRETRAINED_CONFIG_ARCHIVE_MAP, LiltConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_lilt import (
												    LILT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    LiltForQuestionAnswering,
												    LiltForSequenceClassification,
												    LiltForTokenClassification,
												    LiltModel,
												    LiltPreTrainedModel,
												)

else:
						import sys

						A__	:    Optional[Any]			 = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



