




'''simple docstring'''


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				Optional[Any]       							=		logging.get_logger(__name__)


_snake_case				:				List[Any]       							=		{
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_snake_case				:				Dict       							=		1114112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_snake_case				:				str       							=		0

_snake_case				:				List[Any]       							=		0xE_000
_snake_case				:				Optional[int]       							=		0xE_001
_snake_case				:				List[str]       							=		0xE_002
_snake_case				:				Optional[int]       							=		0xE_003
_snake_case				:				int       							=		0xE_004

# Maps special codepoints to human-readable names.
_snake_case				:				Dict[int, str]       							=		{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_snake_case				:				Dict[str, int]       							=		{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}

class       A   (		_a   ):
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







							def __init__(  self	:  List[Any]							,      lowerCAmelCase_	:  List[Any]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Any=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  int=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Optional[int]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  int=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Union[str, Any]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Union[str, Any]=False							,      lowerCAmelCase_	:  List[Any]=20_48							,      **lowerCAmelCase_	:  Optional[int]							,      )      ->							Any:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else bos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else eos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else sep_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else cls_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else pad_token

														# Mask token behave like a normal word, i.e. include the space before it
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      add_prefix_space=lowerCAmelCase_							,      model_max_length=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														# Creates a mapping for looking up the IDs of special symbols.
														_a     					=    {}
														for codepoint, name in SPECIAL_CODEPOINTS.items():
																					_a     					=    codepoint

														# Creates a mapping for looking up the string forms of special symbol IDs.
														_a     					=    {
														    codepoint: name for name, codepoint in self._special_codepoints.items()
														}

														_a     					=    UNICODE_VOCAB_SIZE
														_a     					=    len(self._special_codepoints )







							@property
							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							int:


														"""simple docstring"""


														return self._unicode_vocab_size







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return list(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  str )      ->							int:


														"""simple docstring"""


														try:
																					return ord(lowerCAmelCase_ )
														except TypeError:
																					raise ValueError(F'invalid token: \'{token}\'' )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  int )      ->							str:


														"""simple docstring"""


														try:
																					if index in SPECIAL_CODEPOINTS:
																												return SPECIAL_CODEPOINTS[index]
																					return chr(lowerCAmelCase_ )
														except TypeError:
																					raise ValueError(F'invalid id: {index}' )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[int]:


														"""simple docstring"""


														return "".join(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]

														_a     					=    cls + token_ids_a + sep
														if token_ids_a is not None:
																					result += token_ids_a + sep
														return result







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														_a     					=    [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														if token_ids_a is not None:
																					result += ([0] * len(lowerCAmelCase_ )) + [1]
														return result







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]

														_a     					=    len(cls + token_ids_a + sep ) * [0]
														if token_ids_a is not None:
																					result += len(token_ids_a + sep ) * [1]
														return result







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Any:


														"""simple docstring"""


														return ()







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_snake_case				:				Dict       							=		{
    'configuration_albert': ['ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'AlbertConfig', 'AlbertOnnxConfig'],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				Optional[int]       							=		['AlbertTokenizer']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				str       							=		['AlbertTokenizerFast']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				str       							=		[
				    'ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'AlbertForMaskedLM',
				    'AlbertForMultipleChoice',
				    'AlbertForPreTraining',
				    'AlbertForQuestionAnswering',
				    'AlbertForSequenceClassification',
				    'AlbertForTokenClassification',
				    'AlbertModel',
				    'AlbertPreTrainedModel',
				    'load_tf_weights_in_albert',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				Any       							=		[
				    'TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'TFAlbertForMaskedLM',
				    'TFAlbertForMultipleChoice',
				    'TFAlbertForPreTraining',
				    'TFAlbertForQuestionAnswering',
				    'TFAlbertForSequenceClassification',
				    'TFAlbertForTokenClassification',
				    'TFAlbertMainLayer',
				    'TFAlbertModel',
				    'TFAlbertPreTrainedModel',
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				Any       							=		[
				    'FlaxAlbertForMaskedLM',
				    'FlaxAlbertForMultipleChoice',
				    'FlaxAlbertForPreTraining',
				    'FlaxAlbertForQuestionAnswering',
				    'FlaxAlbertForSequenceClassification',
				    'FlaxAlbertForTokenClassification',
				    'FlaxAlbertModel',
				    'FlaxAlbertPreTrainedModel',
				]

if TYPE_CHECKING:
				from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert import AlbertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert_fast import AlbertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_albert import (
								    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    AlbertForMaskedLM,
								    AlbertForMultipleChoice,
								    AlbertForPreTraining,
								    AlbertForQuestionAnswering,
								    AlbertForSequenceClassification,
								    AlbertForTokenClassification,
								    AlbertModel,
								    AlbertPreTrainedModel,
								    load_tf_weights_in_albert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_albert import (
								    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFAlbertForMaskedLM,
								    TFAlbertForMultipleChoice,
								    TFAlbertForPreTraining,
								    TFAlbertForQuestionAnswering,
								    TFAlbertForSequenceClassification,
								    TFAlbertForTokenClassification,
								    TFAlbertMainLayer,
								    TFAlbertModel,
								    TFAlbertPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_albert import (
								    FlaxAlbertForMaskedLM,
								    FlaxAlbertForMultipleChoice,
								    FlaxAlbertForPreTraining,
								    FlaxAlbertForQuestionAnswering,
								    FlaxAlbertForSequenceClassification,
								    FlaxAlbertForTokenClassification,
								    FlaxAlbertModel,
								    FlaxAlbertPreTrainedModel,
								)
else:
				import sys

				_snake_case				:				Union[str, Any]       							=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

