


"""simple docstring"""


from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


snake_case_    					=	logging.get_logger(__name__)

snake_case_    					=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case_    					=	{
    """vocab_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"""
    },
}

snake_case_    					=	{"""allegro/herbert-base-cased""": 514}
snake_case_    					=	{}







class 				A_      (					__a    ):



						"""simple docstring"""
						__UpperCamelCase		    =							VOCAB_FILES_NAMES
						__UpperCamelCase		    =							PRETRAINED_VOCAB_FILES_MAP
						__UpperCamelCase		    =							PRETRAINED_INIT_CONFIGURATION
						__UpperCamelCase		    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__UpperCamelCase		    =							HerbertTokenizer






						def __init__( self			:Optional[int]       ,  lowercase_			:Dict=None       ,  lowercase_			:Tuple=None       ,  lowercase_			:List[str]=None       ,  lowercase_			:Optional[Any]="<s>"       ,  lowercase_			:str="<unk>"       ,  lowercase_			:str="<pad>"       ,  lowercase_			:str="<mask>"       ,  lowercase_			:Union[str, Any]="</s>"       ,  **lowercase_			:Optional[Any]       ,  )    ->      List[str]:
												super().__init__(
												    a__       ,  a__       ,  tokenizer_file=a__       ,  cls_token=a__       ,  unk_token=a__       ,  pad_token=a__       ,  mask_token=a__       ,  sep_token=a__       ,  **a__       ,  )






						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  lowercase_			:List[int]       ,  lowercase_			:Optional[List[int]] = None )    ->      Union[str, Any]:
												UpperCAmelCase					=			[self.cls_token_id]
												UpperCAmelCase					=			[self.sep_token_id]
												if token_ids_a is None:
																		return cls + token_ids_a + sep

												return cls + token_ids_a + sep + token_ids_a + sep






						def      UpperCAmelCase__					( self			:str       ,  lowercase_			:List[int]       ,  lowercase_			:Optional[List[int]] = None       ,  lowercase_			:bool = False )    ->      Any:
												if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=a__       ,  token_ids_a=a__       ,  already_has_special_tokens=a__ )

												if token_ids_a is None:
																		return [1] + ([0] * len(a__ )) + [1]
												return [1] + ([0] * len(a__ )) + [1] + ([0] * len(a__ )) + [1]






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:List[int]       ,  lowercase_			:Optional[List[int]] = None )    ->      str:
												UpperCAmelCase					=			[self.sep_token_id]
												UpperCAmelCase					=			[self.cls_token_id]

												if token_ids_a is None:
																		return len(cls + token_ids_a + sep ) * [0]
												return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






						def      UpperCAmelCase__					( self			:Tuple       ,  lowercase_			:str       ,  lowercase_			:Optional[str] = None )    ->      Any:
												UpperCAmelCase					=			self._tokenizer.model.save(a__       ,  name=a__ )
												return tuple(a__ )










"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_    					=	{
    """configuration_rembert""": ["""REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RemBertConfig""", """RemBertOnnxConfig"""]
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_    					=	["""RemBertTokenizer"""]

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_    					=	["""RemBertTokenizerFast"""]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_    					=	[
				    """REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
				    """RemBertForCausalLM""",
				    """RemBertForMaskedLM""",
				    """RemBertForMultipleChoice""",
				    """RemBertForQuestionAnswering""",
				    """RemBertForSequenceClassification""",
				    """RemBertForTokenClassification""",
				    """RemBertLayer""",
				    """RemBertModel""",
				    """RemBertPreTrainedModel""",
				    """load_tf_weights_in_rembert""",
				]


try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_    					=	[
				    """TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
				    """TFRemBertForCausalLM""",
				    """TFRemBertForMaskedLM""",
				    """TFRemBertForMultipleChoice""",
				    """TFRemBertForQuestionAnswering""",
				    """TFRemBertForSequenceClassification""",
				    """TFRemBertForTokenClassification""",
				    """TFRemBertLayer""",
				    """TFRemBertModel""",
				    """TFRemBertPreTrainedModel""",
				]


if TYPE_CHECKING:
				from .configuration_rembert import REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RemBertConfig, RemBertOnnxConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_rembert import RemBertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_rembert_fast import RemBertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_rembert import (
								    REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    RemBertForCausalLM,
								    RemBertForMaskedLM,
								    RemBertForMultipleChoice,
								    RemBertForQuestionAnswering,
								    RemBertForSequenceClassification,
								    RemBertForTokenClassification,
								    RemBertLayer,
								    RemBertModel,
								    RemBertPreTrainedModel,
								    load_tf_weights_in_rembert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_rembert import (
								    TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFRemBertForCausalLM,
								    TFRemBertForMaskedLM,
								    TFRemBertForMultipleChoice,
								    TFRemBertForQuestionAnswering,
								    TFRemBertForSequenceClassification,
								    TFRemBertForTokenClassification,
								    TFRemBertLayer,
								    TFRemBertModel,
								    TFRemBertPreTrainedModel,
								)


else:
				import sys

				snake_case_    					=	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





