




"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


SCREAMING_SNAKE_CASE    =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE    =	{'vocab_file': 'vocab.txt'}

SCREAMING_SNAKE_CASE    =	{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

SCREAMING_SNAKE_CASE    =	{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


SCREAMING_SNAKE_CASE    =	{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}




class      UpperCAmelCase_  (						lowerCAmelCase__							):
    lowercase__						 =				VOCAB_FILES_NAMES
    lowercase__						 =				PRETRAINED_VOCAB_FILES_MAP
    lowercase__						 =				PRETRAINED_INIT_CONFIGURATION
    lowercase__						 =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__						 =				ConvBertTokenizer




    def __init__(   self						:	str     ,			snake_case_						:	List[str]=None     ,			snake_case_						:	List[Any]=None     ,			snake_case_						:	Optional[Any]=True     ,			snake_case_						:	Tuple="[UNK]"     ,			snake_case_						:	Union[str, Any]="[SEP]"     ,			snake_case_						:	str="[PAD]"     ,			snake_case_						:	str="[CLS]"     ,			snake_case_						:	Any="[MASK]"     ,			snake_case_						:	List[Any]=True     ,			snake_case_						:	Optional[int]=None     ,			**snake_case_						:	List[Any]     ,			)				->							int:




     '''simple docstring'''

     super().__init__(
         _SCREAMING_SNAKE_CASE     ,			tokenizer_file=_SCREAMING_SNAKE_CASE     ,			do_lower_case=_SCREAMING_SNAKE_CASE     ,			unk_token=_SCREAMING_SNAKE_CASE     ,			sep_token=_SCREAMING_SNAKE_CASE     ,			pad_token=_SCREAMING_SNAKE_CASE     ,			cls_token=_SCREAMING_SNAKE_CASE     ,			mask_token=_SCREAMING_SNAKE_CASE     ,			tokenize_chinese_chars=_SCREAMING_SNAKE_CASE     ,			strip_accents=_SCREAMING_SNAKE_CASE     ,			**_SCREAMING_SNAKE_CASE     ,			)

     A__	=    json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
     if (
         normalizer_state.get("lowercase"     ,			_SCREAMING_SNAKE_CASE				) != do_lower_case
         or normalizer_state.get("strip_accents"     ,			_SCREAMING_SNAKE_CASE				) != strip_accents
         or normalizer_state.get("handle_chinese_chars"     ,			_SCREAMING_SNAKE_CASE				) != tokenize_chinese_chars
     ):
      A__	=    getattr(_SCREAMING_SNAKE_CASE     ,			normalizer_state.pop("type"				)				)
      A__	=    do_lower_case
      A__	=    strip_accents
      A__	=    tokenize_chinese_chars
      A__	=    normalizer_class(**_SCREAMING_SNAKE_CASE				)

     A__	=    do_lower_case




    def 							__magic_name__							(   self						:	Union[str, Any]     ,			snake_case_						:	Dict     ,			snake_case_						:	str=None				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

     return output




    def 							__magic_name__							(   self						:	List[Any]     ,			snake_case_						:	List[str]     ,			snake_case_						:	Dict = None				)				->							List[int]:




     '''simple docstring'''

     A__	=    [self.sep_token_id]
     A__	=    [self.cls_token_id]
     if token_ids_a is None:
      return len(cls + token_ids_a + sep				) * [0]
     return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




    def 							__magic_name__							(   self						:	Tuple     ,			snake_case_						:	List[Any]     ,			snake_case_						:	Tuple = None				)				->							Tuple[str]:




     '''simple docstring'''

     A__	=    self._tokenizer.model.save(_SCREAMING_SNAKE_CASE     ,			name=_SCREAMING_SNAKE_CASE				)
     return tuple(_SCREAMING_SNAKE_CASE				)











"""simple docstring"""



from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE    =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE    =	{
    "SCUT-DLVCLab/lilt-roberta-en-base": (
        "https://huggingface.co/SCUT-DLVCLab/lilt-roberta-en-base/resolve/main/config.json"
    ),
}




class      UpperCAmelCase_  (						A_							):
    lowercase__						 =				'''lilt'''




    def __init__(   self						:	List[str]     ,			snake_case_						:	Any=30_522     ,			snake_case_						:	Optional[Any]=768     ,			snake_case_						:	Union[str, Any]=12     ,			snake_case_						:	Union[str, Any]=12     ,			snake_case_						:	Any=3_072     ,			snake_case_						:	List[str]="gelu"     ,			snake_case_						:	Union[str, Any]=0.1     ,			snake_case_						:	List[Any]=0.1     ,			snake_case_						:	Any=512     ,			snake_case_						:	Optional[Any]=2     ,			snake_case_						:	List[str]=0.02     ,			snake_case_						:	Optional[Any]=1e-12     ,			snake_case_						:	List[Any]=0     ,			snake_case_						:	Any="absolute"     ,			snake_case_						:	str=None     ,			snake_case_						:	int=4     ,			snake_case_						:	int=1_024     ,			**snake_case_						:	Tuple     ,			)				->							List[Any]:




     '''simple docstring'''

     super().__init__(pad_token_id=snake_case_     ,			**snake_case_				)

     A__	=    vocab_size
     A__	=    hidden_size
     A__	=    num_hidden_layers
     A__	=    num_attention_heads
     A__	=    hidden_act
     A__	=    intermediate_size
     A__	=    hidden_dropout_prob
     A__	=    attention_probs_dropout_prob
     A__	=    max_position_embeddings
     A__	=    type_vocab_size
     A__	=    initializer_range
     A__	=    layer_norm_eps
     A__	=    position_embedding_type
     A__	=    classifier_dropout
     A__	=    channel_shrink_ratio
     A__	=    max_ad_position_embeddings





