






"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A__	:    Optional[int]			 = logging.get_logger(__name__)

A__	:    Optional[int]			 = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:    Tuple			 = {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

A__	:    Tuple			 = {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


A__	:    List[str]			 = {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:Optional[int]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:Dict             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:List[str]             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:Union[str, Any]             =			PRETRAINED_INIT_CONFIGURATION
			_UpperCAmelCase		:List[Any]             =			RoFormerTokenizer


			def __init__(				self					:  Union[str, Any]       ,	snake_case__					:  int=None       ,	snake_case__					:  int=None       ,	snake_case__					:  Dict=True       ,	snake_case__					:  str="[UNK]"       ,	snake_case__					:  str="[SEP]"       ,	snake_case__					:  Tuple="[PAD]"       ,	snake_case__					:  Optional[int]="[CLS]"       ,	snake_case__					:  List[Any]="[MASK]"       ,	snake_case__					:  Union[str, Any]=True       ,	snake_case__					:  str=None       ,	**snake_case__					:  List[str]       ,	):
										super().__init__(
										    snake_case__       ,	tokenizer_file=snake_case__       ,	do_lower_case=snake_case__       ,	unk_token=snake_case__       ,	sep_token=snake_case__       ,	pad_token=snake_case__       ,	cls_token=snake_case__       ,	mask_token=snake_case__       ,	tokenize_chinese_chars=snake_case__       ,	strip_accents=snake_case__       ,	**snake_case__       ,	)

										lowerCamelCase_	:							List[Any]									=json.loads(self.backend_tokenizer.normalizer.__getstate__() )
										if (
										    pre_tok_state.get("lowercase"       ,	snake_case__ ) != do_lower_case
										    or pre_tok_state.get("strip_accents"       ,	snake_case__ ) != strip_accents
										):
																	lowerCamelCase_	:							List[Any]									=getattr(snake_case__       ,	pre_tok_state.pop("type" ) )
																	lowerCamelCase_	:							List[str]									=do_lower_case
																	lowerCamelCase_	:							List[str]									=strip_accents
																	lowerCamelCase_	:							List[str]									=pre_tok_class(**snake_case__ )

										lowerCamelCase_	:							Tuple									=do_lower_case


			def __getstate__(				self					:  List[str] ):
										lowerCamelCase_	:							Tuple									=self.__dict__.copy()
										lowerCamelCase_	:							List[str]									=BertPreTokenizer()
										return state


			def __setstate__(				self					:  str       ,	snake_case__					:  Union[str, Any] ):
										lowerCamelCase_	:							List[str]									=d
										lowerCamelCase_	:							Optional[int]									=self.__dict__["_tokenizer"].get_vocab()
										lowerCamelCase_	:							Optional[Any]									=PreTokenizer.custom(JiebaPreTokenizer(snake_case__ ) )


			def      UpperCAmelCase__						(				self					:  Any       ,	snake_case__					:  Union[str, Any]       ,	snake_case__					:  List[str]=None ):
										lowerCamelCase_	:							Dict									=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

										return output


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None ):
										lowerCamelCase_	:							Dict									=[self.sep_token_id]
										lowerCamelCase_	:							Any									=[self.cls_token_id]
										if token_ids_a is None:
																	return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  str       ,	snake_case__					:  Optional[str] = None ):
										lowerCamelCase_	:							int									=self._tokenizer.model.save(snake_case__       ,	name=snake_case__ )
										return tuple(snake_case__ )


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  int       ,	snake_case__					:  List[str]=None       ,	snake_case__					:  Tuple=None       ,	snake_case__					:  str=False       ,	**snake_case__					:  int       ,	):
										lowerCamelCase_	:							str									=BertPreTokenizer()
										return super().save_pretrained(snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__       ,	**snake_case__ )











"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__	:    List[Any]			 = {
    'configuration_lilt': ['LILT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LiltConfig'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    List[str]			 = [
						    'LILT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'LiltForQuestionAnswering',
						    'LiltForSequenceClassification',
						    'LiltForTokenClassification',
						    'LiltModel',
						    'LiltPreTrainedModel',
						]

if TYPE_CHECKING:
						from .configuration_lilt import LILT_PRETRAINED_CONFIG_ARCHIVE_MAP, LiltConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_lilt import (
												    LILT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    LiltForQuestionAnswering,
												    LiltForSequenceClassification,
												    LiltForTokenClassification,
												    LiltModel,
												    LiltPreTrainedModel,
												)

else:
						import sys

						A__	:    Optional[Any]			 = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



