


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase						=		logging.get_logger(__name__)


lowerCamelCase						=		{
    '''nielsr/canine-s''': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
lowerCamelCase						=		111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
lowerCamelCase						=		0

lowerCamelCase						=		0xe_0_0_0
lowerCamelCase						=		0xe_0_0_1
lowerCamelCase						=		0xe_0_0_2
lowerCamelCase						=		0xe_0_0_3
lowerCamelCase						=		0xe_0_0_4

# Maps special codepoints to human-readable names.
lowerCamelCase						=		{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
lowerCamelCase						=		{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}





class  _a							(    _lowercase):
	_a  :  int				   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




	def __init__( self						:		Optional[Any]		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		str=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		str=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		Dict=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=chr(_SCREAMING_SNAKE_CASE		)		, _SCREAMING_SNAKE_CASE						:		List[str]=False		, _SCREAMING_SNAKE_CASE						:		str=2048		, **_SCREAMING_SNAKE_CASE						:		Tuple		, )->	List[Any]:
			lowerCAmelCase__       :      str   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else bos_token
			lowerCAmelCase__       :      Any   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else eos_token
			lowerCAmelCase__       :      Any   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else sep_token
			lowerCAmelCase__       :      Optional[int]   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else cls_token
			lowerCAmelCase__       :      Optional[int]   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else pad_token

			# Mask token behave like a normal word, i.e. include the space before it
			lowerCAmelCase__       :      str   					= AddedToken(_SCREAMING_SNAKE_CASE		, lstrip=_SCREAMING_SNAKE_CASE		, rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		) else mask_token

			super().__init__(
			    bos_token=_SCREAMING_SNAKE_CASE		, eos_token=_SCREAMING_SNAKE_CASE		, sep_token=_SCREAMING_SNAKE_CASE		, cls_token=_SCREAMING_SNAKE_CASE		, pad_token=_SCREAMING_SNAKE_CASE		, mask_token=_SCREAMING_SNAKE_CASE		, add_prefix_space=_SCREAMING_SNAKE_CASE		, model_max_length=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		, )

			# Creates a mapping for looking up the IDs of special symbols.
			lowerCAmelCase__       :      Dict[str, int]   					= {}
			for codepoint, name in SPECIAL_CODEPOINTS.items():
					lowerCAmelCase__       :      Union[str, Any]   					= codepoint

			# Creates a mapping for looking up the string forms of special symbol IDs.
			lowerCAmelCase__       :      Dict[int, str]   					= {
			    codepoint: name for name, codepoint in self._special_codepoints.items()
			}

			lowerCAmelCase__       :      Union[str, Any]   					= UNICODE_VOCAB_SIZE
			lowerCAmelCase__       :      Union[str, Any]   					= len(self._special_codepoints		)




	@property
	def 							UpperCAmelCase__( self						:		str		)->	int:
			return self._unicode_vocab_size




	def 							UpperCAmelCase__( self						:		int		, _SCREAMING_SNAKE_CASE						:		str		)->	List[str]:
			return list(_SCREAMING_SNAKE_CASE		)




	def 							UpperCAmelCase__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		str		)->	int:
			try:
					return ord(_SCREAMING_SNAKE_CASE		)
			except TypeError:
					raise ValueError(F'invalid token: \'{token}\''		)




	def 							UpperCAmelCase__( self						:		Tuple		, _SCREAMING_SNAKE_CASE						:		int		)->	str:
			try:
					if index in SPECIAL_CODEPOINTS:
							return SPECIAL_CODEPOINTS[index]
					return chr(_SCREAMING_SNAKE_CASE		)
			except TypeError:
					raise ValueError(F'invalid id: {index}'		)




	def 							UpperCAmelCase__( self						:		Optional[int]		, _SCREAMING_SNAKE_CASE						:		Tuple		)->	str:
			return "".join(_SCREAMING_SNAKE_CASE		)




	def 							UpperCAmelCase__( self						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      str   					= [self.sep_token_id]
			lowerCAmelCase__       :      Any   					= [self.cls_token_id]

			lowerCAmelCase__       :      Any   					= cls + token_ids_a + sep
			if token_ids_a is not None:
					result += token_ids_a + sep
			return result




	def 							UpperCAmelCase__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		, _SCREAMING_SNAKE_CASE						:		bool = False		)->	List[int]:
			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=_SCREAMING_SNAKE_CASE		, token_ids_a=_SCREAMING_SNAKE_CASE		, already_has_special_tokens=_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      Optional[Any]   					= [1] + ([0] * len(_SCREAMING_SNAKE_CASE		)) + [1]
			if token_ids_a is not None:
					result += ([0] * len(_SCREAMING_SNAKE_CASE		)) + [1]
			return result




	def 							UpperCAmelCase__( self						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      Union[str, Any]   					= [self.sep_token_id]
			lowerCAmelCase__       :      Union[str, Any]   					= [self.cls_token_id]

			lowerCAmelCase__       :      int   					= len(cls + token_ids_a + sep		) * [0]
			if token_ids_a is not None:
					result += len(token_ids_a + sep		) * [1]
			return result




	def 							UpperCAmelCase__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str] = None		)->	List[Any]:
			return ()








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase						=		[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase						=		{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCamelCase						=		{f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}





class  _a							(    _lowercase):
	_a  :  Tuple				   =							VOCAB_FILES_NAMES
	_a  :  Dict				   =							PRETRAINED_VOCAB_FILES_MAP
	_a  :  Dict				   =							PRETRAINED_INIT_CONFIGURATION
	_a  :  Union[str, Any]				   =							FunnelTokenizer
	_a  :  int				   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a  :  int				   =							2




	def __init__( self						:		List[Any]		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		str=None		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Any="<unk>"		, _SCREAMING_SNAKE_CASE						:		Dict="<sep>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"		, _SCREAMING_SNAKE_CASE						:		str="<cls>"		, _SCREAMING_SNAKE_CASE						:		List[str]="<mask>"		, _SCREAMING_SNAKE_CASE						:		Optional[int]="<s>"		, _SCREAMING_SNAKE_CASE						:		Dict="</s>"		, _SCREAMING_SNAKE_CASE						:		Any=True		, _SCREAMING_SNAKE_CASE						:		Dict=True		, _SCREAMING_SNAKE_CASE						:		Tuple=None		, _SCREAMING_SNAKE_CASE						:		str="##"		, **_SCREAMING_SNAKE_CASE						:		List[str]		, )->	List[str]:
			super().__init__(
			    _SCREAMING_SNAKE_CASE		, tokenizer_file=_SCREAMING_SNAKE_CASE		, do_lower_case=_SCREAMING_SNAKE_CASE		, unk_token=_SCREAMING_SNAKE_CASE		, sep_token=_SCREAMING_SNAKE_CASE		, pad_token=_SCREAMING_SNAKE_CASE		, cls_token=_SCREAMING_SNAKE_CASE		, mask_token=_SCREAMING_SNAKE_CASE		, bos_token=_SCREAMING_SNAKE_CASE		, eos_token=_SCREAMING_SNAKE_CASE		, clean_text=_SCREAMING_SNAKE_CASE		, tokenize_chinese_chars=_SCREAMING_SNAKE_CASE		, strip_accents=_SCREAMING_SNAKE_CASE		, wordpieces_prefix=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		, )

			lowerCAmelCase__       :      Any   					= json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
			if (
			    normalizer_state.get('''lowercase'''		, _SCREAMING_SNAKE_CASE		) != do_lower_case
			    or normalizer_state.get('''strip_accents'''		, _SCREAMING_SNAKE_CASE		) != strip_accents
			    or normalizer_state.get('''handle_chinese_chars'''		, _SCREAMING_SNAKE_CASE		) != tokenize_chinese_chars
			):
					lowerCAmelCase__       :      int   					= getattr(_SCREAMING_SNAKE_CASE		, normalizer_state.pop('''type'''		)		)
					lowerCAmelCase__       :      Dict   					= do_lower_case
					lowerCAmelCase__       :      str   					= strip_accents
					lowerCAmelCase__       :      Dict   					= tokenize_chinese_chars
					lowerCAmelCase__       :      str   					= normalizer_class(**_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      List[Any]   					= do_lower_case




	def 							UpperCAmelCase__( self						:		Any		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=None		)->	Optional[int]:
			lowerCAmelCase__       :      Tuple   					= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output




	def 							UpperCAmelCase__( self						:		Optional[Any]		, _SCREAMING_SNAKE_CASE						:		List[int]		, _SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None		)->	List[int]:
			lowerCAmelCase__       :      str   					= [self.sep_token_id]
			lowerCAmelCase__       :      Optional[int]   					= [self.cls_token_id]
			if token_ids_a is None:
					return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
			return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




	def 							UpperCAmelCase__( self						:		str		, _SCREAMING_SNAKE_CASE						:		str		, _SCREAMING_SNAKE_CASE						:		Optional[str] = None		)->	Tuple[str]:
			lowerCAmelCase__       :      Any   					= self._tokenizer.model.save(_SCREAMING_SNAKE_CASE		, name=_SCREAMING_SNAKE_CASE		)
			return tuple(_SCREAMING_SNAKE_CASE		)




