


'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		__snake_case											=					None


__snake_case											=					logging.get_logger(__name__)

__snake_case											=					{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case											=					{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

__snake_case											=					{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

__snake_case											=					'''▁'''

# Segments (not really needed)
__snake_case											=					0
__snake_case											=					1
__snake_case											=					2
__snake_case											=					3
__snake_case											=					4
class        lowercase     (							A__ ):

		"""simple docstring"""

		_a   			=    VOCAB_FILES_NAMES
		_a   			=    PRETRAINED_VOCAB_FILES_MAP
		_a   			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a   			=    'left'
		_a   			=    XLNetTokenizer




		def __init__(		self   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       UpperCamelCase_=False   ,       UpperCamelCase_=True   ,       UpperCamelCase_=False   ,       UpperCamelCase_="<s>"   ,       UpperCamelCase_="</s>"   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_="<sep>"   ,       UpperCamelCase_="<pad>"   ,       UpperCamelCase_="<cls>"   ,       UpperCamelCase_="<mask>"   ,       UpperCamelCase_=["<eop>", "<eod>"]   ,       **UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			AddedToken(UpperCamelCase_   ,       lstrip=UpperCamelCase_   ,       rstrip=UpperCamelCase_							) if isinstance(UpperCamelCase_   ,       UpperCamelCase_							) else mask_token

							super().__init__(
							    vocab_file=UpperCamelCase_   ,       tokenizer_file=UpperCamelCase_   ,       do_lower_case=UpperCamelCase_   ,       remove_space=UpperCamelCase_   ,       keep_accents=UpperCamelCase_   ,       bos_token=UpperCamelCase_   ,       eos_token=UpperCamelCase_   ,       unk_token=UpperCamelCase_   ,       sep_token=UpperCamelCase_   ,       pad_token=UpperCamelCase_   ,       cls_token=UpperCamelCase_   ,       mask_token=UpperCamelCase_   ,       additional_special_tokens=UpperCamelCase_   ,       **UpperCamelCase_   ,       )

							UpperCamelCase__       :Optional[Any]							 =			3
							UpperCamelCase__       :Any							 =			do_lower_case
							UpperCamelCase__       :str							 =			remove_space
							UpperCamelCase__       :Optional[int]							 =			keep_accents
							UpperCamelCase__       :List[str]							 =			vocab_file
							UpperCamelCase__       :Dict							 =			False if not self.vocab_file else True




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			[self.sep_token_id]
							UpperCamelCase__       :Any							 =			[self.cls_token_id]
							if token_ids_a is None:
												return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			[self.sep_token_id]
							UpperCamelCase__       :Optional[Any]							 =			[2]

							if token_ids_a is None:
												return len(token_ids_a + sep							) * [0] + cls_segment_id
							return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							if not self.can_save_slow_tokenizer:
												raise ValueError(
												    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
												    '''tokenizer.'''							)

							if not os.path.isdir(UpperCamelCase_							):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
												return
							UpperCamelCase__       :Optional[Any]							 =			os.path.join(
							    UpperCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase_							):
												copyfile(self.vocab_file   ,       UpperCamelCase_							)

							return (out_vocab_file,)






'''simple docstring'''
from math import ceil


def       a				(      __a			, __a       )  ->     Any:



					'''simple docstring'''






					UpperCamelCase__       :str							 =			list(range(0			, __a       )       )

					UpperCamelCase__       :Optional[int]							 =			[item for sublist in list(device_map.values()       ) for item in sublist]

					# Duplicate check
					UpperCamelCase__       :Optional[int]							 =			[]
					for i in device_map_blocks:
										if device_map_blocks.count(__a       ) > 1 and i not in duplicate_blocks:
															duplicate_blocks.append(__a       )
    # Missing blocks
					UpperCamelCase__       :List[str]							 =			[i for i in blocks if i not in device_map_blocks]
					UpperCamelCase__       :Optional[Any]							 =			[i for i in device_map_blocks if i not in blocks]

					if len(__a       ) != 0:
										raise ValueError(
										    '''Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device.'''
										    ''' These attention blocks were specified more than once: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''There are attention blocks for this model that are not specified in the device_map. Add these attention '''
										    '''blocks to a device on the device_map: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''The device_map contains more attention blocks than this model has. Remove these from the device_map:'''
										    + str(__a       )       )







def       a				(      __a			, __a       )  ->     Tuple:



					'''simple docstring'''






					UpperCamelCase__       :Optional[Any]							 =			list(range(__a       )       )
					UpperCamelCase__       :Any							 =			int(ceil(n_layers / len(__a       )       )       )
					UpperCamelCase__       :List[Any]							 =			[layers[i : i + n_blocks] for i in range(0			, __a			, __a       )]

					return dict(zip(__a			, __a       )       )