




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_xlnet import XLNetTokenizer
else:
			lowerCamelCase__			:       Tuple         =							None


lowerCamelCase__			:       List[Any]         =							logging.get_logger(__name__)

lowerCamelCase__			:       List[str]         =							{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__			:       List[str]         =							{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json',
    },
}

lowerCamelCase__			:       List[str]         =							{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

lowerCamelCase__			:       Optional[Any]         =							'▁'

# Segments (not really needed)
lowerCamelCase__			:       List[str]         =							0
lowerCamelCase__			:       Optional[int]         =							1
lowerCamelCase__			:       Union[str, Any]         =							2
lowerCamelCase__			:       List[str]         =							3
lowerCamelCase__			:       Optional[int]         =							4





class 	lowerCamelCase_							(				_SCREAMING_SNAKE_CASE ):


	'''simple docstring'''







	lowercase_       =   VOCAB_FILES_NAMES
	lowercase_       =   PRETRAINED_VOCAB_FILES_MAP
	lowercase_       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_       =   "left"
	lowercase_       =   XLNetTokenizer



	def __init__(							self  :		Optional[Any]    ,    _lowerCAmelCase  :		str=None    ,    _lowerCAmelCase  :		List[Any]=None    ,    _lowerCAmelCase  :		Dict=False    ,    _lowerCAmelCase  :		str=True    ,    _lowerCAmelCase  :		int=False    ,    _lowerCAmelCase  :		str="<s>"    ,    _lowerCAmelCase  :		Any="</s>"    ,    _lowerCAmelCase  :		Optional[int]="<unk>"    ,    _lowerCAmelCase  :		int="<sep>"    ,    _lowerCAmelCase  :		Dict="<pad>"    ,    _lowerCAmelCase  :		List[Any]="<cls>"    ,    _lowerCAmelCase  :		List[Any]="<mask>"    ,    _lowerCAmelCase  :		Tuple=["<eop>", "<eod>"]    ,    **_lowerCAmelCase  :		Tuple    ,    ):
							# Mask token behave like a normal word, i.e. include the space before it
							SCREAMING_SNAKE_CASE_     						=      AddedToken(_lowerCAmelCase    ,    lstrip=_lowerCAmelCase    ,    rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase    ,    _lowerCAmelCase						) else mask_token

							super().__init__(
							    vocab_file=_lowerCAmelCase    ,    tokenizer_file=_lowerCAmelCase    ,    do_lower_case=_lowerCAmelCase    ,    remove_space=_lowerCAmelCase    ,    keep_accents=_lowerCAmelCase    ,    bos_token=_lowerCAmelCase    ,    eos_token=_lowerCAmelCase    ,    unk_token=_lowerCAmelCase    ,    sep_token=_lowerCAmelCase    ,    pad_token=_lowerCAmelCase    ,    cls_token=_lowerCAmelCase    ,    mask_token=_lowerCAmelCase    ,    additional_special_tokens=_lowerCAmelCase    ,    **_lowerCAmelCase    ,    )

							SCREAMING_SNAKE_CASE_     						=      3
							SCREAMING_SNAKE_CASE_     						=      do_lower_case
							SCREAMING_SNAKE_CASE_     						=      remove_space
							SCREAMING_SNAKE_CASE_     						=      keep_accents
							SCREAMING_SNAKE_CASE_     						=      vocab_file
							SCREAMING_SNAKE_CASE_     						=      False if not self.vocab_file else True



	def 				lowerCAmelCase_  (							self  :		Tuple    ,    _lowerCAmelCase  :		List[int]    ,    _lowerCAmelCase  :		Optional[List[int]] = None						):
							SCREAMING_SNAKE_CASE_     						=      [self.sep_token_id]
							SCREAMING_SNAKE_CASE_     						=      [self.cls_token_id]
							if token_ids_a is None:
													return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls



	def 				lowerCAmelCase_  (							self  :		Optional[Any]    ,    _lowerCAmelCase  :		List[int]    ,    _lowerCAmelCase  :		Optional[List[int]] = None						):
							SCREAMING_SNAKE_CASE_     						=      [self.sep_token_id]
							SCREAMING_SNAKE_CASE_     						=      [2]

							if token_ids_a is None:
													return len(token_ids_a + sep						) * [0] + cls_segment_id
							return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id




	def 				lowerCAmelCase_  (							self  :		int    ,    _lowerCAmelCase  :		str    ,    _lowerCAmelCase  :		Optional[str] = None						):
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
													    'tokenizer.'						)

							if not os.path.isdir(_lowerCAmelCase						):
													logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
													return
							SCREAMING_SNAKE_CASE_     						=      os.path.join(
							    _lowerCAmelCase    ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_lowerCAmelCase						):
													copyfile(self.vocab_file    ,    _lowerCAmelCase						)

							return (out_vocab_file,)





def  UpperCAmelCase_							(				__UpperCAmelCase						:     int      ,      __UpperCAmelCase						:     int							) ->			int:
      return int((input_a, input_a).count(0							) != 0							)

def  UpperCAmelCase_							(				) ->			None:
      assert nand_gate(0      ,      0							) == 1
      assert nand_gate(0      ,      1							) == 1
      assert nand_gate(1      ,      0							) == 1
      assert nand_gate(1      ,      1							) == 0


if __name__ == "__main__":
   print(nand_gate(0, 0))
   print(nand_gate(0, 1))
   print(nand_gate(1, 0))
   print(nand_gate(1, 1))