


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		lowerCAmelCase__  :Any             =							None


lowerCAmelCase__  :List[str]             =							logging.get_logger(__name__)

lowerCAmelCase__  :Any             =							{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase__  :Any             =							{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

lowerCAmelCase__  :Dict             =							{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

lowerCAmelCase__  :Union[str, Any]             =							'''▁'''

# Segments (not really needed)
lowerCAmelCase__  :int             =							0
lowerCAmelCase__  :Optional[Any]             =							1
lowerCAmelCase__  :List[str]             =							2
lowerCAmelCase__  :Optional[int]             =							3
lowerCAmelCase__  :Optional[int]             =							4

class 		__a				(    UpperCAmelCase      ):
		_a     :  int           =     VOCAB_FILES_NAMES
		_a     :  str           =     PRETRAINED_VOCAB_FILES_MAP
		_a     :  List[Any]           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     :  Optional[int]           =     'left'
		_a     :  Optional[Any]           =     XLNetTokenizer
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=None      ,   _SCREAMING_SNAKE_CASE=False      ,   _SCREAMING_SNAKE_CASE=True      ,   _SCREAMING_SNAKE_CASE=False      ,   _SCREAMING_SNAKE_CASE="<s>"      ,   _SCREAMING_SNAKE_CASE="</s>"      ,   _SCREAMING_SNAKE_CASE="<unk>"      ,   _SCREAMING_SNAKE_CASE="<sep>"      ,   _SCREAMING_SNAKE_CASE="<pad>"      ,   _SCREAMING_SNAKE_CASE="<cls>"      ,   _SCREAMING_SNAKE_CASE="<mask>"      ,   _SCREAMING_SNAKE_CASE=["<eop>", "<eod>"]      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      AddedToken(_SCREAMING_SNAKE_CASE      ,   lstrip=_SCREAMING_SNAKE_CASE      ,   rstrip=_SCREAMING_SNAKE_CASE						) if isinstance(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						) else mask_token

									super().__init__(
									    vocab_file=_SCREAMING_SNAKE_CASE      ,   tokenizer_file=_SCREAMING_SNAKE_CASE      ,   do_lower_case=_SCREAMING_SNAKE_CASE      ,   remove_space=_SCREAMING_SNAKE_CASE      ,   keep_accents=_SCREAMING_SNAKE_CASE      ,   bos_token=_SCREAMING_SNAKE_CASE      ,   eos_token=_SCREAMING_SNAKE_CASE      ,   unk_token=_SCREAMING_SNAKE_CASE      ,   sep_token=_SCREAMING_SNAKE_CASE      ,   pad_token=_SCREAMING_SNAKE_CASE      ,   cls_token=_SCREAMING_SNAKE_CASE      ,   mask_token=_SCREAMING_SNAKE_CASE      ,   additional_special_tokens=_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE      ,   )

									_UpperCAmelCase					=      3
									_UpperCAmelCase					=      do_lower_case
									_UpperCAmelCase					=      remove_space
									_UpperCAmelCase					=      keep_accents
									_UpperCAmelCase					=      vocab_file
									_UpperCAmelCase					=      False if not self.vocab_file else True
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]
									if token_ids_a is None:
																return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [2]

									if token_ids_a is None:
																return len(token_ids_a + sep						) * [0] + cls_segment_id
									return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id






		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	Tuple[str]:




									"""simple docstring"""





									if not self.can_save_slow_tokenizer:
																raise ValueError(
																    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
																    'tokenizer.'						)

									if not os.path.isdir(_SCREAMING_SNAKE_CASE						):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																return
									_UpperCAmelCase					=      os.path.join(
									    _SCREAMING_SNAKE_CASE      ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(_SCREAMING_SNAKE_CASE						):
																copyfile(self.vocab_file      ,   _SCREAMING_SNAKE_CASE						)

									return (out_vocab_file,)




from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import torch

from ...utils import BaseOutput, OptionalDependencyNotAvailable, is_torch_available, is_transformers_available

@dataclass
class 		__a				(    UpperCAmelCase      ):
		_a     :  Union[List[np.ndarray], torch.FloatTensor]


try:
		if not (is_transformers_available() and is_torch_available()):
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
		from .pipeline_text_to_video_synth import TextToVideoSDPipeline
		from .pipeline_text_to_video_synth_imgaimg import VideoToVideoSDPipeline  # noqa: F401
		from .pipeline_text_to_video_zero import TextToVideoZeroPipeline
