





'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  lowercase__   :				Dict      =   None

lowercase__   :				Optional[Any]      =   logging.get_logger(__name__)
lowercase__   :				Tuple      =   {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase__   :				Optional[int]      =   {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

lowercase__   :				Union[str, Any]      =   {
    '''albert-base-v1''': 5_12,
    '''albert-large-v1''': 5_12,
    '''albert-xlarge-v1''': 5_12,
    '''albert-xxlarge-v1''': 5_12,
    '''albert-base-v2''': 5_12,
    '''albert-large-v2''': 5_12,
    '''albert-xlarge-v2''': 5_12,
    '''albert-xxlarge-v2''': 5_12,
}

lowercase__   :				Tuple      =   '''▁'''


class 			SCREAMING_SNAKE_CASE   (a__							):
     lowerCAmelCase		      =	VOCAB_FILES_NAMES
     lowerCAmelCase		      =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase		      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase		      =	AlbertTokenizer


     def __init__(						self						,     _UpperCAmelCase=None						,     _UpperCAmelCase=None						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=False						,     _UpperCAmelCase="[CLS]"						,     _UpperCAmelCase="[SEP]"						,     _UpperCAmelCase="<unk>"						,     _UpperCAmelCase="[SEP]"						,     _UpperCAmelCase="<pad>"						,     _UpperCAmelCase="[CLS]"						,     _UpperCAmelCase="[MASK]"						,     **_UpperCAmelCase						,     ):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	(
              AddedToken(_UpperCAmelCase						,     lstrip=_UpperCAmelCase						,     rstrip=_UpperCAmelCase						,     normalized=_UpperCAmelCase)
              if isinstance(_UpperCAmelCase						,     _UpperCAmelCase)
              else mask_token
          )

          super().__init__(
              _UpperCAmelCase						,     tokenizer_file=_UpperCAmelCase						,     do_lower_case=_UpperCAmelCase						,     remove_space=_UpperCAmelCase						,     keep_accents=_UpperCAmelCase						,     bos_token=_UpperCAmelCase						,     eos_token=_UpperCAmelCase						,     unk_token=_UpperCAmelCase						,     sep_token=_UpperCAmelCase						,     pad_token=_UpperCAmelCase						,     cls_token=_UpperCAmelCase						,     mask_token=_UpperCAmelCase						,     **_UpperCAmelCase						,     )

          __A						:			Union[str, Any]								=	do_lower_case
          __A						:			List[str]								=	remove_space
          __A						:			int								=	keep_accents
          __A						:			Dict								=	vocab_file
          __A						:			str								=	False if not self.vocab_file else True


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	[self.sep_token_id]
          __A						:			Union[str, Any]								=	[self.cls_token_id]
          if token_ids_a is None:
               return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	[self.sep_token_id]
          __A						:			Union[str, Any]								=	[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          if not self.can_save_slow_tokenizer:
               raise ValueError(
                   'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                   'tokenizer.')

          if not os.path.isdir(_UpperCAmelCase):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
          __A						:			str								=	os.path.join(
              _UpperCAmelCase						,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

          if os.path.abspath(self.vocab_file) != os.path.abspath(_UpperCAmelCase):
               copyfile(self.vocab_file						,     _UpperCAmelCase)

          return (out_vocab_file,)






'''simple docstring'''




from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding


class 			SCREAMING_SNAKE_CASE   (a__							):
     lowerCAmelCase		      =	['''image_processor''', '''tokenizer''']
     lowerCAmelCase		      =	'''AutoImageProcessor'''
     lowerCAmelCase		      =	'''AutoTokenizer'''


     def __init__(						self						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          super().__init__(_UpperCAmelCase						,     _UpperCAmelCase)
          __A						:			Tuple								=	self.image_processor


     def __call__(						self						,     _UpperCAmelCase=None						,     _UpperCAmelCase=None						,     _UpperCAmelCase=None						,     **_UpperCAmelCase):

          '''simple docstring'''






          if text is None and images is None:
               raise ValueError('You have to specify either text or images. Both cannot be none.')

          if text is not None:
               __A						:			Any								=	self.tokenizer(_UpperCAmelCase						,     return_tensors=_UpperCAmelCase						,     **_UpperCAmelCase)

          if images is not None:
               __A						:			Tuple								=	self.image_processor(_UpperCAmelCase						,     return_tensors=_UpperCAmelCase						,     **_UpperCAmelCase)

          if text is not None and images is not None:
               __A						:			Optional[int]								=	image_features.pixel_values
               return encoding
          elif text is not None:
               return encoding
          else:
               return BatchEncoding(data=dict(**_UpperCAmelCase)						,     tensor_type=_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,     *_UpperCAmelCase						,     **_UpperCAmelCase):

          '''simple docstring'''






          return self.tokenizer.batch_decode(*_UpperCAmelCase						,     **_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,     *_UpperCAmelCase						,     **_UpperCAmelCase):

          '''simple docstring'''






          return self.tokenizer.decode(*_UpperCAmelCase						,     **_UpperCAmelCase)


     @property
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return ["input_ids", "attention_mask", "pixel_values"]