




'''simple docstring'''


import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast

@require_vision
class       A   (		unittest.TestCase   ):







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    tempfile.mkdtemp()

														_a     					=    BlipImageProcessor()
														_a     					=    BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-BertModel''' )

														_a     					=    BlipProcessor(lowerCAmelCase_							,      lowerCAmelCase_ )

														processor.save_pretrained(self.tmpdirname )







							def        __lowerCAmelCase      (  self	:  Dict							,      **lowerCAmelCase_	:  str )      ->							Union[str, Any]:


														"""simple docstring"""


														return AutoProcessor.from_pretrained(self.tmpdirname							,      **lowerCAmelCase_ ).tokenizer







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      **lowerCAmelCase_	:  Optional[int] )      ->							int:


														"""simple docstring"""


														return AutoProcessor.from_pretrained(self.tmpdirname							,      **lowerCAmelCase_ ).image_processor







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							str:


														"""simple docstring"""


														shutil.rmtree(self.tmpdirname )







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    [np.random.randint(2_55							,      size=(3, 30, 4_00)							,      dtype=np.uinta )]

														_a     					=    [Image.fromarray(np.moveaxis(lowerCAmelCase_							,      0							,      -1 ) ) for x in image_inputs]

														return image_inputs







							def        __lowerCAmelCase      (  self	:  int )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    BlipProcessor(tokenizer=self.get_tokenizer()							,      image_processor=self.get_image_processor() )
														processor.save_pretrained(self.tmpdirname )

														_a     					=    self.get_tokenizer(bos_token='''(BOS)'''							,      eos_token='''(EOS)''' )
														_a     					=    self.get_image_processor(do_normalize=lowerCAmelCase_							,      padding_value=1.0 )

														_a     					=    BlipProcessor.from_pretrained(
														    self.tmpdirname							,      bos_token='''(BOS)'''							,      eos_token='''(EOS)'''							,      do_normalize=lowerCAmelCase_							,      padding_value=1.0 )

														self.assertEqual(processor.tokenizer.get_vocab()							,      tokenizer_add_kwargs.get_vocab() )
														self.assertIsInstance(processor.tokenizer							,      lowerCAmelCase_ )

														self.assertEqual(processor.image_processor.to_json_string()							,      image_processor_add_kwargs.to_json_string() )
														self.assertIsInstance(processor.image_processor							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							Dict:


														"""simple docstring"""


														_a     					=    self.get_image_processor()
														_a     					=    self.get_tokenizer()

														_a     					=    BlipProcessor(tokenizer=lowerCAmelCase_							,      image_processor=lowerCAmelCase_ )

														_a     					=    self.prepare_image_inputs()

														_a     					=    image_processor(lowerCAmelCase_							,      return_tensors='''np''' )
														_a     					=    processor(images=lowerCAmelCase_							,      return_tensors='''np''' )

														for key in input_feat_extract.keys():
																					self.assertAlmostEqual(input_feat_extract[key].sum()							,      input_processor[key].sum()							,      delta=1e-2 )







							def        __lowerCAmelCase      (  self	:  str )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    self.get_image_processor()
														_a     					=    self.get_tokenizer()

														_a     					=    BlipProcessor(tokenizer=lowerCAmelCase_							,      image_processor=lowerCAmelCase_ )

														_a     					=    '''lower newer'''

														_a     					=    processor(text=lowerCAmelCase_ )

														_a     					=    tokenizer(lowerCAmelCase_							,      return_token_type_ids=lowerCAmelCase_ )

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]							,      encoded_processor[key] )







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							Any:


														"""simple docstring"""


														_a     					=    self.get_image_processor()
														_a     					=    self.get_tokenizer()

														_a     					=    BlipProcessor(tokenizer=lowerCAmelCase_							,      image_processor=lowerCAmelCase_ )

														_a     					=    '''lower newer'''
														_a     					=    self.prepare_image_inputs()

														_a     					=    processor(text=lowerCAmelCase_							,      images=lowerCAmelCase_ )

														self.assertListEqual(list(inputs.keys() )							,      ['''pixel_values''', '''input_ids''', '''attention_mask'''] )

														# test if it raises when no input is passed
														with pytest.raises(lowerCAmelCase_ ):
																					processor()







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							List[str]:


														"""simple docstring"""


														_a     					=    self.get_image_processor()
														_a     					=    self.get_tokenizer()

														_a     					=    BlipProcessor(tokenizer=lowerCAmelCase_							,      image_processor=lowerCAmelCase_ )

														_a     					=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														_a     					=    processor.batch_decode(lowerCAmelCase_ )
														_a     					=    tokenizer.batch_decode(lowerCAmelCase_ )

														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Tuple:


														"""simple docstring"""


														_a     					=    self.get_image_processor()
														_a     					=    self.get_tokenizer()

														_a     					=    BlipProcessor(tokenizer=lowerCAmelCase_							,      image_processor=lowerCAmelCase_ )

														_a     					=    '''lower newer'''
														_a     					=    self.prepare_image_inputs()

														_a     					=    processor(text=lowerCAmelCase_							,      images=lowerCAmelCase_ )

														# For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask']
														self.assertListEqual(list(inputs.keys() )							,      ['''pixel_values''', '''input_ids''', '''attention_mask'''] )







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_snake_case				:				Any       							=		logging.get_logger(__name__)

_snake_case				:				Any       							=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case				:				int       							=		{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_snake_case				:				int       							=		{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_INIT_CONFIGURATION
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    BertTokenizer







							def __init__(  self	:  str							,      lowerCAmelCase_	:  Union[str, Any]=None							,      lowerCAmelCase_	:  str=None							,      lowerCAmelCase_	:  List[Any]=True							,      lowerCAmelCase_	:  Any="[UNK]"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Tuple="[PAD]"							,      lowerCAmelCase_	:  Tuple="[CLS]"							,      lowerCAmelCase_	:  Optional[int]="[MASK]"							,      lowerCAmelCase_	:  Dict=True							,      lowerCAmelCase_	:  Union[str, Any]=None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							Union[str, Any]:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,      tokenizer_file=lowerCAmelCase_							,      do_lower_case=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      tokenize_chinese_chars=lowerCAmelCase_							,      strip_accents=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														_a     					=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
														if (
														    normalizer_state.get('''lowercase'''							,      lowerCAmelCase_ ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''							,      lowerCAmelCase_ ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''							,      lowerCAmelCase_ ) != tokenize_chinese_chars
														):
																					_a     					=    getattr(lowerCAmelCase_							,      normalizer_state.pop('''type''' ) )
																					_a     					=    do_lower_case
																					_a     					=    strip_accents
																					_a     					=    tokenize_chinese_chars
																					_a     					=    normalizer_class(**lowerCAmelCase_ )

														_a     					=    do_lower_case







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  int=None )      ->							List[str]:


														"""simple docstring"""


														_a     					=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														_a     					=    self._tokenizer.model.save(lowerCAmelCase_							,      name=lowerCAmelCase_ )
														return tuple(lowerCAmelCase_ )

