






'''simple docstring'''
from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
  from PIL import Image

  from transformers import ViTImageProcessor






class        __SCREAMING_SNAKE_CASE :




      def __init__( self		:							Optional[int]   , __lowercase		:							Union[str, Any]   , __lowercase		:							Tuple=13   , __lowercase		:							List[str]=30   , __lowercase		:							Dict=2   , __lowercase		:							Tuple=3   , __lowercase		:							int=True   , __lowercase		:							Dict=True   , __lowercase		:							List[str]=32   , __lowercase		:							Optional[int]=2   , __lowercase		:							int=4   , __lowercase		:							Any=37   , __lowercase		:							Any="gelu"   , __lowercase		:							List[str]=0.1   , __lowercase		:							Union[str, Any]=0.1   , __lowercase		:							Optional[int]=10   , __lowercase		:							Optional[int]=0.02   , __lowercase		:							List[str]=3   , __lowercase		:							Optional[Any]=None   , )		->  Optional[int]:
            SCREAMING_SNAKE_CASE__       :			Dict													=parent
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=batch_size
            SCREAMING_SNAKE_CASE__       :			Dict													=image_size
            SCREAMING_SNAKE_CASE__       :			int													=patch_size
            SCREAMING_SNAKE_CASE__       :			Tuple													=num_channels
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=is_training
            SCREAMING_SNAKE_CASE__       :			str													=use_labels
            SCREAMING_SNAKE_CASE__       :			Dict													=hidden_size
            SCREAMING_SNAKE_CASE__       :			Dict													=num_hidden_layers
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=num_attention_heads
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=intermediate_size
            SCREAMING_SNAKE_CASE__       :			Any													=hidden_act
            SCREAMING_SNAKE_CASE__       :			str													=hidden_dropout_prob
            SCREAMING_SNAKE_CASE__       :			str													=attention_probs_dropout_prob
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=type_sequence_label_size
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=initializer_range
            SCREAMING_SNAKE_CASE__       :			Tuple													=scope

            # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
            SCREAMING_SNAKE_CASE__       :			Dict													=(image_size // patch_size) ** 2
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=num_patches + 1




      def        __magic_name__		( self		:							str )		->  int:
            SCREAMING_SNAKE_CASE__       :			Tuple													=floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

            SCREAMING_SNAKE_CASE__       :			List[Any]													=None
            if self.use_labels:
                  SCREAMING_SNAKE_CASE__       :			Any													=ids_tensor([self.batch_size]   , self.type_sequence_label_size )

            SCREAMING_SNAKE_CASE__       :			str													=self.get_config()

            return config, pixel_values, labels




      def        __magic_name__		( self		:							str )		->  List[str]:
            return ViTConfig(
                image_size=self.image_size   , patch_size=self.patch_size   , num_channels=self.num_channels   , hidden_size=self.hidden_size   , num_hidden_layers=self.num_hidden_layers   , num_attention_heads=self.num_attention_heads   , intermediate_size=self.intermediate_size   , hidden_act=self.hidden_act   , hidden_dropout_prob=self.hidden_dropout_prob   , attention_probs_dropout_prob=self.attention_probs_dropout_prob   , is_decoder=_SCREAMING_SNAKE_CASE   , initializer_range=self.initializer_range   , )




      def        __magic_name__		( self		:							List[str]   , __lowercase		:							List[Any]   , __lowercase		:							Dict   , __lowercase		:							int )		->  Union[str, Any]:
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=TFViTModel(config=_SCREAMING_SNAKE_CASE )
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=model(_SCREAMING_SNAKE_CASE   , training=_SCREAMING_SNAKE_CASE )
            self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size) )

            # Test with an image with different size than the one specified in config.
            SCREAMING_SNAKE_CASE__       :			str													=self.image_size // 2
            SCREAMING_SNAKE_CASE__       :			str													=pixel_values[:, :, :image_size, :image_size]
            SCREAMING_SNAKE_CASE__       :			Any													=model(_SCREAMING_SNAKE_CASE   , interpolate_pos_encoding=_SCREAMING_SNAKE_CASE   , training=_SCREAMING_SNAKE_CASE )
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=(image_size // self.patch_size) ** 2 + 1
            self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, seq_length, self.hidden_size) )




      def        __magic_name__		( self		:							str   , __lowercase		:							List[Any]   , __lowercase		:							str   , __lowercase		:							int )		->  str:
            SCREAMING_SNAKE_CASE__       :			Dict													=self.type_sequence_label_size
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=TFViTForImageClassification(_SCREAMING_SNAKE_CASE )
            SCREAMING_SNAKE_CASE__       :			int													=model(_SCREAMING_SNAKE_CASE   , labels=_SCREAMING_SNAKE_CASE   , training=_SCREAMING_SNAKE_CASE )
            self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.type_sequence_label_size) )

            # Test with an image with different size than the one specified in config.
            SCREAMING_SNAKE_CASE__       :			str													=self.image_size // 2
            SCREAMING_SNAKE_CASE__       :			str													=pixel_values[:, :, :image_size, :image_size]
            SCREAMING_SNAKE_CASE__       :			Any													=model(_SCREAMING_SNAKE_CASE   , interpolate_pos_encoding=_SCREAMING_SNAKE_CASE   , training=_SCREAMING_SNAKE_CASE )
            self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.type_sequence_label_size) )

            # test greyscale images
            SCREAMING_SNAKE_CASE__       :			Dict													=1
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=TFViTForImageClassification(_SCREAMING_SNAKE_CASE )
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=model(_SCREAMING_SNAKE_CASE )
            self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.type_sequence_label_size) )




      def        __magic_name__		( self		:							Optional[int] )		->  str:
            SCREAMING_SNAKE_CASE__       :			Any													=self.prepare_config_and_inputs()
            SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=config_and_inputs
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													={'''pixel_values''': pixel_values}
            return config, inputs_dict






@require_tf
class        __SCREAMING_SNAKE_CASE (     _UpperCamelCase		,			_UpperCamelCase		,			unittest.TestCase			):
      snake_case_												=							(TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
      snake_case_												=							(
          {'feature-extraction': TFViTModel, 'image-classification': TFViTForImageClassification}
          if is_tf_available()
          else {}
      )

      snake_case_												=							False
      snake_case_												=							False
      snake_case_												=							False




      def        __magic_name__		( self		:							List[Any] )		->  List[Any]:
            SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=TFViTModelTester(self )
            SCREAMING_SNAKE_CASE__       :			Dict													=ConfigTester(self   , config_class=_SCREAMING_SNAKE_CASE   , has_text_modality=_SCREAMING_SNAKE_CASE   , hidden_size=37 )




      def        __magic_name__		( self		:							Optional[Any] )		->  Tuple:
            self.config_tester.run_common_tests()




      @unittest.skip(reason='''ViT does not use inputs_embeds''' )
      def        __magic_name__		( self		:							Any )		->  List[str]:
            pass




      @unittest.skip(reason='''ViT does not use inputs_embeds''' )
      def        __magic_name__		( self		:							Union[str, Any] )		->  Optional[Any]:
            pass




      def        __magic_name__		( self		:							List[str] )		->  Optional[int]:
            SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Any													=self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                  SCREAMING_SNAKE_CASE__       :			Any													=model_class(_SCREAMING_SNAKE_CASE )
                  self.assertIsInstance(model.get_input_embeddings()   , (tf.keras.layers.Layer) )
                  SCREAMING_SNAKE_CASE__       :			int													=model.get_output_embeddings()
                  self.assertTrue(x is None or isinstance(_SCREAMING_SNAKE_CASE   , tf.keras.layers.Layer ) )




      def        __magic_name__		( self		:							Any )		->  Tuple:
            SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Optional[int]													=self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                  SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=model_class(_SCREAMING_SNAKE_CASE )
                  SCREAMING_SNAKE_CASE__       :			Any													=inspect.signature(model.call )
                  # signature.parameters is an OrderedDict => so arg_names order is deterministic
                  SCREAMING_SNAKE_CASE__       :			Dict													=[*signature.parameters.keys()]

                  SCREAMING_SNAKE_CASE__       :			Dict													=['''pixel_values''']
                  self.assertListEqual(arg_names[:1]   , _SCREAMING_SNAKE_CASE )




      def        __magic_name__		( self		:							Tuple )		->  List[Any]:
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE )




      def        __magic_name__		( self		:							Tuple )		->  int:
            SCREAMING_SNAKE_CASE__       :			int													=self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*_SCREAMING_SNAKE_CASE )




      @slow
      def        __magic_name__		( self		:							Optional[int] )		->  List[str]:
            SCREAMING_SNAKE_CASE__       :			List[str]													=TFViTModel.from_pretrained('''google/vit-base-patch16-224''' )
            self.assertIsNotNone(_SCREAMING_SNAKE_CASE )





def 	_a(					):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Tuple													=Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
      return image







@require_tf
@require_vision
class        __SCREAMING_SNAKE_CASE (     unittest.TestCase			):




      @cached_property
      def        __magic_name__		( self		:							List[Any] )		->  List[Any]:
            return ViTImageProcessor.from_pretrained('''google/vit-base-patch16-224''' ) if is_vision_available() else None




      @slow
      def        __magic_name__		( self		:							Tuple )		->  Optional[Any]:
            SCREAMING_SNAKE_CASE__       :			Dict													=TFViTForImageClassification.from_pretrained('''google/vit-base-patch16-224''' )

            SCREAMING_SNAKE_CASE__       :			str													=self.default_image_processor
            SCREAMING_SNAKE_CASE__       :			Dict													=prepare_img()
            SCREAMING_SNAKE_CASE__       :			str													=image_processor(images=_SCREAMING_SNAKE_CASE   , return_tensors='''tf''' )

            # forward pass
            SCREAMING_SNAKE_CASE__       :			List[str]													=model(**_SCREAMING_SNAKE_CASE )

            # verify the logits
            SCREAMING_SNAKE_CASE__       :			Dict													=tf.TensorShape((1, 10_00) )
            self.assertEqual(outputs.logits.shape   , _SCREAMING_SNAKE_CASE )

            SCREAMING_SNAKE_CASE__       :			Optional[int]													=tf.constant([-0.2744, 0.8215, -0.0836] )

            tf.debugging.assert_near(outputs.logits[0, :3]   , _SCREAMING_SNAKE_CASE   , atol=1e-4 )







'''simple docstring'''
import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_					=	logging.get_logger(__name__)


a_					=	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

a_					=	{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

a_					=	{'facebook/blenderbot_small-90M': 5_1_2}





def 	_a(					UpperCamelCase__			:  Dict							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=set()
      SCREAMING_SNAKE_CASE__       :			Optional[Any]													=word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)							)
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=char

      SCREAMING_SNAKE_CASE__       :			Any													=set(UpperCamelCase__							)
      return pairs







class        __SCREAMING_SNAKE_CASE (     lowerCamelCase			):
 snake_case_												=							VOCAB_FILES_NAMES
 snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
 snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 snake_case_												=							["""input_ids""", """attention_mask"""]




 def __init__( self		:							Tuple   , __lowercase		:							int   , __lowercase		:							Optional[int]   , __lowercase		:							List[str]="__start__"   , __lowercase		:							Union[str, Any]="__end__"   , __lowercase		:							str="__unk__"   , __lowercase		:							Union[str, Any]="__null__"   , **__lowercase		:							List[str]   , )		->  Optional[int]:
       super().__init__(unk_token=__lowercase   , bos_token=__lowercase   , eos_token=__lowercase   , pad_token=__lowercase   , **__lowercase )

       with open(__lowercase   , encoding='''utf-8''' ) as vocab_handle:
             SCREAMING_SNAKE_CASE__       :			Any													=json.load(__lowercase )
       SCREAMING_SNAKE_CASE__       :			Dict													={v: k for k, v in self.encoder.items()}
       with open(__lowercase   , encoding='''utf-8''' ) as merges_handle:
             SCREAMING_SNAKE_CASE__       :			int													=merges_handle.read().split('''\n''' )[1:-1]
       SCREAMING_SNAKE_CASE__       :			List[Any]													=[tuple(merge.split() ) for merge in merges]
       SCREAMING_SNAKE_CASE__       :			int													=dict(zip(__lowercase   , range(len(__lowercase ) ) ) )
       SCREAMING_SNAKE_CASE__       :			Dict													={}




 @property
 def        __magic_name__		( self		:							Any )		->  int:
       return len(self.encoder )




 def        __magic_name__		( self		:							Tuple )		->  Dict:
       return dict(self.encoder   , **self.added_tokens_encoder )




 def        __magic_name__		( self		:							Optional[Any]   , __lowercase		:							str )		->  str:
       if token in self.cache:
             return self.cache[token]
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=re.sub('''([.,!?()])'''   , r''' \1'''   , __lowercase )
       SCREAMING_SNAKE_CASE__       :			int													=re.sub('''(\')'''   , r''' \1 '''   , __lowercase )
       SCREAMING_SNAKE_CASE__       :			Dict													=re.sub(r'''\s{2,}'''   , ''' '''   , __lowercase )
       if "\n" in token:
             SCREAMING_SNAKE_CASE__       :			List[str]													=token.replace('''\n'''   , ''' __newln__''' )

       SCREAMING_SNAKE_CASE__       :			Dict													=token.split(''' ''' )
       SCREAMING_SNAKE_CASE__       :			Dict													=[]
       for token in tokens:
             if not len(__lowercase ):
                   continue

             SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=token.lower()
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=tuple(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Optional[int]													=tuple(list(word[:-1] ) + [word[-1] + '''</w>'''] )
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=get_pairs(__lowercase )

             if not pairs:
                   words.append(__lowercase )
                   continue

             while True:
                   SCREAMING_SNAKE_CASE__       :			int													=min(__lowercase   , key=lambda __lowercase : self.bpe_ranks.get(__lowercase   , float('''inf''' ) ) )
                   if bigram not in self.bpe_ranks:
                         break
                   SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Any													=bigram
                   SCREAMING_SNAKE_CASE__       :			List[Any]													=[]
                   SCREAMING_SNAKE_CASE__       :			Tuple													=0

                   while i < len(__lowercase ):
                         try:
                               SCREAMING_SNAKE_CASE__       :			Optional[Any]													=word.index(__lowercase   , __lowercase )
                               new_word.extend(word[i:j] )
                               SCREAMING_SNAKE_CASE__       :			Optional[int]													=j
                         except ValueError:
                               new_word.extend(word[i:] )
                               break

                         if word[i] == first and i < len(__lowercase ) - 1 and word[i + 1] == second:
                               new_word.append(first + second )
                               i += 2
                         else:
                               new_word.append(word[i] )
                               i += 1
                   SCREAMING_SNAKE_CASE__       :			int													=tuple(__lowercase )
                   SCREAMING_SNAKE_CASE__       :			int													=new_word
                   if len(__lowercase ) == 1:
                         break
                   else:
                         SCREAMING_SNAKE_CASE__       :			Any													=get_pairs(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Optional[int]													='''@@ '''.join(__lowercase )
             SCREAMING_SNAKE_CASE__       :			List[Any]													=word[:-4]

             SCREAMING_SNAKE_CASE__       :			str													=word
             words.append(__lowercase )
       return " ".join(__lowercase )




 def        __magic_name__		( self		:							str   , __lowercase		:							str )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			Dict													=[]

       SCREAMING_SNAKE_CASE__       :			List[Any]													=re.findall(r'''\S+\n?'''   , __lowercase )

       for token in words:
             split_tokens.extend(list(self.bpe(__lowercase ).split(''' ''' ) ) )
       return split_tokens




 def        __magic_name__		( self		:							List[Any]   , __lowercase		:							str )		->  int:
       SCREAMING_SNAKE_CASE__       :			int													=token.lower()
       return self.encoder.get(__lowercase   , self.encoder.get(self.unk_token ) )




 def        __magic_name__		( self		:							int   , __lowercase		:							int )		->  str:
       return self.decoder.get(__lowercase   , self.unk_token )




 def        __magic_name__		( self		:							List[Any]   , __lowercase		:							List[str] )		->  str:
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=''' '''.join(__lowercase ).replace('''@@ '''   , '''''' ).strip()
       return out_string




 def        __magic_name__		( self		:							Tuple   , __lowercase		:							str   , __lowercase		:							Optional[str] = None )		->  Tuple[str]:
       if not os.path.isdir(__lowercase ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
             return
       SCREAMING_SNAKE_CASE__       :			str													=os.path.join(
           __lowercase   , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
       SCREAMING_SNAKE_CASE__       :			int													=os.path.join(
           __lowercase   , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

       with open(__lowercase   , '''w'''   , encoding='''utf-8''' ) as f:
             f.write(json.dumps(self.encoder   , indent=2   , sort_keys=__lowercase   , ensure_ascii=__lowercase ) + '''\n''' )

       SCREAMING_SNAKE_CASE__       :			List[str]													=0
       with open(__lowercase   , '''w'''   , encoding='''utf-8''' ) as writer:
             writer.write('''#version: 0.2\n''' )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   , key=lambda __lowercase : kv[1] ):
                   if index != token_index:
                         logger.warning(
                             F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             ''' Please check that the tokenizer is not corrupted!''' )
                         SCREAMING_SNAKE_CASE__       :			Any													=token_index
                   writer.write(''' '''.join(__lowercase ) + '''\n''' )
                   index += 1

       return vocab_file, merge_file