'''simple docstring'''




import inspect
import unittest

from transformers import MobileNetVaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import MobileNetVaForImageClassification, MobileNetVaForSemanticSegmentation, MobileNetVaModel
  from transformers.models.mobilenet_va.modeling_mobilenet_va import MOBILENET_V2_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
  from PIL import Image

  from transformers import MobileNetVaImageProcessor




class    a__				(       _UpperCamelCase			):






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  self.config_class(**self.inputs_dict   )
         self.parent.assertTrue(hasattr(_UpperCAmelCase  ,    '''tf_padding'''   )   )
         self.parent.assertTrue(hasattr(_UpperCAmelCase  ,    '''depth_multiplier'''   )   )




class    a__				:






     def __init__(			self		:		Tuple  ,    a		:		Tuple  ,    a		:		List[str]=13  ,    a		:		str=3  ,    a		:		List[str]=32  ,    a		:		Tuple=0.25  ,    a		:		List[str]=8  ,    a		:		int=8  ,    a		:		str=6  ,    a		:		List[Any]=32  ,    a		:		int=True  ,    a		:		List[str]=True  ,    a		:		List[Any]=True  ,    a		:		Dict="relu6"  ,    a		:		Optional[int]=12_80  ,    a		:		str=0.1  ,    a		:		Any=0.02  ,    a		:		int=True  ,    a		:		Optional[Any]=True  ,    a		:		int=10  ,    a		:		List[str]=None  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  parent
         __lowerCamelCase          =  batch_size
         __lowerCamelCase          =  num_channels
         __lowerCamelCase          =  image_size
         __lowerCamelCase          =  depth_multiplier
         __lowerCamelCase          =  depth_divisible_by
         __lowerCamelCase          =  min_depth
         __lowerCamelCase          =  expand_ratio
         __lowerCamelCase          =  tf_padding
         __lowerCamelCase          =  output_stride
         __lowerCamelCase          =  first_layer_is_expansion
         __lowerCamelCase          =  finegrained_output
         __lowerCamelCase          =  hidden_act
         __lowerCamelCase          =  last_hidden_size if finegrained_output else int(last_hidden_size * depth_multiplier   )
         __lowerCamelCase          =  classifier_dropout_prob
         __lowerCamelCase          =  use_labels
         __lowerCamelCase          =  is_training
         __lowerCamelCase          =  num_labels
         __lowerCamelCase          =  initializer_range
         __lowerCamelCase          =  scope






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

         __lowerCamelCase          =  None
         __lowerCamelCase          =  None
         if self.use_labels:
             __lowerCamelCase          =  ids_tensor([self.batch_size]  ,    self.num_labels   )
             __lowerCamelCase          =  ids_tensor([self.batch_size, self.image_size, self.image_size]  ,    self.num_labels   )

         __lowerCamelCase          =  self.get_config()

         return config, pixel_values, labels, pixel_labels






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         return MobileNetVaConfig(
             num_channels=self.num_channels  ,    image_size=self.image_size  ,    depth_multiplier=self.depth_multiplier  ,    depth_divisible_by=self.depth_divisible_by  ,    min_depth=self.min_depth  ,    expand_ratio=self.expand_ratio  ,    output_stride=self.output_stride  ,    first_layer_is_expansion=self.first_layer_is_expansion  ,    finegrained_output=self.finegrained_output  ,    hidden_act=self.hidden_act  ,    tf_padding=self.tf_padding  ,    classifier_dropout_prob=self.classifier_dropout_prob  ,    initializer_range=self.initializer_range  ,    )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		List[Any]  ,    a		:		Union[str, Any]  ,    a		:		str  ,    a		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  MobileNetVaModel(config=_UpperCAmelCase   )
         model.to(_UpperCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_UpperCAmelCase   )
         self.parent.assertEqual(
             result.last_hidden_state.shape  ,    (
                 self.batch_size,
                 self.last_hidden_size,
                 self.image_size // self.output_stride,
                 self.image_size // self.output_stride,
             )  ,    )
         self.parent.assertEqual(
             result.pooler_output.shape  ,    (self.batch_size, self.last_hidden_size)  ,    )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str  ,    a		:		str  ,    a		:		Any  ,    a		:		Optional[Any]  ,    a		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_labels
         __lowerCamelCase          =  MobileNetVaForImageClassification(_UpperCAmelCase   )
         model.to(_UpperCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_UpperCAmelCase  ,    labels=_UpperCAmelCase   )
         self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.num_labels)   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		str  ,    a		:		int  ,    a		:		Union[str, Any]  ,    a		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  self.num_labels
         __lowerCamelCase          =  MobileNetVaForSemanticSegmentation(_UpperCAmelCase   )
         model.to(_UpperCAmelCase   )
         model.eval()
         __lowerCamelCase          =  model(_UpperCAmelCase   )
         self.parent.assertEqual(
             result.logits.shape  ,    (
                 self.batch_size,
                 self.num_labels,
                 self.image_size // self.output_stride,
                 self.image_size // self.output_stride,
             )  ,    )
         __lowerCamelCase          =  model(_UpperCAmelCase  ,    labels=_UpperCAmelCase   )
         self.parent.assertEqual(
             result.logits.shape  ,    (
                 self.batch_size,
                 self.num_labels,
                 self.image_size // self.output_stride,
                 self.image_size // self.output_stride,
             )  ,    )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  self.prepare_config_and_inputs()
         __lowerCamelCase          =  config_and_inputs
         __lowerCamelCase          =  {'pixel_values': pixel_values}
         return config, inputs_dict




@require_torch
class    a__				(       _UpperCamelCase							,       _UpperCamelCase							,       unittest.TestCase			):
     lowerCamelCase  :					List[str]							=(
         (MobileNetVaModel, MobileNetVaForImageClassification, MobileNetVaForSemanticSegmentation)
         if is_torch_available()
         else ()
     )
     lowerCamelCase  :					Tuple							=(
         {
             'feature-extraction': MobileNetVaModel,
             'image-classification': MobileNetVaForImageClassification,
             'image-segmentation': MobileNetVaForSemanticSegmentation,
         }
         if is_torch_available()
         else {}
     )

     lowerCamelCase  :					List[str]							=False
     lowerCamelCase  :					Tuple							=False
     lowerCamelCase  :					List[Any]							=False
     lowerCamelCase  :					List[Any]							=False






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  MobileNetVaModelTester(self   )
         __lowerCamelCase          =  MobileNetVaConfigTester(self  ,    config_class=_UpperCAmelCase  ,    has_text_modality=_UpperCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         self.config_tester.run_common_tests()






     @unittest.skip(reason='''MobileNetV2 does not use inputs_embeds'''   )
     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         pass






     @unittest.skip(reason='''MobileNetV2 does not support input and output embeddings'''   )
     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         pass






     @unittest.skip(reason='''MobileNetV2 does not output attentions'''   )
     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         pass






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
             __lowerCamelCase          =  model_class(_UpperCAmelCase   )
             __lowerCamelCase          =  inspect.signature(model.forward   )
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             __lowerCamelCase          =  [*signature.parameters.keys()]

             __lowerCamelCase          =  ['pixel_values']
             self.assertListEqual(arg_names[:1]  ,    _UpperCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_UpperCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         def check_hidden_states_output(a		:		Dict  ,    a		:		int  ,    a		:		Optional[Any]   ):
             __lowerCamelCase          =  model_class(_UpperCAmelCase   )
             model.to(_UpperCAmelCase   )
             model.eval()

             with torch.no_grad():
                 __lowerCamelCase          =  model(**self._prepare_for_class(_UpperCAmelCase  ,    _UpperCAmelCase   )   )

             __lowerCamelCase          =  outputs.hidden_states

             __lowerCamelCase          =  16
             self.assertEqual(len(_UpperCAmelCase   )  ,    _UpperCAmelCase   )

         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
             __lowerCamelCase          =  True
             check_hidden_states_output(_UpperCAmelCase  ,    _UpperCAmelCase  ,    _UpperCAmelCase   )

             # check that output_hidden_states also work using config
             del inputs_dict["output_hidden_states"]
             __lowerCamelCase          =  True

             check_hidden_states_output(_UpperCAmelCase  ,    _UpperCAmelCase  ,    _UpperCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*_UpperCAmelCase   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_semantic_segmentation(*_UpperCAmelCase   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         for model_name in MOBILENET_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __lowerCamelCase          =  MobileNetVaModel.from_pretrained(_UpperCAmelCase   )
             self.assertIsNotNone(_UpperCAmelCase   )






def __lowerCAmelCase			(							)     ->   Optional[int]:
    __lowerCamelCase          =  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''    )
    return image




@require_torch
@require_vision
class    a__				(       unittest.TestCase			):






     @cached_property
     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         return (
             MobileNetVaImageProcessor.from_pretrained('''google/mobilenet_v2_1.0_224'''   ) if is_vision_available() else None
         )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  MobileNetVaForImageClassification.from_pretrained('''google/mobilenet_v2_1.0_224'''   ).to(_UpperCAmelCase   )

         __lowerCamelCase          =  self.default_image_processor
         __lowerCamelCase          =  prepare_img()
         __lowerCamelCase          =  image_processor(images=_UpperCAmelCase  ,    return_tensors='''pt'''   ).to(_UpperCAmelCase   )

         # forward pass
         with torch.no_grad():
             __lowerCamelCase          =  model(**_UpperCAmelCase   )

         # verify the logits
         __lowerCamelCase          =  torch.Size((1, 10_01)   )
         self.assertEqual(outputs.logits.shape  ,    _UpperCAmelCase   )

         __lowerCamelCase          =  torch.tensor([0.24_45, -1.19_93, 0.19_05]   ).to(_UpperCAmelCase   )

         self.assertTrue(torch.allclose(outputs.logits[0, :3]  ,    _UpperCAmelCase  ,    atol=1e-4   )   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  MobileNetVaForSemanticSegmentation.from_pretrained('''google/deeplabv3_mobilenet_v2_1.0_513'''   )
         __lowerCamelCase          =  model.to(_UpperCAmelCase   )

         __lowerCamelCase          =  MobileNetVaImageProcessor.from_pretrained('''google/deeplabv3_mobilenet_v2_1.0_513'''   )

         __lowerCamelCase          =  prepare_img()
         __lowerCamelCase          =  image_processor(images=_UpperCAmelCase  ,    return_tensors='''pt'''   ).to(_UpperCAmelCase   )

         # forward pass
         with torch.no_grad():
             __lowerCamelCase          =  model(**_UpperCAmelCase   )
         __lowerCamelCase          =  outputs.logits

         # verify the logits
         __lowerCamelCase          =  torch.Size((1, 21, 65, 65)   )
         self.assertEqual(logits.shape  ,    _UpperCAmelCase   )

         __lowerCamelCase          =  torch.tensor(
             [
                 [[17.57_90, 17.75_81, 18.33_55], [18.32_57, 18.42_30, 18.89_73], [18.61_69, 18.86_50, 19.21_87]],
                 [[-2.15_95, -2.09_77, -2.37_41], [-2.42_26, -2.30_28, -2.68_35], [-2.78_19, -2.59_91, -2.77_06]],
                 [[4.20_58, 4.83_17, 4.76_38], [4.41_36, 5.03_61, 4.93_83], [4.50_28, 4.96_44, 4.87_34]],
             ]  ,    device=_UpperCAmelCase  ,    )

         self.assertTrue(torch.allclose(logits[0, :3, :3, :3]  ,    _UpperCAmelCase  ,    atol=1e-4   )   )

'''simple docstring'''




import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												="▁"

__UpperCAmelCase												={"vocab_file": "prophetnet.tokenizer"}

__UpperCAmelCase												={
    "vocab_file": {
        "microsoft/xprophetnet-large-wiki100-cased": (
            "https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer"
        ),
    }
}

__UpperCAmelCase												={
    "microsoft/xprophetnet-large-wiki100-cased": {"do_lower_case": False},
}

__UpperCAmelCase												={
    "microsoft/xprophetnet-large-wiki100-cased": 5_1_2,
}






def __lowerCAmelCase			(							UpperCamelCase__    )     ->   List[str]:
    __lowerCamelCase          =  collections.OrderedDict()
    with open(UpperCamelCase__    ,  '''r'''    ,  encoding='''utf-8'''    ) as reader:
        __lowerCamelCase          =  reader.readlines()
    for index, token in enumerate(UpperCamelCase__    ):
        __lowerCamelCase          =  token.rstrip('''\n'''    )
        __lowerCamelCase          =  index
    return vocab




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					Optional[Any]							=VOCAB_FILES_NAMES
     lowerCamelCase  :					Any							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase  :					Optional[int]							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase  :					Union[str, Any]							=["input_ids", "attention_mask"]






     def __init__(			self		:		int  ,    a		:		List[str]  ,    a		:		Optional[int]="[SEP]"  ,    a		:		int="[SEP]"  ,    a		:		str="[SEP]"  ,    a		:		List[Any]="[UNK]"  ,    a		:		List[Any]="[PAD]"  ,    a		:		str="[CLS]"  ,    a		:		List[str]="[MASK]"  ,    a		:		Optional[Dict[str, Any]] = None  ,    **a		:		str  ,    ):





         """simple docstring"""



         __lowerCamelCase          =  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=a  ,    eos_token=a  ,    sep_token=a  ,    unk_token=a  ,    pad_token=a  ,    cls_token=a  ,    mask_token=a  ,    sp_model_kwargs=self.sp_model_kwargs  ,    **a  ,    )

         try:
             import sentencepiece as spm
         except ImportError:
             logger.warning(
                 '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
                 ''' pip install sentencepiece'''   )
             raise

         __lowerCamelCase          =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
         self.sp_model.Load(str(a   )   )
         __lowerCamelCase          =  vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # put special tokens and [unused] tokens into the vocab
         __lowerCamelCase          =  {'''[PAD]''': 0, '''[CLS]''': 1, '''[SEP]''': 2, '''[UNK]''': 3, '''[MASK]''': 4}

         for i in range(10   ):
             __lowerCamelCase          =  f"""[unused{i}]"""
             __lowerCamelCase          =  5 + i

         # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
         __lowerCamelCase          =  12
         __lowerCamelCase          =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}
         for k in self.fairseq_tokens_to_ids.keys():
             self.unique_no_split_tokens.append(a   )






     def __getstate__(			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.__dict__.copy()
         __lowerCamelCase          =  None
         return state






     def __setstate__(			self		:		int  ,    a		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  d
         try:
             import sentencepiece as spm
         except ImportError:
             logger.warning(
                 '''You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'''
                 ''' pip install sentencepiece'''   )
             raise

         # for backward compatibility
         if not hasattr(self  ,    '''sp_model_kwargs'''   ):
             __lowerCamelCase          =  {}

         __lowerCamelCase          =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
         self.sp_model.Load(self.vocab_file   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None  ,    a		:		bool = False   ):





         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a  ,    token_ids_a=a  ,    already_has_special_tokens=a   )

         if token_ids_a is None:
             return ([0] * len(a   )) + [1]
         return ([0] * len(a   )) + [1] + ([0] * len(a   )) + [1]






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         __lowerCamelCase          =  [self.sep_token_id]

         if token_ids_a is None:
             return len(token_ids_a + sep   ) * [0]
         return len(token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






     @property
     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         return len(self.sp_model   ) + self.fairseq_offset






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  {self.convert_ids_to_tokens(a   ): i for i in range(self.vocab_size   )}
         vocab.update(self.added_tokens_encoder   )
         return vocab






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple  ,    a		:		str   ):





         """simple docstring"""



         return self.sp_model.encode(a  ,    out_type=a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict  ,    a		:		int   ):





         """simple docstring"""



         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
         __lowerCamelCase          =  self.sp_model.PieceToId(a   )

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		Union[str, Any]   ):





         """simple docstring"""



         if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]  ,    a		:		Tuple   ):





         """simple docstring"""



         __lowerCamelCase          =  ''''''.join(a   ).replace(a  ,    ''' '''   ).strip()
         return out_string






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    a		:		str  ,    a		:		Optional[str] = None   ):





         """simple docstring"""



         if not os.path.isdir(a   ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""   )
             return
         __lowerCamelCase          =  os.path.join(
             a  ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(a   ) and os.path.isfile(self.vocab_file   ):
             copyfile(self.vocab_file  ,    a   )
         elif not os.path.isfile(self.vocab_file   ):
             with open(a  ,    '''wb'''   ) as fi:
                 __lowerCamelCase          =  self.sp_model.serialized_model_proto()
                 fi.write(a   )

         return (out_vocab_file,)






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         if token_ids_a is None:
             return token_ids_a + [self.sep_token_id]
         __lowerCamelCase          =  [self.sep_token_id]
         return token_ids_a + sep + token_ids_a + sep
