

'''simple docstring'''
import unittest

from diffusers.models.unet_ad_blocks import *  # noqa F403
from diffusers.utils import torch_device

from .test_unet_blocks_common import UNetBlockTesterMixin


class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							DownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							'down'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       [-0.0_2_3_2, -0.9_8_6_9, 0.8_0_5_4, -0.0_6_3_7, -0.1_6_8_8, -1.4_2_6_4, 0.4_4_7_0, -1.3_3_9_4, 0.0_9_0_4]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							ResnetDownsampleBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					str     	=							'down'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [0.0_7_1_0, 0.2_4_1_0, -0.7_3_2_0, -1.0_7_5_7, -1.1_3_4_3, 0.3_5_4_0, -0.0_1_3_3, -0.2_5_7_6, 0.0_9_4_8]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Dict     	=							AttnDownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					str     	=							'down'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       [0.0_6_3_6, 0.8_9_6_4, -0.6_2_3_4, -1.0_1_3_1, 0.0_8_4_4, 0.4_9_3_5, 0.3_4_3_7, 0.0_9_1_1, -0.2_9_5_7]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Dict     	=							CrossAttnDownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					int     	=							'down'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				Tuple      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				Optional[int]      					=       32
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       [0.2_2_3_8, -0.7_3_9_6, -0.2_2_5_5, -0.3_8_2_9, 0.1_9_2_5, 1.1_6_6_5, 0.0_6_0_3, -0.7_2_9_5, 0.1_9_8_3]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					int     	=							SimpleCrossAttnDownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							'down'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_encoder_hidden_states=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				Union[str, Any]      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				List[Any]      					=       32
      return init_dict, inputs_dict






   @unittest.skipIf(torch_device == "mps"      ,    "MPS result is not consistent"  )
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       [0.7_9_2_1, -0.0_9_9_2, -0.1_9_6_2, -0.7_6_9_5, -0.4_2_4_2, 0.7_8_0_4, 0.4_7_3_7, 0.2_7_6_5, 0.3_3_3_8]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							SkipDownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Any     	=							'down'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_skip_sample=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       [-0.0_8_4_5, -0.2_0_8_7, -0.2_4_6_5, 0.0_9_7_1, 0.1_9_0_0, -0.0_4_8_4, 0.2_6_6_4, 0.4_1_7_9, 0.5_0_6_9]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							AttnSkipDownBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							'down'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_skip_sample=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       [0.5_5_3_9, 0.1_6_0_9, 0.4_9_2_4, 0.0_5_3_7, -0.1_9_9_5, 0.4_0_5_0, 0.0_9_7_9, -0.2_7_2_1, -0.0_6_4_2]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Any     	=							DownEncoderBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							'down'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_temb=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       {
          "in_channels": 32,
          "out_channels": 32,
      }
      _lowercase      :				List[str]      					=       self.dummy_input
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       [1.1_1_0_2, 0.5_3_0_2, 0.4_8_7_2, -0.0_0_2_3, -0.8_0_4_2, 0.0_4_8_3, -0.3_4_8_9, -0.5_6_3_2, 0.7_6_2_6]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Any     	=							AttnDownEncoderBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					int     	=							'down'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_temb=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       {
          "in_channels": 32,
          "out_channels": 32,
      }
      _lowercase      :				List[str]      					=       self.dummy_input
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       [0.8_9_6_6, -0.1_4_8_6, 0.8_5_6_8, 0.8_1_4_1, -0.9_0_4_6, -0.1_3_4_2, -0.0_9_7_2, -0.7_4_1_7, 0.1_5_3_8]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							UNetMidBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Dict     	=							'mid'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       {
          "in_channels": 32,
          "temb_channels": 128,
      }
      _lowercase      :				int      					=       self.dummy_input
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       [-0.1_0_6_2, 1.7_2_4_8, 0.3_4_9_4, 1.4_5_6_9, -0.0_9_1_0, -1.2_4_2_1, -0.9_9_8_4, 0.6_7_3_6, 1.0_0_2_8]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							UNetMidBlockaDCrossAttn  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Any     	=							'mid'


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				str      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				Dict      					=       32
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [0.0_1_8_7, 2.4_2_2_0, 0.4_4_8_4, 1.1_2_0_3, -0.6_1_2_1, -1.5_1_2_2, -0.8_2_7_0, 0.7_8_5_1, 1.8_3_3_5]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							UNetMidBlockaDSimpleCrossAttn  # noqa F405
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							'mid'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_encoder_hidden_states=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				List[str]      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				Optional[Any]      					=       32
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       [0.7_1_4_3, 1.9_9_7_4, 0.5_4_4_8, 1.3_9_7_7, 0.1_2_8_2, -1.1_2_3_7, -1.4_2_3_8, 0.5_5_3_0, 0.8_8_8_0]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							UpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       [-0.2_0_4_1, -0.4_1_6_5, -0.3_0_2_2, 0.0_0_4_1, -0.6_6_2_8, -0.7_0_5_3, 0.1_9_2_8, -0.0_3_2_5, 0.0_5_2_3]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							ResnetUpsampleBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [0.2_2_8_7, 0.3_5_4_9, -0.1_3_4_6, 0.4_7_9_7, -0.1_7_1_5, -0.9_6_4_9, 0.7_3_0_5, -0.5_8_6_4, -0.6_2_4_4]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					str     	=							CrossAttnUpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				Any      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				str      					=       32
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       [-0.1_4_0_3, -0.3_5_1_5, -0.0_4_2_0, -0.1_4_2_5, 0.3_1_6_7, 0.5_0_9_4, -0.2_1_8_1, 0.5_9_3_1, 0.5_5_8_2]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							SimpleCrossAttnUpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					str     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase      ,    include_encoder_hidden_states=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase					,				_lowercase      :				Tuple      					=       super().prepare_init_args_and_inputs_for_common()
      _lowercase      :				int      					=       32
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       [0.2_6_4_5, 0.1_4_8_0, 0.0_9_0_9, 0.8_0_4_4, -0.9_7_5_8, -0.9_0_8_3, 0.0_9_9_4, -1.1_4_5_3, -0.7_4_0_2]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							AttnUpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )






   @unittest.skipIf(torch_device == "mps"      ,    "MPS result is not consistent"  )
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [0.0_9_7_9, 0.1_3_2_6, 0.0_0_2_1, 0.0_6_5_9, 0.2_2_4_9, 0.0_0_5_9, 0.1_1_3_2, 0.5_9_5_2, 0.1_0_3_3]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							SkipUpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [-0.0_8_9_3, -0.1_2_3_4, -0.1_5_0_6, -0.0_3_3_2, 0.0_1_2_3, -0.0_2_1_1, 0.0_5_6_6, 0.0_1_4_3, 0.0_3_6_2]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							AttnSkipUpBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Dict     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_res_hidden_states_tuple=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       [0.0_3_6_1, 0.0_6_1_7, 0.2_7_8_7, -0.0_3_5_0, 0.0_3_4_2, 0.3_4_2_1, -0.0_8_4_3, 0.0_9_1_3, 0.3_0_1_5]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							UpDecoderBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Dict     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_temb=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       {"in_channels": 32, "out_channels": 32}

      _lowercase      :				Optional[Any]      					=       self.dummy_input
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       [0.4_4_0_4, 0.1_9_9_8, -0.9_8_8_6, -0.3_3_2_0, -0.3_1_2_8, -0.7_0_3_4, -0.6_9_5_5, -0.2_3_3_8, -0.3_1_3_7]
      super().test_output(_UpperCamelCase  )




class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							AttnUpDecoderBlockaD  # noqa F405
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							'up'


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return super().get_dummy_input(include_temb=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       {"in_channels": 32, "out_channels": 32}

      _lowercase      :				Any      					=       self.dummy_input
      return init_dict, inputs_dict






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       [0.6_7_3_8, 0.4_4_9_1, 0.1_0_5_5, 1.0_7_1_0, 0.7_3_1_6, 0.3_3_3_9, 0.3_3_5_2, 0.1_0_2_3, 0.3_5_6_8]
      super().test_output(_UpperCamelCase  )









'''simple docstring'''
import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case         =  logging.get_logger(__name__)

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_snake_case         =  {
    'google/bigbird-roberta-base': 4_096,
    'google/bigbird-roberta-large': 4_096,
    'google/bigbird-base-trivia-itc': 4_096,
}


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							['input_ids', 'attention_mask']
   _SCREAMING_SNAKE_CASE   :					List[int]     	=							[]


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<s>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="[SEP]"      ,    _UpperCamelCase="[MASK]"      ,    _UpperCamelCase="[CLS]"      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else bos_token
      _lowercase      :				Optional[int]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else eos_token
      _lowercase      :				Optional[Any]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else unk_token
      _lowercase      :				Union[str, Any]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else pad_token
      _lowercase      :				Any      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else cls_token
      _lowercase      :				str      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      _lowercase      :				Any      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else mask_token

      _lowercase      :				Dict      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=_UpperCamelCase      ,    eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    sep_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    cls_token=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )

      _lowercase      :				str      					=       vocab_file

      _lowercase      :				List[Any]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return self.sp_model.get_piece_size()


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				Optional[Any]      					=       {}

      _lowercase      :				Optional[Any]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.piece_to_id(_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       self.sp_model.IdToPiece(_UpperCamelCase  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				int      					=       ""
      _lowercase      :				Tuple      					=       False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            if not prev_is_special:
               out_string += " "
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Union[str, Any]      					=       True
            _lowercase      :				Optional[int]      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
            _lowercase      :				str      					=       False
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = False      ,    _UpperCamelCase = None      ,    _UpperCamelCase = True      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Any      					=       kwargs.pop("use_source_tokenizer"      ,    _UpperCamelCase  )

      _lowercase      :				Dict      					=       self.convert_ids_to_tokens(_UpperCamelCase      ,    skip_special_tokens=_UpperCamelCase  )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      _lowercase      :				Dict      					=       []
      _lowercase      :				Any      					=       []
      for token in filtered_tokens:
         if skip_special_tokens and token in self.all_special_ids:
            continue
         if token in self.added_tokens_encoder:
            if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase  )  )
               _lowercase      :				Optional[Any]      					=       []
            sub_texts.append(_UpperCamelCase  )
         else:
            current_sub_text.append(_UpperCamelCase  )
      if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase  )  )

      # Mimic the behavior of the Rust tokenizer:
      # No space before [MASK] and [SEP]
      if spaces_between_special_tokens:
         _lowercase      :				List[str]      					=       re.sub(R" (\[(MASK|SEP)\])"      ,    R"\1"      ,    " ".join(_UpperCamelCase  )  )
      else:
         _lowercase      :				List[Any]      					=       "".join(_UpperCamelCase  )

      _lowercase      :				Optional[int]      					=       (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
         _lowercase      :				Tuple      					=       self.clean_up_tokenization(_UpperCamelCase  )
         return clean_text
      else:
         return text


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				Optional[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				List[str]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowercase      :				str      					=       [self.cls_token_id]
      _lowercase      :				List[str]      					=       [self.sep_token_id]
      return cls + token_ids_a + sep + token_ids_a + sep


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=_UpperCamelCase      ,    token_ids_a=_UpperCamelCase      ,    already_has_special_tokens=_UpperCamelCase  )

      if token_ids_a is None:
         return [1] + ([0] * len(_UpperCamelCase  )) + [1]
      return [1] + ([0] * len(_UpperCamelCase  )) + [1] + ([0] * len(_UpperCamelCase  )) + [1]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      _lowercase      :				str      					=       [self.sep_token_id]
      _lowercase      :				Union[str, Any]      					=       [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
      return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






