






'''simple docstring'''

import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_flax,
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import is_flax_available, is_torch_available, is_vision_available

from ...test_modeling_flax_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_flax_bert import FlaxBertModelTester
from ..clip.test_modeling_flax_clip import FlaxCLIPVisionModelTester
from ..vit.test_modeling_flax_vit import FlaxViTModelTester


if is_flax_available():
     from transformers import (
         FlaxBertModel,
         FlaxCLIPVisionModel,
         FlaxVisionTextDualEncoderModel,
         FlaxViTModel,
         VisionTextDualEncoderConfig,
         VisionTextDualEncoderProcessor,
     )
     from transformers.modeling_flax_pytorch_utils import (
         convert_pytorch_state_dict_to_flax,
         load_flax_weights_in_pytorch_model,
     )


if is_torch_available():
     import torch

     from transformers import VisionTextDualEncoderModel

if is_vision_available():
     from PIL import Image







def    lowercase__			(				__UpperCamelCase   )->			str:
      if isinstance(__UpperCamelCase		,						collections.abc.Iterable   ):
            return x
      return (x, x)






@require_flax
class     a_						:

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:



        """simple docstring"""






        pass

  def   A__						(				self				)  ->				Any:



        """simple docstring"""






        pass

  def   A__						(				self				)  ->				Any:



        """simple docstring"""






        pass

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        UpperCamelCase      = np.abs((a - b)				).max()
        self.assertLessEqual(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		F"Difference between torch and flax is {diff} (>= {tol})."				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None      ,		**_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = VisionTextDualEncoderConfig.from_vision_text_configs(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = FlaxVisionTextDualEncoderModel(_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = model(input_ids=_SCREAMING_SNAKE_CASE      ,		pixel_values=_SCREAMING_SNAKE_CASE      ,		attention_mask=_SCREAMING_SNAKE_CASE				)

        self.assertEqual(output["""text_embeds"""].shape      ,		(input_ids.shape[0], config.projection_dim)				)
        self.assertEqual(output["""image_embeds"""].shape      ,		(pixel_values.shape[0], config.projection_dim)				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None      ,		**_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase      = self.get_vision_text_model(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = {"""vision_model""": vision_model, """text_model""": text_model}
        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = model(input_ids=_SCREAMING_SNAKE_CASE      ,		pixel_values=_SCREAMING_SNAKE_CASE      ,		attention_mask=_SCREAMING_SNAKE_CASE				)

        self.assertEqual(output["""text_embeds"""].shape      ,		(input_ids.shape[0], model.config.projection_dim)				)
        self.assertEqual(output["""image_embeds"""].shape      ,		(pixel_values.shape[0], model.config.projection_dim)				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None      ,		**_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase      = self.get_vision_text_model(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = {"""vision_model""": vision_model, """text_model""": text_model}
        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = model(input_ids=_SCREAMING_SNAKE_CASE      ,		pixel_values=_SCREAMING_SNAKE_CASE      ,		attention_mask=_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = output[0]

        with tempfile.TemporaryDirectory() as tmpdirname:
              model.save_pretrained(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_pretrained(_SCREAMING_SNAKE_CASE				)

              UpperCamelCase      = model(input_ids=_SCREAMING_SNAKE_CASE      ,		pixel_values=_SCREAMING_SNAKE_CASE      ,		attention_mask=_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = after_output[0]
              UpperCamelCase      = np.amax(np.abs(out_a - out_a				)				)
              self.assertLessEqual(_SCREAMING_SNAKE_CASE      ,		1e-3				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None      ,		**_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase      = self.get_vision_text_model(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = {"""vision_model""": vision_model, """text_model""": text_model}
        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = model(
            input_ids=_SCREAMING_SNAKE_CASE      ,		pixel_values=_SCREAMING_SNAKE_CASE      ,		attention_mask=_SCREAMING_SNAKE_CASE      ,		output_attentions=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = output.vision_model_output.attentions
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		vision_config.num_hidden_layers				)

        # in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
        UpperCamelCase      = to_atuple(vision_model.config.image_size				)
        UpperCamelCase      = to_atuple(vision_model.config.patch_size				)
        UpperCamelCase      = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
        UpperCamelCase      = num_patches + 1
        self.assertEqual(vision_attentions[0].shape[-3:]      ,		(vision_config.num_attention_heads, seq_len, seq_len)				)

        UpperCamelCase      = output.text_model_output.attentions
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		text_config.num_hidden_layers				)

        self.assertEqual(
            text_attentions[0].shape[-3:]      ,		(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])      ,		)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        pt_model.to(_SCREAMING_SNAKE_CASE				)
        pt_model.eval()

        # prepare inputs
        UpperCamelCase      = inputs_dict
        UpperCamelCase      = {k: torch.tensor(v.tolist()				) for k, v in flax_inputs.items()}

        with torch.no_grad():
              UpperCamelCase      = pt_model(**_SCREAMING_SNAKE_CASE				).to_tuple()

        UpperCamelCase      = fx_model(**_SCREAMING_SNAKE_CASE				).to_tuple()
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		len(_SCREAMING_SNAKE_CASE				)      ,		"""Output lengths differ between Flax and PyTorch"""				)
        for fx_output, pt_output in zip(fx_outputs[:4]      ,		pt_outputs[:4]				):
              self.assert_almost_equals(_SCREAMING_SNAKE_CASE      ,		pt_output.numpy()      ,		4e-2				)

        # PT -> Flax
        with tempfile.TemporaryDirectory() as tmpdirname:
              pt_model.save_pretrained(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_pretrained(_SCREAMING_SNAKE_CASE      ,		from_pt=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = fx_model_loaded(**_SCREAMING_SNAKE_CASE				).to_tuple()
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		len(_SCREAMING_SNAKE_CASE				)      ,		"""Output lengths differ between Flax and PyTorch"""				)
        for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4]      ,		pt_outputs[:4]				):
              self.assert_almost_equals(_SCREAMING_SNAKE_CASE      ,		pt_output.numpy()      ,		4e-2				)

        # Flax -> PT
        with tempfile.TemporaryDirectory() as tmpdirname:
              fx_model.save_pretrained(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = VisionTextDualEncoderModel.from_pretrained(_SCREAMING_SNAKE_CASE      ,		from_flax=_SCREAMING_SNAKE_CASE				)

        pt_model_loaded.to(_SCREAMING_SNAKE_CASE				)
        pt_model_loaded.eval()

        with torch.no_grad():
              UpperCamelCase      = pt_model_loaded(**_SCREAMING_SNAKE_CASE				).to_tuple()

        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		len(_SCREAMING_SNAKE_CASE				)      ,		"""Output lengths differ between Flax and PyTorch"""				)
        for fx_output, pt_output_loaded in zip(fx_outputs[:4]      ,		pt_outputs_loaded[:4]				):
              self.assert_almost_equals(_SCREAMING_SNAKE_CASE      ,		pt_output_loaded.numpy()      ,		4e-2				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = VisionTextDualEncoderConfig.from_vision_text_configs(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = VisionTextDualEncoderModel(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = FlaxVisionTextDualEncoderModel(_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = convert_pytorch_state_dict_to_flax(pt_model.state_dict()      ,		_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = fx_state

        self.check_pt_flax_equivalence(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = VisionTextDualEncoderConfig.from_vision_text_configs(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = VisionTextDualEncoderModel(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = FlaxVisionTextDualEncoderModel(_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = load_flax_weights_in_pytorch_model(_SCREAMING_SNAKE_CASE      ,		fx_model.params				)

        self.check_pt_flax_equivalence(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.prepare_config_and_inputs()
        self.check_model_from_pretrained_configs(**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				Any:



        """simple docstring"""






        UpperCamelCase      = self.prepare_config_and_inputs()
        self.check_vision_text_dual_encoder_from_pretrained(**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        UpperCamelCase      = self.prepare_config_and_inputs()
        self.check_save_load(**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.prepare_config_and_inputs()
        self.check_vision_text_output_attention(**_SCREAMING_SNAKE_CASE				)

  @is_pt_flax_cross_test
  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        UpperCamelCase      = self.prepare_config_and_inputs()
        UpperCamelCase      = config_inputs_dict.pop("""vision_config"""				)
        UpperCamelCase      = config_inputs_dict.pop("""text_config"""				)

        UpperCamelCase      = config_inputs_dict

        self.check_equivalence_pt_to_flax(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        self.check_equivalence_flax_to_pt(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)


  @slow
  def   A__						(				self				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase      = self.get_pretrained_model_and_inputs()

        UpperCamelCase      = model_a(**_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = outputs[0]

        with tempfile.TemporaryDirectory() as tmp_dirname:
              model_a.save_pretrained(_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_pretrained(_SCREAMING_SNAKE_CASE				)

              UpperCamelCase      = model_a(**_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = after_outputs[0]
              UpperCamelCase      = np.amax(np.abs(out_a - out_a				)				)
              self.assertLessEqual(_SCREAMING_SNAKE_CASE      ,		1e-5				)






@require_flax
class     a_						(      lowerCamelCase     ,	unittest.TestCase    ):

  def   A__						(				self				)  ->				int:



        """simple docstring"""






        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
            """hf-internal-testing/tiny-random-vit"""      ,		"""hf-internal-testing/tiny-bert"""      ,		vision_from_pt=_SCREAMING_SNAKE_CASE      ,		text_from_pt=_SCREAMING_SNAKE_CASE      ,		)
        UpperCamelCase      = 13
        UpperCamelCase      = floats_tensor(
            [
                batch_size,
                model.config.vision_config.num_channels,
                model.config.vision_config.image_size,
                model.config.vision_config.image_size,
            ]				)
        UpperCamelCase      = ids_tensor([batch_size, 4]      ,		model.config.text_config.vocab_size				)
        UpperCamelCase      = random_attention_mask([batch_size, 4]				)
        UpperCamelCase      = {"""pixel_values""": pixel_values, """input_ids""": input_ids, """attention_mask""": attention_mask}

        return model, inputs

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase      = FlaxViTModel(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = FlaxBertModel(_SCREAMING_SNAKE_CASE				)
        return vision_model, text_model


  def   A__						(				self				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = FlaxViTModelTester(self				)
        UpperCamelCase      = FlaxBertModelTester(self				)
        UpperCamelCase      = vit_model_tester.prepare_config_and_inputs()
        UpperCamelCase      = bert_model_tester.prepare_config_and_inputs()

        UpperCamelCase  ,UpperCamelCase      = vision_config_and_inputs

        UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase      = text_config_and_inputs

        # make sure that cross attention layers are added
        return {
            "text_config": text_config,
            "vision_config": vision_config,
            "pixel_values": pixel_values,
            "attention_mask": attention_mask,
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
        }






@require_torch
class     a_						(      lowerCamelCase     ,	unittest.TestCase    ):

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
            """hf-internal-testing/tiny-random-clip"""      ,		"""hf-internal-testing/tiny-bert"""      ,		vision_from_pt=_SCREAMING_SNAKE_CASE      ,		text_from_pt=_SCREAMING_SNAKE_CASE      ,		)
        UpperCamelCase      = 13
        UpperCamelCase      = floats_tensor(
            [
                batch_size,
                model.config.vision_config.num_channels,
                model.config.vision_config.image_size,
                model.config.vision_config.image_size,
            ]				)
        UpperCamelCase      = ids_tensor([batch_size, 4]      ,		model.config.text_config.vocab_size				)
        UpperCamelCase      = random_attention_mask([batch_size, 4]				)
        UpperCamelCase      = {"""pixel_values""": pixel_values, """input_ids""": input_ids, """attention_mask""": attention_mask}

        return model, inputs

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = FlaxCLIPVisionModel(_SCREAMING_SNAKE_CASE				)
        UpperCamelCase      = FlaxBertModel(_SCREAMING_SNAKE_CASE				)
        return vision_model, text_model


  def   A__						(				self				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = FlaxCLIPVisionModelTester(self				)
        UpperCamelCase      = FlaxBertModelTester(self				)
        UpperCamelCase      = clip_model_tester.prepare_config_and_inputs()
        UpperCamelCase      = bert_model_tester.prepare_config_and_inputs()

        UpperCamelCase  ,UpperCamelCase      = vision_config_and_inputs

        UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase      = text_config_and_inputs

        # make sure that cross attention layers are added
        return {
            "text_config": text_config,
            "vision_config": vision_config,
            "pixel_values": pixel_values,
            "attention_mask": attention_mask,
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
        }






@require_flax
@require_vision
class     a_						(      unittest.TestCase    ):

  @slow
  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = FlaxVisionTextDualEncoderModel.from_pretrained("""clip-italian/clip-italian"""      ,		logit_scale_init_value=1.0				)
        UpperCamelCase      = VisionTextDualEncoderProcessor.from_pretrained("""clip-italian/clip-italian"""				)

        UpperCamelCase      = Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""				)
        UpperCamelCase      = processor(
            text=["""una foto di un gatto""", """una foto di un cane"""]      ,		images=_SCREAMING_SNAKE_CASE      ,		padding=_SCREAMING_SNAKE_CASE      ,		return_tensors="""np"""				)

        UpperCamelCase      = model(**_SCREAMING_SNAKE_CASE				)

        # verify the logits
        self.assertEqual(outputs.logits_per_image.shape      ,		(inputs.pixel_values.shape[0], inputs.input_ids.shape[0])				)
        self.assertEqual(
            outputs.logits_per_text.shape      ,		(inputs.input_ids.shape[0], inputs.pixel_values.shape[0])      ,		)

        UpperCamelCase      = np.array([[1.2_2_8_4_7_2_7, 0.3_1_0_4_1_2_2]]				)

        self.assertTrue(np.allclose(outputs.logits_per_image      ,		_SCREAMING_SNAKE_CASE      ,		atol=1e-3				)				)








'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'spm_char.model'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

SCREAMING_SNAKE_CASE__     =       {
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}






class     a_						(      lowerCamelCase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				None:



        """simple docstring"""






        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=_SCREAMING_SNAKE_CASE      ,		eos_token=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**_SCREAMING_SNAKE_CASE      ,		)

        UpperCamelCase      = vocab_file

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.get_piece_size()

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def __getstate__(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        return self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,		out_type=_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        return self.sp_model.piece_to_id(_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE				)
        return token

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = """"""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				) + token
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE				)
        out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				)
        return out_string.strip()

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=None				)  ->				List[int]:



        """simple docstring"""






        if token_ids_a is None:
              return token_ids_a + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return token_ids_a + token_ids_a + [self.eos_token_id]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_SCREAMING_SNAKE_CASE      ,		token_ids_a=_SCREAMING_SNAKE_CASE      ,		already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = [1]
        if token_ids_a is None:
              return ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones
        return ([0] * len(_SCREAMING_SNAKE_CASE				)) + ([0] * len(_SCREAMING_SNAKE_CASE				)) + suffix_ones


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            _SCREAMING_SNAKE_CASE      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(_SCREAMING_SNAKE_CASE				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		_SCREAMING_SNAKE_CASE				)
        elif not os.path.isfile(self.vocab_file				):
              with open(_SCREAMING_SNAKE_CASE      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE				)

        return (out_vocab_file,)
