

'''simple docstring'''
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case         =  get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Any     	=							XLMRobertaTokenizer
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							XLMRobertaTokenizerFast
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							True
   _SCREAMING_SNAKE_CASE   :					Dict     	=							True


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().setUp()

      # We have a SentencePiece fixture for testing
      _lowercase      :				Dict      					=       XLMRobertaTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )
      tokenizer.save_pretrained(self.tmpdirname  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       "<pad>"
      _lowercase      :				Optional[Any]      					=       1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase  )      ,    _UpperCamelCase  )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase  )      ,    _UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       list(self.get_tokenizer().get_vocab().keys()  )

      self.assertEqual(vocab_keys[0]      ,    "<s>"  )
      self.assertEqual(vocab_keys[1]      ,    "<pad>"  )
      self.assertEqual(vocab_keys[-1]      ,    "<mask>"  )
      self.assertEqual(len(_UpperCamelCase  )      ,    1002  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.assertEqual(self.get_tokenizer().vocab_size      ,    1002  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       XLMRobertaTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )

      _lowercase      :				Dict      					=       tokenizer.tokenize("This is a test"  )
      self.assertListEqual(_UpperCamelCase      ,    ["▁This", "▁is", "▁a", "▁t", "est"]  )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(_UpperCamelCase  )      ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,    )

      _lowercase      :				List[str]      					=       tokenizer.tokenize("I was born in 92000, and this is falsé."  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]      ,    )
      _lowercase      :				int      					=       tokenizer.convert_tokens_to_ids(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
          ]      ,    )

      _lowercase      :				Union[str, Any]      					=       tokenizer.convert_ids_to_tokens(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]      ,    )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if not self.test_slow_tokenizer:
         # as we don't have a slow version, we can't compare the outputs between slow and fast versions
         return

      _lowercase      :				Tuple      					=       (self.rust_tokenizer_class, "hf-internal-testing/tiny-xlm-roberta", {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
            _lowercase      :				int      					=       self.rust_tokenizer_class.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )
            _lowercase      :				Any      					=       self.tokenizer_class.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )

            _lowercase      :				Any      					=       tempfile.mkdtemp()

            _lowercase      :				List[Any]      					=       tokenizer_r.save_pretrained(_UpperCamelCase  )
            _lowercase      :				Optional[int]      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files  )  )
            _lowercase      :				List[Any]      					=       tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f  )
            self.assertSequenceEqual(_UpperCamelCase      ,    _UpperCamelCase  )

            # Checks everything loads correctly in the same way
            _lowercase      :				Dict      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				int      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )
               # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
               # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

            shutil.rmtree(_UpperCamelCase  )

            # Save tokenizer rust, legacy_format=True
            _lowercase      :				List[str]      					=       tempfile.mkdtemp()

            _lowercase      :				Dict      					=       tokenizer_r.save_pretrained(_UpperCamelCase      ,    legacy_format=_UpperCamelCase  )
            _lowercase      :				Dict      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it save with the same files
            self.assertSequenceEqual(_UpperCamelCase      ,    _UpperCamelCase  )

            # Checks everything loads correctly in the same way
            _lowercase      :				Optional[Any]      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				Union[str, Any]      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )

            shutil.rmtree(_UpperCamelCase  )

            # Save tokenizer rust, legacy_format=False
            _lowercase      :				List[Any]      					=       tempfile.mkdtemp()

            _lowercase      :				int      					=       tokenizer_r.save_pretrained(_UpperCamelCase      ,    legacy_format=_UpperCamelCase  )
            _lowercase      :				Optional[Any]      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files  )  )

            # Checks everything loads correctly in the same way
            _lowercase      :				int      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				Optional[int]      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )

            shutil.rmtree(_UpperCamelCase  )


   @cached_property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XLMRobertaTokenizer.from_pretrained("xlm-roberta-base"  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      with tempfile.NamedTemporaryFile() as f:
         shutil.copyfile(_UpperCamelCase      ,    f.name  )
         _lowercase      :				Dict      					=       XLMRobertaTokenizer(f.name      ,    keep_accents=_UpperCamelCase  )
         _lowercase      :				Any      					=       pickle.dumps(_UpperCamelCase  )
      pickle.loads(_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if not self.test_rust_tokenizer:
         return

      _lowercase      :				Optional[Any]      					=       self.get_tokenizer()
      _lowercase      :				int      					=       self.get_rust_tokenizer()

      _lowercase      :				Optional[int]      					=       "I was born in 92000, and this is falsé."

      _lowercase      :				Optional[int]      					=       tokenizer.tokenize(_UpperCamelCase  )
      _lowercase      :				Tuple      					=       rust_tokenizer.tokenize(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				int      					=       tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      _lowercase      :				Optional[Any]      					=       rust_tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				List[str]      					=       self.get_rust_tokenizer()
      _lowercase      :				Optional[Any]      					=       tokenizer.encode(_UpperCamelCase  )
      _lowercase      :				str      					=       rust_tokenizer.encode(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       "Hello World!"
      _lowercase      :				int      					=       [0, 35378, 6661, 38, 2]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(_UpperCamelCase      ,    self.big_tokenizer.encode(_UpperCamelCase  )  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       (
          "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
          " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
      )
      _lowercase      :				int      					=       [
          0,
          3293,
          83,
          10,
          4552,
          4989,
          7986,
          678,
          10,
          5915,
          111,
          179459,
          124850,
          4,
          6044,
          237,
          12,
          6,
          5,
          6,
          4,
          6780,
          705,
          15,
          1388,
          44,
          378,
          10114,
          711,
          152,
          20,
          6,
          5,
          22376,
          642,
          1221,
          15190,
          34153,
          450,
          5608,
          959,
          1119,
          57702,
          136,
          186,
          47,
          1098,
          29367,
          47,
          # 4426, # What fairseq tokenizes from "<unk>": "_<"
          # 3678, # What fairseq tokenizes from "<unk>": "unk"
          # 2740, # What fairseq tokenizes from "<unk>": ">"
          3,  # What we tokenize from "<unk>": "<unk>"
          6,  # Residue from the tokenization: an extra sentencepiece underline
          4,
          6044,
          237,
          6284,
          50901,
          528,
          31,
          90,
          34,
          927,
          2,
      ]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(_UpperCamelCase      ,    self.big_tokenizer.encode(_UpperCamelCase  )  )






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       {"input_ids": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_UpperCamelCase      ,    model_name="xlm-roberta-base"      ,    revision="d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"      ,    )









'''simple docstring'''
import random
import unittest

import torch

from diffusers import IFImgaImgSuperResolutionPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin


@skip_mps
class        a__  (							lowerCamelCase_  ,				lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							IFImgaImgSuperResolutionPipeline
   _SCREAMING_SNAKE_CASE   :					int     	=							TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'width', 'height'}
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS.union({'original_image'}  )
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							PipelineTesterMixin.required_optional_params - {'latents'}


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return self._get_superresolution_dummy_components()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=0  ):





      """simple docstring"""





      if str(_UpperCamelCase  ).startswith("mps"  ):
         _lowercase      :				Tuple      					=       torch.manual_seed(_UpperCamelCase  )
      else:
         _lowercase      :				int      					=       torch.Generator(device=_UpperCamelCase  ).manual_seed(_UpperCamelCase  )

      _lowercase      :				List[Any]      					=       floats_tensor((1, 3, 32, 32)      ,    rng=random.Random(_UpperCamelCase  )  ).to(_UpperCamelCase  )
      _lowercase      :				Optional[int]      					=       floats_tensor((1, 3, 16, 16)      ,    rng=random.Random(_UpperCamelCase  )  ).to(_UpperCamelCase  )

      _lowercase      :				List[str]      					=       {
          "prompt": "A painting of a squirrel eating a burger",
          "image": image,
          "original_image": original_image,
          "generator": generator,
          "num_inference_steps": 2,
          "output_type": "numpy",
      }

      return inputs


   @unittest.skipIf(
       torch_device != "cuda" or not is_xformers_available()      ,    reason="XFormers attention is only available with CUDA and `xformers` installed"      ,    )
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1E-3  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self._test_save_load_optional_components()


   @unittest.skipIf(torch_device != "cuda"      ,    reason="float16 requires CUDA"  )
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().test_save_load_floataa(expected_max_diff=1E-1  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self._test_attention_slicing_forward_pass(expected_max_diff=1E-2  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self._test_save_load_local()






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self._test_inference_batch_single_identical(
          expected_max_diff=1E-2      ,    )






