

'''simple docstring'''
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case         =  get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Any     	=							XLMRobertaTokenizer
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							XLMRobertaTokenizerFast
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							True
   _SCREAMING_SNAKE_CASE   :					Dict     	=							True


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().setUp()

      # We have a SentencePiece fixture for testing
      _lowercase      :				Dict      					=       XLMRobertaTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )
      tokenizer.save_pretrained(self.tmpdirname  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       "<pad>"
      _lowercase      :				Optional[Any]      					=       1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase  )      ,    _UpperCamelCase  )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase  )      ,    _UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       list(self.get_tokenizer().get_vocab().keys()  )

      self.assertEqual(vocab_keys[0]      ,    "<s>"  )
      self.assertEqual(vocab_keys[1]      ,    "<pad>"  )
      self.assertEqual(vocab_keys[-1]      ,    "<mask>"  )
      self.assertEqual(len(_UpperCamelCase  )      ,    1002  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.assertEqual(self.get_tokenizer().vocab_size      ,    1002  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       XLMRobertaTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )

      _lowercase      :				Dict      					=       tokenizer.tokenize("This is a test"  )
      self.assertListEqual(_UpperCamelCase      ,    ["▁This", "▁is", "▁a", "▁t", "est"]  )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(_UpperCamelCase  )      ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,    )

      _lowercase      :				List[str]      					=       tokenizer.tokenize("I was born in 92000, and this is falsé."  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]      ,    )
      _lowercase      :				int      					=       tokenizer.convert_tokens_to_ids(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
          ]      ,    )

      _lowercase      :				Union[str, Any]      					=       tokenizer.convert_ids_to_tokens(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]      ,    )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if not self.test_slow_tokenizer:
         # as we don't have a slow version, we can't compare the outputs between slow and fast versions
         return

      _lowercase      :				Tuple      					=       (self.rust_tokenizer_class, "hf-internal-testing/tiny-xlm-roberta", {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
            _lowercase      :				int      					=       self.rust_tokenizer_class.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )
            _lowercase      :				Any      					=       self.tokenizer_class.from_pretrained(_UpperCamelCase      ,    **_UpperCamelCase  )

            _lowercase      :				Any      					=       tempfile.mkdtemp()

            _lowercase      :				List[Any]      					=       tokenizer_r.save_pretrained(_UpperCamelCase  )
            _lowercase      :				Optional[int]      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it save with the same files + the tokenizer.json file for the fast one
            self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files  )  )
            _lowercase      :				List[Any]      					=       tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f  )
            self.assertSequenceEqual(_UpperCamelCase      ,    _UpperCamelCase  )

            # Checks everything loads correctly in the same way
            _lowercase      :				Dict      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				int      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )
               # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
               # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

            shutil.rmtree(_UpperCamelCase  )

            # Save tokenizer rust, legacy_format=True
            _lowercase      :				List[str]      					=       tempfile.mkdtemp()

            _lowercase      :				Dict      					=       tokenizer_r.save_pretrained(_UpperCamelCase      ,    legacy_format=_UpperCamelCase  )
            _lowercase      :				Dict      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it save with the same files
            self.assertSequenceEqual(_UpperCamelCase      ,    _UpperCamelCase  )

            # Checks everything loads correctly in the same way
            _lowercase      :				Optional[Any]      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				Union[str, Any]      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )

            shutil.rmtree(_UpperCamelCase  )

            # Save tokenizer rust, legacy_format=False
            _lowercase      :				List[Any]      					=       tempfile.mkdtemp()

            _lowercase      :				int      					=       tokenizer_r.save_pretrained(_UpperCamelCase      ,    legacy_format=_UpperCamelCase  )
            _lowercase      :				Optional[Any]      					=       tokenizer_p.save_pretrained(_UpperCamelCase  )

            # Checks it saved the tokenizer.json file
            self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files  )  )

            # Checks everything loads correctly in the same way
            _lowercase      :				int      					=       tokenizer_r.from_pretrained(_UpperCamelCase  )
            _lowercase      :				Optional[int]      					=       tokenizer_p.from_pretrained(_UpperCamelCase  )

            # Check special tokens are set accordingly on Rust and Python
            for key in tokenizer_pp.special_tokens_map:
               self.assertTrue(hasattr(_UpperCamelCase      ,    _UpperCamelCase  )  )

            shutil.rmtree(_UpperCamelCase  )


   @cached_property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XLMRobertaTokenizer.from_pretrained("xlm-roberta-base"  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      with tempfile.NamedTemporaryFile() as f:
         shutil.copyfile(_UpperCamelCase      ,    f.name  )
         _lowercase      :				Dict      					=       XLMRobertaTokenizer(f.name      ,    keep_accents=_UpperCamelCase  )
         _lowercase      :				Any      					=       pickle.dumps(_UpperCamelCase  )
      pickle.loads(_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if not self.test_rust_tokenizer:
         return

      _lowercase      :				Optional[Any]      					=       self.get_tokenizer()
      _lowercase      :				int      					=       self.get_rust_tokenizer()

      _lowercase      :				Optional[int]      					=       "I was born in 92000, and this is falsé."

      _lowercase      :				Optional[int]      					=       tokenizer.tokenize(_UpperCamelCase  )
      _lowercase      :				Tuple      					=       rust_tokenizer.tokenize(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				int      					=       tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      _lowercase      :				Optional[Any]      					=       rust_tokenizer.encode(_UpperCamelCase      ,    add_special_tokens=_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )

      _lowercase      :				List[str]      					=       self.get_rust_tokenizer()
      _lowercase      :				Optional[Any]      					=       tokenizer.encode(_UpperCamelCase  )
      _lowercase      :				str      					=       rust_tokenizer.encode(_UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       "Hello World!"
      _lowercase      :				int      					=       [0, 35378, 6661, 38, 2]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(_UpperCamelCase      ,    self.big_tokenizer.encode(_UpperCamelCase  )  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       (
          "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
          " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
      )
      _lowercase      :				int      					=       [
          0,
          3293,
          83,
          10,
          4552,
          4989,
          7986,
          678,
          10,
          5915,
          111,
          179459,
          124850,
          4,
          6044,
          237,
          12,
          6,
          5,
          6,
          4,
          6780,
          705,
          15,
          1388,
          44,
          378,
          10114,
          711,
          152,
          20,
          6,
          5,
          22376,
          642,
          1221,
          15190,
          34153,
          450,
          5608,
          959,
          1119,
          57702,
          136,
          186,
          47,
          1098,
          29367,
          47,
          # 4426, # What fairseq tokenizes from "<unk>": "_<"
          # 3678, # What fairseq tokenizes from "<unk>": "unk"
          # 2740, # What fairseq tokenizes from "<unk>": ">"
          3,  # What we tokenize from "<unk>": "<unk>"
          6,  # Residue from the tokenization: an extra sentencepiece underline
          4,
          6044,
          237,
          6284,
          50901,
          528,
          31,
          90,
          34,
          927,
          2,
      ]
      # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
      # xlmr.eval()
      # xlmr.encode(symbols)

      self.assertListEqual(_UpperCamelCase      ,    self.big_tokenizer.encode(_UpperCamelCase  )  )






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Any      					=       {"input_ids": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_UpperCamelCase      ,    model_name="xlm-roberta-base"      ,    revision="d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"      ,    )









'''simple docstring'''
from sklearn.metrics import mean_squared_error

import datasets


_snake_case         =  '\\n@article{scikit-learn,\n title={Scikit-learn: Machine Learning in {P}ython},\n author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n journal={Journal of Machine Learning Research},\n volume={12},\n pages={2825--2830},\n year={2011}\n}\n'

_snake_case         =  '\\nMean Squared Error(MSE) is the average of the square of difference between the predicted\nand actual values.\n'


_snake_case         =  '\nArgs:\n    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)\n        Estimated target values.\n    references: array-like of shape (n_samples,) or (n_samples, n_outputs)\n        Ground truth (correct) target values.\n    sample_weight: array-like of shape (n_samples,), default=None\n        Sample weights.\n    multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average"\n        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.\n\n        "raw_values" : Returns a full set of errors in case of multioutput input.\n\n        "uniform_average" : Errors of all outputs are averaged with uniform weight.\n\n    squared : bool, default=True\n        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.\n\nReturns:\n    mse : mean squared error.\nExamples:\n\n    >>> mse_metric = datasets.load_metric("mse")\n    >>> predictions = [2.5, 0.0, 2, 8]\n    >>> references = [3, -0.5, 2, 7]\n    >>> results = mse_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'mse\': 0.375}\n    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)\n    >>> print(rmse_result)\n    {\'mse\': 0.6123724356957945}\n\n    If you\'re using multi-dimensional lists, then set the config as follows :\n\n    >>> mse_metric = datasets.load_metric("mse", "multilist")\n    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]\n    >>> references = [[0, 2], [-1, 2], [8, -5]]\n    >>> results = mse_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'mse\': 0.7083333333333334}\n    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput=\'raw_values\')\n    >>> print(results) # doctest: +NORMALIZE_WHITESPACE\n    {\'mse\': array([0.41666667, 1. ])}\n'


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,				_KWARGS_DESCRIPTION  )
class        a__  (							datasets.Metric  ):


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return datasets.MetricInfo(
          description=_DESCRIPTION      ,    citation=_CITATION      ,    inputs_description=_KWARGS_DESCRIPTION      ,    features=datasets.Features(self._get_feature_types()  )      ,    reference_urls=[
              "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html"
          ]      ,    )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      if self.config_name == "multilist":
         return {
             "predictions": datasets.Sequence(datasets.Value("float"  )  ),
             "references": datasets.Sequence(datasets.Value("float"  )  ),
         }
      else:
         return {
             "predictions": datasets.Value("float"  ),
             "references": datasets.Value("float"  ),
         }






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase=None      ,    _UpperCamelCase="uniform_average"      ,    _UpperCamelCase=True  ):





      """simple docstring"""





      _lowercase      :				str      					=       mean_squared_error(
          _UpperCamelCase      ,    _UpperCamelCase      ,    sample_weight=_UpperCamelCase      ,    multioutput=_UpperCamelCase      ,    squared=_UpperCamelCase  )

      return {"mse": mse}






