

'''simple docstring'''
import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case         =  get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
class        a__  (							lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							XLMProphetNetTokenizer
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							False
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							True


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().setUp()

      # We have a SentencePiece fixture for testing
      _lowercase      :				int      					=       XLMProphetNetTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )
      tokenizer.save_pretrained(self.tmpdirname  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       "[PAD]"
      _lowercase      :				List[str]      					=       0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase  )      ,    _UpperCamelCase  )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase  )      ,    _UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       list(self.get_tokenizer().get_vocab().keys()  )

      self.assertEqual(vocab_keys[0]      ,    "[PAD]"  )
      self.assertEqual(vocab_keys[1]      ,    "[CLS]"  )
      self.assertEqual(vocab_keys[-1]      ,    "j"  )
      self.assertEqual(len(_UpperCamelCase  )      ,    1012  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.assertEqual(self.get_tokenizer().vocab_size      ,    1012  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       XLMProphetNetTokenizer(_UpperCamelCase      ,    keep_accents=_UpperCamelCase  )

      _lowercase      :				Optional[Any]      					=       tokenizer.tokenize("This is a test"  )
      self.assertListEqual(_UpperCamelCase      ,    ["▁This", "▁is", "▁a", "▁t", "est"]  )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(_UpperCamelCase  )      ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,    )

      _lowercase      :				Tuple      					=       tokenizer.tokenize("I was born in 92000, and this is falsé."  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]      ,    )
      _lowercase      :				int      					=       tokenizer.convert_tokens_to_ids(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
          ]      ,    )

      _lowercase      :				List[str]      					=       tokenizer.convert_ids_to_tokens(_UpperCamelCase  )
      self.assertListEqual(
          _UpperCamelCase      ,    [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "[UNK]",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "[UNK]",
              ".",
          ]      ,    )


   @cached_property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       "Hello World!"
      _lowercase      :				List[Any]      					=       [35389, 6672, 49, 2]
      self.assertListEqual(_UpperCamelCase      ,    self.big_tokenizer.encode(_UpperCamelCase  )  )






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       {"input_ids": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=_UpperCamelCase      ,    model_name="microsoft/xprophetnet-large-wiki100-cased"      ,    revision="1acad1643ddd54a44df6a1b797ada8373685d90e"      ,    )









'''simple docstring'''
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


_snake_case         =  logging.getLogger()


@unittest.skip('Temporarily disable the doc tests.'  )
@require_torch
@require_tf
@slow
class        a__  (							unittest.TestCase  ):


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = True      ,    ):





      """simple docstring"""





      _lowercase      :				str      					=       [file for file in os.listdir(_UpperCamelCase  ) if os.path.isfile(os.path.join(_UpperCamelCase      ,    _UpperCamelCase  )  )]

      if identifier is not None:
         _lowercase      :				str      					=       [file for file in files if identifier in file]

      if n_identifier is not None:
         if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            for n_ in n_identifier:
               _lowercase      :				Dict      					=       [file for file in files if n_ not in file]
         else:
            _lowercase      :				Optional[Any]      					=       [file for file in files if n_identifier not in file]

      _lowercase      :				Dict      					=       ignore_files or []
      ignore_files.append("__init__.py"  )
      _lowercase      :				List[str]      					=       [file for file in files if file not in ignore_files]

      for file in files:
         # Open all files
         print("Testing"      ,    _UpperCamelCase  )

         if only_modules:
            _lowercase      :				Optional[Any]      					=       file.split("."  )[0]
            try:
               _lowercase      :				Union[str, Any]      					=       getattr(_UpperCamelCase      ,    _UpperCamelCase  )
               _lowercase      :				Optional[int]      					=       doctest.DocTestSuite(_UpperCamelCase  )
               _lowercase      :				Tuple      					=       unittest.TextTestRunner().run(_UpperCamelCase  )
               self.assertIs(len(result.failures  )      ,    0  )
            except AttributeError:
               logger.info(f'''{module_identifier} is not a module.'''  )
         else:
            _lowercase      :				Optional[Any]      					=       doctest.testfile(str(".." / directory / file  )      ,    optionflags=doctest.ELLIPSIS  )
            self.assertIs(result.failed      ,    0  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				str      					=       "modeling"
      _lowercase      :				Tuple      					=       [
          "modeling_ctrl.py",
          "modeling_tf_ctrl.py",
      ]
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase      ,    ignore_files=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       Path("src/transformers"  )
      _lowercase      :				Any      					=       "tokenization"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				Optional[Any]      					=       "configuration"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       Path("src/transformers"  )
      _lowercase      :				List[Any]      					=       ["configuration", "modeling", "tokenization"]
      self.analyze_directory(_UpperCamelCase      ,    n_identifier=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       Path("docs/source"  )
      _lowercase      :				int      					=       ["favicon.ico"]
      self.analyze_directory(_UpperCamelCase      ,    ignore_files=_UpperCamelCase      ,    only_modules=_UpperCamelCase  )






