






'''simple docstring'''

import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     =       get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
class     a_						(      lowerCamelCase     ,	unittest.TestCase    ):
  lowercase							=				XLMProphetNetTokenizer
  lowercase							=				False
  lowercase							=				True

  def   A__						(				self				)  ->				Any:



        """simple docstring"""






        super().setUp()

        # We have a SentencePiece fixture for testing
        UpperCamelCase      = XLMProphetNetTokenizer(_SCREAMING_SNAKE_CASE      ,		keep_accents=_SCREAMING_SNAKE_CASE				)
        tokenizer.save_pretrained(self.tmpdirname				)

  def   A__						(				self				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = """[PAD]"""
        UpperCamelCase      = 0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE				)      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE				)      ,		_SCREAMING_SNAKE_CASE				)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = list(self.get_tokenizer().get_vocab().keys()				)

        self.assertEqual(vocab_keys[0]      ,		"""[PAD]"""				)
        self.assertEqual(vocab_keys[1]      ,		"""[CLS]"""				)
        self.assertEqual(vocab_keys[-1]      ,		"""j"""				)
        self.assertEqual(len(_SCREAMING_SNAKE_CASE				)      ,		1012				)

  def   A__						(				self				)  ->				List[Any]:



        """simple docstring"""






        self.assertEqual(self.get_tokenizer().vocab_size      ,		1012				)

  def   A__						(				self				)  ->				Optional[int]:



        """simple docstring"""






        UpperCamelCase      = XLMProphetNetTokenizer(_SCREAMING_SNAKE_CASE      ,		keep_accents=_SCREAMING_SNAKE_CASE				)

        UpperCamelCase      = tokenizer.tokenize("""This is a test"""				)
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)      ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,		)

        UpperCamelCase      = tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]      ,		)
        UpperCamelCase      = tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
            ]      ,		)

        UpperCamelCase      = tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE      ,		[
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """[UNK]""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """[UNK]""",
                """.""",
            ]      ,		)

  @cached_property
  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""				)

  @slow
  def   A__						(				self				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = """Hello World!"""
        UpperCamelCase      = [35389, 6672, 49, 2]
        self.assertListEqual(_SCREAMING_SNAKE_CASE      ,		self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE				)				)


  @slow
  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase      = {"""input_ids""": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=_SCREAMING_SNAKE_CASE      ,		model_name="""microsoft/xprophetnet-large-wiki100-cased"""      ,		revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""      ,		)








'''simple docstring'''

import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


SCREAMING_SNAKE_CASE__     =       'scheduler_config.json'






class     a_						(      lowerCamelCase    ):
  lowercase							=				1
  lowercase							=				2
  lowercase							=				3
  lowercase							=				4
  lowercase							=				5
  lowercase							=				6
  lowercase							=				7
  lowercase							=				8
  lowercase							=				9
  lowercase							=				10
  lowercase							=				11
  lowercase							=				12
  lowercase							=				13
  lowercase							=				14






@dataclass
class     a_						(      lowerCamelCase    ):
  lowercase							=				42






class     a_						:
  lowercase							=				SCHEDULER_CONFIG_NAME
  lowercase							=				[]
  lowercase							=				True

  @classmethod
  def   A__						(				cls      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE=False      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase  ,UpperCamelCase  ,UpperCamelCase      = cls.load_config(
            pretrained_model_name_or_path=_SCREAMING_SNAKE_CASE      ,		subfolder=_SCREAMING_SNAKE_CASE      ,		return_unused_kwargs=_SCREAMING_SNAKE_CASE      ,		return_commit_hash=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE      ,		)
        return cls.from_config(_SCREAMING_SNAKE_CASE      ,		return_unused_kwargs=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = False      ,		**_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        self.save_config(save_directory=_SCREAMING_SNAKE_CASE      ,		push_to_hub=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        return self._get_compatibles()


  @classmethod
  def   A__						(				cls				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = list(set([cls.__name__] + cls._compatibles				)				)
        UpperCamelCase      = importlib.import_module(__name__.split("""."""				)[0]				)
        UpperCamelCase      = [
            getattr(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				) for c in compatible_classes_str if hasattr(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        ]
        return compatible_classes
