import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase      :			Dict							       =		get_tests_dir('''fixtures/spiece.model''')







@require_sentencepiece
@require_tokenizers
class __snake_case    (						lowerCamelCase_       ,		unittest.TestCase		):
   lowerCAmelCase_														=							DebertaVaTokenizer
   lowerCAmelCase_														=							DebertaVaTokenizerFast
   lowerCAmelCase_														=							True
   lowerCAmelCase_														=							True




   def      __a						(   self      :    Union[str, Any]				):



     """simple docstring"""






     super().setUp()

     # We have a SentencePiece fixture for testing
     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     unk_token="""<unk>"""				)
     tokenizer.save_pretrained(self.tmpdirname				)




   def      __a						(   self      :    int	,     _lowercase      :    Any				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""this is a test"""
     SCREAMING_SNAKE_CASE__      							=				"""this is a test"""
     return input_text, output_text




   def      __a						(   self      :    Tuple				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""<pad>"""
     SCREAMING_SNAKE_CASE__      							=				0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase				)	,     _lowercase				)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase				)	,     _lowercase				)




   def      __a						(   self      :    Dict				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				list(self.get_tokenizer().get_vocab().keys()				)
     self.assertEqual(vocab_keys[0]	,     """<pad>"""				)
     self.assertEqual(vocab_keys[1]	,     """<unk>"""				)
     self.assertEqual(vocab_keys[-1]	,     """[PAD]"""				)
     self.assertEqual(len(_lowercase				)	,     3_00_01				)




   def      __a						(   self      :    Tuple				):



     """simple docstring"""






     self.assertEqual(self.get_tokenizer().vocab_size	,     3_00_00				)




   def      __a						(   self      :    Optional[int]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				""" \tHeLLo!how  \n Are yoU?  """
     SCREAMING_SNAKE_CASE__      							=				["""▁hello""", """!""", """how""", """▁are""", """▁you""", """?"""]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     do_lower_case=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     do_lower_case=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)




   @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""				)
   def      __a						(   self      :    Dict				):



     """simple docstring"""






     pass




   @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""				)
   def      __a						(   self      :    str				):



     """simple docstring"""






     pass




   def      __a						(   self      :    Tuple				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Tuple				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""
     SCREAMING_SNAKE_CASE__      							=				["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)
     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Optional[Any]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""
     SCREAMING_SNAKE_CASE__      							=				["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Dict				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    List[str]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				""" \tHeLLo!how  \n Are yoU?  """
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """<unk>""", """e""", """<unk>""", """o""", """!""", """how""", """▁""", """<unk>""", """re""", """▁yo""", """<unk>""", """?"""]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     do_lower_case=_lowercase	,     split_by_punct=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)

     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Any				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				self.get_tokenizer()
     SCREAMING_SNAKE_CASE__      							=				self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""

     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode(_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.encode(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Tuple				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				"""This is a test"""
     SCREAMING_SNAKE_CASE__      							=				[13, 1, 43_98, 25, 21, 12_89]
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """T""", """his""", """▁is""", """▁a""", """▁test"""]
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """<unk>""", """his""", """▁is""", """▁a""", """▁test"""]

     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase	,     keep_accents=_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizerFast(_lowercase	,     keep_accents=_lowercase				)

     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.tokenize(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.tokenize(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     # fmt: off
     SCREAMING_SNAKE_CASE__      							=				"""I was born in 92000, and this is falsé."""
     SCREAMING_SNAKE_CASE__      							=				[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """I""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """.""", ]
     SCREAMING_SNAKE_CASE__      							=				["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
     # fmt: on

     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.tokenize(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.convert_ids_to_tokens(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)

     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.encode(_lowercase	,     add_special_tokens=_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.tokenize(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)
     SCREAMING_SNAKE_CASE__      							=				rust_tokenizer.convert_ids_to_tokens(_lowercase				)
     self.assertListEqual(_lowercase	,     _lowercase				)




   def      __a						(   self      :    Optional[int]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				DebertaVaTokenizer(_lowercase				)

     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode("""sequence builders"""				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.encode("""multi-sequence build"""				)

     SCREAMING_SNAKE_CASE__      							=				tokenizer.build_inputs_with_special_tokens(_lowercase				)
     SCREAMING_SNAKE_CASE__      							=				tokenizer.build_inputs_with_special_tokens(_lowercase	,     _lowercase				)

     self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,     _lowercase				)
     self.assertEqual(
         [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,     _lowercase	,     )




   @slow
   def      __a						(   self      :    Optional[int]				):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__      							=				{"""input_ids""": [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_lowercase	,     model_name="""microsoft/deberta-v2-xlarge"""	,     revision="""ad6e42c1532ddf3a15c39246b63f5559d558b670"""	,     )

from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels
from .squad import SquadExample, SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features
from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor
from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels
