






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase     :							Optional[Any]    =get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
@require_tokenizers
class       __a  (   A__	,       unittest.TestCase	):
 _lowerCAmelCase	:	List[str]				=			XLNetTokenizer
 _lowerCAmelCase	:	Union[str, Any]				=			XLNetTokenizerFast
 _lowerCAmelCase	:	Optional[int]				=			True
 _lowerCAmelCase	:	List[str]				=			True







 def  __lowercase (		self :       Dict						):


      '''simple docstring'''






      super().setUp()

      # We have a SentencePiece fixture for testing
      UpperCamelCase__      :       Tuple	    =  XLNetTokenizer(SCREAMING_SNAKE_CASE							,       keep_accents=SCREAMING_SNAKE_CASE						)
      tokenizer.sanitize_special_tokens()
      tokenizer.save_pretrained(self.tmpdirname						)







 def  __lowercase (		self :       int						):


      '''simple docstring'''






      UpperCamelCase__      :       Any	    =  "<s>"
      UpperCamelCase__      :       List[str]	    =  1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE						)							,       SCREAMING_SNAKE_CASE						)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE						)							,       SCREAMING_SNAKE_CASE						)







 def  __lowercase (		self :       str						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[Any]	    =  list(self.get_tokenizer().get_vocab().keys()						)

      self.assertEqual(vocab_keys[0]							,       "<unk>"						)
      self.assertEqual(vocab_keys[1]							,       "<s>"						)
      self.assertEqual(vocab_keys[-1]							,       "<eod>"						)
      self.assertEqual(len(SCREAMING_SNAKE_CASE						)							,       10_06						)







 def  __lowercase (		self :       List[Any]						):


      '''simple docstring'''






      self.assertEqual(self.get_tokenizer().vocab_size							,       10_00						)







 def  __lowercase (		self :       int						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  XLNetTokenizer(SCREAMING_SNAKE_CASE							,       keep_accents=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Union[str, Any]	    =  tokenizer.tokenize("This is a test"						)
      self.assertListEqual(SCREAMING_SNAKE_CASE							,       ["▁This", "▁is", "▁a", "▁t", "est"]						)

      self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE						)							,       [2_85, 46, 10, 1_70, 3_82]						)

      UpperCamelCase__      :       Optional[Any]	    =  tokenizer.tokenize("I was born in 92000, and this is falsé."						)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE							,       [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "é",
              ".",
          ]							,       )
      UpperCamelCase__      :       str	    =  tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE						)
      self.assertListEqual(SCREAMING_SNAKE_CASE							,       [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

      UpperCamelCase__      :       int	    =  tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE						)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE							,       [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "<unk>",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "s",
              "<unk>",
              ".",
          ]							,       )







 def  __lowercase (		self :       Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Any	    =  XLNetTokenizer(SCREAMING_SNAKE_CASE							,       do_lower_case=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       str	    =  tokenizer.tokenize("I was born in 92000, and this is falsé."						)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE							,       [
              SPIECE_UNDERLINE + "",
              "i",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "se",
              ".",
          ]							,       )
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)							,       ["▁he", "ll", "o"]						)







 def  __lowercase (		self :       List[str]						):


      '''simple docstring'''






      UpperCamelCase__      :       str	    =  XLNetTokenizer(SCREAMING_SNAKE_CASE							,       do_lower_case=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       Optional[Any]	    =  tokenizer.tokenize("I was born in 92000, and this is falsé."						)
      self.assertListEqual(
          SCREAMING_SNAKE_CASE							,       [
              SPIECE_UNDERLINE + "I",
              SPIECE_UNDERLINE + "was",
              SPIECE_UNDERLINE + "b",
              "or",
              "n",
              SPIECE_UNDERLINE + "in",
              SPIECE_UNDERLINE + "",
              "9",
              "2",
              "0",
              "0",
              "0",
              ",",
              SPIECE_UNDERLINE + "and",
              SPIECE_UNDERLINE + "this",
              SPIECE_UNDERLINE + "is",
              SPIECE_UNDERLINE + "f",
              "al",
              "se",
              ".",
          ]							,       )







 @slow
 def  __lowercase (		self :       Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Dict	    =  XLNetTokenizer.from_pretrained("xlnet-base-cased"						)

      UpperCamelCase__      :       Optional[Any]	    =  tokenizer.encode("sequence builders"							,       add_special_tokens=SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       Any	    =  tokenizer.encode("multi-sequence build"							,       add_special_tokens=SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       Tuple	    =  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__      :       Tuple	    =  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE							,       SCREAMING_SNAKE_CASE						)

      assert encoded_sentence == text + [4, 3]
      assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def  __lowercase (		self :       str						):


      '''simple docstring'''






      UpperCamelCase__      :       str	    =  {"input_ids": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=SCREAMING_SNAKE_CASE							,       model_name="xlnet-base-cased"							,       revision="c841166438c31ec7ca9a106dee7bb312b73ae511"							,       )







from __future__ import annotations







def 		SCREAMING_SNAKE_CASE							( __lowerCAmelCase ,   __lowerCAmelCase = None ,   __lowerCAmelCase = None      )			->     None:
     if start is None:
          UpperCamelCase__      :       Union[str, Any]	    =  0

     if end is None:
          UpperCamelCase__      :       List[Any]	    =  len(__lowerCAmelCase      ) - 1

     if start >= end:
          return

     UpperCamelCase__      :       Union[str, Any]	    =  (start + end) // 2

     slowsort(__lowerCAmelCase ,   __lowerCAmelCase ,   __lowerCAmelCase      )
     slowsort(__lowerCAmelCase ,   mid + 1 ,   __lowerCAmelCase      )

     if sequence[end] < sequence[mid]:
          UpperCamelCase__    ,					UpperCamelCase__      :       Optional[int]	    =  sequence[mid], sequence[end]

     slowsort(__lowerCAmelCase ,   __lowerCAmelCase ,   end - 1      )


if __name__ == "__main__":
 from doctest import testmod

 testmod()