





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case           =			get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase							(       lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      XLNetTokenizer
  __lowerCamelCase		=      XLNetTokenizerFast
  __lowerCamelCase		=      True
  __lowerCamelCase		=      True







  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__			       =  XLNetTokenizer(_lowercase   ,							keep_accents=_lowercase )
     tokenizer.sanitize_special_tokens()
     tokenizer.save_pretrained(self.tmpdirname )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  "<s>"
     lowercase__			       =  1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase )   ,							_lowercase )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase )   ,							_lowercase )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  list(self.get_tokenizer().get_vocab().keys() )

     self.assertEqual(vocab_keys[0]   ,							"<unk>" )
     self.assertEqual(vocab_keys[1]   ,							"<s>" )
     self.assertEqual(vocab_keys[-1]   ,							"<eod>" )
     self.assertEqual(len(_lowercase )   ,							10_06 )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     self.assertEqual(self.get_tokenizer().vocab_size   ,							10_00 )







  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     lowercase__			       =  XLNetTokenizer(_lowercase   ,							keep_accents=_lowercase )

     lowercase__			       =  tokenizer.tokenize("This is a test" )
     self.assertListEqual(_lowercase   ,							["▁This", "▁is", "▁a", "▁t", "est"] )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase )   ,							[2_85, 46, 10, 1_70, 3_82] )

     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "é",
             ".",
         ]   ,							)
     lowercase__			       =  tokenizer.convert_tokens_to_ids(_lowercase )
     self.assertListEqual(_lowercase   ,							[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

     lowercase__			       =  tokenizer.convert_ids_to_tokens(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "<unk>",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "<unk>",
             ".",
         ]   ,							)







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  XLNetTokenizer(_lowercase   ,							do_lower_case=_lowercase )
     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "",
             "i",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "se",
             ".",
         ]   ,							)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )   ,							["▁he", "ll", "o"] )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     lowercase__			       =  XLNetTokenizer(_lowercase   ,							do_lower_case=_lowercase )
     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "se",
             ".",
         ]   ,							)







  @slow
  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  XLNetTokenizer.from_pretrained("xlnet-base-cased" )

     lowercase__			       =  tokenizer.encode("sequence builders"   ,							add_special_tokens=_lowercase )
     lowercase__			       =  tokenizer.encode("multi-sequence build"   ,							add_special_tokens=_lowercase )

     lowercase__			       =  tokenizer.build_inputs_with_special_tokens(_lowercase )
     lowercase__			       =  tokenizer.build_inputs_with_special_tokens(_lowercase   ,							_lowercase )

     assert encoded_sentence == text + [4, 3]
     assert encoded_pair == text + [4] + text_a + [4, 3]







  @slow
  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  {"input_ids": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_lowercase   ,							model_name="xlnet-base-cased"   ,							revision="c841166438c31ec7ca9a106dee7bb312b73ae511"   ,							)













def _A       (   __magic_name__	):
   try:
      lowercase__			       =  float(__magic_name__	)
   except ValueError:
      raise ValueError("Please enter a valid number"	)
   lowercase__			       =  decimal - int(__magic_name__	)
   if fractional_part == 0:
      return int(__magic_name__	), 1
   else:
      lowercase__			       =  len(str(__magic_name__	).split("."	)[1]	)
      lowercase__			       =  int(decimal * (10**number_of_frac_digits)	)
      lowercase__			       =  10**number_of_frac_digits
      lowercase__       ,							lowercase__			       =  denominator, numerator
      while True:
         lowercase__			       =  dividend % divisor
         if remainder == 0:
            break
         lowercase__       ,							lowercase__			       =  divisor, remainder
      lowercase__       ,							lowercase__			       =  numerator / divisor, denominator / divisor
      return int(__magic_name__	), int(__magic_name__	)


if __name__ == "__main__":
  print(F"""{decimal_to_fraction(2) = }""")
  print(F"""{decimal_to_fraction(89.0) = }""")
  print(F"""{decimal_to_fraction("67") = }""")
  print(F"""{decimal_to_fraction("45.0") = }""")
  print(F"""{decimal_to_fraction(1.5) = }""")
  print(F"""{decimal_to_fraction("6.25") = }""")
  print(F"""{decimal_to_fraction("78td") = }""")






