




"""simple docstring"""


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A													=      get_tests_dir("""fixtures/spiece.model""")


@require_sentencepiece
@require_tokenizers
class _lowerCamelCase    (					a_       ,	unittest.TestCase						):
   _lowerCamelCase    :Optional[Any]     				=     AlbertTokenizer
   _lowerCamelCase    :Optional[int]     				=     AlbertTokenizerFast
   _lowerCamelCase    :List[Any]     				=     True
   _lowerCamelCase    :Optional[int]     				=     True
   _lowerCamelCase    :Dict     				=     True






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						Tuple:

         """simple docstring"""
         super().setUp()

         # We have a SentencePiece fixture for testing
         lowerCAmelCase__      :			Optional[int]   =	AlbertTokenizer(UpperCamelCase   )
         tokenizer.save_pretrained(self.tmpdirname   )






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		Dict   )			->						Tuple:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	"""this is a test"""
         lowerCAmelCase__      :			List[str]   =	"""this is a test"""
         return input_text, output_text






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						Dict:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	"""<pad>"""
         lowerCAmelCase__      :			Optional[int]   =	0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase   )   ,     UpperCamelCase   )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase   )   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		int   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	list(self.get_tokenizer().get_vocab().keys()   )

         self.assertEqual(vocab_keys[0]   ,     """<pad>"""   )
         self.assertEqual(vocab_keys[1]   ,     """<unk>"""   )
         self.assertEqual(vocab_keys[-1]   ,     """▁eloquent"""   )
         self.assertEqual(len(UpperCamelCase   )   ,     3_00_00   )






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   )			->						Optional[int]:

         """simple docstring"""
         self.assertEqual(self.get_tokenizer().vocab_size   ,     3_00_00   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						List[str]:

         """simple docstring"""
         if not self.test_rust_tokenizer:
               return

         lowerCAmelCase__      :			Union[str, Any]   =	self.get_tokenizer()
         lowerCAmelCase__      :			Dict   =	self.get_rust_tokenizer()

         lowerCAmelCase__      :			int   =	"""I was born in 92000, and this is falsé."""

         lowerCAmelCase__      :			Tuple   =	tokenizer.tokenize(UpperCamelCase   )
         lowerCAmelCase__      :			Optional[Any]   =	rust_tokenizer.tokenize(UpperCamelCase   )
         self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )

         lowerCAmelCase__      :			int   =	tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
         lowerCAmelCase__      :			Dict   =	rust_tokenizer.encode(UpperCamelCase   ,     add_special_tokens=UpperCamelCase   )
         self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )

         lowerCAmelCase__      :			str   =	self.get_rust_tokenizer()
         lowerCAmelCase__      :			Union[str, Any]   =	tokenizer.encode(UpperCamelCase   )
         lowerCAmelCase__      :			str   =	rust_tokenizer.encode(UpperCamelCase   )
         self.assertListEqual(UpperCamelCase   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[int]   =	AlbertTokenizer(UpperCamelCase   ,     keep_accents=UpperCamelCase   )

         lowerCAmelCase__      :			Tuple   =	tokenizer.tokenize("""This is a test"""   )
         self.assertListEqual(UpperCamelCase   ,     ["""▁this""", """▁is""", """▁a""", """▁test"""]   )

         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase   )   ,     [48, 25, 21, 12_89]   )

         lowerCAmelCase__      :			Optional[int]   =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
         self.assertListEqual(
             UpperCamelCase   ,     ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]   )
         lowerCAmelCase__      :			Tuple   =	tokenizer.convert_tokens_to_ids(UpperCamelCase   )
         self.assertListEqual(UpperCamelCase   ,     [31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]   )

         lowerCAmelCase__      :			int   =	tokenizer.convert_ids_to_tokens(UpperCamelCase   )
         self.assertListEqual(
             UpperCamelCase   ,     ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]   ,     )






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Optional[int]:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	AlbertTokenizer(UpperCamelCase   )

         lowerCAmelCase__      :			Any   =	tokenizer.encode("""sequence builders"""   )
         lowerCAmelCase__      :			List[Any]   =	tokenizer.encode("""multi-sequence build"""   )

         lowerCAmelCase__      :			List[str]   =	tokenizer.build_inputs_with_special_tokens(UpperCamelCase   )
         lowerCAmelCase__      :			Optional[int]   =	tokenizer.build_inputs_with_special_tokens(UpperCamelCase   ,     UpperCamelCase   )

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
             tokenizer.sep_token_id
         ]






   @slow
   def 	_lowerCAmelCase   (		self       :		int   )			->						str:

         """simple docstring"""
         # fmt: off
         lowerCAmelCase__      :			int   =	{"""attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """input_ids""": [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=UpperCamelCase   ,     model_name="""albert-base-v2"""   ,     revision="""6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"""   ,     )











"""simple docstring"""


# This is the module that test_patching.py uses to test patch_submodule()

import os  # noqa: this is just for tests
import os as renamed_os  # noqa: this is just for tests
from os import path  # noqa: this is just for tests
from os import path as renamed_path  # noqa: this is just for tests
from os.path import join  # noqa: this is just for tests
from os.path import join as renamed_join  # noqa: this is just for tests


_A													=      open  # noqa: we just need to have a builtin inside this module to test it properly





