




'''simple docstring'''


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__				:				int       							=		get_tests_dir('fixtures/spiece.model')

@require_sentencepiece
@require_tokenizers
class       A   (		_a					,unittest.TestCase   ):
							lowercase_          =    AlbertTokenizer
							lowercase_          =    AlbertTokenizerFast
							lowercase_          =    True
							lowercase_          =    True
							lowercase_          =    True







							def        __lowerCAmelCase      (  self	:  str )      ->							List[str]:


														"""simple docstring"""


														super().setUp()

														# We have a SentencePiece fixture for testing
														_a     					=    AlbertTokenizer(lowerCAmelCase_ )
														tokenizer.save_pretrained(self.tmpdirname )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  int )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    '''this is a test'''
														_a     					=    '''this is a test'''
														return input_text, output_text







							def        __lowerCAmelCase      (  self	:  Dict )      ->							str:


														"""simple docstring"""


														_a     					=    '''<pad>'''
														_a     					=    0

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_ )							,      lowerCAmelCase_ )
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_ )							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Dict:


														"""simple docstring"""


														_a     					=    list(self.get_tokenizer().get_vocab().keys() )

														self.assertEqual(vocab_keys[0]							,      '''<pad>''' )
														self.assertEqual(vocab_keys[1]							,      '''<unk>''' )
														self.assertEqual(vocab_keys[-1]							,      '''▁eloquent''' )
														self.assertEqual(len(lowerCAmelCase_ )							,      3_00_00 )







							def        __lowerCAmelCase      (  self	:  Dict )      ->							List[Any]:


														"""simple docstring"""


														self.assertEqual(self.get_tokenizer().vocab_size							,      3_00_00 )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							int:


														"""simple docstring"""


														if not self.test_rust_tokenizer:
																					return

														_a     					=    self.get_tokenizer()
														_a     					=    self.get_rust_tokenizer()

														_a     					=    '''I was born in 92000, and this is falsé.'''

														_a     					=    tokenizer.tokenize(lowerCAmelCase_ )
														_a     					=    rust_tokenizer.tokenize(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )

														_a     					=    tokenizer.encode(lowerCAmelCase_							,      add_special_tokens=lowerCAmelCase_ )
														_a     					=    rust_tokenizer.encode(lowerCAmelCase_							,      add_special_tokens=lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )

														_a     					=    self.get_rust_tokenizer()
														_a     					=    tokenizer.encode(lowerCAmelCase_ )
														_a     					=    rust_tokenizer.encode(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    AlbertTokenizer(lowerCAmelCase_							,      keep_accents=lowerCAmelCase_ )

														_a     					=    tokenizer.tokenize('''This is a test''' )
														self.assertListEqual(lowerCAmelCase_							,      ['''▁this''', '''▁is''', '''▁a''', '''▁test'''] )

														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )							,      [48, 25, 21, 12_89] )

														_a     					=    tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
														self.assertListEqual(
														    lowerCAmelCase_							,      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.'''] )
														_a     					=    tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      [31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9] )

														_a     					=    tokenizer.convert_ids_to_tokens(lowerCAmelCase_ )
														self.assertListEqual(
														    lowerCAmelCase_							,      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']							,      )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    AlbertTokenizer(lowerCAmelCase_ )

														_a     					=    tokenizer.encode('''sequence builders''' )
														_a     					=    tokenizer.encode('''multi-sequence build''' )

														_a     					=    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_ )
														_a     					=    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_							,      lowerCAmelCase_ )

														assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
														assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
														    tokenizer.sep_token_id
														]







							@slow
							def        __lowerCAmelCase      (  self	:  str )      ->							Any:


														"""simple docstring"""


														_a     					=    {'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
														# fmt: on

														self.tokenizer_integration_test_util(
														    expected_encoding=lowerCAmelCase_							,      model_name='''albert-base-v2'''							,      revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''							,      )







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_snake_case				:				List[str]       							=		{'configuration_xlnet': ['XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLNetConfig']}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				List[Any]       							=		['XLNetTokenizer']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				int       							=		['XLNetTokenizerFast']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				Optional[int]       							=		[
				    'XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'XLNetForMultipleChoice',
				    'XLNetForQuestionAnswering',
				    'XLNetForQuestionAnsweringSimple',
				    'XLNetForSequenceClassification',
				    'XLNetForTokenClassification',
				    'XLNetLMHeadModel',
				    'XLNetModel',
				    'XLNetPreTrainedModel',
				    'load_tf_weights_in_xlnet',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				_snake_case				:				str       							=		[
				    'TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'TFXLNetForMultipleChoice',
				    'TFXLNetForQuestionAnsweringSimple',
				    'TFXLNetForSequenceClassification',
				    'TFXLNetForTokenClassification',
				    'TFXLNetLMHeadModel',
				    'TFXLNetMainLayer',
				    'TFXLNetModel',
				    'TFXLNetPreTrainedModel',
				]


if TYPE_CHECKING:
				from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_xlnet import XLNetTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_xlnet_fast import XLNetTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_xlnet import (
								    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
								    XLNetForMultipleChoice,
								    XLNetForQuestionAnswering,
								    XLNetForQuestionAnsweringSimple,
								    XLNetForSequenceClassification,
								    XLNetForTokenClassification,
								    XLNetLMHeadModel,
								    XLNetModel,
								    XLNetPreTrainedModel,
								    load_tf_weights_in_xlnet,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_xlnet import (
								    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFXLNetForMultipleChoice,
								    TFXLNetForQuestionAnsweringSimple,
								    TFXLNetForSequenceClassification,
								    TFXLNetForTokenClassification,
								    TFXLNetLMHeadModel,
								    TFXLNetMainLayer,
								    TFXLNetModel,
								    TFXLNetPreTrainedModel,
								)

else:
				import sys

				_snake_case				:				Union[str, Any]       							=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

