




'''simple docstring'''


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case				:				int       							=		get_tests_dir('fixtures/spiece.model')

@require_sentencepiece
@require_tokenizers
class       A   (		_a					,unittest.TestCase   ):
							lowercase_          =    AlbertTokenizer
							lowercase_          =    AlbertTokenizerFast
							lowercase_          =    True
							lowercase_          =    True
							lowercase_          =    True







							def        __lowerCAmelCase      (  self	:  str )      ->							List[str]:


														"""simple docstring"""


														super().setUp()

														# We have a SentencePiece fixture for testing
														_a     					=    AlbertTokenizer(lowerCAmelCase_ )
														tokenizer.save_pretrained(self.tmpdirname )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  int )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    '''this is a test'''
														_a     					=    '''this is a test'''
														return input_text, output_text







							def        __lowerCAmelCase      (  self	:  Dict )      ->							str:


														"""simple docstring"""


														_a     					=    '''<pad>'''
														_a     					=    0

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_ )							,      lowerCAmelCase_ )
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_ )							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Dict:


														"""simple docstring"""


														_a     					=    list(self.get_tokenizer().get_vocab().keys() )

														self.assertEqual(vocab_keys[0]							,      '''<pad>''' )
														self.assertEqual(vocab_keys[1]							,      '''<unk>''' )
														self.assertEqual(vocab_keys[-1]							,      '''▁eloquent''' )
														self.assertEqual(len(lowerCAmelCase_ )							,      3_00_00 )







							def        __lowerCAmelCase      (  self	:  Dict )      ->							List[Any]:


														"""simple docstring"""


														self.assertEqual(self.get_tokenizer().vocab_size							,      3_00_00 )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							int:


														"""simple docstring"""


														if not self.test_rust_tokenizer:
																					return

														_a     					=    self.get_tokenizer()
														_a     					=    self.get_rust_tokenizer()

														_a     					=    '''I was born in 92000, and this is falsé.'''

														_a     					=    tokenizer.tokenize(lowerCAmelCase_ )
														_a     					=    rust_tokenizer.tokenize(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )

														_a     					=    tokenizer.encode(lowerCAmelCase_							,      add_special_tokens=lowerCAmelCase_ )
														_a     					=    rust_tokenizer.encode(lowerCAmelCase_							,      add_special_tokens=lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )

														_a     					=    self.get_rust_tokenizer()
														_a     					=    tokenizer.encode(lowerCAmelCase_ )
														_a     					=    rust_tokenizer.encode(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    AlbertTokenizer(lowerCAmelCase_							,      keep_accents=lowerCAmelCase_ )

														_a     					=    tokenizer.tokenize('''This is a test''' )
														self.assertListEqual(lowerCAmelCase_							,      ['''▁this''', '''▁is''', '''▁a''', '''▁test'''] )

														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )							,      [48, 25, 21, 12_89] )

														_a     					=    tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
														self.assertListEqual(
														    lowerCAmelCase_							,      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.'''] )
														_a     					=    tokenizer.convert_tokens_to_ids(lowerCAmelCase_ )
														self.assertListEqual(lowerCAmelCase_							,      [31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9] )

														_a     					=    tokenizer.convert_ids_to_tokens(lowerCAmelCase_ )
														self.assertListEqual(
														    lowerCAmelCase_							,      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']							,      )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    AlbertTokenizer(lowerCAmelCase_ )

														_a     					=    tokenizer.encode('''sequence builders''' )
														_a     					=    tokenizer.encode('''multi-sequence build''' )

														_a     					=    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_ )
														_a     					=    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_							,      lowerCAmelCase_ )

														assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
														assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
														    tokenizer.sep_token_id
														]







							@slow
							def        __lowerCAmelCase      (  self	:  str )      ->							Any:


														"""simple docstring"""


														_a     					=    {'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
														# fmt: on

														self.tokenizer_integration_test_util(
														    expected_encoding=lowerCAmelCase_							,      model_name='''albert-base-v2'''							,      revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''							,      )







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_snake_case				:				Any       							=		logging.get_logger(__name__)

_snake_case				:				Any       							=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case				:				int       							=		{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_snake_case				:				int       							=		{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_INIT_CONFIGURATION
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    BertTokenizer







							def __init__(  self	:  str							,      lowerCAmelCase_	:  Union[str, Any]=None							,      lowerCAmelCase_	:  str=None							,      lowerCAmelCase_	:  List[Any]=True							,      lowerCAmelCase_	:  Any="[UNK]"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Tuple="[PAD]"							,      lowerCAmelCase_	:  Tuple="[CLS]"							,      lowerCAmelCase_	:  Optional[int]="[MASK]"							,      lowerCAmelCase_	:  Dict=True							,      lowerCAmelCase_	:  Union[str, Any]=None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							Union[str, Any]:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,      tokenizer_file=lowerCAmelCase_							,      do_lower_case=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      tokenize_chinese_chars=lowerCAmelCase_							,      strip_accents=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														_a     					=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
														if (
														    normalizer_state.get('''lowercase'''							,      lowerCAmelCase_ ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''							,      lowerCAmelCase_ ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''							,      lowerCAmelCase_ ) != tokenize_chinese_chars
														):
																					_a     					=    getattr(lowerCAmelCase_							,      normalizer_state.pop('''type''' ) )
																					_a     					=    do_lower_case
																					_a     					=    strip_accents
																					_a     					=    tokenize_chinese_chars
																					_a     					=    normalizer_class(**lowerCAmelCase_ )

														_a     					=    do_lower_case







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  int=None )      ->							List[str]:


														"""simple docstring"""


														_a     					=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														_a     					=    self._tokenizer.model.save(lowerCAmelCase_							,      name=lowerCAmelCase_ )
														return tuple(lowerCAmelCase_ )

