'''simple docstring'''




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    a__				(       UpperCAmelCase__							,       unittest.TestCase			):
     lowerCamelCase  :					Tuple							=BertTokenizer
     lowerCamelCase  :					Tuple							=BertTokenizerFast
     lowerCamelCase  :					Tuple							=True
     lowerCamelCase  :					List[Any]							=True
     lowerCamelCase  :					Tuple							=filter_non_english






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         super().setUp()

         __lowerCamelCase          =  [
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         __lowerCamelCase          =  os.path.join(self.tmpdirname  ,    VOCAB_FILES_NAMES['''vocab_file''']   )
         with open(self.vocab_file  ,    '''w'''  ,    encoding='''utf-8'''   ) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]  ,    a		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  '''UNwant\u00E9d,running'''
         __lowerCamelCase          =  '''unwanted, running'''
         return input_text, output_text






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.tokenizer_class(self.vocab_file   )

         __lowerCamelCase          =  tokenizer.tokenize('''UNwant\u00E9d,running'''   )
         self.assertListEqual(a  ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )  ,    [9, 6, 7, 12, 10, 11]   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         if not self.test_rust_tokenizer:
             return

         __lowerCamelCase          =  self.get_tokenizer()
         __lowerCamelCase          =  self.get_rust_tokenizer()

         __lowerCamelCase          =  '''UNwant\u00E9d,running'''

         __lowerCamelCase          =  tokenizer.tokenize(a   )
         __lowerCamelCase          =  rust_tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  tokenizer.encode(a  ,    add_special_tokens=a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  self.get_rust_tokenizer()
         __lowerCamelCase          =  tokenizer.encode(a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a   )
         self.assertListEqual(a  ,    a   )

         # With lower casing
         __lowerCamelCase          =  self.get_tokenizer(do_lower_case=a   )
         __lowerCamelCase          =  self.get_rust_tokenizer(do_lower_case=a   )

         __lowerCamelCase          =  '''UNwant\u00E9d,running'''

         __lowerCamelCase          =  tokenizer.tokenize(a   )
         __lowerCamelCase          =  rust_tokenizer.tokenize(a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  tokenizer.encode(a  ,    add_special_tokens=a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a  ,    add_special_tokens=a   )
         self.assertListEqual(a  ,    a   )

         __lowerCamelCase          =  self.get_rust_tokenizer()
         __lowerCamelCase          =  tokenizer.encode(a   )
         __lowerCamelCase          =  rust_tokenizer.encode(a   )
         self.assertListEqual(a  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )  ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )  ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''h\u00E9llo''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )  ,    ['''hello''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )  ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a  ,    strip_accents=a   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )  ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer(do_lower_case=a  ,    never_split=['''[UNK]''']   )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )  ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  BasicTokenizer()
         __lowerCamelCase          =  '''a\n\'ll !!to?\'d of, can\'t.'''
         __lowerCamelCase          =  ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
         self.assertListEqual(tokenizer.tokenize(a   )  ,    a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         __lowerCamelCase          =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         __lowerCamelCase          =  {}
         for i, token in enumerate(a   ):
             __lowerCamelCase          =  i
         __lowerCamelCase          =  WordpieceTokenizer(vocab=a  ,    unk_token='''[UNK]'''   )

         self.assertListEqual(tokenizer.tokenize(''''''   )  ,    []   )

         self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )  ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

         self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )  ,    ['''[UNK]''', '''runn''', '''##ing''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		str   ):





         """simple docstring"""



         self.assertTrue(_is_whitespace(''' '''   )   )
         self.assertTrue(_is_whitespace('''\t'''   )   )
         self.assertTrue(_is_whitespace('''\r'''   )   )
         self.assertTrue(_is_whitespace('''\n'''   )   )
         self.assertTrue(_is_whitespace('''\u00A0'''   )   )

         self.assertFalse(_is_whitespace('''A'''   )   )
         self.assertFalse(_is_whitespace('''-'''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Dict   ):





         """simple docstring"""



         self.assertTrue(_is_control('''\u0005'''   )   )

         self.assertFalse(_is_control('''A'''   )   )
         self.assertFalse(_is_control(''' '''   )   )
         self.assertFalse(_is_control('''\t'''   )   )
         self.assertFalse(_is_control('''\r'''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         self.assertTrue(_is_punctuation('''-'''   )   )
         self.assertTrue(_is_punctuation('''$'''   )   )
         self.assertTrue(_is_punctuation('''`'''   )   )
         self.assertTrue(_is_punctuation('''.'''   )   )

         self.assertFalse(_is_punctuation('''A'''   )   )
         self.assertFalse(_is_punctuation(''' '''   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.get_tokenizer()
         __lowerCamelCase          =  self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]  ,    [['''[UNK]'''], [], ['''[UNK]''']]   )

         self.assertListEqual(
             [rust_tokenizer.tokenize(a   ) for t in ['''Test''', '''\xad''', '''test''']]  ,    [['''[UNK]'''], [], ['''[UNK]''']]   )






     @slow
     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

         __lowerCamelCase          =  tokenizer.encode('''sequence builders'''  ,    add_special_tokens=a   )
         __lowerCamelCase          =  tokenizer.encode('''multi-sequence build'''  ,    add_special_tokens=a   )

         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a   )
         __lowerCamelCase          =  tokenizer.build_inputs_with_special_tokens(a  ,    a   )

         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[int]   ):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                 __lowerCamelCase          =  tokenizer_r.encode_plus(
                     a  ,    return_attention_mask=a  ,    return_token_type_ids=a  ,    return_offsets_mapping=a  ,    add_special_tokens=a  ,    )

                 __lowerCamelCase          =  tokenizer_r.do_lower_case if hasattr(a  ,    '''do_lower_case'''   ) else False
                 __lowerCamelCase          =  (
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''A'''),
                         ((1, 2), ''','''),
                         ((3, 5), '''na'''),
                         ((5, 6), '''##ï'''),
                         ((6, 8), '''##ve'''),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), '''Allen'''),
                         ((21, 23), '''##NL'''),
                         ((23, 24), '''##P'''),
                         ((25, 33), '''sentence'''),
                         ((33, 34), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''a'''),
                         ((1, 2), ''','''),
                         ((3, 8), '''naive'''),
                         ((9, 15), tokenizer_r.mask_token),
                         ((16, 21), '''allen'''),
                         ((21, 23), '''##nl'''),
                         ((23, 24), '''##p'''),
                         ((25, 33), '''sentence'''),
                         ((33, 34), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]  ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
                 self.assertEqual([e[0] for e in expected_results]  ,    tokens['''offset_mapping''']   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]   ):





         """simple docstring"""



         __lowerCamelCase          =  ['''的''', '''人''', '''有''']
         __lowerCamelCase          =  ''''''.join(a   )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                 __lowerCamelCase          =  True
                 __lowerCamelCase          =  self.tokenizer_class.from_pretrained(a  ,    **a   )
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  tokenizer_p.encode(a  ,    add_special_tokens=a   )
                 __lowerCamelCase          =  tokenizer_r.encode(a  ,    add_special_tokens=a   )

                 __lowerCamelCase          =  tokenizer_r.convert_ids_to_tokens(a   )
                 __lowerCamelCase          =  tokenizer_p.convert_ids_to_tokens(a   )

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(a  ,    a   )
                 self.assertListEqual(a  ,    a   )

                 __lowerCamelCase          =  False
                 __lowerCamelCase          =  self.rust_tokenizer_class.from_pretrained(a  ,    **a   )
                 __lowerCamelCase          =  self.tokenizer_class.from_pretrained(a  ,    **a   )

                 __lowerCamelCase          =  tokenizer_r.encode(a  ,    add_special_tokens=a   )
                 __lowerCamelCase          =  tokenizer_p.encode(a  ,    add_special_tokens=a   )

                 __lowerCamelCase          =  tokenizer_r.convert_ids_to_tokens(a   )
                 __lowerCamelCase          =  tokenizer_p.convert_ids_to_tokens(a   )

                 # it is expected that only the first Chinese character is not preceded by "##".
                 __lowerCamelCase          =  [
                     f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a   )
                 ]
                 self.assertListEqual(a  ,    a   )
                 self.assertListEqual(a  ,    a   )

'''simple docstring'''




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_ta import TaTokenizer
else:
  __UpperCAmelCase												=None


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												={"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase												={
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/tokenizer.json",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/tokenizer.json",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/tokenizer.json",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/tokenizer.json",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/tokenizer.json",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCAmelCase												={
    "t5-small": 5_1_2,
    "t5-base": 5_1_2,
    "t5-large": 5_1_2,
    "t5-3b": 5_1_2,
    "t5-11b": 5_1_2,
}




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					Dict							=VOCAB_FILES_NAMES
     lowerCamelCase  :					Optional[Any]							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase  :					Dict							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase  :					int							=["input_ids", "attention_mask"]
     lowerCamelCase  :					Tuple							=TaTokenizer

     lowerCamelCase  :					List[int]							=[]






     def __init__(			self		:		int  ,    a		:		List[str]=None  ,    a		:		List[str]=None  ,    a		:		Dict="</s>"  ,    a		:		Optional[int]="<unk>"  ,    a		:		Any="<pad>"  ,    a		:		Optional[Any]=1_00  ,    a		:		List[Any]=None  ,    **a		:		Union[str, Any]  ,    ):





         """simple docstring"""



         if extra_ids > 0 and additional_special_tokens is None:
             __lowerCamelCase          =  [f"""<extra_id_{i}>""" for i in range(a   )]
         elif extra_ids > 0 and additional_special_tokens is not None:
             # Check that we have the right number of extra special tokens
             __lowerCamelCase          =  len(set(filter(lambda a   : bool('''extra_id_''' in str(a   )   )  ,    a   )   )   )
             if extra_tokens != extra_ids:
                 raise ValueError(
                     f"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
                     ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                     ''' tokens'''   )

         super().__init__(
             a  ,    tokenizer_file=a  ,    eos_token=a  ,    unk_token=a  ,    pad_token=a  ,    extra_ids=a  ,    additional_special_tokens=a  ,    **a  ,    )

         __lowerCamelCase          =  vocab_file
         __lowerCamelCase          =  False if not self.vocab_file else True
         __lowerCamelCase          =  extra_ids






     @staticmethod
     def        SCREAMING_SNAKE_CASE__       (			a		:		Optional[int]  ,    a		:		List[str]  ,    a		:		Union[str, Any]   ):





         """simple docstring"""



         if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
             __lowerCamelCase          =  TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
             if init_max_model_length is not None and init_max_model_length != max_model_length:
                 return init_max_model_length
             elif init_max_model_length is None:
                 warnings.warn(
                     '''This tokenizer was incorrectly instantiated with a model max length of'''
                     f""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
                     ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                     ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                     f""" {pretrained_model_name_or_path} automatically truncating your input to"""
                     f""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
                     f""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
                     ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                     ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''  ,    a  ,    )

         return max_model_length






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple  ,    a		:		str  ,    a		:		Optional[str] = None   ):





         """simple docstring"""



         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                 '''tokenizer.'''   )

         if not os.path.isdir(a   ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""   )
             return
         __lowerCamelCase          =  os.path.join(
             a  ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(a   ):
             copyfile(self.vocab_file  ,    a   )
             logger.info(f"""Copy vocab file to {out_vocab_file}"""   )

         return (out_vocab_file,)






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         __lowerCamelCase          =  token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a
         else:
             __lowerCamelCase          =  token_ids_a + [self.eos_token_id]
             return self.prefix_tokens + token_ids_a + token_ids_a






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         __lowerCamelCase          =  [self.eos_token_id]

         if token_ids_a is None:
             return len(token_ids_a + eos   ) * [0]
         return len(token_ids_a + eos + token_ids_a + eos   ) * [0]






     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         return list(
             set(filter(lambda a   : bool(re.search(R'''<extra_id_\d+>'''  ,    a   )   ) is not None  ,    self.additional_special_tokens   )   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]   ):





         """simple docstring"""



         return [self.convert_tokens_to_ids(a   ) for token in self.get_sentinel_tokens()]
