





"""simple docstring"""




from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest

@require_tokenizers
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    List[Any]							=   DistilBertTokenizer
      A_						:    int							=   DistilBertTokenizerFast
      A_						:    str							=   True


      @slow
      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         __snake_case								=       DistilBertTokenizer.from_pretrained('''distilbert-base-uncased'''				)

         __snake_case								=       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=a__				)
         __snake_case								=       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=a__				)

         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__				)
         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__    ,      a__				)

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
             tokenizer.sep_token_id
         ]








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_           =      {}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_           =      ['NllbTokenizer']

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_           =      ['NllbTokenizerFast']


if TYPE_CHECKING:
    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_nllb import NllbTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_nllb_fast import NllbTokenizerFast

else:
    import sys

    snake_case_           =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

