


'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


A				:					Union[str, Any]    						=			logging.get_logger(__name__)

A				:					Any    						=			{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A				:					Optional[int]    						=			[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

A				:					Any    						=			{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
A				:					Tuple    						=			{f"""funnel-transformer/{name}""": 512 for name in _model_names}
A				:					Tuple    						=			{f"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}


class 			__lowerCamelCase			(	a_	):
    """simple docstring"""
    a													=   VOCAB_FILES_NAMES
    a													=   PRETRAINED_VOCAB_FILES_MAP
    a													=   PRETRAINED_INIT_CONFIGURATION
    a													=   FunnelTokenizer
    a													=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a													=   2


    def __init__(  self      :							Dict				,							SCREAMING_SNAKE_CASE      :							List[str]=None				,							SCREAMING_SNAKE_CASE      :							List[Any]=None				,							SCREAMING_SNAKE_CASE      :							str=True				,							SCREAMING_SNAKE_CASE      :							Union[str, Any]="<unk>"				,							SCREAMING_SNAKE_CASE      :							int="<sep>"				,							SCREAMING_SNAKE_CASE      :							List[Any]="<pad>"				,							SCREAMING_SNAKE_CASE      :							Tuple="<cls>"				,							SCREAMING_SNAKE_CASE      :							int="<mask>"				,							SCREAMING_SNAKE_CASE      :							Any="<s>"				,							SCREAMING_SNAKE_CASE      :							List[Any]="</s>"				,							SCREAMING_SNAKE_CASE      :							Optional[Any]=True				,							SCREAMING_SNAKE_CASE      :							str=True				,							SCREAMING_SNAKE_CASE      :							str=None				,							SCREAMING_SNAKE_CASE      :							Optional[Any]="##"				,							**SCREAMING_SNAKE_CASE      :							str				,							):
           super().__init__(
               SCREAMING_SNAKE_CASE				,							tokenizer_file=SCREAMING_SNAKE_CASE				,							do_lower_case=SCREAMING_SNAKE_CASE				,							unk_token=SCREAMING_SNAKE_CASE				,							sep_token=SCREAMING_SNAKE_CASE				,							pad_token=SCREAMING_SNAKE_CASE				,							cls_token=SCREAMING_SNAKE_CASE				,							mask_token=SCREAMING_SNAKE_CASE				,							bos_token=SCREAMING_SNAKE_CASE				,							eos_token=SCREAMING_SNAKE_CASE				,							clean_text=SCREAMING_SNAKE_CASE				,							tokenize_chinese_chars=SCREAMING_SNAKE_CASE				,							strip_accents=SCREAMING_SNAKE_CASE				,							wordpieces_prefix=SCREAMING_SNAKE_CASE				,							**SCREAMING_SNAKE_CASE				,							)

           _A					:     Tuple     = json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('lowercase'				,							SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('strip_accents'				,							SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,							SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
                  _A					:     Optional[int]     = getattr(SCREAMING_SNAKE_CASE				,							normalizer_state.pop('type'))
                  _A					:     Optional[int]     = do_lower_case
                  _A					:     Tuple     = strip_accents
                  _A					:     str     = tokenize_chinese_chars
                  _A					:     Dict     = normalizer_class(**SCREAMING_SNAKE_CASE)

           _A					:     Union[str, Any]     = do_lower_case


    def        A   (  self      :							Optional[Any]				,							SCREAMING_SNAKE_CASE      :							str				,							SCREAMING_SNAKE_CASE      :							Any=None):
           _A					:     Optional[Any]     = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output


    def        A   (  self      :							Union[str, Any]				,							SCREAMING_SNAKE_CASE      :							List[int]				,							SCREAMING_SNAKE_CASE      :							Optional[List[int]] = None):
           _A					:     str     = [self.sep_token_id]
           _A					:     Optional[Any]     = [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
           return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


    def        A   (  self      :							Union[str, Any]				,							SCREAMING_SNAKE_CASE      :							str				,							SCREAMING_SNAKE_CASE      :							Optional[str] = None):
           _A					:     Tuple     = self._tokenizer.model.save(SCREAMING_SNAKE_CASE				,							name=SCREAMING_SNAKE_CASE)
           return tuple(SCREAMING_SNAKE_CASE)










'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A				:					Optional[int]    						=			{
    '''configuration_mask2former''': [
        '''MASK2FORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''Mask2FormerConfig''',
    ],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					Tuple    						=			['''Mask2FormerImageProcessor''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A				:					Dict    						=			[
       '''MASK2FORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''Mask2FormerForUniversalSegmentation''',
       '''Mask2FormerModel''',
       '''Mask2FormerPreTrainedModel''',
   ]

if TYPE_CHECKING:
   from .configuration_maskaformer import MASK2FORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskaFormerConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_maskaformer import MaskaFormerImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_maskaformer import (
          MASK2FORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          MaskaFormerForUniversalSegmentation,
          MaskaFormerModel,
          MaskaFormerPreTrainedModel,
      )


else:
   import sys

   A				:					Optional[Any]    						=			_LazyModule(__name__, globals()['''__file__'''], _import_structure)






