




'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase__   :				List[Any]      =   logging.get_logger(__name__)

lowercase__   :				Dict      =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__   :				Optional[int]      =   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowercase__   :				Dict      =   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowercase__   :				Dict      =   {f"""funnel-transformer/{name}""": 5_12 for name in _model_names}
lowercase__   :				List[str]      =   {f"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}


class 			SCREAMING_SNAKE_CASE   (lowerCamelCase__							):
     lowerCAmelCase		      =	VOCAB_FILES_NAMES
     lowerCAmelCase		      =	PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase		      =	PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase		      =	FunnelTokenizer
     lowerCAmelCase		      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase		      =	2


     def __init__(						self						,     _UpperCAmelCase=None						,     _UpperCAmelCase=None						,     _UpperCAmelCase=True						,     _UpperCAmelCase="<unk>"						,     _UpperCAmelCase="<sep>"						,     _UpperCAmelCase="<pad>"						,     _UpperCAmelCase="<cls>"						,     _UpperCAmelCase="<mask>"						,     _UpperCAmelCase="<s>"						,     _UpperCAmelCase="</s>"						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=None						,     _UpperCAmelCase="##"						,     **_UpperCAmelCase						,     ):

          '''simple docstring'''






          super().__init__(
              __snake_case						,     tokenizer_file=__snake_case						,     do_lower_case=__snake_case						,     unk_token=__snake_case						,     sep_token=__snake_case						,     pad_token=__snake_case						,     cls_token=__snake_case						,     mask_token=__snake_case						,     bos_token=__snake_case						,     eos_token=__snake_case						,     clean_text=__snake_case						,     tokenize_chinese_chars=__snake_case						,     strip_accents=__snake_case						,     wordpieces_prefix=__snake_case						,     **__snake_case						,     )

          __A						:			Tuple								=	json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              normalizer_state.get('lowercase'						,     __snake_case) != do_lower_case
              or normalizer_state.get('strip_accents'						,     __snake_case) != strip_accents
              or normalizer_state.get('handle_chinese_chars'						,     __snake_case) != tokenize_chinese_chars
          ):
               __A						:			Tuple								=	getattr(__snake_case						,     normalizer_state.pop('type'))
               __A						:			Dict								=	do_lower_case
               __A						:			Any								=	strip_accents
               __A						:			str								=	tokenize_chinese_chars
               __A						:			str								=	normalizer_class(**__snake_case)

          __A						:			Optional[Any]								=	do_lower_case


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase=None):

          '''simple docstring'''






          __A						:			str								=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			Any								=	[self.sep_token_id]
          __A						:			List[str]								=	[self.cls_token_id]
          if token_ids_a is None:
               return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
          return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase = None):

          '''simple docstring'''






          __A						:			List[str]								=	self._tokenizer.model.save(__snake_case						,     name=__snake_case)
          return tuple(__snake_case)






'''simple docstring'''




from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
  if not (is_transformers_available() and is_torch_available()):
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline
else:
  from .camera import create_pan_cameras
  from .pipeline_shap_e import ShapEPipeline
  from .pipeline_shap_e_img2img import ShapEImgaImgPipeline
  from .renderer import (
      BoundingBoxVolume,
      ImportanceRaySampler,
      MLPNeRFModelOutput,
      MLPNeRSTFModel,
      ShapEParamsProjModel,
      ShapERenderer,
      StratifiedRaySampler,
      VoidNeRFModel,
  )