






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase     :							Dict    =logging.get_logger(__name__)

lowerCamelCase     :							Union[str, Any]    ={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase     :							int    =[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase     :							Dict    ={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase     :							str    ={F"""funnel-transformer/{name}""": 512 for name in _model_names}
lowerCamelCase     :							int    ={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}





class       __a  (   A__	):
 _lowerCAmelCase	:	Union[str, Any]				=			VOCAB_FILES_NAMES
 _lowerCAmelCase	:	Optional[int]				=			PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase	:	List[str]				=			PRETRAINED_INIT_CONFIGURATION
 _lowerCAmelCase	:	Optional[int]				=			FunnelTokenizer
 _lowerCAmelCase	:	Optional[Any]				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase	:	int				=			2







 def __init__(		self :       Union[str, Any]							,       SCREAMING_SNAKE_CASE :       Tuple=None							,       SCREAMING_SNAKE_CASE :       Union[str, Any]=None							,       SCREAMING_SNAKE_CASE :       Any=True							,       SCREAMING_SNAKE_CASE :       Dict="<unk>"							,       SCREAMING_SNAKE_CASE :       str="<sep>"							,       SCREAMING_SNAKE_CASE :       Union[str, Any]="<pad>"							,       SCREAMING_SNAKE_CASE :       Any="<cls>"							,       SCREAMING_SNAKE_CASE :       List[Any]="<mask>"							,       SCREAMING_SNAKE_CASE :       List[str]="<s>"							,       SCREAMING_SNAKE_CASE :       Tuple="</s>"							,       SCREAMING_SNAKE_CASE :       List[Any]=True							,       SCREAMING_SNAKE_CASE :       List[str]=True							,       SCREAMING_SNAKE_CASE :       Optional[Any]=None							,       SCREAMING_SNAKE_CASE :       Optional[Any]="##"							,       **SCREAMING_SNAKE_CASE :       Optional[Any]							,       ):


      '''simple docstring'''






      super().__init__(
          SCREAMING_SNAKE_CASE							,       tokenizer_file=SCREAMING_SNAKE_CASE							,       do_lower_case=SCREAMING_SNAKE_CASE							,       unk_token=SCREAMING_SNAKE_CASE							,       sep_token=SCREAMING_SNAKE_CASE							,       pad_token=SCREAMING_SNAKE_CASE							,       cls_token=SCREAMING_SNAKE_CASE							,       mask_token=SCREAMING_SNAKE_CASE							,       bos_token=SCREAMING_SNAKE_CASE							,       eos_token=SCREAMING_SNAKE_CASE							,       clean_text=SCREAMING_SNAKE_CASE							,       tokenize_chinese_chars=SCREAMING_SNAKE_CASE							,       strip_accents=SCREAMING_SNAKE_CASE							,       wordpieces_prefix=SCREAMING_SNAKE_CASE							,       **SCREAMING_SNAKE_CASE							,       )

      UpperCamelCase__      :       Union[str, Any]	    =  json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
      if (
          normalizer_state.get("lowercase"							,       SCREAMING_SNAKE_CASE						) != do_lower_case
          or normalizer_state.get("strip_accents"							,       SCREAMING_SNAKE_CASE						) != strip_accents
          or normalizer_state.get("handle_chinese_chars"							,       SCREAMING_SNAKE_CASE						) != tokenize_chinese_chars
      ):
           UpperCamelCase__      :       str	    =  getattr(SCREAMING_SNAKE_CASE							,       normalizer_state.pop("type"						)						)
           UpperCamelCase__      :       str	    =  do_lower_case
           UpperCamelCase__      :       Optional[int]	    =  strip_accents
           UpperCamelCase__      :       Union[str, Any]	    =  tokenize_chinese_chars
           UpperCamelCase__      :       Optional[Any]	    =  normalizer_class(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__      :       str	    =  do_lower_case







 def  __lowercase (		self :       Optional[Any]							,       SCREAMING_SNAKE_CASE :       Dict							,       SCREAMING_SNAKE_CASE :       List[Any]=None						):


      '''simple docstring'''






      UpperCamelCase__      :       Any	    =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output







 def  __lowercase (		self :       int							,       SCREAMING_SNAKE_CASE :       List[int]							,       SCREAMING_SNAKE_CASE :       Optional[List[int]] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  [self.sep_token_id]
      UpperCamelCase__      :       Tuple	    =  [self.cls_token_id]
      if token_ids_a is None:
           return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
      return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







 def  __lowercase (		self :       Optional[int]							,       SCREAMING_SNAKE_CASE :       str							,       SCREAMING_SNAKE_CASE :       Optional[str] = None						):


      '''simple docstring'''






      UpperCamelCase__      :       Tuple	    =  self._tokenizer.model.save(SCREAMING_SNAKE_CASE							,       name=SCREAMING_SNAKE_CASE						)
      return tuple(SCREAMING_SNAKE_CASE						)







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import GLPNImageProcessor





class       __a  (   unittest.TestCase	):







 def __init__(		self :       Tuple							,       SCREAMING_SNAKE_CASE :       Optional[int]							,       SCREAMING_SNAKE_CASE :       List[Any]=7							,       SCREAMING_SNAKE_CASE :       str=3							,       SCREAMING_SNAKE_CASE :       Dict=18							,       SCREAMING_SNAKE_CASE :       str=30							,       SCREAMING_SNAKE_CASE :       Union[str, Any]=4_00							,       SCREAMING_SNAKE_CASE :       int=True							,       SCREAMING_SNAKE_CASE :       Any=32							,       SCREAMING_SNAKE_CASE :       int=True							,       ):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[int]	    =  parent
      UpperCamelCase__      :       Dict	    =  batch_size
      UpperCamelCase__      :       Optional[Any]	    =  num_channels
      UpperCamelCase__      :       int	    =  image_size
      UpperCamelCase__      :       Dict	    =  min_resolution
      UpperCamelCase__      :       List[str]	    =  max_resolution
      UpperCamelCase__      :       Union[str, Any]	    =  do_resize
      UpperCamelCase__      :       Optional[Any]	    =  size_divisor
      UpperCamelCase__      :       int	    =  do_rescale







 def  __lowercase (		self :       Union[str, Any]						):


      '''simple docstring'''






      return {
          "do_resize": self.do_resize,
          "size_divisor": self.size_divisor,
          "do_rescale": self.do_rescale,
      }







@require_torch
@require_vision
class       __a  (   A__	,       unittest.TestCase	):
 _lowerCAmelCase	:	List[Any]				=			GLPNImageProcessor if is_vision_available() else None







 def  __lowercase (		self :       Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__      :       Tuple	    =  GLPNImageProcessingTester(self						)







 @property
 def  __lowercase (		self :       int						):


      '''simple docstring'''






      return self.image_processor_tester.prepare_image_processor_dict()







 def  __lowercase (		self :       List[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[Any]	    =  self.image_processing_class(**self.image_processor_dict						)
      self.assertTrue(hasattr(SCREAMING_SNAKE_CASE							,       "do_resize"						)						)
      self.assertTrue(hasattr(SCREAMING_SNAKE_CASE							,       "size_divisor"						)						)
      self.assertTrue(hasattr(SCREAMING_SNAKE_CASE							,       "resample"						)						)
      self.assertTrue(hasattr(SCREAMING_SNAKE_CASE							,       "do_rescale"						)						)







 def  __lowercase (		self :       List[str]						):


      '''simple docstring'''






      pass







 def  __lowercase (		self :       Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       Any	    =  self.image_processing_class(**self.image_processor_dict						)
      # create random PIL images
      UpperCamelCase__      :       Tuple	    =  prepare_image_inputs(self.image_processor_tester							,       equal_resolution=SCREAMING_SNAKE_CASE						)
      for image in image_inputs:
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,       Image.Image						)

      # Test not batched input (GLPNImageProcessor doesn't support batching)
      UpperCamelCase__      :       List[Any]	    =  image_processing(image_inputs[0]							,       return_tensors="pt"						).pixel_values
      self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
      self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)







 def  __lowercase (		self :       Any						):


      '''simple docstring'''






      UpperCamelCase__      :       Optional[Any]	    =  self.image_processing_class(**self.image_processor_dict						)
      # create random numpy tensors
      UpperCamelCase__      :       Optional[Any]	    =  prepare_image_inputs(self.image_processor_tester							,       equal_resolution=SCREAMING_SNAKE_CASE							,       numpify=SCREAMING_SNAKE_CASE						)
      for image in image_inputs:
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,       np.ndarray						)

      # Test not batched input (GLPNImageProcessor doesn't support batching)
      UpperCamelCase__      :       str	    =  image_processing(image_inputs[0]							,       return_tensors="pt"						).pixel_values
      self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
      self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)







 def  __lowercase (		self :       Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__      :       int	    =  self.image_processing_class(**self.image_processor_dict						)
      # create random PyTorch tensors
      UpperCamelCase__      :       Tuple	    =  prepare_image_inputs(self.image_processor_tester							,       equal_resolution=SCREAMING_SNAKE_CASE							,       torchify=SCREAMING_SNAKE_CASE						)
      for image in image_inputs:
           self.assertIsInstance(SCREAMING_SNAKE_CASE							,       torch.Tensor						)

      # Test not batched input (GLPNImageProcessor doesn't support batching)
      UpperCamelCase__      :       Dict	    =  image_processing(image_inputs[0]							,       return_tensors="pt"						).pixel_values
      self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
      self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)