
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase_						     =					logging.get_logger(__name__)

lowerCamelCase_						     =					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_						     =					[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase_						     =					{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase_						     =					{f'funnel-transformer/{name}': 5_12 for name in _model_names}
lowerCamelCase_						     =					{f'funnel-transformer/{name}': {'''do_lower_case''': True} for name in _model_names}




class      __A(    __lowerCamelCase			):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__         =      VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__         =      PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__         =      PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE__         =      FunnelTokenizer
  SCREAMING_SNAKE_CASE__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__         =      2







  def __init__(self	,      SCREAMING_SNAKE_CASE_=None	,      SCREAMING_SNAKE_CASE_=None	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_="<unk>"	,      SCREAMING_SNAKE_CASE_="<sep>"	,      SCREAMING_SNAKE_CASE_="<pad>"	,      SCREAMING_SNAKE_CASE_="<cls>"	,      SCREAMING_SNAKE_CASE_="<mask>"	,      SCREAMING_SNAKE_CASE_="<s>"	,      SCREAMING_SNAKE_CASE_="</s>"	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_=None	,      SCREAMING_SNAKE_CASE_="##"	,      **SCREAMING_SNAKE_CASE_	,      ):
       super().__init__(
           SCREAMING_SNAKE_CASE_	,      tokenizer_file=SCREAMING_SNAKE_CASE_	,      do_lower_case=SCREAMING_SNAKE_CASE_	,      unk_token=SCREAMING_SNAKE_CASE_	,      sep_token=SCREAMING_SNAKE_CASE_	,      pad_token=SCREAMING_SNAKE_CASE_	,      cls_token=SCREAMING_SNAKE_CASE_	,      mask_token=SCREAMING_SNAKE_CASE_	,      bos_token=SCREAMING_SNAKE_CASE_	,      eos_token=SCREAMING_SNAKE_CASE_	,      clean_text=SCREAMING_SNAKE_CASE_	,      tokenize_chinese_chars=SCREAMING_SNAKE_CASE_	,      strip_accents=SCREAMING_SNAKE_CASE_	,      wordpieces_prefix=SCREAMING_SNAKE_CASE_	,      **SCREAMING_SNAKE_CASE_	,      )

       UpperCamelCase__				=    json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
       if (
           normalizer_state.get("""lowercase"""	,      SCREAMING_SNAKE_CASE_						) != do_lower_case
           or normalizer_state.get("""strip_accents"""	,      SCREAMING_SNAKE_CASE_						) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""	,      SCREAMING_SNAKE_CASE_						) != tokenize_chinese_chars
       ):
            UpperCamelCase__				=    getattr(SCREAMING_SNAKE_CASE_	,      normalizer_state.pop("""type"""						)						)
            UpperCamelCase__				=    do_lower_case
            UpperCamelCase__				=    strip_accents
            UpperCamelCase__				=    tokenize_chinese_chars
            UpperCamelCase__				=    normalizer_class(**SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    do_lower_case







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_=None						):
       UpperCamelCase__				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=    [self.sep_token_id]
       UpperCamelCase__				=    [self.cls_token_id]
       if token_ids_a is None:
            return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
       return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=    self._tokenizer.model.save(SCREAMING_SNAKE_CASE_	,      name=SCREAMING_SNAKE_CASE_						)
       return tuple(SCREAMING_SNAKE_CASE_						)







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import GLPNImageProcessor




class      __A(    unittest.TestCase			):

  """simple docstring"""







  def __init__(self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_=7	,      SCREAMING_SNAKE_CASE_=3	,      SCREAMING_SNAKE_CASE_=18	,      SCREAMING_SNAKE_CASE_=30	,      SCREAMING_SNAKE_CASE_=4_00	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_=32	,      SCREAMING_SNAKE_CASE_=True	,      ):
       UpperCamelCase__				=    parent
       UpperCamelCase__				=    batch_size
       UpperCamelCase__				=    num_channels
       UpperCamelCase__				=    image_size
       UpperCamelCase__				=    min_resolution
       UpperCamelCase__				=    max_resolution
       UpperCamelCase__				=    do_resize
       UpperCamelCase__				=    size_divisor
       UpperCamelCase__				=    do_rescale







  def 	UpperCAmelCase_ (self						):
       return {
           "do_resize": self.do_resize,
           "size_divisor": self.size_divisor,
           "do_rescale": self.do_rescale,
       }






@require_torch
@require_vision
class      __A(    __lowerCamelCase    ,				unittest.TestCase			):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__         =      GLPNImageProcessor if is_vision_available() else None







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    GLPNImageProcessingTester(self						)







  @property
  def 	UpperCAmelCase_ (self						):
       return self.image_processor_tester.prepare_image_processor_dict()







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.image_processing_class(**self.image_processor_dict						)
       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_	,      """do_resize"""						)						)
       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_	,      """size_divisor"""						)						)
       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_	,      """resample"""						)						)
       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_	,      """do_rescale"""						)						)







  def 	UpperCAmelCase_ (self						):
       pass







  def 	UpperCAmelCase_ (self						):
       # Initialize image_processing
       UpperCamelCase__				=    self.image_processing_class(**self.image_processor_dict						)
       # create random PIL images
       UpperCamelCase__				=    prepare_image_inputs(self.image_processor_tester	,      equal_resolution=SCREAMING_SNAKE_CASE_						)
       for image in image_inputs:
            self.assertIsInstance(SCREAMING_SNAKE_CASE_	,      Image.Image						)

       # Test not batched input (GLPNImageProcessor doesn't support batching)
       UpperCamelCase__				=    image_processing(image_inputs[0]	,      return_tensors="""pt"""						).pixel_values
       self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
       self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)







  def 	UpperCAmelCase_ (self						):
       # Initialize image_processing
       UpperCamelCase__				=    self.image_processing_class(**self.image_processor_dict						)
       # create random numpy tensors
       UpperCamelCase__				=    prepare_image_inputs(self.image_processor_tester	,      equal_resolution=SCREAMING_SNAKE_CASE_	,      numpify=SCREAMING_SNAKE_CASE_						)
       for image in image_inputs:
            self.assertIsInstance(SCREAMING_SNAKE_CASE_	,      np.ndarray						)

       # Test not batched input (GLPNImageProcessor doesn't support batching)
       UpperCamelCase__				=    image_processing(image_inputs[0]	,      return_tensors="""pt"""						).pixel_values
       self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
       self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)







  def 	UpperCAmelCase_ (self						):
       # Initialize image_processing
       UpperCamelCase__				=    self.image_processing_class(**self.image_processor_dict						)
       # create random PyTorch tensors
       UpperCamelCase__				=    prepare_image_inputs(self.image_processor_tester	,      equal_resolution=SCREAMING_SNAKE_CASE_	,      torchify=SCREAMING_SNAKE_CASE_						)
       for image in image_inputs:
            self.assertIsInstance(SCREAMING_SNAKE_CASE_	,      torch.Tensor						)

       # Test not batched input (GLPNImageProcessor doesn't support batching)
       UpperCamelCase__				=    image_processing(image_inputs[0]	,      return_tensors="""pt"""						).pixel_values
       self.assertTrue(encoded_images.shape[-1] % self.image_processor_tester.size_divisor == 0						)
       self.assertTrue(encoded_images.shape[-2] % self.image_processor_tester.size_divisor == 0						)





