



'''simple docstring'''
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast






class 							lowercase__     (							unittest.TestCase       ):





	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       mock.Mock()
							_UpperCamelCase :		List[Any]			    =       500
							_UpperCamelCase :		Optional[Any]			    =       {}
							_UpperCamelCase :		Tuple			    =       HTTPError
							_UpperCamelCase :		Any			    =       {}

							# Download this model to make sure it's in the cache.
							_UpperCamelCase :		Optional[int]			    =       BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'			)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'			,return_value=lowerCamelCase__			) as mock_head:
													_UpperCamelCase :		Dict			    =       BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'			)
													# This check we did call the fake head request
													mock_head.assert_called()





	@require_tokenizers
	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       mock.Mock()
							_UpperCamelCase :		Optional[int]			    =       500
							_UpperCamelCase :		Any			    =       {}
							_UpperCamelCase :		Optional[Any]			    =       HTTPError
							_UpperCamelCase :		Union[str, Any]			    =       {}

							# Download this model to make sure it's in the cache.
							_UpperCamelCase :		Optional[Any]			    =       GPTaTokenizerFast.from_pretrained('gpt2'			)

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'			,return_value=lowerCamelCase__			) as mock_head:
													_UpperCamelCase :		List[str]			    =       GPTaTokenizerFast.from_pretrained('gpt2'			)
													# This check we did call the fake head request
													mock_head.assert_called()





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							# This test is for deprecated behavior and can be removed in v5
							try:
													_UpperCamelCase :		str			    =       tempfile.mktemp()
													with open(lowerCamelCase__			,'wb'			) as f:
																			http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'			,lowerCamelCase__			)

													_UpperCamelCase :		Optional[int]			    =       AlbertTokenizer.from_pretrained(lowerCamelCase__			)
							finally:
													os.remove(lowerCamelCase__			)

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('tokenizer.json'			):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
							try:
													with open('tokenizer.json'			,'wb'			) as f:
																			http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'			,lowerCamelCase__			)
													_UpperCamelCase :		List[Any]			    =       AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'			)
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size			,1000			)
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
													os.remove('tokenizer.json'			)





	def      UpperCamelCase_  (			self :    int			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'			)






@is_staging_test
class 							lowercase__     (							unittest.TestCase       ):
	lowercase__      =      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]





	@classmethod
	def      UpperCamelCase_  (			cls :    int			):







							'''simple docstring'''
							_UpperCamelCase :		List[str]			    =       TOKEN
							HfFolder.save_token(lowerCamelCase__			)





	@classmethod
	def      UpperCamelCase_  (			cls :    Tuple			):







							'''simple docstring'''
							try:
													delete_repo(token=cls._token			,repo_id='test-tokenizer'			)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token			,repo_id='valid_org/test-tokenizer-org'			)
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token			,repo_id='test-dynamic-tokenizer'			)
							except HTTPError:
													pass





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCamelCase :		List[str]			    =       os.path.join(lowerCamelCase__			,'vocab.txt'			)
													with open(lowerCamelCase__			,'w'			,encoding='utf-8'			) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]			)			)
													_UpperCamelCase :		List[Any]			    =       BertTokenizer(lowerCamelCase__			)

							tokenizer.push_to_hub('test-tokenizer'			,use_auth_token=self._token			)
							_UpperCamelCase :		List[Any]			    =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'			)
							self.assertDictEqual(new_tokenizer.vocab			,tokenizer.vocab			)

							# Reset repo
							delete_repo(token=self._token			,repo_id='test-tokenizer'			)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(lowerCamelCase__			,repo_id='test-tokenizer'			,push_to_hub=lowerCamelCase__			,use_auth_token=self._token			)

							_UpperCamelCase :		int			    =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'			)
							self.assertDictEqual(new_tokenizer.vocab			,tokenizer.vocab			)





	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCamelCase :		List[Any]			    =       os.path.join(lowerCamelCase__			,'vocab.txt'			)
													with open(lowerCamelCase__			,'w'			,encoding='utf-8'			) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]			)			)
													_UpperCamelCase :		Any			    =       BertTokenizer(lowerCamelCase__			)

							tokenizer.push_to_hub('valid_org/test-tokenizer-org'			,use_auth_token=self._token			)
							_UpperCamelCase :		Optional[int]			    =       BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'			)
							self.assertDictEqual(new_tokenizer.vocab			,tokenizer.vocab			)

							# Reset repo
							delete_repo(token=self._token			,repo_id='valid_org/test-tokenizer-org'			)

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    lowerCamelCase__			,repo_id='valid_org/test-tokenizer-org'			,push_to_hub=lowerCamelCase__			,use_auth_token=self._token			)

							_UpperCamelCase :		Dict			    =       BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'			)
							self.assertDictEqual(new_tokenizer.vocab			,tokenizer.vocab			)





	@require_tokenizers
	def      UpperCamelCase_  (			self :    str			):







							'''simple docstring'''
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCamelCase :		List[Any]			    =       os.path.join(lowerCamelCase__			,'vocab.txt'			)
													with open(lowerCamelCase__			,'w'			,encoding='utf-8'			) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]			)			)
													_UpperCamelCase :		List[Any]			    =       CustomTokenizer(lowerCamelCase__			)

							# No fast custom tokenizer
							tokenizer.push_to_hub('test-dynamic-tokenizer'			,use_auth_token=self._token			)

							_UpperCamelCase :		Tuple			    =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,trust_remote_code=lowerCamelCase__			)
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__			,'CustomTokenizer'			)

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCamelCase :		Optional[int]			    =       os.path.join(lowerCamelCase__			,'vocab.txt'			)
													with open(lowerCamelCase__			,'w'			,encoding='utf-8'			) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]			)			)

													_UpperCamelCase :		Dict			    =       BertTokenizerFast.from_pretrained(lowerCamelCase__			)
													bert_tokenizer.save_pretrained(lowerCamelCase__			)
													_UpperCamelCase :		Any			    =       CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

							tokenizer.push_to_hub('test-dynamic-tokenizer'			,use_auth_token=self._token			)

							_UpperCamelCase :		Optional[Any]			    =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,trust_remote_code=lowerCamelCase__			)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__			,'CustomTokenizerFast'			)
							_UpperCamelCase :		Any			    =       AutoTokenizer.from_pretrained(
							    F'{USER}/test-dynamic-tokenizer'			,use_fast=lowerCamelCase__			,trust_remote_code=lowerCamelCase__			)
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__			,'CustomTokenizer'			)






class 							lowercase__     (							unittest.TestCase       ):





	def      UpperCamelCase_  (			self :    Tuple			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       Trie()
							trie.add('Hello 友達'			)
							self.assertEqual(trie.data			,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}			)
							trie.add('Hello'			)
							trie.data
							self.assertEqual(trie.data			,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}			)





	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[int]			    =       Trie()
							self.assertEqual(trie.split('[CLS] This is a extra_id_100'			)			,['[CLS] This is a extra_id_100']			)
							trie.add('[CLS]'			)
							trie.add('extra_id_1'			)
							trie.add('extra_id_100'			)
							self.assertEqual(trie.split('[CLS] This is a extra_id_100'			)			,['[CLS]', ' This is a ', 'extra_id_100']			)





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       Trie()
							trie.add('A'			)
							self.assertEqual(trie.split('ABC'			)			,['A', 'BC']			)
							self.assertEqual(trie.split('BCA'			)			,['BC', 'A']			)





	def      UpperCamelCase_  (			self :    List[str]			):







							'''simple docstring'''
							_UpperCamelCase :		str			    =       Trie()
							trie.add('TOKEN]'			)
							trie.add('[SPECIAL_TOKEN]'			)
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'			)			,['This is something ', '[SPECIAL_TOKEN]']			)





	def      UpperCamelCase_  (			self :    str			):







							'''simple docstring'''
							_UpperCamelCase :		Union[str, Any]			    =       Trie()
							trie.add('A'			)
							trie.add('P'			)
							trie.add('[SPECIAL_TOKEN]'			)
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'			)			,['This is something ', '[SPECIAL_TOKEN]']			)





	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       Trie()
							trie.add('AB'			)
							trie.add('B'			)
							trie.add('C'			)
							self.assertEqual(trie.split('ABC'			)			,['AB', 'C']			)





	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		str			    =       Trie()
							trie.add('ABC'			)
							trie.add('B'			)
							trie.add('CD'			)
							self.assertEqual(trie.split('ABCD'			)			,['ABC', 'D']			)





	def      UpperCamelCase_  (			self :    str			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       Trie()
							_UpperCamelCase :		Any			    =       trie.cut_text('ABC'			,[0, 0, 2, 1, 2, 3]			)
							self.assertEqual(lowerCamelCase__			,['AB', 'C']			)










'''simple docstring'''
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import MobileViTImageProcessor






class 							lowercase__     (							unittest.TestCase       ):





	def __init__(			self :    List[str]			,lowerCamelCase__ :    Optional[Any]			,lowerCamelCase__ :    Tuple=7			,lowerCamelCase__ :    Tuple=3			,lowerCamelCase__ :    Any=18			,lowerCamelCase__ :    Tuple=30			,lowerCamelCase__ :    Tuple=400			,lowerCamelCase__ :    int=True			,lowerCamelCase__ :    int=None			,lowerCamelCase__ :    str=True			,lowerCamelCase__ :    str=None			,lowerCamelCase__ :    Any=True			,):







							'''simple docstring'''
							_UpperCamelCase :		Any			    =       size if size is not None else {'shortest_edge': 20}
							_UpperCamelCase :		int			    =       crop_size if crop_size is not None else {'height': 18, 'width': 18}
							_UpperCamelCase :		List[str]			    =       parent
							_UpperCamelCase :		List[Any]			    =       batch_size
							_UpperCamelCase :		int			    =       num_channels
							_UpperCamelCase :		Optional[int]			    =       image_size
							_UpperCamelCase :		List[Any]			    =       min_resolution
							_UpperCamelCase :		List[str]			    =       max_resolution
							_UpperCamelCase :		int			    =       do_resize
							_UpperCamelCase :		Optional[int]			    =       size
							_UpperCamelCase :		str			    =       do_center_crop
							_UpperCamelCase :		List[Any]			    =       crop_size
							_UpperCamelCase :		Optional[Any]			    =       do_flip_channel_order





	def      UpperCamelCase_  (			self :    Optional[Any]			):







							'''simple docstring'''
							return {
							    "do_resize": self.do_resize,
							    "size": self.size,
							    "do_center_crop": self.do_center_crop,
							    "crop_size": self.crop_size,
							    "do_flip_channel_order": self.do_flip_channel_order,
							}






@require_torch
@require_vision
class 							lowercase__     (							lowercase						, unittest.TestCase       ):
	lowercase__      =      MobileViTImageProcessor if is_vision_available() else None





	def      UpperCamelCase_  (			self :    Optional[Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       MobileViTImageProcessingTester(self			)





	@property
	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							return self.image_processor_tester.prepare_image_processor_dict()





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							_UpperCamelCase :		Any			    =       self.image_processing_class(**self.image_processor_dict			)
							self.assertTrue(hasattr(lowerCamelCase__			,'do_resize'			)			)
							self.assertTrue(hasattr(lowerCamelCase__			,'size'			)			)
							self.assertTrue(hasattr(lowerCamelCase__			,'do_center_crop'			)			)
							self.assertTrue(hasattr(lowerCamelCase__			,'center_crop'			)			)
							self.assertTrue(hasattr(lowerCamelCase__			,'do_flip_channel_order'			)			)





	def      UpperCamelCase_  (			self :    List[str]			):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       self.image_processing_class.from_dict(self.image_processor_dict			)
							self.assertEqual(image_processor.size			,{'shortest_edge': 20}			)
							self.assertEqual(image_processor.crop_size			,{'height': 18, 'width': 18}			)

							_UpperCamelCase :		Union[str, Any]			    =       self.image_processing_class.from_dict(self.image_processor_dict			,size=42			,crop_size=84			)
							self.assertEqual(image_processor.size			,{'shortest_edge': 42}			)
							self.assertEqual(image_processor.crop_size			,{'height': 84, 'width': 84}			)





	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							pass





	def      UpperCamelCase_  (			self :    List[str]			):







							'''simple docstring'''
							# Initialize image_processing
							_UpperCamelCase :		int			    =       self.image_processing_class(**self.image_processor_dict			)
							# create random PIL images
							_UpperCamelCase :		int			    =       prepare_image_inputs(self.image_processor_tester			,equal_resolution=lowerCamelCase__			)
							for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__			,Image.Image			)

							# Test not batched input
							_UpperCamelCase :		Optional[Any]			    =       image_processing(image_inputs[0]			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)

							# Test batched
							_UpperCamelCase :		int			    =       image_processing(lowerCamelCase__			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							# Initialize image_processing
							_UpperCamelCase :		int			    =       self.image_processing_class(**self.image_processor_dict			)
							# create random numpy tensors
							_UpperCamelCase :		List[Any]			    =       prepare_image_inputs(self.image_processor_tester			,equal_resolution=lowerCamelCase__			,numpify=lowerCamelCase__			)
							for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__			,np.ndarray			)

							# Test not batched input
							_UpperCamelCase :		Optional[int]			    =       image_processing(image_inputs[0]			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)

							# Test batched
							_UpperCamelCase :		List[str]			    =       image_processing(lowerCamelCase__			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)





	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							# Initialize image_processing
							_UpperCamelCase :		Optional[Any]			    =       self.image_processing_class(**self.image_processor_dict			)
							# create random PyTorch tensors
							_UpperCamelCase :		str			    =       prepare_image_inputs(self.image_processor_tester			,equal_resolution=lowerCamelCase__			,torchify=lowerCamelCase__			)
							for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__			,torch.Tensor			)

							# Test not batched input
							_UpperCamelCase :		str			    =       image_processing(image_inputs[0]			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)

							# Test batched
							_UpperCamelCase :		Optional[int]			    =       image_processing(lowerCamelCase__			,return_tensors='pt'			).pixel_values
							self.assertEqual(
							    encoded_images.shape			,(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,)


