


"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast





class  _a							(    unittest.TestCase):




	def 							UpperCAmelCase__( self						:		Tuple		)->	Union[str, Any]:
			lowerCAmelCase__       :      str   					= mock.Mock()
			lowerCAmelCase__       :      Dict   					= 500
			lowerCAmelCase__       :      Dict   					= {}
			lowerCAmelCase__       :      List[Any]   					= HTTPError
			lowerCAmelCase__       :      Tuple   					= {}

			# Download this model to make sure it's in the cache.
			lowerCAmelCase__       :      Any   					= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)

			# Under the mock environment we get a 500 error when trying to reach the tokenizer.
			with mock.patch('''requests.Session.request'''		, return_value=lowerCamelCase__		) as mock_head:
					lowerCAmelCase__       :      Optional[Any]   					= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
					# This check we did call the fake head request
					mock_head.assert_called()




	@require_tokenizers
	def 							UpperCAmelCase__( self						:		str		)->	List[Any]:
			lowerCAmelCase__       :      Optional[Any]   					= mock.Mock()
			lowerCAmelCase__       :      List[Any]   					= 500
			lowerCAmelCase__       :      Optional[Any]   					= {}
			lowerCAmelCase__       :      Any   					= HTTPError
			lowerCAmelCase__       :      Dict   					= {}

			# Download this model to make sure it's in the cache.
			lowerCAmelCase__       :      List[Any]   					= GPTaTokenizerFast.from_pretrained('''gpt2'''		)

			# Under the mock environment we get a 500 error when trying to reach the tokenizer.
			with mock.patch('''requests.Session.request'''		, return_value=lowerCamelCase__		) as mock_head:
					lowerCAmelCase__       :      List[str]   					= GPTaTokenizerFast.from_pretrained('''gpt2'''		)
					# This check we did call the fake head request
					mock_head.assert_called()




	def 							UpperCAmelCase__( self						:		Tuple		)->	Optional[int]:






			try:
					lowerCAmelCase__       :      List[Any]   					= tempfile.mktemp()
					with open(lowerCamelCase__		, '''wb'''		) as f:
							http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		, lowerCamelCase__		)

					lowerCAmelCase__       :      List[Any]   					= AlbertTokenizer.from_pretrained(lowerCamelCase__		)
			finally:
					os.remove(lowerCamelCase__		)

			# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
			# the current folder and have the right name.
			if os.path.isfile('''tokenizer.json'''		):
					# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
					return
			try:
					with open('''tokenizer.json'''		, '''wb'''		) as f:
							http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''		, lowerCamelCase__		)
					lowerCAmelCase__       :      List[str]   					= AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''		)
					# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
					self.assertEqual(tokenizer.vocab_size		, 1000		)
					# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

			finally:
					os.remove('''tokenizer.json'''		)




	def 							UpperCAmelCase__( self						:		Optional[Any]		)->	List[str]:
			lowerCAmelCase__       :      Optional[int]   					= AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		)





@is_staging_test
class  _a							(    unittest.TestCase):
	_a  :  Optional[int]				   =							["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




	@classmethod
	def 							UpperCAmelCase__( cls						:		Optional[Any]		)->	Dict:
			lowerCAmelCase__       :      Any   					= TOKEN
			HfFolder.save_token(lowerCamelCase__		)




	@classmethod
	def 							UpperCAmelCase__( cls						:		Optional[Any]		)->	List[Any]:






			try:
					delete_repo(token=cls._token		, repo_id='''test-tokenizer'''		)
			except HTTPError:
					pass

			try:
					delete_repo(token=cls._token		, repo_id='''valid_org/test-tokenizer-org'''		)
			except HTTPError:
					pass

			try:
					delete_repo(token=cls._token		, repo_id='''test-dynamic-tokenizer'''		)
			except HTTPError:
					pass




	def 							UpperCAmelCase__( self						:		Optional[Any]		)->	List[Any]:






			with tempfile.TemporaryDirectory() as tmp_dir:
					lowerCAmelCase__       :      Optional[int]   					= os.path.join(lowerCamelCase__		, '''vocab.txt'''		)
					with open(lowerCamelCase__		, '''w'''		, encoding='''utf-8'''		) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
					lowerCAmelCase__       :      Optional[Any]   					= BertTokenizer(lowerCamelCase__		)

			tokenizer.push_to_hub('''test-tokenizer'''		, use_auth_token=self._token		)
			lowerCAmelCase__       :      Any   					= BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab		)

			# Reset repo
			delete_repo(token=self._token		, repo_id='''test-tokenizer'''		)

			# Push to hub via save_pretrained
			with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(lowerCamelCase__		, repo_id='''test-tokenizer'''		, push_to_hub=lowerCamelCase__		, use_auth_token=self._token		)

			lowerCAmelCase__       :      Dict   					= BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab		)




	def 							UpperCAmelCase__( self						:		Optional[int]		)->	str:






			with tempfile.TemporaryDirectory() as tmp_dir:
					lowerCAmelCase__       :      Optional[int]   					= os.path.join(lowerCamelCase__		, '''vocab.txt'''		)
					with open(lowerCamelCase__		, '''w'''		, encoding='''utf-8'''		) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
					lowerCAmelCase__       :      List[str]   					= BertTokenizer(lowerCamelCase__		)

			tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''		, use_auth_token=self._token		)
			lowerCAmelCase__       :      Optional[int]   					= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab		)

			# Reset repo
			delete_repo(token=self._token		, repo_id='''valid_org/test-tokenizer-org'''		)

			# Push to hub via save_pretrained
			with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(
					    lowerCamelCase__		, repo_id='''valid_org/test-tokenizer-org'''		, push_to_hub=lowerCamelCase__		, use_auth_token=self._token		)

			lowerCAmelCase__       :      Any   					= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab		)




	@require_tokenizers
	def 							UpperCAmelCase__( self						:		List[str]		)->	Optional[Any]:
			CustomTokenizer.register_for_auto_class()
			with tempfile.TemporaryDirectory() as tmp_dir:
					lowerCAmelCase__       :      Dict   					= os.path.join(lowerCamelCase__		, '''vocab.txt'''		)
					with open(lowerCamelCase__		, '''w'''		, encoding='''utf-8'''		) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
					lowerCAmelCase__       :      int   					= CustomTokenizer(lowerCamelCase__		)

			# No fast custom tokenizer
			tokenizer.push_to_hub('''test-dynamic-tokenizer'''		, use_auth_token=self._token		)

			lowerCAmelCase__       :      Dict   					= AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		, trust_remote_code=lowerCamelCase__		)
			# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizer'''		)

			# Fast and slow custom tokenizer
			CustomTokenizerFast.register_for_auto_class()
			with tempfile.TemporaryDirectory() as tmp_dir:
					lowerCAmelCase__       :      int   					= os.path.join(lowerCamelCase__		, '''vocab.txt'''		)
					with open(lowerCamelCase__		, '''w'''		, encoding='''utf-8'''		) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)

					lowerCAmelCase__       :      Any   					= BertTokenizerFast.from_pretrained(lowerCamelCase__		)
					bert_tokenizer.save_pretrained(lowerCamelCase__		)
					lowerCAmelCase__       :      str   					= CustomTokenizerFast.from_pretrained(lowerCamelCase__		)

			tokenizer.push_to_hub('''test-dynamic-tokenizer'''		, use_auth_token=self._token		)

			lowerCAmelCase__       :      Optional[int]   					= AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		, trust_remote_code=lowerCamelCase__		)
			# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizerFast'''		)
			lowerCAmelCase__       :      List[str]   					= AutoTokenizer.from_pretrained(
			    F'{USER}/test-dynamic-tokenizer'		, use_fast=lowerCamelCase__		, trust_remote_code=lowerCamelCase__		)
			# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizer'''		)





class  _a							(    unittest.TestCase):




	def 							UpperCAmelCase__( self						:		Tuple		)->	Optional[int]:
			lowerCAmelCase__       :      str   					= Trie()
			trie.add('''Hello 友達'''		)
			self.assertEqual(trie.data		, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)
			trie.add('''Hello'''		)
			trie.data
			self.assertEqual(trie.data		, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)




	def 							UpperCAmelCase__( self						:		str		)->	List[str]:
			lowerCAmelCase__       :      Tuple   					= Trie()
			self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)		, ['''[CLS] This is a extra_id_100''']		)
			trie.add('''[CLS]'''		)
			trie.add('''extra_id_1'''		)
			trie.add('''extra_id_100'''		)
			self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)		, ['''[CLS]''', ''' This is a ''', '''extra_id_100''']		)




	def 							UpperCAmelCase__( self						:		Tuple		)->	str:
			lowerCAmelCase__       :      int   					= Trie()
			trie.add('''A'''		)
			self.assertEqual(trie.split('''ABC'''		)		, ['''A''', '''BC''']		)
			self.assertEqual(trie.split('''BCA'''		)		, ['''BC''', '''A''']		)




	def 							UpperCAmelCase__( self						:		Any		)->	Any:
			lowerCAmelCase__       :      str   					= Trie()
			trie.add('''TOKEN]'''		)
			trie.add('''[SPECIAL_TOKEN]'''		)
			self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)		, ['''This is something ''', '''[SPECIAL_TOKEN]''']		)




	def 							UpperCAmelCase__( self						:		List[str]		)->	List[Any]:
			lowerCAmelCase__       :      List[Any]   					= Trie()
			trie.add('''A'''		)
			trie.add('''P'''		)
			trie.add('''[SPECIAL_TOKEN]'''		)
			self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)		, ['''This is something ''', '''[SPECIAL_TOKEN]''']		)




	def 							UpperCAmelCase__( self						:		int		)->	Optional[int]:
			lowerCAmelCase__       :      Any   					= Trie()
			trie.add('''AB'''		)
			trie.add('''B'''		)
			trie.add('''C'''		)
			self.assertEqual(trie.split('''ABC'''		)		, ['''AB''', '''C''']		)




	def 							UpperCAmelCase__( self						:		Optional[int]		)->	Optional[Any]:
			lowerCAmelCase__       :      List[Any]   					= Trie()
			trie.add('''ABC'''		)
			trie.add('''B'''		)
			trie.add('''CD'''		)
			self.assertEqual(trie.split('''ABCD'''		)		, ['''ABC''', '''D''']		)




	def 							UpperCAmelCase__( self						:		Union[str, Any]		)->	Optional[Any]:
			lowerCAmelCase__       :      Union[str, Any]   					= Trie()
			lowerCAmelCase__       :      Union[str, Any]   					= trie.cut_text('''ABC'''		, [0, 0, 2, 1, 2, 3]		)
			self.assertEqual(lowerCamelCase__		, ['''AB''', '''C''']		)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase						=		{'''configuration_beit''': ['''BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BeitConfig''', '''BeitOnnxConfig''']}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase						=		['''BeitFeatureExtractor''']
		lowerCamelCase						=		['''BeitImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase						=		[
		    '''BEIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''BeitForImageClassification''',
		    '''BeitForMaskedImageModeling''',
		    '''BeitForSemanticSegmentation''',
		    '''BeitModel''',
		    '''BeitPreTrainedModel''',
		]


try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase						=		[
		    '''FlaxBeitForImageClassification''',
		    '''FlaxBeitForMaskedImageModeling''',
		    '''FlaxBeitModel''',
		    '''FlaxBeitPreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig, BeitOnnxConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_beit import BeitFeatureExtractor
				from .image_processing_beit import BeitImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_beit import (
				    BEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BeitForImageClassification,
				    BeitForMaskedImageModeling,
				    BeitForSemanticSegmentation,
				    BeitModel,
				    BeitPreTrainedModel,
				)

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_beit import (
				    FlaxBeitForImageClassification,
				    FlaxBeitForMaskedImageModeling,
				    FlaxBeitModel,
				    FlaxBeitPreTrainedModel,
				)


else:
		import sys

		lowerCamelCase						=		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




