
'''simple docstring'''



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast




class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			mock.Mock()
												_lowerCAmelCase  						=			500
												_lowerCAmelCase  						=			{}
												_lowerCAmelCase  						=			HTTPError
												_lowerCAmelCase  						=			{}

												# Download this model to make sure it's in the cache.
												_lowerCAmelCase  						=			BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("""requests.Session.request"""   ,       return_value=__snake_case				) as mock_head:
																		_lowerCAmelCase  						=			BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)
																		# This check we did call the fake head request
																		mock_head.assert_called()




						@require_tokenizers
						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			mock.Mock()
												_lowerCAmelCase  						=			500
												_lowerCAmelCase  						=			{}
												_lowerCAmelCase  						=			HTTPError
												_lowerCAmelCase  						=			{}

												# Download this model to make sure it's in the cache.
												_lowerCAmelCase  						=			GPTaTokenizerFast.from_pretrained("""gpt2"""				)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("""requests.Session.request"""   ,       return_value=__snake_case				) as mock_head:
																		_lowerCAmelCase  						=			GPTaTokenizerFast.from_pretrained("""gpt2"""				)
																		# This check we did call the fake head request
																		mock_head.assert_called()




						def  _lowercase     ( self				):

												"""simple docstring"""



												try:
																		_lowerCAmelCase  						=			tempfile.mktemp()
																		with open(__snake_case   ,       """wb"""				) as f:
																								http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""   ,       __snake_case				)

																		_lowerCAmelCase  						=			AlbertTokenizer.from_pretrained(__snake_case				)
												finally:
																		os.remove(__snake_case				)

												# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
												# the current folder and have the right name.
												if os.path.isfile("""tokenizer.json"""				):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
												try:
																		with open("""tokenizer.json"""   ,       """wb"""				) as f:
																								http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""   ,       __snake_case				)
																		_lowerCAmelCase  						=			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""				)
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size   ,       1_000				)
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

												finally:
																		os.remove("""tokenizer.json"""				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""				)






@is_staging_test
class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						_lowercase		:				Union[str, Any]									=       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




						@classmethod
						def  _lowercase     ( cls				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TOKEN
												HfFolder.save_token(__snake_case				)




						@classmethod
						def  _lowercase     ( cls				):

												"""simple docstring"""



												try:
																		delete_repo(token=cls._token   ,       repo_id="""test-tokenizer"""				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,       repo_id="""valid_org/test-tokenizer-org"""				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,       repo_id="""test-dynamic-tokenizer"""				)
												except HTTPError:
																		pass




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmp_dir:
																		_lowerCAmelCase  						=			os.path.join(__snake_case   ,       """vocab.txt"""				)
																		with open(__snake_case   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
																		_lowerCAmelCase  						=			BertTokenizer(__snake_case				)

												tokenizer.push_to_hub("""test-tokenizer"""   ,       use_auth_token=self._token				)
												_lowerCAmelCase  						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'				)
												self.assertDictEqual(new_tokenizer.vocab   ,       tokenizer.vocab				)

												# Reset repo
												delete_repo(token=self._token   ,       repo_id="""test-tokenizer"""				)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__snake_case   ,       repo_id="""test-tokenizer"""   ,       push_to_hub=__snake_case   ,       use_auth_token=self._token				)

												_lowerCAmelCase  						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'				)
												self.assertDictEqual(new_tokenizer.vocab   ,       tokenizer.vocab				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmp_dir:
																		_lowerCAmelCase  						=			os.path.join(__snake_case   ,       """vocab.txt"""				)
																		with open(__snake_case   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
																		_lowerCAmelCase  						=			BertTokenizer(__snake_case				)

												tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""   ,       use_auth_token=self._token				)
												_lowerCAmelCase  						=			BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""				)
												self.assertDictEqual(new_tokenizer.vocab   ,       tokenizer.vocab				)

												# Reset repo
												delete_repo(token=self._token   ,       repo_id="""valid_org/test-tokenizer-org"""				)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    __snake_case   ,       repo_id="""valid_org/test-tokenizer-org"""   ,       push_to_hub=__snake_case   ,       use_auth_token=self._token				)

												_lowerCAmelCase  						=			BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""				)
												self.assertDictEqual(new_tokenizer.vocab   ,       tokenizer.vocab				)




						@require_tokenizers
						def  _lowercase     ( self				):

												"""simple docstring"""



												CustomTokenizer.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		_lowerCAmelCase  						=			os.path.join(__snake_case   ,       """vocab.txt"""				)
																		with open(__snake_case   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
																		_lowerCAmelCase  						=			CustomTokenizer(__snake_case				)

												# No fast custom tokenizer
												tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,       use_auth_token=self._token				)

												_lowerCAmelCase  						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'   ,       trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,       """CustomTokenizer"""				)

												# Fast and slow custom tokenizer
												CustomTokenizerFast.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		_lowerCAmelCase  						=			os.path.join(__snake_case   ,       """vocab.txt"""				)
																		with open(__snake_case   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)

																		_lowerCAmelCase  						=			BertTokenizerFast.from_pretrained(__snake_case				)
																		bert_tokenizer.save_pretrained(__snake_case				)
																		_lowerCAmelCase  						=			CustomTokenizerFast.from_pretrained(__snake_case				)

												tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,       use_auth_token=self._token				)

												_lowerCAmelCase  						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'   ,       trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,       """CustomTokenizerFast"""				)
												_lowerCAmelCase  						=			AutoTokenizer.from_pretrained(
												    F'{USER}/test-dynamic-tokenizer'   ,       use_fast=__snake_case   ,       trust_remote_code=__snake_case				)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,       """CustomTokenizer"""				)






class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""Hello 友達"""				)
												self.assertEqual(trie.data   ,       {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}				)
												trie.add("""Hello"""				)
												trie.data
												self.assertEqual(trie.data   ,       {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""				)   ,       ["""[CLS] This is a extra_id_100"""]				)
												trie.add("""[CLS]"""				)
												trie.add("""extra_id_1"""				)
												trie.add("""extra_id_100"""				)
												self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""				)   ,       ["""[CLS]""", """ This is a """, """extra_id_100"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""A"""				)
												self.assertEqual(trie.split("""ABC"""				)   ,       ["""A""", """BC"""]				)
												self.assertEqual(trie.split("""BCA"""				)   ,       ["""BC""", """A"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""TOKEN]"""				)
												trie.add("""[SPECIAL_TOKEN]"""				)
												self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""				)   ,       ["""This is something """, """[SPECIAL_TOKEN]"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""A"""				)
												trie.add("""P"""				)
												trie.add("""[SPECIAL_TOKEN]"""				)
												self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""				)   ,       ["""This is something """, """[SPECIAL_TOKEN]"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""AB"""				)
												trie.add("""B"""				)
												trie.add("""C"""				)
												self.assertEqual(trie.split("""ABC"""				)   ,       ["""AB""", """C"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												trie.add("""ABC"""				)
												trie.add("""B"""				)
												trie.add("""CD"""				)
												self.assertEqual(trie.split("""ABCD"""				)   ,       ["""ABC""", """D"""]				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			Trie()
												_lowerCAmelCase  						=			trie.cut_text("""ABC"""   ,       [0, 0, 2, 1, 2, 3]				)
												self.assertEqual(__snake_case   ,       ["""AB""", """C"""]				)


'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_lowercase    =		{"""configuration_deit""": ["""DEIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DeiTConfig""", """DeiTOnnxConfig"""]}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		["""DeiTFeatureExtractor"""]
						_lowercase    =		["""DeiTImageProcessor"""]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		[
						    """DEIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """DeiTForImageClassification""",
						    """DeiTForImageClassificationWithTeacher""",
						    """DeiTForMaskedImageModeling""",
						    """DeiTModel""",
						    """DeiTPreTrainedModel""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		[
						    """TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFDeiTForImageClassification""",
						    """TFDeiTForImageClassificationWithTeacher""",
						    """TFDeiTForMaskedImageModeling""",
						    """TFDeiTModel""",
						    """TFDeiTPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_deit import DEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, DeiTConfig, DeiTOnnxConfig

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_deit import DeiTFeatureExtractor
												from .image_processing_deit import DeiTImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_deit import (
												    DEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    DeiTForImageClassification,
												    DeiTForImageClassificationWithTeacher,
												    DeiTForMaskedImageModeling,
												    DeiTModel,
												    DeiTPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_deit import (
												    TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFDeiTForImageClassification,
												    TFDeiTForImageClassificationWithTeacher,
												    TFDeiTForMaskedImageModeling,
												    TFDeiTModel,
												    TFDeiTPreTrainedModel,
												)


else:
						import sys

						_lowercase    =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
