
'''simple docstring'''



import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_lowercase    =		get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_lowercase    =		get_tests_dir("""fixtures/vocab.json""")
_lowercase    =		get_tests_dir("""fixtures""")




class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						_lowercase		:				Dict									=       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			0




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""				)
												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  						=			WavaVecaConfig()
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""				)

																		# save in new folder
																		model_config.save_pretrained(_lowercase				)
																		processor.save_pretrained(_lowercase				)

																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)

												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmpdirname:
																		# copy relevant files
																		copyfile(_lowercase   ,       os.path.join(_lowercase   ,       _lowercase				)				)
																		copyfile(_lowercase   ,       os.path.join(_lowercase   ,       """vocab.json"""				)				)

																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)

												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  						=			WavaVecaFeatureExtractor()
																		_lowerCAmelCase  						=			AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""				)

																		_lowerCAmelCase  						=			WavaVecaProcessor(_lowercase   ,       _lowercase				)

																		# save in new folder
																		processor.save_pretrained(_lowercase				)

																		# drop `processor_class` in tokenizer
																		with open(os.path.join(_lowercase   ,       _lowercase				)   ,       """r"""				) as f:
																								_lowerCAmelCase  						=			json.load(_lowercase				)
																								config_dict.pop("""processor_class"""				)

																		with open(os.path.join(_lowercase   ,       _lowercase				)   ,       """w"""				) as f:
																								f.write(json.dumps(_lowercase				)				)

																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)

												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  						=			WavaVecaFeatureExtractor()
																		_lowerCAmelCase  						=			AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""				)

																		_lowerCAmelCase  						=			WavaVecaProcessor(_lowercase   ,       _lowercase				)

																		# save in new folder
																		processor.save_pretrained(_lowercase				)

																		# drop `processor_class` in feature extractor
																		with open(os.path.join(_lowercase   ,       _lowercase				)   ,       """r"""				) as f:
																								_lowerCAmelCase  						=			json.load(_lowercase				)
																								config_dict.pop("""processor_class"""				)

																		with open(os.path.join(_lowercase   ,       _lowercase				)   ,       """w"""				) as f:
																								f.write(json.dumps(_lowercase				)				)

																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)

												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  						=			WavaVecaConfig(processor_class="""Wav2Vec2Processor"""				)
																		model_config.save_pretrained(_lowercase				)
																		# copy relevant files
																		copyfile(_lowercase   ,       os.path.join(_lowercase   ,       """vocab.json"""				)				)
																		# create emtpy sample processor
																		with open(os.path.join(_lowercase   ,       _lowercase				)   ,       """w"""				) as f:
																								f.write("""{}"""				)

																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)

												self.assertIsInstance(_lowercase   ,       _lowercase				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												with self.assertRaises(_lowercase				):
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""				)
												# If remote code is disabled, we can't load this config.
												with self.assertRaises(_lowercase				):
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(
																		    """hf-internal-testing/test_dynamic_processor"""   ,       trust_remote_code=_lowercase				)

												_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""   ,       trust_remote_code=_lowercase				)
												self.assertTrue(processor.special_attribute_present				)
												self.assertEqual(processor.__class__.__name__   ,       """NewProcessor"""				)

												_lowerCAmelCase  						=			processor.feature_extractor
												self.assertTrue(feature_extractor.special_attribute_present				)
												self.assertEqual(feature_extractor.__class__.__name__   ,       """NewFeatureExtractor"""				)

												_lowerCAmelCase  						=			processor.tokenizer
												self.assertTrue(tokenizer.special_attribute_present				)
												if is_tokenizers_available():
																		self.assertEqual(tokenizer.__class__.__name__   ,       """NewTokenizerFast"""				)

																		# Test we can also load the slow version
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(
																		    """hf-internal-testing/test_dynamic_processor"""   ,       trust_remote_code=_lowercase   ,       use_fast=_lowercase				)
																		_lowerCAmelCase  						=			new_processor.tokenizer
																		self.assertTrue(new_tokenizer.special_attribute_present				)
																		self.assertEqual(new_tokenizer.__class__.__name__   ,       """NewTokenizer"""				)
												else:
																		self.assertEqual(tokenizer.__class__.__name__   ,       """NewTokenizer"""				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												try:
																		AutoConfig.register("""custom"""   ,       _lowercase				)
																		AutoFeatureExtractor.register(_lowercase   ,       _lowercase				)
																		AutoTokenizer.register(_lowercase   ,       slow_tokenizer_class=_lowercase				)
																		AutoProcessor.register(_lowercase   ,       _lowercase				)
																		# Trying to register something existing in the Transformers library will raise an error
																		with self.assertRaises(_lowercase				):
																								AutoProcessor.register(_lowercase   ,       _lowercase				)

																		# Now that the config is registered, it can be used as any other config with the auto-API
																		_lowerCAmelCase  						=			CustomFeatureExtractor.from_pretrained(_lowercase				)

																		with tempfile.TemporaryDirectory() as tmp_dir:
																								_lowerCAmelCase  						=			os.path.join(_lowercase   ,       """vocab.txt"""				)
																								with open(_lowercase   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
																								_lowerCAmelCase  						=			CustomTokenizer(_lowercase				)

																		_lowerCAmelCase  						=			CustomProcessor(_lowercase   ,       _lowercase				)

																		with tempfile.TemporaryDirectory() as tmp_dir:
																								processor.save_pretrained(_lowercase				)
																								_lowerCAmelCase  						=			AutoProcessor.from_pretrained(_lowercase				)
																								self.assertIsInstance(_lowercase   ,       _lowercase				)

												finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																		if CustomConfig in TOKENIZER_MAPPING._extra_content:
																								del TOKENIZER_MAPPING._extra_content[CustomConfig]
																		if CustomConfig in PROCESSOR_MAPPING._extra_content:
																								del PROCESSOR_MAPPING._extra_content[CustomConfig]




						def  _lowercase     ( self				):

												"""simple docstring"""



												class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






																		'''simple docstring'''






																		_lowercase		:				List[str]									=       False






												class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






																		'''simple docstring'''






																		_lowercase		:				int									=       False






												class   UpperCAmelCase_					(		_SCREAMING_SNAKE_CASE			):






																		'''simple docstring'''






																		_lowercase		:				Optional[int]									=       "AutoFeatureExtractor"
																		_lowercase		:				List[Any]									=       "AutoTokenizer"
																		_lowercase		:				Any									=       False

												try:
																		AutoConfig.register("""custom"""   ,       _lowercase				)
																		AutoFeatureExtractor.register(_lowercase   ,       _lowercase				)
																		AutoTokenizer.register(_lowercase   ,       slow_tokenizer_class=_lowercase				)
																		AutoProcessor.register(_lowercase   ,       _lowercase				)
																		# If remote code is not set, the default is to use local classes.
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""				)
																		self.assertEqual(processor.__class__.__name__   ,       """NewProcessor"""				)
																		self.assertFalse(processor.special_attribute_present				)
																		self.assertFalse(processor.feature_extractor.special_attribute_present				)
																		self.assertFalse(processor.tokenizer.special_attribute_present				)

																		# If remote code is disabled, we load the local ones.
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(
																		    """hf-internal-testing/test_dynamic_processor"""   ,       trust_remote_code=_lowercase				)
																		self.assertEqual(processor.__class__.__name__   ,       """NewProcessor"""				)
																		self.assertFalse(processor.special_attribute_present				)
																		self.assertFalse(processor.feature_extractor.special_attribute_present				)
																		self.assertFalse(processor.tokenizer.special_attribute_present				)

																		# If remote is enabled, we load from the Hub.
																		_lowerCAmelCase  						=			AutoProcessor.from_pretrained(
																		    """hf-internal-testing/test_dynamic_processor"""   ,       trust_remote_code=_lowercase				)
																		self.assertEqual(processor.__class__.__name__   ,       """NewProcessor"""				)
																		self.assertTrue(processor.special_attribute_present				)
																		self.assertTrue(processor.feature_extractor.special_attribute_present				)
																		self.assertTrue(processor.tokenizer.special_attribute_present				)

												finally:
																		if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																		if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
																		if CustomConfig in TOKENIZER_MAPPING._extra_content:
																								del TOKENIZER_MAPPING._extra_content[CustomConfig]
																		if CustomConfig in PROCESSOR_MAPPING._extra_content:
																								del PROCESSOR_MAPPING._extra_content[CustomConfig]




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-bert"""				)
												self.assertEqual(processor.__class__.__name__   ,       """BertTokenizerFast"""				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-convnext"""				)
												self.assertEqual(processor.__class__.__name__   ,       """ConvNextImageProcessor"""				)






@is_staging_test
class   UpperCAmelCase_					(		unittest.TestCase			):






						'''simple docstring'''






						_lowercase		:				str									=       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




						@classmethod
						def  _lowercase     ( cls				):

												"""simple docstring"""



												_lowerCAmelCase  						=			TOKEN
												HfFolder.save_token(_lowercase				)




						@classmethod
						def  _lowercase     ( cls				):

												"""simple docstring"""



												try:
																		delete_repo(token=cls._token   ,       repo_id="""test-processor"""				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,       repo_id="""valid_org/test-processor-org"""				)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,       repo_id="""test-dynamic-processor"""				)
												except HTTPError:
																		pass




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			WavaVecaProcessor.from_pretrained(_lowercase				)
												with tempfile.TemporaryDirectory() as tmp_dir:
																		processor.save_pretrained(
																		    os.path.join(_lowercase   ,       """test-processor"""				)   ,       push_to_hub=_lowercase   ,       use_auth_token=self._token				)

																		_lowerCAmelCase  						=			WavaVecaProcessor.from_pretrained(F'{USER}/test-processor'				)
																		for k, v in processor.feature_extractor.__dict__.items():
																								self.assertEqual(_lowercase   ,       getattr(new_processor.feature_extractor   ,       _lowercase				)				)
																		self.assertDictEqual(new_processor.tokenizer.get_vocab()   ,       processor.tokenizer.get_vocab()				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												_lowerCAmelCase  						=			WavaVecaProcessor.from_pretrained(_lowercase				)

												with tempfile.TemporaryDirectory() as tmp_dir:
																		processor.save_pretrained(
																		    os.path.join(_lowercase   ,       """test-processor-org"""				)   ,       push_to_hub=_lowercase   ,       use_auth_token=self._token   ,       organization="""valid_org"""   ,       )

																		_lowerCAmelCase  						=			WavaVecaProcessor.from_pretrained("""valid_org/test-processor-org"""				)
																		for k, v in processor.feature_extractor.__dict__.items():
																								self.assertEqual(_lowercase   ,       getattr(new_processor.feature_extractor   ,       _lowercase				)				)
																		self.assertDictEqual(new_processor.tokenizer.get_vocab()   ,       processor.tokenizer.get_vocab()				)




						def  _lowercase     ( self				):

												"""simple docstring"""



												CustomFeatureExtractor.register_for_auto_class()
												CustomTokenizer.register_for_auto_class()
												CustomProcessor.register_for_auto_class()

												_lowerCAmelCase  						=			CustomFeatureExtractor.from_pretrained(_lowercase				)

												with tempfile.TemporaryDirectory() as tmp_dir:
																		_lowerCAmelCase  						=			os.path.join(_lowercase   ,       """vocab.txt"""				)
																		with open(_lowercase   ,       """w"""   ,       encoding="""utf-8"""				) as vocab_writer:
																								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]				)				)
																		_lowerCAmelCase  						=			CustomTokenizer(_lowercase				)

												_lowerCAmelCase  						=			CustomProcessor(_lowercase   ,       _lowercase				)

												with tempfile.TemporaryDirectory() as tmp_dir:
																		create_repo(F'{USER}/test-dynamic-processor'   ,       token=self._token				)
																		_lowerCAmelCase  						=			Repository(_lowercase   ,       clone_from=F'{USER}/test-dynamic-processor'   ,       token=self._token				)
																		processor.save_pretrained(_lowercase				)

																		# This has added the proper auto_map field to the feature extractor config
																		self.assertDictEqual(
																		    processor.feature_extractor.auto_map   ,       {
																		        """AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor""",
																		        """AutoProcessor""": """custom_processing.CustomProcessor""",
																		    }   ,       )

																		# This has added the proper auto_map field to the tokenizer config
																		with open(os.path.join(_lowercase   ,       """tokenizer_config.json"""				)				) as f:
																								_lowerCAmelCase  						=			json.load(_lowercase				)
																		self.assertDictEqual(
																		    tokenizer_config["""auto_map"""]   ,       {
																		        """AutoTokenizer""": ["""custom_tokenization.CustomTokenizer""", None],
																		        """AutoProcessor""": """custom_processing.CustomProcessor""",
																		    }   ,       )

																		# The code has been copied from fixtures
																		self.assertTrue(os.path.isfile(os.path.join(_lowercase   ,       """custom_feature_extraction.py"""				)				)				)
																		self.assertTrue(os.path.isfile(os.path.join(_lowercase   ,       """custom_tokenization.py"""				)				)				)
																		self.assertTrue(os.path.isfile(os.path.join(_lowercase   ,       """custom_processing.py"""				)				)				)

																		repo.push_to_hub()

												_lowerCAmelCase  						=			AutoProcessor.from_pretrained(F'{USER}/test-dynamic-processor'   ,       trust_remote_code=_lowercase				)
												# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
												self.assertEqual(new_processor.__class__.__name__   ,       """CustomProcessor"""				)


'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_lowercase    =		{"""configuration_deit""": ["""DEIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DeiTConfig""", """DeiTOnnxConfig"""]}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		["""DeiTFeatureExtractor"""]
						_lowercase    =		["""DeiTImageProcessor"""]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		[
						    """DEIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """DeiTForImageClassification""",
						    """DeiTForImageClassificationWithTeacher""",
						    """DeiTForMaskedImageModeling""",
						    """DeiTModel""",
						    """DeiTPreTrainedModel""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase    =		[
						    """TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFDeiTForImageClassification""",
						    """TFDeiTForImageClassificationWithTeacher""",
						    """TFDeiTForMaskedImageModeling""",
						    """TFDeiTModel""",
						    """TFDeiTPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_deit import DEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, DeiTConfig, DeiTOnnxConfig

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_deit import DeiTFeatureExtractor
												from .image_processing_deit import DeiTImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_deit import (
												    DEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    DeiTForImageClassification,
												    DeiTForImageClassificationWithTeacher,
												    DeiTForMaskedImageModeling,
												    DeiTModel,
												    DeiTPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_deit import (
												    TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFDeiTForImageClassification,
												    TFDeiTForImageClassificationWithTeacher,
												    TFDeiTForMaskedImageModeling,
												    TFDeiTModel,
												    TFDeiTPreTrainedModel,
												)


else:
						import sys

						_lowercase    =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
