import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase_    :		Optional[int]							=	get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")


@require_sentencepiece
@require_tokenizers
class 						a__							(       __snake_case				,						unittest.TestCase		):
			A__    :       Optional[int]        =      PegasusTokenizer
			A__    :       str        =      PegasusTokenizerFast
			A__    :       Optional[int]        =      True
			A__    :       Union[str, Any]        =      True



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Dict:
								super().setUp()

								# We have a SentencePiece fixture for testing
								__a		 =       PegasusTokenizer(UpperCAmelCase   )
								tokenizer.save_pretrained(self.tmpdirname   )



			@cached_property
			def __SCREAMING_SNAKE_CASE							(						self   )			->   Any:
								return PegasusTokenizer.from_pretrained('google/pegasus-large'   )



			def __SCREAMING_SNAKE_CASE							(						self				,						**UpperCAmelCase   )			->   PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname				,						**UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self				,						UpperCAmelCase   )			->   List[str]:
								return ("This is a test", "This is a test")



			def __SCREAMING_SNAKE_CASE							(						self   )			->   int:
								__a		 =       '</s>'
								__a		 =       1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase   )				,						UpperCAmelCase   )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase   )				,						UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Tuple:
								__a		 =       list(self.get_tokenizer().get_vocab().keys()   )

								self.assertEqual(vocab_keys[0]				,						'<pad>'   )
								self.assertEqual(vocab_keys[1]				,						'</s>'   )
								self.assertEqual(vocab_keys[-1]				,						'v'   )
								self.assertEqual(len(UpperCAmelCase   )				,						1_1_0_3   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Optional[Any]:
								self.assertEqual(self.get_tokenizer().vocab_size				,						1_1_0_3   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Optional[int]:
								__a		 =       self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
								__a		 =       self.tokenizer_class.from_pretrained(self.tmpdirname   )
								__a		 =       (
								    'Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'
								    ' </s> <pad> <pad> <pad>'
								)
								__a		 =       rust_tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase				,						add_special_tokens=UpperCAmelCase   ).input_ids[0]
								__a		 =       py_tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase				,						add_special_tokens=UpperCAmelCase   ).input_ids[0]
								self.assertListEqual(UpperCAmelCase				,						UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   List[Any]:
								__a		 =       self._large_tokenizer
								# <mask_1> masks whole sentence while <mask_2> masks single word
								__a		 =       '<mask_1> To ensure a <mask_2> flow of bank resolutions.'
								__a		 =       [2, 4_1_3, 6_1_5, 1_1_4, 3, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
								__a		 =       tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase   ).input_ids[0]
								self.assertListEqual(UpperCAmelCase				,						UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Tuple:
								__a		 =       self._large_tokenizer
								# The tracebacks for the following asserts are **better** without messages or self.assertEqual
								assert tokenizer.vocab_size == 9_6_1_0_3
								assert tokenizer.pad_token_id == 0
								assert tokenizer.eos_token_id == 1
								assert tokenizer.offset == 1_0_3
								assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_0_5
								assert tokenizer.unk_token == "<unk>"
								assert tokenizer.model_max_length == 1_0_2_4
								__a		 =       'To ensure a smooth flow of bank resolutions.'
								__a		 =       [4_1_3, 6_1_5, 1_1_4, 2_2_9_1, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
								__a		 =       tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase   ).input_ids[0]
								self.assertListEqual(UpperCAmelCase				,						UpperCAmelCase   )
								assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]   ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]



			@require_torch
			def __SCREAMING_SNAKE_CASE							(						self   )			->   List[str]:
								__a		 =       ['This is going to be way too long.' * 1_5_0, 'short example']
								__a		 =       ['not super long but more than 5 tokens', 'tiny']
								__a		 =       self._large_tokenizer(UpperCAmelCase				,						padding=UpperCAmelCase				,						truncation=UpperCAmelCase				,						return_tensors='pt'   )
								__a		 =       self._large_tokenizer(
								    text_target=UpperCAmelCase				,						max_length=5				,						padding=UpperCAmelCase				,						truncation=UpperCAmelCase				,						return_tensors='pt'   )

								assert batch.input_ids.shape == (2, 1_0_2_4)
								assert batch.attention_mask.shape == (2, 1_0_2_4)
								assert targets["input_ids"].shape == (2, 5)
								assert len(UpperCAmelCase   ) == 2  # input_ids, attention_mask.







			@slow
			def __SCREAMING_SNAKE_CASE							(						self   )			->   int:
								# fmt: off
								__a		 =       {'input_ids': [[3_8_9_7_9, 1_4_3, 1_8_4_8_5, 6_0_6, 1_3_0, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 5_4_1_8_9, 1_1_2_9, 1_1_1, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 9_1_1_4, 1_4_7_8_7, 1_2_1, 1_3_2_4_9, 1_5_8, 5_9_2, 9_5_6, 1_2_1, 1_4_6_2_1, 3_1_5_7_6, 1_4_3, 6_2_6_1_3, 1_0_8, 9_6_8_8, 9_3_0, 4_3_4_3_0, 1_1_5_6_2, 6_2_6_1_3, 3_0_4, 1_0_8, 1_1_4_4_3, 8_9_7, 1_0_8, 9_3_1_4, 1_7_4_1_5, 6_3_3_9_9, 1_0_8, 1_1_4_4_3, 7_6_1_4, 1_8_3_1_6, 1_1_8, 4_2_8_4, 7_1_4_8, 1_2_4_3_0, 1_4_3, 1_4_0_0, 2_5_7_0_3, 1_5_8, 1_1_1, 4_2_8_4, 7_1_4_8, 1_1_7_7_2, 1_4_3, 2_1_2_9_7, 1_0_6_4, 1_5_8, 1_2_2, 2_0_4, 3_5_0_6, 1_7_5_4, 1_1_3_3, 1_4_7_8_7, 1_5_8_1, 1_1_5, 3_3_2_2_4, 4_4_8_2, 1_1_1, 1_3_5_5, 1_1_0, 2_9_1_7_3, 3_1_7, 5_0_8_3_3, 1_0_8, 2_0_1_4_7, 9_4_6_6_5, 1_1_1, 7_7_1_9_8, 1_0_7, 1], [1_1_0, 6_2_6_1_3, 1_1_7, 6_3_8, 1_1_2, 1_1_3_3, 1_2_1, 2_0_0_9_8, 1_3_5_5, 7_9_0_5_0, 1_3_8_7_2, 1_3_5, 1_5_9_6, 5_3_5_4_1, 1_3_5_2, 1_4_1, 1_3_0_3_9, 5_5_4_2, 1_2_4, 3_0_2, 5_1_8, 1_1_1, 2_6_8, 2_9_5_6, 1_1_5, 1_4_9, 4_4_2_7, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_3_9, 1_2_3_5, 2_7_9_9, 1_8_2_8_9, 1_7_7_8_0, 2_0_4, 1_0_9, 9_4_7_4, 1_2_9_6, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=UpperCAmelCase				,						model_name='google/bigbird-pegasus-large-arxiv'				,						revision='ba85d0851d708441f91440d509690f1ab6353415'				,						)






@require_sentencepiece
@require_tokenizers
class 						a__							(       __snake_case				,						unittest.TestCase		):
			A__    :       int        =      PegasusTokenizer
			A__    :       Dict        =      PegasusTokenizerFast
			A__    :       Any        =      True
			A__    :       str        =      True



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Dict:
								super().setUp()

								# We have a SentencePiece fixture for testing
								__a		 =       PegasusTokenizer(UpperCAmelCase				,						offset=0				,						mask_token_sent=UpperCAmelCase				,						mask_token='[MASK]'   )
								tokenizer.save_pretrained(self.tmpdirname   )



			@cached_property
			def __SCREAMING_SNAKE_CASE							(						self   )			->   Optional[int]:
								return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv'   )



			def __SCREAMING_SNAKE_CASE							(						self				,						**UpperCAmelCase   )			->   PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname				,						**UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self				,						UpperCAmelCase   )			->   List[Any]:
								return ("This is a test", "This is a test")



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Any:
								__a		 =       self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
								__a		 =       self.tokenizer_class.from_pretrained(self.tmpdirname   )
								__a		 =       (
								    'Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'
								    ' <pad> <pad> <pad>'
								)
								__a		 =       rust_tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase				,						add_special_tokens=UpperCAmelCase   ).input_ids[0]
								__a		 =       py_tokenizer([raw_input_str]				,						return_tensors=UpperCAmelCase				,						add_special_tokens=UpperCAmelCase   ).input_ids[0]
								self.assertListEqual(UpperCAmelCase				,						UpperCAmelCase   )



			@require_torch
			def __SCREAMING_SNAKE_CASE							(						self   )			->   int:
								__a		 =       ['This is going to be way too long.' * 1_0_0_0, 'short example']
								__a		 =       ['not super long but more than 5 tokens', 'tiny']
								__a		 =       self._large_tokenizer(UpperCAmelCase				,						padding=UpperCAmelCase				,						truncation=UpperCAmelCase				,						return_tensors='pt'   )
								__a		 =       self._large_tokenizer(
								    text_target=UpperCAmelCase				,						max_length=5				,						padding=UpperCAmelCase				,						truncation=UpperCAmelCase				,						return_tensors='pt'   )

								assert batch.input_ids.shape == (2, 4_0_9_6)
								assert batch.attention_mask.shape == (2, 4_0_9_6)
								assert targets["input_ids"].shape == (2, 5)
								assert len(UpperCAmelCase   ) == 2  # input_ids, attention_mask.







			def __SCREAMING_SNAKE_CASE							(						self   )			->   str:
								__a		 =       (
								    'This is an example string that is used to test the original TF implementation against the HF'
								    ' implementation'
								)

								__a		 =       self._large_tokenizer(UpperCAmelCase   ).input_ids

								self.assertListEqual(
								    UpperCAmelCase				,						[1_8_2, 1_1_7, 1_4_2, 5_8_7, 4_2_1_1, 1_2_0, 1_1_7, 2_6_3, 1_1_2, 8_0_4, 1_0_9, 8_5_6, 2_5_0_1_6, 3_1_3_7, 4_6_4, 1_0_9, 2_6_9_5_5, 3_1_3_7, 1]				,						)





import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs


class 						a__							(       __snake_case		):



			def __init__(						self				,						UpperCAmelCase				,						UpperCAmelCase=None				,						UpperCAmelCase=True				,						UpperCAmelCase=None				,						**UpperCAmelCase   )			->   List[Any]:
								__a		 =       parent
								__a		 =       config_class
								__a		 =       has_text_modality
								__a		 =       kwargs
								__a		 =       common_properties



			def __SCREAMING_SNAKE_CASE							(						self   )			->   List[str]:
								__a		 =       self.config_class(**self.inputs_dict   )
								__a		 =       (
								    ['hidden_size', 'num_attention_heads', 'num_hidden_layers']
								    if self.common_properties is None
								    else self.common_properties
								)

								# Add common fields for text models
								if self.has_text_modality:
													common_properties.extend(['vocab_size']   )

								# Test that config has the common properties as getters
								for prop in common_properties:
													self.parent.assertTrue(hasattr(UpperCAmelCase				,						UpperCAmelCase   )				,						msg=f'''`{prop}` does not exist'''   )

								# Test that config has the common properties as setter
								for idx, name in enumerate(UpperCAmelCase   ):
													try:
																		setattr(UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase   )
																		self.parent.assertEqual(
																		    getattr(UpperCAmelCase				,						UpperCAmelCase   )				,						UpperCAmelCase				,						msg=f'''`{name} value {idx} expected, but was {getattr(UpperCAmelCase				,						UpperCAmelCase   )}'''   )
													except NotImplementedError:
																		# Some models might not be able to implement setters for common_properties
																		# In that case, a NotImplementedError is raised
																		pass

        # Test if config class can be called with Config(prop_name=..)
								for idx, name in enumerate(UpperCAmelCase   ):
													try:
																		__a		 =       self.config_class(**{name: idx}   )
																		self.parent.assertEqual(
																		    getattr(UpperCAmelCase				,						UpperCAmelCase   )				,						UpperCAmelCase				,						msg=f'''`{name} value {idx} expected, but was {getattr(UpperCAmelCase				,						UpperCAmelCase   )}'''   )
													except NotImplementedError:
																		# Some models might not be able to implement setters for common_properties
																		# In that case, a NotImplementedError is raised
																		pass



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Tuple:
								__a		 =       self.config_class(**self.inputs_dict   )
								__a		 =       json.loads(config.to_json_string()   )
								for key, value in self.inputs_dict.items():
													self.parent.assertEqual(obj[key]				,						UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Tuple:
								__a		 =       self.config_class(**self.inputs_dict   )

								with tempfile.TemporaryDirectory() as tmpdirname:
													__a		 =       os.path.join(UpperCAmelCase				,						'config.json'   )
													config_first.to_json_file(UpperCAmelCase   )
													__a		 =       self.config_class.from_json_file(UpperCAmelCase   )

								self.parent.assertEqual(config_second.to_dict()				,						config_first.to_dict()   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Tuple:
								__a		 =       self.config_class(**self.inputs_dict   )

								with tempfile.TemporaryDirectory() as tmpdirname:
													config_first.save_pretrained(UpperCAmelCase   )
													__a		 =       self.config_class.from_pretrained(UpperCAmelCase   )

								self.parent.assertEqual(config_second.to_dict()				,						config_first.to_dict()   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   str:
								__a		 =       self.config_class(**self.inputs_dict   )

								__a		 =       'test'
								with tempfile.TemporaryDirectory() as tmpdirname:
													__a		 =       os.path.join(UpperCAmelCase				,						UpperCAmelCase   )
													config_first.save_pretrained(UpperCAmelCase   )
													__a		 =       self.config_class.from_pretrained(UpperCAmelCase				,						subfolder=UpperCAmelCase   )

								self.parent.assertEqual(config_second.to_dict()				,						config_first.to_dict()   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Union[str, Any]:
								__a		 =       self.config_class(**self.inputs_dict				,						num_labels=5   )
								self.parent.assertEqual(len(config.idalabel   )				,						5   )
								self.parent.assertEqual(len(config.labelaid   )				,						5   )

								__a		 =       3
								self.parent.assertEqual(len(config.idalabel   )				,						3   )
								self.parent.assertEqual(len(config.labelaid   )				,						3   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   List[str]:
								if self.config_class.is_composition:
													return
								__a		 =       self.config_class()
								self.parent.assertIsNotNone(UpperCAmelCase   )



			def __SCREAMING_SNAKE_CASE							(						self   )			->   Union[str, Any]:
								__a		 =       copy.deepcopy(UpperCAmelCase   )
								__a		 =       self.config_class(**UpperCAmelCase   )
								__a		 =       []
								for key, value in config_common_kwargs.items():
													if key == "torch_dtype":
																		if not is_torch_available():
																							continue
																		else:
																							import torch

																							if config.torch_dtype != torch.floataa:
																												wrong_values.append(('torch_dtype', config.torch_dtype, torch.floataa)   )
													elif getattr(UpperCAmelCase				,						UpperCAmelCase   ) != value:
																		wrong_values.append((key, getattr(UpperCAmelCase				,						UpperCAmelCase   ), value)   )

								if len(UpperCAmelCase   ) > 0:
													__a		 =       '\n'.join([f'''- {v[0]}: got {v[1]} instead of {v[2]}''' for v in wrong_values]   )
													raise ValueError(f'''The following keys were not properly set in the config:\n{errors}'''   )







			def __SCREAMING_SNAKE_CASE							(						self   )			->   Optional[Any]:
								self.create_and_test_config_common_properties()
								self.create_and_test_config_to_json_string()
								self.create_and_test_config_to_json_file()
								self.create_and_test_config_from_and_save_pretrained()
								self.create_and_test_config_from_and_save_pretrained_subfolder()
								self.create_and_test_config_with_num_labels()
								self.check_config_can_be_init_without_params()
								self.check_config_arguments_init()




