





'''simple docstring'''



import argparse

import requests
import torch
from PIL import Image
from torchvision.transforms import Compose, Normalize, Resize, ToTensor

from transformers import SwinaSRConfig, SwinaSRForImageSuperResolution, SwinaSRImageProcessor




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Tuple			):
			__lowercase			  =							SwinaSRConfig()

			if "Swin2SR_ClassicalSR_X4_64" in checkpoint_url:
						__lowercase			  =							4
			elif "Swin2SR_CompressedSR_X4_48" in checkpoint_url:
						__lowercase			  =							4
						__lowercase			  =							4_8
						__lowercase			  =							'''pixelshuffle_aux'''
			elif "Swin2SR_Lightweight_X2_64" in checkpoint_url:
						__lowercase			  =							[6, 6, 6, 6]
						__lowercase			  =							6_0
						__lowercase			  =							[6, 6, 6, 6]
						__lowercase			  =							'''pixelshuffledirect'''
			elif "Swin2SR_RealworldSR_X4_64_BSRGAN_PSNR" in checkpoint_url:
						__lowercase			  =							4
						__lowercase			  =							'''nearest+conv'''
			elif "Swin2SR_Jpeg_dynamic" in checkpoint_url:
						__lowercase			  =							1
						__lowercase			  =							1
						__lowercase			  =							1_2_6
						__lowercase			  =							7
						__lowercase			  =							2_55.0
						__lowercase			  =							''''''

			return config




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Dict						,    lowerCamelCase_     :					Optional[int]			):
			if "patch_embed.proj" in name and "layers" not in name:
						__lowercase			  =							name.replace('''patch_embed.proj'''						,    '''embeddings.patch_embeddings.projection'''			)
			if "patch_embed.norm" in name:
						__lowercase			  =							name.replace('''patch_embed.norm'''						,    '''embeddings.patch_embeddings.layernorm'''			)
			if "layers" in name:
						__lowercase			  =							name.replace('''layers'''						,    '''encoder.stages'''			)
			if "residual_group.blocks" in name:
						__lowercase			  =							name.replace('''residual_group.blocks'''						,    '''layers'''			)
			if "attn.proj" in name:
						__lowercase			  =							name.replace('''attn.proj'''						,    '''attention.output.dense'''			)
			if "attn" in name:
						__lowercase			  =							name.replace('''attn'''						,    '''attention.self'''			)
			if "norm1" in name:
						__lowercase			  =							name.replace('''norm1'''						,    '''layernorm_before'''			)
			if "norm2" in name:
						__lowercase			  =							name.replace('''norm2'''						,    '''layernorm_after'''			)
			if "mlp.fc1" in name:
						__lowercase			  =							name.replace('''mlp.fc1'''						,    '''intermediate.dense'''			)
			if "mlp.fc2" in name:
						__lowercase			  =							name.replace('''mlp.fc2'''						,    '''output.dense'''			)
			if "q_bias" in name:
						__lowercase			  =							name.replace('''q_bias'''						,    '''query.bias'''			)
			if "k_bias" in name:
						__lowercase			  =							name.replace('''k_bias'''						,    '''key.bias'''			)
			if "v_bias" in name:
						__lowercase			  =							name.replace('''v_bias'''						,    '''value.bias'''			)
			if "cpb_mlp" in name:
						__lowercase			  =							name.replace('''cpb_mlp'''						,    '''continuous_position_bias_mlp'''			)
			if "patch_embed.proj" in name:
						__lowercase			  =							name.replace('''patch_embed.proj'''						,    '''patch_embed.projection'''			)

			if name == "norm.weight":
						__lowercase			  =							'''layernorm.weight'''
			if name == "norm.bias":
						__lowercase			  =							'''layernorm.bias'''

			if "conv_first" in name:
						__lowercase			  =							name.replace('''conv_first'''						,    '''first_convolution'''			)

			if (
			    "upsample" in name
			    or "conv_before_upsample" in name
			    or "conv_bicubic" in name
			    or "conv_up" in name
			    or "conv_hr" in name
			    or "conv_last" in name
			    or "aux" in name
			):
						# heads
						if "conv_last" in name:
									__lowercase			  =							name.replace('''conv_last'''						,    '''final_convolution'''			)
						if config.upsampler in ["pixelshuffle", "pixelshuffle_aux", "nearest+conv"]:
									if "conv_before_upsample.0" in name:
												__lowercase			  =							name.replace('''conv_before_upsample.0'''						,    '''conv_before_upsample'''			)
									if "upsample.0" in name:
												__lowercase			  =							name.replace('''upsample.0'''						,    '''upsample.convolution_0'''			)
									if "upsample.2" in name:
												__lowercase			  =							name.replace('''upsample.2'''						,    '''upsample.convolution_1'''			)
									__lowercase			  =							'''upsample.''' + name
						elif config.upsampler == "pixelshuffledirect":
									__lowercase			  =							name.replace('''upsample.0.weight'''						,    '''upsample.conv.weight'''			)
									__lowercase			  =							name.replace('''upsample.0.bias'''						,    '''upsample.conv.bias'''			)
						else:
									pass
			else:
						__lowercase			  =							'''swin2sr.''' + name

			return name




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Tuple						,    lowerCamelCase_     :					List[str]			):
			for key in orig_state_dict.copy().keys():
						__lowercase			  =							orig_state_dict.pop(lowerCamelCase_			)

						if "qkv" in key:
									__lowercase			  =							key.split('''.'''			)
									__lowercase			  =							int(key_split[1]			)
									__lowercase			  =							int(key_split[4]			)
									__lowercase			  =							config.embed_dim

									if "weight" in key:
												__lowercase			  =							val[:dim, :]
												__lowercase			  =							val[dim : dim * 2, :]
												__lowercase			  =							val[-dim:, :]
									else:
												__lowercase			  =							val[:dim]
												__lowercase			  =							val[dim : dim * 2]
												__lowercase			  =							val[-dim:]
									pass
						else:
									__lowercase			  =							val

			return orig_state_dict




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Any						,    lowerCamelCase_     :					int						,    lowerCamelCase_     :					int			):
			__lowercase			  =							get_config(lowerCamelCase_			)
			__lowercase			  =							SwinaSRForImageSuperResolution(lowerCamelCase_			)
			model.eval()

			__lowercase			  =							torch.hub.load_state_dict_from_url(lowerCamelCase_						,    map_location='''cpu'''			)
			__lowercase			  =							convert_state_dict(lowerCamelCase_						,    lowerCamelCase_			)
			__lowercase  ,						__lowercase			  =							model.load_state_dict(lowerCamelCase_						,    strict=lowerCamelCase_			)

			if len(lowerCamelCase_			) > 0:
						raise ValueError('''Missing keys when converting: {}'''.format(lowerCamelCase_			)			)
			for key in unexpected_keys:
						if not ("relative_position_index" in key or "relative_coords_table" in key or "self_mask" in key):
									raise ValueError(f"Unexpected key {key} in state_dict"			)

    # verify values
			__lowercase			  =							'''https://github.com/mv-lab/swin2sr/blob/main/testsets/real-inputs/shanghai.jpg?raw=true'''
			__lowercase			  =							Image.open(requests.get(lowerCamelCase_						,    stream=lowerCamelCase_			).raw			).convert('''RGB'''			)
			__lowercase			  =							SwinaSRImageProcessor()
			# pixel_values = processor(image, return_tensors="pt").pixel_values

			__lowercase			  =							1_2_6 if '''Jpeg''' in checkpoint_url else 2_5_6
			__lowercase			  =							Compose(
			    [
			        Resize((image_size, image_size)			),
			        ToTensor(),
			        Normalize(mean=[0.4_85, 0.4_56, 0.4_06]						,    std=[0.2_29, 0.2_24, 0.2_25]			),
			    ]			)
			__lowercase			  =							transforms(lowerCamelCase_			).unsqueeze(0			)

			if config.num_channels == 1:
						__lowercase			  =							pixel_values[:, 0, :, :].unsqueeze(1			)

			__lowercase			  =							model(lowerCamelCase_			)

			# assert values
			if "Swin2SR_ClassicalSR_X2_64" in checkpoint_url:
						__lowercase			  =							torch.Size([1, 3, 5_1_2, 5_1_2]			)
						__lowercase			  =							torch.tensor(
						    [[-0.70_87, -0.71_38, -0.67_21], [-0.83_40, -0.80_95, -0.72_98], [-0.91_49, -0.84_14, -0.79_40]]			)
			elif "Swin2SR_ClassicalSR_X4_64" in checkpoint_url:
						__lowercase			  =							torch.Size([1, 3, 1_0_2_4, 1_0_2_4]			)
						__lowercase			  =							torch.tensor(
						    [[-0.77_75, -0.81_05, -0.89_33], [-0.77_64, -0.83_56, -0.92_25], [-0.79_76, -0.86_86, -0.95_79]]			)
			elif "Swin2SR_CompressedSR_X4_48" in checkpoint_url:
						# TODO values didn't match exactly here
						__lowercase			  =							torch.Size([1, 3, 1_0_2_4, 1_0_2_4]			)
						__lowercase			  =							torch.tensor(
						    [[-0.80_35, -0.75_04, -0.74_91], [-0.85_38, -0.81_24, -0.77_82], [-0.88_04, -0.86_51, -0.84_93]]			)
			elif "Swin2SR_Lightweight_X2_64" in checkpoint_url:
						__lowercase			  =							torch.Size([1, 3, 5_1_2, 5_1_2]			)
						__lowercase			  =							torch.tensor(
						    [[-0.76_69, -0.86_62, -0.87_67], [-0.88_10, -0.99_62, -0.98_20], [-0.93_40, -1.03_22, -1.11_49]]			)
			elif "Swin2SR_RealworldSR_X4_64_BSRGAN_PSNR" in checkpoint_url:
						__lowercase			  =							torch.Size([1, 3, 1_0_2_4, 1_0_2_4]			)
						__lowercase			  =							torch.tensor(
						    [[-0.52_38, -0.55_57, -0.63_21], [-0.60_16, -0.59_03, -0.63_91], [-0.62_44, -0.63_34, -0.68_89]]			)

			assert (
			    outputs.reconstruction.shape == expected_shape
			), f"Shape of reconstruction should be {expected_shape}, but is {outputs.reconstruction.shape}"
			assert torch.allclose(outputs.reconstruction[0, 0, :3, :3]						,    lowerCamelCase_						,    atol=1E-3			)
			print('''Looks ok!'''			)

			__lowercase			  =							{
			    '''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_ClassicalSR_X2_64.pth''': (
			        '''swin2SR-classical-sr-x2-64'''
			    ),
			    '''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_ClassicalSR_X4_64.pth''': (
			        '''swin2SR-classical-sr-x4-64'''
			    ),
			    '''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_CompressedSR_X4_48.pth''': (
			        '''swin2SR-compressed-sr-x4-48'''
			    ),
			    '''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_Lightweight_X2_64.pth''': (
			        '''swin2SR-lightweight-x2-64'''
			    ),
			    '''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_RealworldSR_X4_64_BSRGAN_PSNR.pth''': (
			        '''swin2SR-realworld-sr-x4-64-bsrgan-psnr'''
			    ),
			}
			__lowercase			  =							url_to_name[checkpoint_url]

			if pytorch_dump_folder_path is not None:
						print(f"Saving model {model_name} to {pytorch_dump_folder_path}"			)
						model.save_pretrained(lowerCamelCase_			)
						print(f"Saving image processor to {pytorch_dump_folder_path}"			)
						processor.save_pretrained(lowerCamelCase_			)

			if push_to_hub:
						model.push_to_hub(f"caidas/{model_name}"			)
						processor.push_to_hub(f"caidas/{model_name}"			)


if __name__ == "__main__":
	_SCREAMING_SNAKE_CASE   =      argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--checkpoint_url''',
	    default='''https://github.com/mv-lab/swin2sr/releases/download/v0.0.1/Swin2SR_ClassicalSR_X2_64.pth''',
	    type=str,
	    help='''URL of the original Swin2SR checkpoint you\'d like to convert.''',
	)
	parser.add_argument(
	    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
	)
	parser.add_argument('''--push_to_hub''', action='''store_true''', help='''Whether to push the converted model to the hub.''')

	_SCREAMING_SNAKE_CASE   =      parser.parse_args()
	convert_swinasr_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.push_to_hub)












'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class 							__lowercase    ( lowerCAmelCase__	, unittest.TestCase  ):







	'''simple docstring'''

	a      :			List[Any]     =    CanineTokenizer
	a      :			Union[str, Any]     =    False


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				super().setUp()
				__lowercase			  =							CanineTokenizer()
				tokenizer.save_pretrained(self.tmpdirname	)


	@cached_property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return CanineTokenizer.from_pretrained('''google/canine-s'''	)


	def   _UpperCAmelCase       (self       ,**_lowerCamelCase	)							->							CanineTokenizer:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class.from_pretrained(self.tmpdirname       ,**_lowerCamelCase	)
				__lowercase			  =							1024
				return tokenizer


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
				# fmt: off
				__lowercase			  =							[57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
				# fmt: on
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertIsInstance(_lowerCamelCase       ,_lowerCamelCase	)

				__lowercase			  =							list(batch.input_ids.numpy()[0]	)

				self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

				self.assertEqual((2, 39)       ,batch.input_ids.shape	)
				self.assertEqual((2, 39)       ,batch.attention_mask.shape	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				# check if input_ids, attention_mask and token_type_ids are returned
				self.assertIn('''input_ids'''       ,_lowerCamelCase	)
				self.assertIn('''attention_mask'''       ,_lowerCamelCase	)
				self.assertIn('''token_type_ids'''       ,_lowerCamelCase	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							[
				    '''What\'s the weater?''',
				    '''It\'s about 25 degrees.''',
				]
				__lowercase			  =							tokenizer(
				    text_target=_lowerCamelCase       ,max_length=32       ,padding='''max_length'''       ,truncation=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertEqual(32       ,targets['''input_ids'''].shape[1]	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										self.assertNotEqual(tokenizer.model_max_length       ,42	)

        # Now let's start the test
				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

										shutil.rmtree(_lowerCamelCase	)

				__lowercase			  =							self.get_tokenizers(model_max_length=42	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''

										__lowercase			  =							tokenizer.additional_special_tokens

										# We can add a new special token for Canine as follows:
										__lowercase			  =							chr(0xe_0_0_7	)
										additional_special_tokens.append(_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertIn(_lowerCamelCase       ,after_tokenizer.additional_special_tokens	)
										self.assertEqual(after_tokenizer.model_max_length       ,42	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase       ,model_max_length=43	)
										self.assertEqual(tokenizer.model_max_length       ,43	)

										shutil.rmtree(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase  ,						__lowercase			  =							self.get_clean_sequence(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_5
										__lowercase			  =							chr(_lowerCamelCase	)

										tokenizer.add_special_tokens({'''cls_token''': special_token}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)

										__lowercase			  =							tokenizer.decode(ids + encoded_special_token       ,clean_up_tokenization_spaces=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(_lowerCamelCase       ,input_encoded + special_token_id	)

										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,skip_special_tokens=_lowerCamelCase	)
										self.assertTrue(special_token not in decoded	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							chr(0xe_0_0_5	)
										__lowercase			  =							chr(0xe_0_0_6	)

										# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
										tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,special_tokens=_lowerCamelCase	)
										# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
										# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}	)

										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)
										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)

										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}	)

										with tempfile.TemporaryDirectory() as tmp_dir_name:
													tokenizer.save_pretrained(_lowerCamelCase	)
													tokenizer.from_pretrained(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							[]
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							[new_token_a]
										__lowercase			  =							[new_token_a]

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)
										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										__lowercase			  =							tokenizer_class.from_pretrained(_lowerCamelCase       ,extra_ids=0	)
										self.assertIn(_lowerCamelCase       ,tokenizer_without_change_in_init.additional_special_tokens	)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]	)	)       ,)

										__lowercase			  =							0xe_0_0_7
										__lowercase			  =							chr(_lowerCamelCase	)
										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										__lowercase			  =							[AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)]
										__lowercase			  =							tokenizer_class.from_pretrained(
										    _lowerCamelCase       ,additional_special_tokens=_lowerCamelCase       ,extra_ids=0	)

										self.assertIn(_lowerCamelCase       ,tokenizer.additional_special_tokens	)
										# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]	)	)	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							'''hello world'''
										if self.space_between_special_tokens:
													__lowercase			  =							'''[CLS] hello world [SEP]'''
										else:
													__lowercase			  =							input
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,spaces_between_special_tokens=self.space_between_special_tokens	)
										self.assertIn(_lowerCamelCase       ,[output, output.lower()]	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							[
										    '''bos_token''',
										    '''eos_token''',
										    '''unk_token''',
										    '''sep_token''',
										    '''pad_token''',
										    '''cls_token''',
										    '''mask_token''',
										]

				__lowercase			  =							'''a'''
				__lowercase			  =							ord(_lowerCamelCase	)

				for attr in attributes_list:
							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[]	)

				__lowercase			  =							0xe_0_0_6
				__lowercase			  =							chr(_lowerCamelCase	)
				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[additional_special_token_id]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[additional_special_token]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[additional_special_token_id]	)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass





