





'''simple docstring'''



import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(4_2)

_SCREAMING_SNAKE_CASE   =      '''bert-base-cased'''
_SCREAMING_SNAKE_CASE   =      '''fp16'''
_SCREAMING_SNAKE_CASE   =      '''bf16'''
_SCREAMING_SNAKE_CASE   =      [FPaa, BFaa]






@require_fsdp
@require_cuda
class 							__lowercase    ( __snake_case  ):







	'''simple docstring'''

	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				super().setUp()

				__lowercase			  =							dict(
				    ACCELERATE_USE_FSDP='''true'''       ,MASTER_ADDR='''localhost'''       ,MASTER_PORT='''10999'''       ,RANK='''0'''       ,LOCAL_RANK='''0'''       ,WORLD_SIZE='''1'''       ,)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

				for i, strategy in enumerate(lowerCamelCase_	):
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							f"{i + 1}"
							__lowercase			  =							strategy
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							FullyShardedDataParallelPlugin()
										self.assertEqual(fsdp_plugin.sharding_strategy       ,ShardingStrategy(i + 1	)	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

				for i, prefetch_policy in enumerate(lowerCamelCase_	):
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							prefetch_policy
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							FullyShardedDataParallelPlugin()
										if prefetch_policy == "NO_PREFETCH":
													self.assertIsNone(fsdp_plugin.backward_prefetch	)
										else:
													self.assertEqual(fsdp_plugin.backward_prefetch       ,BackwardPrefetch(i + 1	)	)


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

				for i, state_dict_type in enumerate(lowerCamelCase_	):
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							state_dict_type
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							FullyShardedDataParallelPlugin()
										self.assertEqual(fsdp_plugin.state_dict_type       ,StateDictType(i + 1	)	)
										if state_dict_type == "FULL_STATE_DICT":
													self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu	)
													self.assertTrue(fsdp_plugin.state_dict_config.ranka_only	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							AutoModel.from_pretrained(lowerCamelCase_	)
				for policy in FSDP_AUTO_WRAP_POLICY:
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							policy
							if policy == "TRANSFORMER_BASED_WRAP":
										__lowercase			  =							"""BertLayer"""
							elif policy == "SIZE_BASED_WRAP":
										__lowercase			  =							"""2000"""
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							FullyShardedDataParallelPlugin()
										fsdp_plugin.set_auto_wrap_policy(lowerCamelCase_	)
										if policy == "NO_WRAP":
													self.assertIsNone(fsdp_plugin.auto_wrap_policy	)
										else:
													self.assertIsNotNone(fsdp_plugin.auto_wrap_policy	)

				__lowercase			  =							self.dist_env.copy()
				__lowercase			  =							"""TRANSFORMER_BASED_WRAP"""
				__lowercase			  =							"""T5Layer"""
				with mockenv_context(**lowerCamelCase_	):
							__lowercase			  =							FullyShardedDataParallelPlugin()
							with self.assertRaises(lowerCamelCase_	) as cm:
										fsdp_plugin.set_auto_wrap_policy(lowerCamelCase_	)
							self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception	)	)

				__lowercase			  =							self.dist_env.copy()
				__lowercase			  =							"""SIZE_BASED_WRAP"""
				__lowercase			  =							"""0"""
				with mockenv_context(**lowerCamelCase_	):
							__lowercase			  =							FullyShardedDataParallelPlugin()
							fsdp_plugin.set_auto_wrap_policy(lowerCamelCase_	)
							self.assertIsNone(fsdp_plugin.auto_wrap_policy	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
				from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

				for mp_dtype in dtypes:
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							mp_dtype
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							Accelerator()
										if mp_dtype == "fp16":
													__lowercase			  =							torch.floataa
										elif mp_dtype == "bf16":
													__lowercase			  =							torch.bfloataa
										__lowercase			  =							MixedPrecision(param_dtype=lowerCamelCase_       ,reduce_dtype=lowerCamelCase_       ,buffer_dtype=lowerCamelCase_	)
										self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy       ,lowerCamelCase_	)
										if mp_dtype == FPaa:
													self.assertTrue(isinstance(accelerator.scaler       ,lowerCamelCase_	)	)
										elif mp_dtype == BFaa:
													self.assertIsNone(accelerator.scaler	)
										AcceleratorState._reset_state(lowerCamelCase_	)


	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

				for flag in [True, False]:
							__lowercase			  =							self.dist_env.copy()
							__lowercase			  =							str(lowerCamelCase_	).lower()
							with mockenv_context(**lowerCamelCase_	):
										__lowercase			  =							FullyShardedDataParallelPlugin()
										self.assertEqual(fsdp_plugin.cpu_offload       ,CPUOffload(offload_params=lowerCamelCase_	)	)






@require_fsdp
@require_multi_gpu
@slow
class 							__lowercase    ( __snake_case  ):







	'''simple docstring'''

	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				super().setUp()
				__lowercase			  =							0.8_2
				__lowercase			  =							[
				    """fsdp_shard_grad_op_transformer_based_wrap""",
				    """fsdp_full_shard_transformer_based_wrap""",
				]
				__lowercase			  =							{
				    """multi_gpu_fp16""": 3200,
				    """fsdp_shard_grad_op_transformer_based_wrap_fp16""": 2000,
				    """fsdp_full_shard_transformer_based_wrap_fp16""": 1900,
				    # Disabling below test as it overwhelms the RAM memory usage
				    # on CI self-hosted runner leading to tests getting killed.
				    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
				}
				__lowercase			  =							160
				__lowercase			  =							160

				__lowercase			  =							inspect.getfile(accelerate.test_utils	)
				__lowercase			  =							os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ['''scripts''', '''external_deps''']	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							os.path.join(self.test_scripts_folder       ,'''test_performance.py'''	)
				__lowercase			  =							["""accelerate""", """launch""", """--num_processes=2""", """--num_machines=1""", """--machine_rank=0""", """--use_fsdp"""]
				for config in self.performance_configs:
							__lowercase			  =							cmd.copy()
							for i, strategy in enumerate(lowerCamelCase_	):
										if strategy.lower() in config:
													cmd_config.append(f"--fsdp_sharding_strategy={i+1}"	)
													break

							if "fp32" in config:
										cmd_config.append('''--mixed_precision=no'''	)
							else:
										cmd_config.append('''--mixed_precision=fp16'''	)

							if "cpu_offload" in config:
										cmd_config.append('''--fsdp_offload_params=True'''	)

							for policy in FSDP_AUTO_WRAP_POLICY:
										if policy.lower() in config:
													cmd_config.append(f"--fsdp_auto_wrap_policy={policy}"	)
													break

							if policy == "TRANSFORMER_BASED_WRAP":
										cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''	)
							elif policy == "SIZE_BASED_WRAP":
										cmd_config.append('''--fsdp_min_num_params=2000'''	)

							cmd_config.extend(
							    [
							        self.test_file_path,
							        f"--output_dir={self.tmpdir}",
							        f"--performance_lower_bound={self.performance_lower_bound}",
							    ]	)
							with patch_environment(omp_num_threads=1	):
										execute_subprocess_async(lowerCamelCase_       ,env=os.environ.copy()	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							os.path.join(self.test_scripts_folder       ,'''test_checkpointing.py'''	)
				__lowercase			  =							[
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				    """--use_fsdp""",
				    """--mixed_precision=fp16""",
				    """--fsdp_transformer_layer_cls_to_wrap=BertLayer""",
				]

				for i, strategy in enumerate(lowerCamelCase_	):
							__lowercase			  =							cmd.copy()
							cmd_config.append(f"--fsdp_sharding_strategy={i+1}"	)
							if strategy != "FULL_SHARD":
										continue
							__lowercase			  =							len(lowerCamelCase_	)
							for state_dict_type in FSDP_STATE_DICT_TYPE:
										__lowercase			  =							cmd_config[:state_dict_config_index]
										cmd_config.append(f"--fsdp_state_dict_type={state_dict_type}"	)
										cmd_config.extend(
										    [
										        self.test_file_path,
										        f"--output_dir={self.tmpdir}",
										        '''--partial_train_epoch=1''',
										    ]	)
										with patch_environment(omp_num_threads=1	):
													execute_subprocess_async(lowerCamelCase_       ,env=os.environ.copy()	)

										__lowercase			  =							cmd_config[:-1]
										__lowercase			  =							os.path.join(self.tmpdir       ,'''epoch_0'''	)
										cmd_config.extend(
										    [
										        f"--resume_from_checkpoint={resume_from_checkpoint}",
										    ]	)
										with patch_environment(omp_num_threads=1	):
													execute_subprocess_async(lowerCamelCase_       ,env=os.environ.copy()	)


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				__lowercase			  =							os.path.join(self.test_scripts_folder       ,'''test_peak_memory_usage.py'''	)
				__lowercase			  =							[
				    """accelerate""",
				    """launch""",
				    """--num_processes=2""",
				    """--num_machines=1""",
				    """--machine_rank=0""",
				]
				for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
							__lowercase			  =							cmd.copy()
							if "fp16" in spec:
										cmd_config.extend(['''--mixed_precision=fp16''']	)
							else:
										cmd_config.extend(['''--mixed_precision=no''']	)

							if "multi_gpu" in spec:
										continue
							else:
										cmd_config.extend(['''--use_fsdp''']	)
										for i, strategy in enumerate(lowerCamelCase_	):
													if strategy.lower() in spec:
																cmd_config.append(f"--fsdp_sharding_strategy={i+1}"	)
																break

										if "cpu_offload" in spec:
													cmd_config.append('''--fsdp_offload_params=True'''	)

										for policy in FSDP_AUTO_WRAP_POLICY:
													if policy.lower() in spec:
																cmd_config.append(f"--fsdp_auto_wrap_policy={policy}"	)
																break

										if policy == "TRANSFORMER_BASED_WRAP":
													cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''	)
										elif policy == "SIZE_BASED_WRAP":
													cmd_config.append('''--fsdp_min_num_params=2000'''	)

							cmd_config.extend(
							    [
							        self.test_file_path,
							        f"--output_dir={self.tmpdir}",
							        f"--peak_memory_upper_bound={peak_mem_upper_bound}",
							        f"--n_train={self.n_train}",
							        f"--n_val={self.n_val}",
							    ]	)
							with patch_environment(omp_num_threads=1	):
										execute_subprocess_async(lowerCamelCase_       ,env=os.environ.copy()	)












'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class 							__lowercase    ( lowerCAmelCase__	, unittest.TestCase  ):







	'''simple docstring'''

	a      :			List[Any]     =    CanineTokenizer
	a      :			Union[str, Any]     =    False


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				super().setUp()
				__lowercase			  =							CanineTokenizer()
				tokenizer.save_pretrained(self.tmpdirname	)


	@cached_property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return CanineTokenizer.from_pretrained('''google/canine-s'''	)


	def   _UpperCAmelCase       (self       ,**_lowerCamelCase	)							->							CanineTokenizer:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class.from_pretrained(self.tmpdirname       ,**_lowerCamelCase	)
				__lowercase			  =							1024
				return tokenizer


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
				# fmt: off
				__lowercase			  =							[57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
				# fmt: on
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertIsInstance(_lowerCamelCase       ,_lowerCamelCase	)

				__lowercase			  =							list(batch.input_ids.numpy()[0]	)

				self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

				self.assertEqual((2, 39)       ,batch.input_ids.shape	)
				self.assertEqual((2, 39)       ,batch.attention_mask.shape	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				# check if input_ids, attention_mask and token_type_ids are returned
				self.assertIn('''input_ids'''       ,_lowerCamelCase	)
				self.assertIn('''attention_mask'''       ,_lowerCamelCase	)
				self.assertIn('''token_type_ids'''       ,_lowerCamelCase	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							[
				    '''What\'s the weater?''',
				    '''It\'s about 25 degrees.''',
				]
				__lowercase			  =							tokenizer(
				    text_target=_lowerCamelCase       ,max_length=32       ,padding='''max_length'''       ,truncation=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertEqual(32       ,targets['''input_ids'''].shape[1]	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										self.assertNotEqual(tokenizer.model_max_length       ,42	)

        # Now let's start the test
				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

										shutil.rmtree(_lowerCamelCase	)

				__lowercase			  =							self.get_tokenizers(model_max_length=42	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''

										__lowercase			  =							tokenizer.additional_special_tokens

										# We can add a new special token for Canine as follows:
										__lowercase			  =							chr(0xe_0_0_7	)
										additional_special_tokens.append(_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertIn(_lowerCamelCase       ,after_tokenizer.additional_special_tokens	)
										self.assertEqual(after_tokenizer.model_max_length       ,42	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase       ,model_max_length=43	)
										self.assertEqual(tokenizer.model_max_length       ,43	)

										shutil.rmtree(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase  ,						__lowercase			  =							self.get_clean_sequence(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_5
										__lowercase			  =							chr(_lowerCamelCase	)

										tokenizer.add_special_tokens({'''cls_token''': special_token}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)

										__lowercase			  =							tokenizer.decode(ids + encoded_special_token       ,clean_up_tokenization_spaces=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(_lowerCamelCase       ,input_encoded + special_token_id	)

										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,skip_special_tokens=_lowerCamelCase	)
										self.assertTrue(special_token not in decoded	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							chr(0xe_0_0_5	)
										__lowercase			  =							chr(0xe_0_0_6	)

										# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
										tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,special_tokens=_lowerCamelCase	)
										# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
										# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}	)

										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)
										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)

										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}	)

										with tempfile.TemporaryDirectory() as tmp_dir_name:
													tokenizer.save_pretrained(_lowerCamelCase	)
													tokenizer.from_pretrained(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							[]
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							[new_token_a]
										__lowercase			  =							[new_token_a]

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)
										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										__lowercase			  =							tokenizer_class.from_pretrained(_lowerCamelCase       ,extra_ids=0	)
										self.assertIn(_lowerCamelCase       ,tokenizer_without_change_in_init.additional_special_tokens	)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]	)	)       ,)

										__lowercase			  =							0xe_0_0_7
										__lowercase			  =							chr(_lowerCamelCase	)
										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										__lowercase			  =							[AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)]
										__lowercase			  =							tokenizer_class.from_pretrained(
										    _lowerCamelCase       ,additional_special_tokens=_lowerCamelCase       ,extra_ids=0	)

										self.assertIn(_lowerCamelCase       ,tokenizer.additional_special_tokens	)
										# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]	)	)	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							'''hello world'''
										if self.space_between_special_tokens:
													__lowercase			  =							'''[CLS] hello world [SEP]'''
										else:
													__lowercase			  =							input
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,spaces_between_special_tokens=self.space_between_special_tokens	)
										self.assertIn(_lowerCamelCase       ,[output, output.lower()]	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							[
										    '''bos_token''',
										    '''eos_token''',
										    '''unk_token''',
										    '''sep_token''',
										    '''pad_token''',
										    '''cls_token''',
										    '''mask_token''',
										]

				__lowercase			  =							'''a'''
				__lowercase			  =							ord(_lowerCamelCase	)

				for attr in attributes_list:
							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[]	)

				__lowercase			  =							0xe_0_0_6
				__lowercase			  =							chr(_lowerCamelCase	)
				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[additional_special_token_id]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[additional_special_token]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[additional_special_token_id]	)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass





