





'''simple docstring'''



import argparse
import random

import joblib
import numpy as np
import torch
from igf.igf import (
    SecondaryLearner,
    collect_objective_set,
    compute_perplexity,
    generate_datasets,
    load_gpta,
    recopy_gpta,
    set_seed,
    train_secondary_learner,
)
from torch.utils.data import DataLoader, RandomSampler

from transformers import GPTaLMHeadModel




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Dict=3_2						,    lowerCamelCase_     :					Optional[int]=1_0						,    lowerCamelCase_     :					List[str]=1_0_0						,    lowerCamelCase_     :					Optional[int]=1_0_2_6						,    lowerCamelCase_     :					Any=True						,    lowerCamelCase_     :					Optional[int]="data/tokenized_stories_train_wikitext103.jbl"						,    lowerCamelCase_     :					Dict="igf_context_pairs.jbl"						,    ):
			set_seed(3			)
			# generate train_data and objective_set
			__lowercase  ,						__lowercase			  =							generate_datasets(
			    lowerCamelCase_						,    lowerCamelCase_						,    number=lowerCamelCase_						,    min_len=1_0_2_6						,    trim=lowerCamelCase_			)
			# keeps model same across runs
			set_seed(4			)
			# model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights
			# can we train on GPU?
			__lowercase			  =							torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''			)

			# load pretrained model
			__lowercase			  =							load_gpta('''gpt2'''			).to(lowerCamelCase_			)
			print('''computing perplexity on objective set'''			)
			__lowercase			  =							compute_perplexity(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			).item()
			print('''perplexity on objective set:'''						,    lowerCamelCase_			)

			# collect igf pairs and save to file demo.jbl
			collect_objective_set(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			)

			# clean up, delete model and data we don't need anymore
			del model, train_data, objective_set
			torch.cuda.empty_cache()




def 				_lowerCAmelCase  (			lowerCamelCase_     :					int						,    lowerCamelCase_     :					str=1_5						,    lowerCamelCase_     :					List[Any]=1_2_8						,    lowerCamelCase_     :					Union[str, Any]=1_0_0						,    lowerCamelCase_     :					Tuple="igf_model.pt"						,    ):
			set_seed(4_2			)

			# Load pre-trained model
			__lowercase			  =							GPTaLMHeadModel.from_pretrained('''gpt2'''			)

			# Initialize secondary learner to use embedding weights of model
			__lowercase			  =							SecondaryLearner(lowerCamelCase_			)

			# Train secondary learner
			__lowercase			  =							train_secondary_learner(
			    lowerCamelCase_						,    lowerCamelCase_						,    max_epochs=lowerCamelCase_						,    batch_size=lowerCamelCase_						,    eval_freq=1_0_0						,    igf_model_path=lowerCamelCase_						,    )

			del model, secondary_learner_train_data
			torch.cuda.empty_cache()

			return secondary_learner




def 				_lowerCAmelCase  (			lowerCamelCase_     :					str						,    lowerCamelCase_     :					Union[str, Any]						,    lowerCamelCase_     :					Any						,    lowerCamelCase_     :					List[Any]=3_2						,    lowerCamelCase_     :					Optional[int]=1_0_0_0						,    lowerCamelCase_     :					int=1_6						,    lowerCamelCase_     :					Union[str, Any]=1.0						,    lowerCamelCase_     :					Dict=recopy_gpta						,    lowerCamelCase_     :					Tuple=None						,    lowerCamelCase_     :					int=1_0						,    lowerCamelCase_     :					Union[str, Any]="gpt2_finetuned.pt"						,    ):
			__lowercase			  =							torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''			)
			__lowercase			  =							RandomSampler(lowerCamelCase_			)
			__lowercase			  =							DataLoader(lowerCamelCase_						,    sampler=lowerCamelCase_			)

			__lowercase			  =							max_steps // (len(lowerCamelCase_			)) + 1
			__lowercase			  =							0
			__lowercase			  =							torch.zeros((1, context_len)						,    dtype=torch.long						,    device=lowerCamelCase_			)
			__lowercase  ,						__lowercase  ,						__lowercase			  =							recopy_model(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			)

			model.train()
			if secondary_learner is not None:
						secondary_learner.to(lowerCamelCase_			)
						secondary_learner.eval()
			__lowercase			  =							[]
			__lowercase			  =							0

			__lowercase			  =							[]
			__lowercase			  =							[]

			# Compute the performance of the transformer model at the beginning
			__lowercase			  =							compute_perplexity(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			)
			test_perps.append(lowerCamelCase_			)
			print('''Test perplexity, step'''						,    lowerCamelCase_						,    ''':'''						,    lowerCamelCase_			)
			for epoch in range(int(lowerCamelCase_			)			):
						for step, example in enumerate(lowerCamelCase_			):
									torch.cuda.empty_cache()
									__lowercase			  =							random.randint(0						,    example.size(2			) - context_len - 1			)
									__lowercase			  =							example[0, 0, start : start + context_len]
									lm_optimizer.zero_grad()
									__lowercase			  =							model(lowerCamelCase_						,    labels=lowerCamelCase_			)
									__lowercase			  =							True

									if secondary_learner is not None:
												__lowercase			  =							secondary_learner.forward(
												    torch.tensor(lowerCamelCase_						,    dtype=torch.long						,    device=lowerCamelCase_			).unsqueeze(0			)			)[0].item()
												observed_qs.append(float(lowerCamelCase_			)			)

												# Here we implement the simple non-constant threshold for the predicted IG(X) value
												# We will decay the selectivity of our secondary learner filter from
												# 1 standard deviation above average to 1 below average after 10 batches.

												if global_step == 1_0:
															__lowercase			  =							-1
												if predicted_q < threshold:
															__lowercase			  =							False

            # If we passed the filter, add the context to the batch!
									if do_backprop:
												contexts.append(np.array(context.cpu()			)			)
												__lowercase			  =							outputs[0]
												lm_loss.backward()
												examples += 1

									del outputs

									# Once the batch is filled with enough contexts, backprop on the batch.
									if examples == batch_size:
												torch.cuda.empty_cache()
												__lowercase			  =							0
												# Do LM backprop
												torch.nn.utils.clip_grad_norm_(model.parameters()						,    3.0			)
												lm_optimizer.step()
												lm_scheduler.step()  # Update learning rate schedule
												global_step += 1
												# Compute the performance of the transformer model at this batch
												if global_step % eval_interval == 0:
															__lowercase			  =							compute_perplexity(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			)
															test_perps.append(lowerCamelCase_			)

															print('''Test perplexity, step'''						,    lowerCamelCase_						,    ''':'''						,    lowerCamelCase_			)
            # Break out of the loop after 60 batches
									if max_steps > 0 and global_step > 6_0:
												break
						if max_steps > 0 and global_step > 6_0:
									break

    # save finetuned transformer model
			torch.save(model.state_dict()						,    lowerCamelCase_			)
			torch.cuda.empty_cache()
			# Do some cleaning up so we can reinitialize for the next run of this function
			del lm_optimizer
			del lm_scheduler
			return model




def 				_lowerCAmelCase  (			):
			__lowercase			  =							argparse.ArgumentParser(description='''Fine-tune a transformer model with IGF on a language modeling task'''			)

			# Required parameters
			parser.add_argument(
			    '''--data_dir'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    required=lowerCamelCase_						,    help='''The input data dir. Should contain data files for WikiText.'''						,    )
			parser.add_argument(
			    '''--model_name_or_path'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    required=lowerCamelCase_						,    help='''Path to pretrained model or model identifier from huggingface.co/models'''						,    )
			parser.add_argument(
			    '''--data_file'''						,    type=lowerCamelCase_						,    default=lowerCamelCase_						,    help=(
			        '''A jbl file containing tokenized data which can be split as objective dataset, '''
			        '''train_dataset and test_dataset.'''
			    )						,    )

			parser.add_argument(
			    '''--igf_data_file'''						,    type=lowerCamelCase_						,    default=lowerCamelCase_						,    help='''A jbl file containing the context and information gain pairs to train secondary learner.'''						,    )

			parser.add_argument(
			    '''--output_dir'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    required=lowerCamelCase_						,    help='''The output directory where the final fine-tuned model is stored.'''						,    )

			parser.add_argument(
			    '''--tokenizer_name'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    help='''Pretrained tokenizer name or path if not the same as model_name'''						,    )
			parser.add_argument('''--seed'''						,    type=lowerCamelCase_						,    default=lowerCamelCase_						,    help='''A seed for reproducible training.'''			)

			parser.add_argument(
			    '''--context_len'''						,    default=3_2						,    type=lowerCamelCase_						,    help=(
			        '''The maximum total input sequence length after tokenization. Sequences longer '''
			        '''than this will be truncated, sequences shorter will be padded.'''
			    )						,    )

			parser.add_argument(
			    '''--size_objective_set'''						,    default=1_0_0						,    type=lowerCamelCase_						,    help='''number of articles that are long enough to be used as our objective set'''						,    )
			parser.add_argument(
			    '''--eval_freq'''						,    default=1_0_0						,    type=lowerCamelCase_						,    help='''secondary model evaluation is triggered at eval_freq'''			)

			parser.add_argument('''--max_steps'''						,    default=1_0_0_0						,    type=lowerCamelCase_						,    help='''To calculate training epochs'''			)

			parser.add_argument(
			    '''--secondary_learner_batch_size'''						,    default=1_2_8						,    type=lowerCamelCase_						,    help='''batch size of training data for secondary learner'''						,    )

			parser.add_argument(
			    '''--batch_size'''						,    default=1_6						,    type=lowerCamelCase_						,    help='''batch size of training data of language model(gpt2) '''			)

			parser.add_argument(
			    '''--eval_interval'''						,    default=1_0						,    type=lowerCamelCase_						,    help=(
			        '''decay the selectivity of our secondary learner filter from'''
			        '''1 standard deviation above average to 1 below average after 10 batches'''
			    )						,    )

			parser.add_argument(
			    '''--number'''						,    default=1_0_0						,    type=lowerCamelCase_						,    help='''The number of examples split to be used as objective_set/test_data'''			)

			parser.add_argument(
			    '''--min_len'''						,    default=1_0_2_6						,    type=lowerCamelCase_						,    help='''The minimum length of the article to be used as objective set'''			)

			parser.add_argument(
			    '''--secondary_learner_max_epochs'''						,    default=1_5						,    type=lowerCamelCase_						,    help='''number of epochs to train secondary learner'''			)

			parser.add_argument('''--trim'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    help='''truncate the example if it exceeds context length'''			)

			parser.add_argument(
			    '''--threshold'''						,    default=1.0						,    type=lowerCamelCase_						,    help=(
			        '''The threshold value used by secondary learner to filter the train_data and allow only'''
			        ''' informative data as input to the model'''
			    )						,    )

			parser.add_argument('''--finetuned_model_name'''						,    default='''gpt2_finetuned.pt'''						,    type=lowerCamelCase_						,    help='''finetuned_model_name'''			)

			parser.add_argument(
			    '''--recopy_model'''						,    default=lowerCamelCase_						,    type=lowerCamelCase_						,    help='''Reset the model to the original pretrained GPT-2 weights after each iteration'''						,    )

			# function calls
			# Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner
			generate_n_pairs(
			    context_len=3_2						,    max_steps=1_0						,    size_objective_set=1_0_0						,    min_len=1_0_2_6						,    trim=lowerCamelCase_						,    data_file='''data/tokenized_stories_train_wikitext103.jbl'''						,    igf_data_file='''igf_context_pairs.jbl'''						,    )

			# Load train data for secondary learner
			__lowercase			  =							joblib.load('''data/IGF_values.jbl'''			)

			# Train secondary learner
			__lowercase			  =							training_secondary_learner(
			    lowerCamelCase_						,    secondary_learner_max_epochs=1_5						,    secondary_learner_batch_size=1_2_8						,    eval_freq=1_0_0						,    igf_model_path='''igf_model.pt'''						,    )

			# load pretrained gpt2 model
			__lowercase			  =							GPTaLMHeadModel.from_pretrained('''gpt2'''			)
			set_seed(4_2			)

			# Generate train and test data to train and evaluate gpt2 model
			__lowercase  ,						__lowercase			  =							generate_datasets(
			    context_len=3_2						,    file='''data/tokenized_stories_train_wikitext103.jbl'''						,    number=1_0_0						,    min_len=1_0_2_6						,    trim=lowerCamelCase_			)

			# fine-tuning of the gpt2 model using igf (Information Gain Filtration)
			finetune(
			    lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_						,    context_len=3_2						,    max_steps=1_0_0_0						,    batch_size=1_6						,    threshold=1.0						,    recopy_model=lowerCamelCase_						,    secondary_learner=lowerCamelCase_						,    eval_interval=1_0						,    finetuned_model_name='''gpt2_finetuned.pt'''						,    )


if __name__ == "__main__":
	main()












'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class 							__lowercase    ( lowerCAmelCase__	, unittest.TestCase  ):







	'''simple docstring'''

	a      :			List[Any]     =    CanineTokenizer
	a      :			Union[str, Any]     =    False


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				super().setUp()
				__lowercase			  =							CanineTokenizer()
				tokenizer.save_pretrained(self.tmpdirname	)


	@cached_property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return CanineTokenizer.from_pretrained('''google/canine-s'''	)


	def   _UpperCAmelCase       (self       ,**_lowerCamelCase	)							->							CanineTokenizer:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class.from_pretrained(self.tmpdirname       ,**_lowerCamelCase	)
				__lowercase			  =							1024
				return tokenizer


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
				# fmt: off
				__lowercase			  =							[57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
				# fmt: on
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertIsInstance(_lowerCamelCase       ,_lowerCamelCase	)

				__lowercase			  =							list(batch.input_ids.numpy()[0]	)

				self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

				self.assertEqual((2, 39)       ,batch.input_ids.shape	)
				self.assertEqual((2, 39)       ,batch.attention_mask.shape	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				# check if input_ids, attention_mask and token_type_ids are returned
				self.assertIn('''input_ids'''       ,_lowerCamelCase	)
				self.assertIn('''attention_mask'''       ,_lowerCamelCase	)
				self.assertIn('''token_type_ids'''       ,_lowerCamelCase	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							[
				    '''What\'s the weater?''',
				    '''It\'s about 25 degrees.''',
				]
				__lowercase			  =							tokenizer(
				    text_target=_lowerCamelCase       ,max_length=32       ,padding='''max_length'''       ,truncation=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertEqual(32       ,targets['''input_ids'''].shape[1]	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										self.assertNotEqual(tokenizer.model_max_length       ,42	)

        # Now let's start the test
				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

										shutil.rmtree(_lowerCamelCase	)

				__lowercase			  =							self.get_tokenizers(model_max_length=42	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''

										__lowercase			  =							tokenizer.additional_special_tokens

										# We can add a new special token for Canine as follows:
										__lowercase			  =							chr(0xe_0_0_7	)
										additional_special_tokens.append(_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertIn(_lowerCamelCase       ,after_tokenizer.additional_special_tokens	)
										self.assertEqual(after_tokenizer.model_max_length       ,42	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase       ,model_max_length=43	)
										self.assertEqual(tokenizer.model_max_length       ,43	)

										shutil.rmtree(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase  ,						__lowercase			  =							self.get_clean_sequence(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_5
										__lowercase			  =							chr(_lowerCamelCase	)

										tokenizer.add_special_tokens({'''cls_token''': special_token}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)

										__lowercase			  =							tokenizer.decode(ids + encoded_special_token       ,clean_up_tokenization_spaces=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(_lowerCamelCase       ,input_encoded + special_token_id	)

										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,skip_special_tokens=_lowerCamelCase	)
										self.assertTrue(special_token not in decoded	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							chr(0xe_0_0_5	)
										__lowercase			  =							chr(0xe_0_0_6	)

										# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
										tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,special_tokens=_lowerCamelCase	)
										# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
										# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}	)

										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)
										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)

										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}	)

										with tempfile.TemporaryDirectory() as tmp_dir_name:
													tokenizer.save_pretrained(_lowerCamelCase	)
													tokenizer.from_pretrained(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							[]
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							[new_token_a]
										__lowercase			  =							[new_token_a]

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)
										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										__lowercase			  =							tokenizer_class.from_pretrained(_lowerCamelCase       ,extra_ids=0	)
										self.assertIn(_lowerCamelCase       ,tokenizer_without_change_in_init.additional_special_tokens	)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]	)	)       ,)

										__lowercase			  =							0xe_0_0_7
										__lowercase			  =							chr(_lowerCamelCase	)
										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										__lowercase			  =							[AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)]
										__lowercase			  =							tokenizer_class.from_pretrained(
										    _lowerCamelCase       ,additional_special_tokens=_lowerCamelCase       ,extra_ids=0	)

										self.assertIn(_lowerCamelCase       ,tokenizer.additional_special_tokens	)
										# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]	)	)	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							'''hello world'''
										if self.space_between_special_tokens:
													__lowercase			  =							'''[CLS] hello world [SEP]'''
										else:
													__lowercase			  =							input
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,spaces_between_special_tokens=self.space_between_special_tokens	)
										self.assertIn(_lowerCamelCase       ,[output, output.lower()]	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							[
										    '''bos_token''',
										    '''eos_token''',
										    '''unk_token''',
										    '''sep_token''',
										    '''pad_token''',
										    '''cls_token''',
										    '''mask_token''',
										]

				__lowercase			  =							'''a'''
				__lowercase			  =							ord(_lowerCamelCase	)

				for attr in attributes_list:
							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[]	)

				__lowercase			  =							0xe_0_0_6
				__lowercase			  =							chr(_lowerCamelCase	)
				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[additional_special_token_id]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[additional_special_token]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[additional_special_token_id]	)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass





