





'''simple docstring'''




import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the experiment tracking capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

lowercase__   :				Tuple      =   16
lowercase__   :				List[Any]      =   32






def 	_lowerCAmelCase      (  __snake_case  :			Accelerator			,							__snake_case  :			int = 16      )					->					Tuple:
     __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('bert-base-cased'      )
     __A						:			str								=	load_dataset('glue'			,							'mrpc'      )

     def tokenize_function(__snake_case  :			Tuple      ):
          # max_length=None => use the model max length (it's actually the default)
          __A						:			Union[str, Any]								=	tokenizer(examples['sentence1']			,							examples['sentence2']			,							truncation=__snake_case			,							max_length=__snake_case      )
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     # starting with the main process first:
     with accelerator.main_process_first():
          __A						:			Optional[Any]								=	datasets.map(
              __snake_case			,							batched=__snake_case			,							remove_columns=['idx', 'sentence1', 'sentence2']			,							)

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     __A						:			Any								=	tokenized_datasets.rename_column('label'			,							'labels'      )

     def collate_fn(__snake_case  :			Any      ):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          __A						:			Union[str, Any]								=	1_28 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
               __A						:			str								=	16
          elif accelerator.mixed_precision != "no":
               __A						:			Union[str, Any]								=	8
          else:
               __A						:			int								=	None

          return tokenizer.pad(
              __snake_case			,							padding='longest'			,							max_length=__snake_case			,							pad_to_multiple_of=__snake_case			,							return_tensors='pt'			,							)

     # Instantiate dataloaders.
     __A						:			Tuple								=	DataLoader(
         tokenized_datasets['train']			,							shuffle=__snake_case			,							collate_fn=__snake_case			,							batch_size=__snake_case      )
     __A						:			Any								=	DataLoader(
         tokenized_datasets['validation']			,							shuffle=__snake_case			,							collate_fn=__snake_case			,							batch_size=__snake_case      )

     return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  lowercase__   :				str      =   mocked_dataloaders  # noqa: F811






def 	_lowerCAmelCase      (  __snake_case  :			Optional[int]			,							__snake_case  :			List[Any]      )					->					List[Any]:
     # For testing only
     if os.environ.get('TESTING_MOCKED_DATALOADERS'			,							__snake_case      ) == "1":
          __A						:			Union[str, Any]								=	2
     # Initialize Accelerator

     # New Code #
     # We pass in "all" to `log_with` to grab all available trackers in the environment
     # Note: If using a custom `Tracker` class, should be passed in here such as:
     # >>> log_with = ["all", MyCustomTrackerClassInstance()]
     if args.with_tracking:
          __A						:			int								=	Accelerator(
              cpu=args.cpu			,							mixed_precision=args.mixed_precision			,							log_with='all'			,							project_dir=args.project_dir      )
     else:
          __A						:			List[str]								=	Accelerator(cpu=args.cpu			,							mixed_precision=args.mixed_precision      )
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     __A						:			str								=	config['lr']
     __A						:			str								=	int(config['num_epochs']      )
     __A						:			str								=	int(config['seed']      )
     __A						:			Optional[int]								=	int(config['batch_size']      )
     set_seed(__snake_case      )

     __A			,__A						:			List[str]								=	get_dataloaders(__snake_case			,							__snake_case      )
     __A						:			List[Any]								=	evaluate.load('glue'			,							'mrpc'      )

     # If the batch size is too big we use gradient accumulation
     __A						:			Optional[int]								=	1
     if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
          __A						:			Any								=	batch_size // MAX_GPU_BATCH_SIZE
          __A						:			Dict								=	MAX_GPU_BATCH_SIZE

     # Instantiate the model (we build the model here so that the seed also control new weights initialization)
     __A						:			Optional[int]								=	AutoModelForSequenceClassification.from_pretrained('bert-base-cased'			,							return_dict=__snake_case      )

     # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
     # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
     # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
     __A						:			List[Any]								=	model.to(accelerator.device      )

     # Instantiate optimizer
     __A						:			Optional[Any]								=	AdamW(params=model.parameters()			,							lr=__snake_case      )

     # Instantiate scheduler
     __A						:			Tuple								=	get_linear_schedule_with_warmup(
         optimizer=__snake_case			,							num_warmup_steps=1_00			,							num_training_steps=(len(__snake_case      ) * num_epochs) // gradient_accumulation_steps			,							)

     # Prepare everything
     # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
     # prepare method.
     __A			,__A			,__A			,__A			,__A						:			Dict								=	accelerator.prepare(
         __snake_case			,							__snake_case			,							__snake_case			,							__snake_case			,							__snake_case      )

     # New Code #
     # We need to initialize the trackers we use. Overall configurations can also be stored
     if args.with_tracking:
          __A						:			Tuple								=	os.path.split(__snake_case      )[-1].split('.'      )[0]
          accelerator.init_trackers(__snake_case			,							__snake_case      )

     # Now we train the model
     for epoch in range(__snake_case      ):
          model.train()
          # New Code #
          # For our tracking example, we will log the total loss of each epoch
          if args.with_tracking:
               __A						:			str								=	0
          for step, batch in enumerate(__snake_case      ):
               # We could avoid this line since we set the accelerator with `device_placement=True`.
               batch.to(accelerator.device      )
               __A						:			Tuple								=	model(**__snake_case      )
               __A						:			Optional[Any]								=	outputs.loss
               # New Code #
               if args.with_tracking:
                    total_loss += loss.detach().float()
               __A						:			Dict								=	loss / gradient_accumulation_steps
               accelerator.backward(__snake_case      )
               if step % gradient_accumulation_steps == 0:
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

          model.eval()
          for step, batch in enumerate(__snake_case      ):
               # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
               batch.to(accelerator.device      )
               with torch.no_grad():
                    __A						:			str								=	model(**__snake_case      )
               __A						:			List[str]								=	outputs.logits.argmax(dim=-1      )
               __A			,__A						:			Dict								=	accelerator.gather_for_metrics((predictions, batch['labels'])      )
               metric.add_batch(
                   predictions=__snake_case			,							references=__snake_case			,							)

          __A						:			int								=	metric.compute()
          # Use accelerator.print to print only on the main process.
          accelerator.print(f'epoch {epoch}:'			,							__snake_case      )

          # New Code #
          # To actually log, we call `Accelerator.log`
          # The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
          if args.with_tracking:
               accelerator.log(
                   {
                       'accuracy': eval_metric['accuracy'],
                       'f1': eval_metric['f1'],
                       'train_loss': total_loss.item() / len(__snake_case      ),
                       'epoch': epoch,
                   }			,							step=__snake_case			,							)

    # New Code #
    # When a run is finished, you should call `accelerator.end_training()`
    # to close all of the open trackers
     if args.with_tracking:
          accelerator.end_training()






def 	_lowerCAmelCase      (  )					->					Optional[int]:
     __A						:			Optional[int]								=	argparse.ArgumentParser(description='Simple example of training script.'      )
     parser.add_argument(
         '--mixed_precision'			,							type=__snake_case			,							default=__snake_case			,							choices=['no', 'fp16', 'bf16', 'fp8']			,							help='Whether to use mixed precision. Choose'
         'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
         'and an Nvidia Ampere GPU.'			,							)
     parser.add_argument('--cpu'			,							action='store_true'			,							help='If passed, will train on the CPU.'      )
     parser.add_argument(
         '--with_tracking'			,							action='store_true'			,							help='Whether to load in all available experiment trackers from the environment and use them for logging.'			,							)
     parser.add_argument(
         '--project_dir'			,							type=__snake_case			,							default='logs'			,							help='Location on where to store experiment tracking logs` and relevent project information'			,							)
     __A						:			str								=	parser.parse_args()
     __A						:			Any								=	{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
     training_function(__snake_case			,							__snake_case      )


if __name__ == "__main__":
  main()






'''simple docstring'''




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 			SCREAMING_SNAKE_CASE   (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	0


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)

          for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (GPTaTokenizer, GPTaTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (RobertaTokenizer, RobertaTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     20)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	AutoConfig.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)
          # Check that tokenizer_type ≠ model_type
          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     config=_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			str								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with pytest.raises(_UpperCAmelCase):
               AutoTokenizer.from_pretrained('./'						,     tokenizer_type='xxx')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               __A						:			List[Any]								=	tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased')
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))

               if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):
                    self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,     _UpperCAmelCase)
               else:
                    self.assertEqual(tokenizer.do_lower_case						,     _UpperCAmelCase)

               self.assertEqual(tokenizer.model_max_length						,     512)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   _UpperCAmelCase						,     'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'						,     ):
                    __A						:			str								=	tokenizer_class.from_pretrained('julien-c/herlolip-not-exists')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	TOKENIZER_MAPPING.values()
          __A						:			Union[str, Any]								=	[]

          for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                    tokenizer_names.append(slow_tok.__name__)

               if fast_tok is not None:
                    tokenizer_names.append(fast_tok.__name__)

          for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(_UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'						,     use_fast=_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased')						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[str]								=	AutoTokenizer.from_pretrained('distilbert-base-uncased'						,     do_lower_case=_UpperCAmelCase)
          __A						:			str								=	'Hello, world. How are you?'
          __A						:			List[str]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])

          __A						:			Dict								=	AutoTokenizer.from_pretrained('microsoft/mpnet-base'						,     do_lower_case=_UpperCAmelCase)
          __A						:			List[Any]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config')
          self.assertEqual(type(_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertEqual(tokenizer.model_max_length						,     512)
          self.assertEqual(tokenizer.vocab_size						,     3_0000)
          self.assertEqual(tokenizer.unk_token						,     '[UNK]')
          self.assertEqual(tokenizer.padding_side						,     'right')
          self.assertEqual(tokenizer.truncation_side						,     'right')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)

          self.assertIsInstance(_UpperCAmelCase						,     tokenizer.__class__)
          self.assertEqual(tokenizera.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	AutoTokenizer.from_pretrained('ctrl')
          # There is no fast CTRL so this always gives us a slow tokenizer.
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[Any]								=	get_tokenizer_config('bert-base-cased')
          __A						:			Optional[int]								=	config.pop('_commit_hash'						,     _UpperCAmelCase)
          # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
          self.assertEqual(_UpperCAmelCase						,     {'do_lower_case': False})

          # This model does not have a tokenizer_config so we get back an empty dict.
          __A						:			Dict								=	get_tokenizer_config(_UpperCAmelCase)
          self.assertDictEqual(_UpperCAmelCase						,     {})

          # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
          __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	get_tokenizer_config(_UpperCAmelCase)

          # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
          self.assertEqual(config['tokenizer_class']						,     'BertTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)

               __A						:			Optional[Any]								=	CustomTokenizer.from_pretrained(_UpperCAmelCase)
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			int								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               # Can register in two steps
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, None))
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   _UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                    __A						:			Optional[int]								=	BertTokenizerFast.from_pretrained(_UpperCAmelCase)
                    bert_tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Dict								=	CustomTokenizerFast.from_pretrained(_UpperCAmelCase)

               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

                    __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaises(_UpperCAmelCase):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
          # If remote code is disabled, we can't load this config.
          with self.assertRaises(_UpperCAmelCase):
               __A						:			Dict								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)

          __A						:			str								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          # Test tokenizer can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Dict								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(reloaded_tokenizer.special_attribute_present)

          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(reloaded_tokenizer.special_attribute_present)
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	False




          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	NewTokenizer
               lowerCAmelCase		      =	False

          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               # If remote code is not set, the default is to use local
               __A						:			List[Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Dict								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote code is disabled, we load the local one.
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Any								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote is enabled, we load from the Hub
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertTrue(tokenizer.special_attribute_present)
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(tokenizer.special_attribute_present)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			int								=	AutoTokenizer.from_pretrained(
              'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     'bert-base is not a local folder and is not a valid model identifier'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('bert-base')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     revision='aaaaaa')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
          with RequestCounter() as counter:
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
               self.assertEqual(counter.get_request_count						,     0)
               self.assertEqual(counter.head_request_count						,     1)
               self.assertEqual(counter.other_request_count						,     0)