





'''simple docstring'''




import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


lowercase__   :				Optional[int]      =   logging.getLogger(__name__)


class 			SCREAMING_SNAKE_CASE   (a__							):
     lowerCAmelCase		      =	'''sequence-classification'''


     def __init__(						self						,     _UpperCAmelCase):

          '''simple docstring'''






          if type(_UpperCAmelCase) == dict:
               __A						:			Any								=	Namespace(**_UpperCAmelCase)
          __A						:			Union[str, Any]								=	glue_output_modes[hparams.task]
          __A						:			List[Any]								=	glue_tasks_num_labels[hparams.task]

          super().__init__(_UpperCAmelCase						,     _UpperCAmelCase						,     self.mode)


     def 							SCREAMING_SNAKE_CASE  (						self						,     **_UpperCAmelCase):

          '''simple docstring'''






          return self.model(**_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Any								=	{'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

          if self.config.model_type not in ["distilbert", "bart"]:
               __A						:			List[Any]								=	batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

          __A						:			Dict								=	self(**_UpperCAmelCase)
          __A						:			str								=	outputs[0]

          __A						:			str								=	self.trainer.lr_schedulers[0]['scheduler']
          __A						:			List[Any]								=	{'loss': loss, 'rate': lr_scheduler.get_last_lr()[-1]}
          return {"loss": loss, "log": tensorboard_logs}


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[Any]								=	self.hparams
          __A						:			Optional[Any]								=	processors[args.task]()
          __A						:			Union[str, Any]								=	processor.get_labels()

          for mode in ["train", "dev"]:
               __A						:			int								=	self._feature_file(_UpperCAmelCase)
               if os.path.exists(_UpperCAmelCase) and not args.overwrite_cache:
                    logger.info('Loading features from cached file %s'						,     _UpperCAmelCase)
               else:
                    logger.info('Creating features from dataset file at %s'						,     args.data_dir)
                    __A						:			Optional[int]								=	(
                        processor.get_dev_examples(args.data_dir)
                        if mode == 'dev'
                        else processor.get_train_examples(args.data_dir)
                    )
                    __A						:			Any								=	convert_examples_to_features(
                        _UpperCAmelCase						,     self.tokenizer						,     max_length=args.max_seq_length						,     label_list=self.labels						,     output_mode=args.glue_output_mode						,     )
                    logger.info('Saving features into cached file %s'						,     _UpperCAmelCase)
                    torch.save(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase = False):

          '''simple docstring'''






          __A						:			Dict								=	'dev' if mode == 'test' else mode

          __A						:			Optional[Any]								=	self._feature_file(_UpperCAmelCase)
          logger.info('Loading features from cached file %s'						,     _UpperCAmelCase)
          __A						:			Optional[Any]								=	torch.load(_UpperCAmelCase)
          __A						:			Union[str, Any]								=	torch.tensor([f.input_ids for f in features]						,     dtype=torch.long)
          __A						:			Any								=	torch.tensor([f.attention_mask for f in features]						,     dtype=torch.long)
          __A						:			Optional[Any]								=	torch.tensor([f.token_type_ids for f in features]						,     dtype=torch.long)
          if self.hparams.glue_output_mode == "classification":
               __A						:			Union[str, Any]								=	torch.tensor([f.label for f in features]						,     dtype=torch.long)
          elif self.hparams.glue_output_mode == "regression":
               __A						:			int								=	torch.tensor([f.label for f in features]						,     dtype=torch.float)

          return DataLoader(
              TensorDataset(_UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)						,     batch_size=_UpperCAmelCase						,     shuffle=_UpperCAmelCase						,     )


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Tuple								=	{'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

          if self.config.model_type not in ["distilbert", "bart"]:
               __A						:			Dict								=	batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

          __A						:			Any								=	self(**_UpperCAmelCase)
          __A			,__A						:			str								=	outputs[:2]
          __A						:			Optional[Any]								=	logits.detach().cpu().numpy()
          __A						:			List[str]								=	inputs['labels'].detach().cpu().numpy()

          return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Dict								=	torch.stack([x['val_loss'] for x in outputs]).mean().detach().cpu().item()
          __A						:			str								=	np.concatenate([x['pred'] for x in outputs]						,     axis=0)

          if self.hparams.glue_output_mode == "classification":
               __A						:			int								=	np.argmax(_UpperCAmelCase						,     axis=1)
          elif self.hparams.glue_output_mode == "regression":
               __A						:			Dict								=	np.squeeze(_UpperCAmelCase)

          __A						:			Union[str, Any]								=	np.concatenate([x['target'] for x in outputs]						,     axis=0)
          __A						:			Dict								=	[[] for _ in range(out_label_ids.shape[0])]
          __A						:			int								=	[[] for _ in range(out_label_ids.shape[0])]

          __A						:			Dict								=	{**{'val_loss': val_loss_mean}, **compute_metrics(self.hparams.task						,     _UpperCAmelCase						,     _UpperCAmelCase)}

          __A						:			Tuple								=	dict(results.items())
          __A						:			Optional[int]								=	results
          return ret, preds_list, out_label_list


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          __A			,__A			,__A						:			int								=	self._eval_end(_UpperCAmelCase)
          __A						:			Dict								=	ret['log']
          return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase):

          '''simple docstring'''






          __A			,__A			,__A						:			Optional[Any]								=	self._eval_end(_UpperCAmelCase)
          __A						:			Tuple								=	ret['log']
          # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
          return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}


     @staticmethod
     def 							SCREAMING_SNAKE_CASE  (						_UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          BaseTransformer.add_model_specific_args(_UpperCAmelCase						,     _UpperCAmelCase)
          parser.add_argument(
              '--max_seq_length'						,     default=128						,     type=_UpperCAmelCase						,     help=(
                  'The maximum total input sequence length after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )						,     )

          parser.add_argument(
              '--task'						,     default=''						,     type=_UpperCAmelCase						,     required=_UpperCAmelCase						,     help='The GLUE task to run'						,     )
          parser.add_argument(
              '--gpus'						,     default=0						,     type=_UpperCAmelCase						,     help='The number of GPUs allocated for this, it is by default 0 meaning none'						,     )

          parser.add_argument(
              '--overwrite_cache'						,     action='store_true'						,     help='Overwrite the cached training and evaluation sets')

          return parser






def 	_lowerCAmelCase      (  )					->					Dict:
     __A						:			int								=	argparse.ArgumentParser()
     add_generic_args(__snake_case			,							os.getcwd()      )
     __A						:			Tuple								=	GLUETransformer.add_model_specific_args(__snake_case			,							os.getcwd()      )
     __A						:			Union[str, Any]								=	parser.parse_args()

     # If output_dir not provided, a folder will be generated in pwd
     if args.output_dir is None:
          __A						:			List[str]								=	os.path.join(
              './results'			,							f'{args.task}_{time.strftime("%Y%m%d_%H%M%S"      )}'			,							)
          os.makedirs(args.output_dir      )

     __A						:			Any								=	GLUETransformer(__snake_case      )
     __A						:			List[Any]								=	generic_train(__snake_case			,							__snake_case      )

     # Optionally, predict on dev set and write to output_dir
     if args.do_predict:
          __A						:			List[str]								=	sorted(glob.glob(os.path.join(args.output_dir			,							'checkpoint-epoch=*.ckpt'      )			,							recursive=__snake_case      )      )
          __A						:			List[Any]								=	model.load_from_checkpoint(checkpoints[-1]      )
          return trainer.test(__snake_case      )


if __name__ == "__main__":
  main()






'''simple docstring'''




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 			SCREAMING_SNAKE_CASE   (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	0


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)

          for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (GPTaTokenizer, GPTaTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (RobertaTokenizer, RobertaTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     20)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	AutoConfig.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)
          # Check that tokenizer_type ≠ model_type
          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     config=_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			str								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with pytest.raises(_UpperCAmelCase):
               AutoTokenizer.from_pretrained('./'						,     tokenizer_type='xxx')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               __A						:			List[Any]								=	tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased')
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))

               if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):
                    self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,     _UpperCAmelCase)
               else:
                    self.assertEqual(tokenizer.do_lower_case						,     _UpperCAmelCase)

               self.assertEqual(tokenizer.model_max_length						,     512)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   _UpperCAmelCase						,     'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'						,     ):
                    __A						:			str								=	tokenizer_class.from_pretrained('julien-c/herlolip-not-exists')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	TOKENIZER_MAPPING.values()
          __A						:			Union[str, Any]								=	[]

          for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                    tokenizer_names.append(slow_tok.__name__)

               if fast_tok is not None:
                    tokenizer_names.append(fast_tok.__name__)

          for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(_UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'						,     use_fast=_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased')						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[str]								=	AutoTokenizer.from_pretrained('distilbert-base-uncased'						,     do_lower_case=_UpperCAmelCase)
          __A						:			str								=	'Hello, world. How are you?'
          __A						:			List[str]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])

          __A						:			Dict								=	AutoTokenizer.from_pretrained('microsoft/mpnet-base'						,     do_lower_case=_UpperCAmelCase)
          __A						:			List[Any]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config')
          self.assertEqual(type(_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertEqual(tokenizer.model_max_length						,     512)
          self.assertEqual(tokenizer.vocab_size						,     3_0000)
          self.assertEqual(tokenizer.unk_token						,     '[UNK]')
          self.assertEqual(tokenizer.padding_side						,     'right')
          self.assertEqual(tokenizer.truncation_side						,     'right')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)

          self.assertIsInstance(_UpperCAmelCase						,     tokenizer.__class__)
          self.assertEqual(tokenizera.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	AutoTokenizer.from_pretrained('ctrl')
          # There is no fast CTRL so this always gives us a slow tokenizer.
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[Any]								=	get_tokenizer_config('bert-base-cased')
          __A						:			Optional[int]								=	config.pop('_commit_hash'						,     _UpperCAmelCase)
          # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
          self.assertEqual(_UpperCAmelCase						,     {'do_lower_case': False})

          # This model does not have a tokenizer_config so we get back an empty dict.
          __A						:			Dict								=	get_tokenizer_config(_UpperCAmelCase)
          self.assertDictEqual(_UpperCAmelCase						,     {})

          # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
          __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	get_tokenizer_config(_UpperCAmelCase)

          # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
          self.assertEqual(config['tokenizer_class']						,     'BertTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)

               __A						:			Optional[Any]								=	CustomTokenizer.from_pretrained(_UpperCAmelCase)
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			int								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               # Can register in two steps
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, None))
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   _UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                    __A						:			Optional[int]								=	BertTokenizerFast.from_pretrained(_UpperCAmelCase)
                    bert_tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Dict								=	CustomTokenizerFast.from_pretrained(_UpperCAmelCase)

               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

                    __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaises(_UpperCAmelCase):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
          # If remote code is disabled, we can't load this config.
          with self.assertRaises(_UpperCAmelCase):
               __A						:			Dict								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)

          __A						:			str								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          # Test tokenizer can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Dict								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(reloaded_tokenizer.special_attribute_present)

          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(reloaded_tokenizer.special_attribute_present)
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	False




          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	NewTokenizer
               lowerCAmelCase		      =	False

          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               # If remote code is not set, the default is to use local
               __A						:			List[Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Dict								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote code is disabled, we load the local one.
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Any								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote is enabled, we load from the Hub
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertTrue(tokenizer.special_attribute_present)
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(tokenizer.special_attribute_present)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			int								=	AutoTokenizer.from_pretrained(
              'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     'bert-base is not a local folder and is not a valid model identifier'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('bert-base')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     revision='aaaaaa')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
          with RequestCounter() as counter:
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
               self.assertEqual(counter.get_request_count						,     0)
               self.assertEqual(counter.head_request_count						,     1)
               self.assertEqual(counter.other_request_count						,     0)