




'''simple docstring'''


import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

_snake_case				:				Dict       							=		[
    'cross_validation.py',
    'gradient_accumulation.py',
    'local_sgd.py',
    'multi_process_metrics.py',
    'memory.py',
    'automatic_gradient_accumulation.py',
    'fsdp_with_peak_mem_tracking.py',
    'deepspeed_with_config_support.py',
    'megatron_lm_gpt_pretraining.py',
]

class       A   (		unittest.TestCase   ):







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  bool							,      lowerCAmelCase_	:  str = None							,      lowerCAmelCase_	:  list = None )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    None
														_a     					=    os.path.abspath(os.path.join('''examples'''							,      '''by_feature''' ) )
														_a     					=    os.path.abspath('''examples''' )
														for item in os.listdir(lowerCAmelCase_ ):
																					if item not in EXCLUDE_EXAMPLES:
																												_a     					=    os.path.join(lowerCAmelCase_							,      lowerCAmelCase_ )
																												if os.path.isfile(lowerCAmelCase_ ) and ".py" in item_path:
																																			with self.subTest(
																																			    tested_script=lowerCAmelCase_							,      feature_script=lowerCAmelCase_							,      tested_section='''main()''' if parser_only else '''training_function()'''							,      ):
																																										_a     					=    compare_against_test(
																																										    os.path.join(lowerCAmelCase_							,      lowerCAmelCase_ )							,      lowerCAmelCase_							,      lowerCAmelCase_							,      lowerCAmelCase_ )
																																										_a     					=    '''\n'''.join(lowerCAmelCase_ )
																																										if special_strings is not None:
																																																	for string in special_strings:
																																																								_a     					=    diff.replace(lowerCAmelCase_							,      '''''' )
																																										self.assertEqual(lowerCAmelCase_							,      '''''' )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							Optional[Any]:


														"""simple docstring"""


														self.one_complete_example('''complete_nlp_example.py'''							,      lowerCAmelCase_ )
														self.one_complete_example('''complete_nlp_example.py'''							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  int )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    os.path.abspath(os.path.join('''examples'''							,      '''cv_example.py''' ) )
														_a     					=    [
														    ''' ''' * 16 + '''{\n\n''',
														    ''' ''' * 20 + '''"accuracy": eval_metric["accuracy"],\n\n''',
														    ''' ''' * 20 + '''"f1": eval_metric["f1"],\n\n''',
														    ''' ''' * 20 + '''"train_loss": total_loss.item() / len(train_dataloader),\n\n''',
														    ''' ''' * 20 + '''"epoch": epoch,\n\n''',
														    ''' ''' * 16 + '''},\n\n''',
														    ''' ''' * 16 + '''step=epoch,\n''',
														    ''' ''' * 12,
														    ''' ''' * 8 + '''for step, batch in enumerate(active_dataloader):\n''',
														]
														self.one_complete_example('''complete_cv_example.py'''							,      lowerCAmelCase_							,      lowerCAmelCase_							,      lowerCAmelCase_ )
														self.one_complete_example('''complete_cv_example.py'''							,      lowerCAmelCase_							,      lowerCAmelCase_							,      lowerCAmelCase_ )



@mock.patch.dict(os.environ					,{'TESTING_MOCKED_DATALOADERS': '1'}   )
class       A   (		_a   ):
							lowercase_          =    False







							@classmethod
							def        __lowerCAmelCase      (  cls	:  Dict )      ->							Any:


														"""simple docstring"""


														super().setUpClass()
														_a     					=    tempfile.mkdtemp()
														_a     					=    os.path.join(cls._tmpdir							,      '''default_config.yml''' )

														write_basic_config(save_location=cls.configPath )
														_a     					=    ['''accelerate''', '''launch''', '''--config_file''', cls.configPath]







							@classmethod
							def        __lowerCAmelCase      (  cls	:  List[Any] )      ->							Dict:


														"""simple docstring"""


														super().tearDownClass()
														shutil.rmtree(cls._tmpdir )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    F'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps epoch\n        --output_dir {self.tmpdir}\n        '.split()
														run_command(self._launch_args + testargs )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir							,      '''epoch_0''' ) ) )







							def        __lowerCAmelCase      (  self	:  int )      ->							str:


														"""simple docstring"""


														_a     					=    F'\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps 1\n        --output_dir {self.tmpdir}\n        '.split()
														_a     					=    run_command(self._launch_args + testargs )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir							,      '''step_2''' ) ) )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							List[str]:


														"""simple docstring"""


														_a     					=    F'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir							,      "epoch_0" )}\n        '.split()
														_a     					=    run_command(self._launch_args + testargs							,      return_stdout=lowerCAmelCase_ )
														self.assertNotIn('''epoch 0:'''							,      lowerCAmelCase_ )
														self.assertIn('''epoch 1:'''							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    F'\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir							,      "step_2" )}\n        '.split()
														_a     					=    run_command(self._launch_args + testargs							,      return_stdout=lowerCAmelCase_ )
														if torch.cuda.is_available():
																					_a     					=    torch.cuda.device_count()
														else:
																					_a     					=    1
														if num_processes > 1:
																					self.assertNotIn('''epoch 0:'''							,      lowerCAmelCase_ )
																					self.assertIn('''epoch 1:'''							,      lowerCAmelCase_ )
														else:
																					self.assertIn('''epoch 0:'''							,      lowerCAmelCase_ )
																					self.assertIn('''epoch 1:'''							,      lowerCAmelCase_ )







							@slow
							def        __lowerCAmelCase      (  self	:  Any )      ->							Tuple:


														"""simple docstring"""


														_a     					=    '''
        examples/by_feature/cross_validation.py
        --num_folds 2
        '''.split()
														with mock.patch.dict(os.environ							,      {'''TESTING_MOCKED_DATALOADERS''': '''0'''} ):
																					_a     					=    run_command(self._launch_args + testargs							,      return_stdout=lowerCAmelCase_ )
																					_a     					=    re.findall('''({.+})'''							,      lowerCAmelCase_ )
																					_a     					=    [r for r in results if '''accuracy''' in r][-1]
																					_a     					=    ast.literal_eval(lowerCAmelCase_ )
																					self.assertGreaterEqual(results['''accuracy''']							,      0.7_5 )







							def        __lowerCAmelCase      (  self	:  Optional[Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    ['''examples/by_feature/multi_process_metrics.py''']
														run_command(self._launch_args + testargs )







							@require_trackers
							@mock.patch.dict(os.environ							,      {'''WANDB_MODE''': '''offline'''} )
							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Optional[Any]:


														"""simple docstring"""


														with tempfile.TemporaryDirectory() as tmpdir:
																					_a     					=    F'\n            examples/by_feature/tracking.py\n            --with_tracking\n            --project_dir {tmpdir}\n            '.split()
																					run_command(self._launch_args + testargs )
																					self.assertTrue(os.path.exists(os.path.join(lowerCAmelCase_							,      '''tracking''' ) ) )







							def        __lowerCAmelCase      (  self	:  str )      ->							Any:


														"""simple docstring"""


														_a     					=    ['''examples/by_feature/gradient_accumulation.py''']
														run_command(self._launch_args + testargs )







							def        __lowerCAmelCase      (  self	:  str )      ->							Dict:


														"""simple docstring"""


														_a     					=    ['''examples/by_feature/local_sgd.py''']
														run_command(self._launch_args + testargs )







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_snake_case				:				Any       							=		logging.get_logger(__name__)

_snake_case				:				Any       							=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case				:				int       							=		{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_snake_case				:				int       							=		{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_INIT_CONFIGURATION
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    BertTokenizer







							def __init__(  self	:  str							,      lowerCAmelCase_	:  Union[str, Any]=None							,      lowerCAmelCase_	:  str=None							,      lowerCAmelCase_	:  List[Any]=True							,      lowerCAmelCase_	:  Any="[UNK]"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Tuple="[PAD]"							,      lowerCAmelCase_	:  Tuple="[CLS]"							,      lowerCAmelCase_	:  Optional[int]="[MASK]"							,      lowerCAmelCase_	:  Dict=True							,      lowerCAmelCase_	:  Union[str, Any]=None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							Union[str, Any]:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,      tokenizer_file=lowerCAmelCase_							,      do_lower_case=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      tokenize_chinese_chars=lowerCAmelCase_							,      strip_accents=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														_a     					=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
														if (
														    normalizer_state.get('''lowercase'''							,      lowerCAmelCase_ ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''							,      lowerCAmelCase_ ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''							,      lowerCAmelCase_ ) != tokenize_chinese_chars
														):
																					_a     					=    getattr(lowerCAmelCase_							,      normalizer_state.pop('''type''' ) )
																					_a     					=    do_lower_case
																					_a     					=    strip_accents
																					_a     					=    tokenize_chinese_chars
																					_a     					=    normalizer_class(**lowerCAmelCase_ )

														_a     					=    do_lower_case







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  int=None )      ->							List[str]:


														"""simple docstring"""


														_a     					=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														_a     					=    self._tokenizer.model.save(lowerCAmelCase_							,      name=lowerCAmelCase_ )
														return tuple(lowerCAmelCase_ )

