


"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							:      Tuple						=      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							:      List[Any]						=      '▁'

__SCREAMING_SNAKE_CASE							:      Union[str, Any]						=      {'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

__SCREAMING_SNAKE_CASE							:      int						=      {
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

__SCREAMING_SNAKE_CASE							:      Tuple						=      {'vinai/bartpho-syllable': 1_024}





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						Optional[Any]							 =      VOCAB_FILES_NAMES
							__lowercase:						Optional[int]							 =      PRETRAINED_VOCAB_FILES_MAP
							__lowercase:						Tuple							 =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase:						Optional[int]							 =      ["""input_ids""", """attention_mask"""]



							def __init__(			self							: Dict						,    UpperCAmelCase_							: Tuple						,    UpperCAmelCase_							: int						,    UpperCAmelCase_							: Dict="<s>"						,    UpperCAmelCase_							: int="</s>"						,    UpperCAmelCase_							: Tuple="</s>"						,    UpperCAmelCase_							: int="<s>"						,    UpperCAmelCase_							: int="<unk>"						,    UpperCAmelCase_							: Union[str, Any]="<pad>"						,    UpperCAmelCase_							: Optional[int]="<mask>"						,    UpperCAmelCase_							: Optional[Dict[str, Any]] = None						,    **UpperCAmelCase_							: Tuple						,    )   ->None:



														"""simple docstring"""



														snake_case_          =						AddedToken(UpperCAmelCase_						,    lstrip=UpperCAmelCase_						,    rstrip=UpperCAmelCase_    ) if isinstance(UpperCAmelCase_						,    UpperCAmelCase_    ) else mask_token

														snake_case_          =						{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=UpperCAmelCase_						,    eos_token=UpperCAmelCase_						,    unk_token=UpperCAmelCase_						,    sep_token=UpperCAmelCase_						,    cls_token=UpperCAmelCase_						,    pad_token=UpperCAmelCase_						,    mask_token=UpperCAmelCase_						,    sp_model_kwargs=self.sp_model_kwargs						,    **UpperCAmelCase_						,    )

														snake_case_          =						vocab_file
														snake_case_          =						monolingual_vocab_file
														snake_case_          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(str(UpperCAmelCase_    )    )

														# Load the reduced vocab

														# Keep order of special tokens for backward compatibility
														snake_case_          =						{}
														snake_case_          =						0
														for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
																					if str(UpperCAmelCase_    ) not in self.fairseq_tokens_to_ids:
																												snake_case_          =						cnt
																												cnt += 1
														with open(UpperCAmelCase_						,    """r"""						,    encoding="""utf-8"""    ) as f:
																					for line in f.readlines():
																												snake_case_          =						line.strip().split()[0]
																												snake_case_          =						len(self.fairseq_tokens_to_ids    )
														if str(UpperCAmelCase_    ) not in self.fairseq_tokens_to_ids:
																					snake_case_          =						len(self.fairseq_tokens_to_ids    )

														snake_case_          =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}



							def __getstate__(			self							: Optional[Any]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						self.__dict__.copy()
														snake_case_          =						None
														snake_case_          =						self.sp_model.serialized_model_proto()
														return state



							def __setstate__(			self							: Optional[int]						,    UpperCAmelCase_							: Dict    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						d

														# for backward compatibility
														if not hasattr(self						,    """sp_model_kwargs"""    ):
																					snake_case_          =						{}

														snake_case_          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )



							def 						lowerCAmelCase    (			self							: List[str]						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None    )   ->List[int]:



														"""simple docstring"""



														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														snake_case_          =						[self.cls_token_id]
														snake_case_          =						[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep



							def 						lowerCAmelCase    (			self							: int						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None						,    UpperCAmelCase_							: bool = False    )   ->List[int]:



														"""simple docstring"""



														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase_						,    token_ids_a=UpperCAmelCase_						,    already_has_special_tokens=UpperCAmelCase_    )

														if token_ids_a is None:
																					return [1] + ([0] * len(UpperCAmelCase_    )) + [1]
														return [1] + ([0] * len(UpperCAmelCase_    )) + [1, 1] + ([0] * len(UpperCAmelCase_    )) + [1]



							def 						lowerCAmelCase    (			self							: Optional[int]						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None    )   ->List[int]:



														"""simple docstring"""



														snake_case_          =						[self.sep_token_id]
														snake_case_          =						[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep    ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]



							@property
							def 						lowerCAmelCase    (			self							: List[str]    )   ->str:



														"""simple docstring"""



														return len(self.fairseq_ids_to_tokens    )



							def 						lowerCAmelCase    (			self							: Dict    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						{self.convert_ids_to_tokens(UpperCAmelCase_    ): i for i in range(self.vocab_size    )}
														vocab.update(self.added_tokens_encoder    )
														return vocab



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: str    )   ->List[str]:



														"""simple docstring"""



														return self.sp_model.encode(UpperCAmelCase_						,    out_type=UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: List[str]						,    UpperCAmelCase_							: Union[str, Any]    )   ->Dict:



														"""simple docstring"""



														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														else:
																					return self.unk_token_id



							def 						lowerCAmelCase    (			self							: List[str]						,    UpperCAmelCase_							: List[Any]    )   ->Tuple:



														"""simple docstring"""



														return self.fairseq_ids_to_tokens[index]



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: Optional[int]    )   ->str:



														"""simple docstring"""



														snake_case_          =						"""""".join(UpperCAmelCase_    ).replace(UpperCAmelCase_						,    """ """    ).strip()
														return out_string



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[str] = None    )   ->Tuple[str]:



														"""simple docstring"""



														if not os.path.isdir(UpperCAmelCase_    ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
																					return
														snake_case_          =						os.path.join(
														    UpperCAmelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
														snake_case_          =						os.path.join(
														    UpperCAmelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""monolingual_vocab_file"""]						,    )

														if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCAmelCase_    ) and os.path.isfile(self.vocab_file    ):
																					copyfile(self.vocab_file						,    UpperCAmelCase_    )
														elif not os.path.isfile(self.vocab_file    ):
																					with open(UpperCAmelCase_						,    """wb"""    ) as fi:
																												snake_case_          =						self.sp_model.serialized_model_proto()
																												fi.write(UpperCAmelCase_    )

														if os.path.abspath(self.monolingual_vocab_file    ) != os.path.abspath(
														    UpperCAmelCase_    ) and os.path.isfile(self.monolingual_vocab_file    ):
																					copyfile(self.monolingual_vocab_file						,    UpperCAmelCase_    )
														elif not os.path.isfile(self.monolingual_vocab_file    ):
																					with open(UpperCAmelCase_						,    """w"""						,    encoding="""utf-8"""    ) as fp:
																												for token in self.fairseq_tokens_to_ids:
																																			if token not in self.all_special_tokens:
																																										fp.write(F"""{str(UpperCAmelCase_    )} \n"""    )

														return out_vocab_file, out_monolingual_vocab_file







"""simple docstring"""
import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

__SCREAMING_SNAKE_CASE							:      Tuple						=      [
    'cross_validation.py',
    'gradient_accumulation.py',
    'local_sgd.py',
    'multi_process_metrics.py',
    'memory.py',
    'automatic_gradient_accumulation.py',
    'fsdp_with_peak_mem_tracking.py',
    'deepspeed_with_config_support.py',
    'megatron_lm_gpt_pretraining.py',
]





class        __A       (unittest.TestCase):





							'''simple docstring'''

							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: bool						,    UpperCAmelCase_							: str = None						,    UpperCAmelCase_							: list = None    )   ->List[Any]:



														"""simple docstring"""



														snake_case_          =						None
														snake_case_          =						os.path.abspath(os.path.join("""examples"""						,    """by_feature"""    )    )
														snake_case_          =						os.path.abspath("""examples"""    )
														for item in os.listdir(UpperCAmelCase_    ):
																					if item not in EXCLUDE_EXAMPLES:
																												snake_case_          =						os.path.join(UpperCAmelCase_						,    UpperCAmelCase_    )
																												if os.path.isfile(UpperCAmelCase_    ) and ".py" in item_path:
																																			with self.subTest(
																																			    tested_script=UpperCAmelCase_						,    feature_script=UpperCAmelCase_						,    tested_section="""main()""" if parser_only else """training_function()"""						,    ):
																																										snake_case_          =						compare_against_test(
																																										    os.path.join(UpperCAmelCase_						,    UpperCAmelCase_    )						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )
																																										snake_case_          =						"""\n""".join(UpperCAmelCase_    )
																																										if special_strings is not None:
																																																	for string in special_strings:
																																																								snake_case_          =						diff.replace(UpperCAmelCase_						,    """"""    )
																																										self.assertEqual(UpperCAmelCase_						,    """"""    )



							def 						lowerCAmelCase    (			self							: Tuple    )   ->List[Any]:



														"""simple docstring"""



														self.one_complete_example("""complete_nlp_example.py"""						,    UpperCAmelCase_    )
														self.one_complete_example("""complete_nlp_example.py"""						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Tuple    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						os.path.abspath(os.path.join("""examples"""						,    """cv_example.py"""    )    )
														snake_case_          =						[
														    """ """ * 16 + """{\n\n""",
														    """ """ * 20 + """\"accuracy\": eval_metric[\"accuracy\"],\n\n""",
														    """ """ * 20 + """\"f1\": eval_metric[\"f1\"],\n\n""",
														    """ """ * 20 + """\"train_loss\": total_loss.item() / len(train_dataloader),\n\n""",
														    """ """ * 20 + """\"epoch\": epoch,\n\n""",
														    """ """ * 16 + """},\n\n""",
														    """ """ * 16 + """step=epoch,\n""",
														    """ """ * 12,
														    """ """ * 8 + """for step, batch in enumerate(active_dataloader):\n""",
														]
														self.one_complete_example("""complete_cv_example.py"""						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )
														self.one_complete_example("""complete_cv_example.py"""						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_    )





@mock.patch.dict(os.environ  ,						{"""TESTING_MOCKED_DATALOADERS""": """1"""})
class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						str							 =      False



							@classmethod
							def 						lowerCAmelCase    (			cls							: Any    )   ->List[str]:



														"""simple docstring"""



														super().setUpClass()
														snake_case_          =						tempfile.mkdtemp()
														snake_case_          =						os.path.join(cls._tmpdir						,    """default_config.yml"""    )

														write_basic_config(save_location=cls.configPath    )
														snake_case_          =						["""accelerate""", """launch""", """--config_file""", cls.configPath]



							@classmethod
							def 						lowerCAmelCase    (			cls							: List[str]    )   ->int:



														"""simple docstring"""



														super().tearDownClass()
														shutil.rmtree(cls._tmpdir    )



							def 						lowerCAmelCase    (			self							: List[str]    )   ->Optional[Any]:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps epoch
        --output_dir {self.tmpdir}
        """.split()
														run_command(self._launch_args + testargs    )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir						,    """epoch_0"""    )    )    )



							def 						lowerCAmelCase    (			self							: Any    )   ->Optional[Any]:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps 1
        --output_dir {self.tmpdir}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs    )
														self.assertTrue(os.path.exists(os.path.join(self.tmpdir						,    """step_2"""    )    )    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->str:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir						,    "epoch_0"    )}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
														self.assertNotIn("""epoch 0:"""						,    UpperCAmelCase_    )
														self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir						,    "step_2"    )}
        """.split()
														snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
														if torch.cuda.is_available():
																					snake_case_          =						torch.cuda.device_count()
														else:
																					snake_case_          =						1
														if num_processes > 1:
																					self.assertNotIn("""epoch 0:"""						,    UpperCAmelCase_    )
																					self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )
														else:
																					self.assertIn("""epoch 0:"""						,    UpperCAmelCase_    )
																					self.assertIn("""epoch 1:"""						,    UpperCAmelCase_    )



							@slow
							def 						lowerCAmelCase    (			self							: Dict    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						"""
        examples/by_feature/cross_validation.py
        --num_folds 2
        """.split()
														with mock.patch.dict(os.environ						,    {"""TESTING_MOCKED_DATALOADERS""": """0"""}    ):
																					snake_case_          =						run_command(self._launch_args + testargs						,    return_stdout=UpperCAmelCase_    )
																					snake_case_          =						re.findall("""({.+})"""						,    UpperCAmelCase_    )
																					snake_case_          =						[r for r in results if """accuracy""" in r][-1]
																					snake_case_          =						ast.literal_eval(UpperCAmelCase_    )
																					self.assertGreaterEqual(results["""accuracy"""]						,    0.75    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->int:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/multi_process_metrics.py"""]
														run_command(self._launch_args + testargs    )



							@require_trackers
							@mock.patch.dict(os.environ						,    {"""WANDB_MODE""": """offline"""}    )
							def 						lowerCAmelCase    (			self							: Dict    )   ->Optional[Any]:



														"""simple docstring"""



														with tempfile.TemporaryDirectory() as tmpdir:
																					snake_case_          =						F"""
            examples/by_feature/tracking.py
            --with_tracking
            --project_dir {tmpdir}
            """.split()
																					run_command(self._launch_args + testargs    )
																					self.assertTrue(os.path.exists(os.path.join(UpperCAmelCase_						,    """tracking"""    )    )    )



							def 						lowerCAmelCase    (			self							: List[str]    )   ->Union[str, Any]:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/gradient_accumulation.py"""]
														run_command(self._launch_args + testargs    )



							def 						lowerCAmelCase    (			self							: str    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						["""examples/by_feature/local_sgd.py"""]
														run_command(self._launch_args + testargs    )



