


"""simple docstring"""
import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test





class        __A       (unittest.TestCase):





							'''simple docstring'''

							@parameterized.expand([(None,), ("""foo.json""",)]    )
							def 						lowerCAmelCase    (			self							: Any						,    UpperCAmelCase_							: Any    )   ->Any:



														"""simple docstring"""



														snake_case_          =						GenerationConfig(
														    do_sample=UpperCAmelCase_						,    temperature=0.7						,    length_penalty=1.0						,    bad_words_ids=[[1, 2, 3], [4, 5]]						,    )
														with tempfile.TemporaryDirectory() as tmp_dir:
																					config.save_pretrained(UpperCAmelCase_						,    config_name=UpperCAmelCase_    )
																					snake_case_          =						GenerationConfig.from_pretrained(UpperCAmelCase_						,    config_name=UpperCAmelCase_    )

														# Checks parameters that were specified
														self.assertEqual(loaded_config.do_sample						,    UpperCAmelCase_    )
														self.assertEqual(loaded_config.temperature						,    0.7    )
														self.assertEqual(loaded_config.length_penalty						,    1.0    )
														self.assertEqual(loaded_config.bad_words_ids						,    [[1, 2, 3], [4, 5]]    )

														# Checks parameters that were not specified (defaults)
														self.assertEqual(loaded_config.top_k						,    50    )
														self.assertEqual(loaded_config.max_length						,    20    )
														self.assertEqual(loaded_config.max_time						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						AutoConfig.from_pretrained("""gpt2"""    )
														snake_case_          =						GenerationConfig.from_model_config(UpperCAmelCase_    )
														snake_case_          =						GenerationConfig()

														# The generation config has loaded a few non-default parameters from the model config
														self.assertNotEqual(UpperCAmelCase_						,    UpperCAmelCase_    )

														# One of those parameters is eos_token_id -- check if it matches
														self.assertNotEqual(generation_config_from_model.eos_token_id						,    default_generation_config.eos_token_id    )
														self.assertEqual(generation_config_from_model.eos_token_id						,    model_config.eos_token_id    )



							def 						lowerCAmelCase    (			self							: str    )   ->Optional[int]:



														"""simple docstring"""



														snake_case_          =						GenerationConfig()
														snake_case_          =						{
														    """max_new_tokens""": 1_024,
														    """foo""": """bar""",
														}
														snake_case_          =						copy.deepcopy(UpperCAmelCase_    )
														snake_case_          =						generation_config.update(**UpperCAmelCase_    )

														# update_kwargs was not modified (no side effects)
														self.assertEqual(UpperCAmelCase_						,    UpperCAmelCase_    )

														# update_kwargs was used to update the config on valid attributes
														self.assertEqual(generation_config.max_new_tokens						,    1_024    )

														# `.update()` returns a dictionary of unused kwargs
														self.assertEqual(UpperCAmelCase_						,    {"""foo""": """bar"""}    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->List[Any]:



														"""simple docstring"""



														snake_case_          =						GenerationConfig()
														snake_case_          =						"""bar"""

														with tempfile.TemporaryDirectory("""test-generation-config"""    ) as tmp_dir:
																					generation_config.save_pretrained(UpperCAmelCase_    )

																					snake_case_          =						GenerationConfig.from_pretrained(UpperCAmelCase_    )
														# update_kwargs was used to update the config on valid attributes
														self.assertEqual(new_config.foo						,    """bar"""    )

														snake_case_          =						GenerationConfig.from_model_config(UpperCAmelCase_    )
														assert not hasattr(UpperCAmelCase_						,    """foo"""    )  # no new kwargs should be initialized if from config



							def 						lowerCAmelCase    (			self							: int    )   ->int:



														"""simple docstring"""



														snake_case_          =						GenerationConfig()
														self.assertEqual(default_config.temperature						,    1.0    )
														self.assertEqual(default_config.do_sample						,    UpperCAmelCase_    )
														self.assertEqual(default_config.num_beams						,    1    )

														snake_case_          =						GenerationConfig(
														    do_sample=UpperCAmelCase_						,    temperature=0.7						,    length_penalty=1.0						,    bad_words_ids=[[1, 2, 3], [4, 5]]						,    )
														self.assertEqual(config.temperature						,    0.7    )
														self.assertEqual(config.do_sample						,    UpperCAmelCase_    )
														self.assertEqual(config.num_beams						,    1    )

														with tempfile.TemporaryDirectory() as tmp_dir:
																					config.save_pretrained(UpperCAmelCase_    )
																					snake_case_          =						GenerationConfig.from_pretrained(UpperCAmelCase_						,    temperature=1.0    )

														self.assertEqual(loaded_config.temperature						,    1.0    )
														self.assertEqual(loaded_config.do_sample						,    UpperCAmelCase_    )
														self.assertEqual(loaded_config.num_beams						,    1    )  # default value





@is_staging_test
class        __A       (unittest.TestCase):





							'''simple docstring'''

							@classmethod
							def 						lowerCAmelCase    (			cls							: List[Any]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						TOKEN
														HfFolder.save_token(UpperCAmelCase_    )



							@classmethod
							def 						lowerCAmelCase    (			cls							: str    )   ->List[Any]:



														"""simple docstring"""



														try:
																					delete_repo(token=cls._token						,    repo_id="""test-generation-config"""    )
														except HTTPError:
																					pass

														try:
																					delete_repo(token=cls._token						,    repo_id="""valid_org/test-generation-config-org"""    )
														except HTTPError:
																					pass



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->int:



														"""simple docstring"""



														snake_case_          =						GenerationConfig(
														    do_sample=UpperCAmelCase_						,    temperature=0.7						,    length_penalty=1.0						,    )
														config.push_to_hub("""test-generation-config"""						,    use_auth_token=self._token    )

														snake_case_          =						GenerationConfig.from_pretrained(F"""{USER}/test-generation-config"""    )
														for k, v in config.to_dict().items():
																					if k != "transformers_version":
																												self.assertEqual(UpperCAmelCase_						,    getattr(UpperCAmelCase_						,    UpperCAmelCase_    )    )

        # Reset repo
														delete_repo(token=self._token						,    repo_id="""test-generation-config"""    )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					config.save_pretrained(
																					    UpperCAmelCase_						,    repo_id="""test-generation-config"""						,    push_to_hub=UpperCAmelCase_						,    use_auth_token=self._token    )

														snake_case_          =						GenerationConfig.from_pretrained(F"""{USER}/test-generation-config"""    )
														for k, v in config.to_dict().items():
																					if k != "transformers_version":
																												self.assertEqual(UpperCAmelCase_						,    getattr(UpperCAmelCase_						,    UpperCAmelCase_    )    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->Union[str, Any]:



														"""simple docstring"""



														snake_case_          =						GenerationConfig(
														    do_sample=UpperCAmelCase_						,    temperature=0.7						,    length_penalty=1.0						,    )
														config.push_to_hub("""valid_org/test-generation-config-org"""						,    use_auth_token=self._token    )

														snake_case_          =						GenerationConfig.from_pretrained("""valid_org/test-generation-config-org"""    )
														for k, v in config.to_dict().items():
																					if k != "transformers_version":
																												self.assertEqual(UpperCAmelCase_						,    getattr(UpperCAmelCase_						,    UpperCAmelCase_    )    )

        # Reset repo
														delete_repo(token=self._token						,    repo_id="""valid_org/test-generation-config-org"""    )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					config.save_pretrained(
																					    UpperCAmelCase_						,    repo_id="""valid_org/test-generation-config-org"""						,    push_to_hub=UpperCAmelCase_						,    use_auth_token=self._token    )

														snake_case_          =						GenerationConfig.from_pretrained("""valid_org/test-generation-config-org"""    )
														for k, v in config.to_dict().items():
																					if k != "transformers_version":
																												self.assertEqual(UpperCAmelCase_						,    getattr(UpperCAmelCase_						,    UpperCAmelCase_    )    )







"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be trained."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""./"""  ,						metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-train"""  ,						metadata={"""help""": """Name or path of training dataset."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-valid"""  ,						metadata={"""help""": """Name or path of validation dataset."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size for training."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size for evaluation."""})
							__lowercase:						Optional[float]							 =      field(default=0.1  ,						metadata={"""help""": """Value of weight decay."""})
							__lowercase:						Optional[int]							 =      field(
							    default=1_00_00  ,						metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""})
							__lowercase:						Optional[float]							 =      field(default=2E-4  ,						metadata={"""help""": """Learning rate fo training."""})
							__lowercase:						Optional[str]							 =      field(default="""cosine"""  ,						metadata={"""help""": """Learning rate."""})
							__lowercase:						Optional[int]							 =      field(
							    default=7_50  ,						metadata={"""help""": """Number of warmup steps in the learning rate schedule."""})
							__lowercase:						Optional[int]							 =      field(
							    default=16  ,						metadata={"""help""": """Number of gradient accumulation steps."""})
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""})
							__lowercase:						Optional[int]							 =      field(default=5_00_00  ,						metadata={"""help""": """Maximum number of training steps."""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=10_24  ,						metadata={"""help""": """Sequence lengths used for training."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Training seed."""})
							__lowercase:						Optional[int]							 =      field(
							    default=10_24  ,						metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}  ,						)
							__lowercase:						Optional[str]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """States path if the training should continue from a checkpoint folder."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """If True the data is pretokenized."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be evaluated."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-valid"""  ,						metadata={"""help""": """Name or path of validation dataset."""})
							__lowercase:						Optional[int]							 =      field(default=2  ,						metadata={"""help""": """Batch size used for evaluation."""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=10_24  ,						metadata={"""help""": """Length of sequences to be evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Random seed used for evaluation."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Model name or path of model to be evaluated."""})
							__lowercase:						Optional[int]							 =      field(default=snake_case__  ,						metadata={"""help""": """Number of workers used for code evaluation."""})
							__lowercase:						Optional[int]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}  ,						)
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """Sample from the language model's output distribution."""})
							__lowercase:						Optional[float]							 =      field(default=0.2  ,						metadata={"""help""": """Sampling temperature used for generation."""})
							__lowercase:						Optional[int]							 =      field(default=2_56  ,						metadata={"""help""": """Maximum number of newly generated tokens."""})
							__lowercase:						Optional[int]							 =      field(default=0  ,						metadata={"""help""": """Top-k parameter used for generation."""})
							__lowercase:						Optional[float]							 =      field(default=0.9_5  ,						metadata={"""help""": """Top-p parameter used for nucleus sampling."""})
							__lowercase:						Optional[int]							 =      field(default=10  ,						metadata={"""help""": """Number of generations to run in parallel."""})
							__lowercase:						Optional[int]							 =      field(
							    default=2_00  ,						metadata={"""help""": """Number of completions to generate for each sample."""})
							__lowercase:						Optional[int]							 =      field(default=1  ,						metadata={"""help""": """Random seed used for evaluation."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""eval_results.json"""  ,						metadata={"""help""": """Random seed used for evaluation."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""0"""  ,						metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""})
							__lowercase:						Optional[int]							 =      field(
							    default=-1  ,						metadata={
							        """help""": (
							            """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
							            """ number corresponds to which GPU device id to run on."""
							        )
							    }  ,						)





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[int]							 =      field(
							    default=snake_case__  ,						metadata={
							        """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
							    }  ,						)
							__lowercase:						Optional[str]							 =      field(
							    default="""transformersbook/codeparrot"""  ,						metadata={"""help""": """Folder or name of dataset to process."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot-clean"""  ,						metadata={"""help""": """Folder to save processed processed dataset."""})
							__lowercase:						Optional[int]							 =      field(
							    default=10_00_00  ,						metadata={"""help""": """Number of files to save per JSON output file."""})
							__lowercase:						Optional[str]							 =      field(default="""content"""  ,						metadata={"""help""": """Column containing text data to process."""})
							__lowercase:						Optional[float]							 =      field(
							    default=10_00  ,						metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=1_00  ,						metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.2_5  ,						metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=1.5  ,						metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.7  ,						metadata={"""help""": """Probability for filtering config, test and uncommon files."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Name or path to the tokenizer."""}  ,						)
							__lowercase:						Optional[bool]							 =      field(
							    default=snake_case__  ,						metadata={"""help""": """If True, near-duplicate samples are removed."""})
							__lowercase:						Optional[float]							 =      field(
							    default=0.8_5  ,						metadata={"""help""": """Jaccard threshold for near-duplicate samples."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""gpt2"""  ,						metadata={"""help""": """Base tokenizer to build new tokenizer from."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""transformersbook/codeparrot-train"""  ,						metadata={"""help""": """Dataset to train tokenizer on."""})
							__lowercase:						Optional[str]							 =      field(default="""content"""  ,						metadata={"""help""": """Column containing text data to process."""})
							__lowercase:						Optional[int]							 =      field(default=20_00_00  ,						metadata={"""help""": """Number of examples to train tokenizer on."""})
							__lowercase:						Optional[int]							 =      field(
							    default=3_27_68  ,						metadata={"""help""": """Number of examples to train the tokenizer on."""})
							__lowercase:						Optional[str]							 =      field(default="""codeparrot"""  ,						metadata={"""help""": """Name of new tokenizer."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """Push saved tokenizer to the hub."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Name or path to the tokenizer."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot-clean-train"""  ,						metadata={"""help""": """Name or path to the dataset to pretokenize."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""tokenized-codeparrot-train"""  ,						metadata={"""help""": """Repo name of the pretokenized data."""})
							__lowercase:						Optional[int]							 =      field(default=snake_case__  ,						metadata={"""help""": """Number of workers used for code evaluation."""})





@dataclass
class        __A       :





							'''simple docstring'''

							__lowercase:						Optional[str]							 =      field(
							    default="""gpt2-large"""  ,						metadata={"""help""": """Configuration to use for model initialization."""})
							__lowercase:						Optional[str]							 =      field(
							    default="""codeparrot/codeparrot"""  ,						metadata={"""help""": """Tokenizer attached to model."""})
							__lowercase:						Optional[str]							 =      field(default="""codeparrot"""  ,						metadata={"""help""": """Name of the created model."""})
							__lowercase:						Optional[bool]							 =      field(default=snake_case__  ,						metadata={"""help""": """Push saved tokenizer to the hub."""})



