


"""simple docstring"""







import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
 from transformers.trainer import DEFAULT_CALLBACKS

 from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel






class 				__A    (	SCREAMING_SNAKE_CASE__      ):






 """simple docstring"""




 def __init__(		self						) ->					int:
     a					=[]


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Any:
     self.events.append('''on_init_end'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					List[Any]:
     self.events.append('''on_train_begin'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Dict:
     self.events.append('''on_train_end'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					str:
     self.events.append('''on_epoch_begin'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Union[str, Any]:
     self.events.append('''on_epoch_end'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Optional[int]:
     self.events.append('''on_step_begin'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Optional[int]:
     self.events.append('''on_step_end'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Optional[int]:
     self.events.append('''on_evaluate'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					List[str]:
     self.events.append('''on_predict'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Tuple:
     self.events.append('''on_save'''						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					Union[str, Any]:
     self.events.append('''on_log'''						)





 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A			,   __A			,   **__A						) ->					int:
     self.events.append('''on_prediction_step'''						)






@require_torch
class 				__A    (	unittest.TestCase      ):






 """simple docstring"""




 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Any:
     a					=tempfile.mkdtemp()


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Any:
     shutil.rmtree(self.output_dir						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A=0			,   __A=0			,   __A=64			,   __A=64			,   __A=None			,   __A=False			,   **__A						) ->					Tuple:
     a					=RegressionDataset(length=a_						)
     a					=RegressionDataset(length=a_						)
     a					=RegressionModelConfig(a=a_			,   b=a_						)
     a					=RegressionPreTrainedModel(a_						)

     a					=TrainingArguments(self.output_dir			,   disable_tqdm=a_			,   report_to=[]			,   **a_						)
     return Trainer(
         a_			,   a_			,   train_dataset=a_			,   eval_dataset=a_			,   callbacks=a_			,   )


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A			,   __A						) ->					Union[str, Any]:
     self.assertEqual(len(a_						)			,   len(a_						)						)

     # Order doesn't matter
     a					=sorted(a_			,   key=lambda __A						: cb.__name__ if isinstance(a_			,   a_						) else cb.__class__.__name__						)
     a					=sorted(a_			,   key=lambda __A						: cb.__name__ if isinstance(a_			,   a_						) else cb.__class__.__name__						)

     for cba, cba in zip(a_			,   a_						):
         if isinstance(a_			,   a_						) and isinstance(a_			,   a_						):
             self.assertEqual(a_			,   a_						)
         elif isinstance(a_			,   a_						) and not isinstance(a_			,   a_						):
             self.assertEqual(a_			,   cba.__class__						)
         elif not isinstance(a_			,   a_						) and isinstance(a_			,   a_						):
             self.assertEqual(cba.__class__			,   a_						)
         else:
             self.assertEqual(a_			,   a_						)


 def 				SCREAMING_SNAKE_CASE							(		self			,   __A						) ->					List[Any]:
     a					=['''on_init_end''', '''on_train_begin''']
     a					=0
     a					=len(trainer.get_eval_dataloader()						)
     a					=['''on_prediction_step'''] * len(trainer.get_eval_dataloader()						) + ['''on_log''', '''on_evaluate''']
     for _ in range(trainer.state.num_train_epochs						):
         expected_events.append('''on_epoch_begin'''						)
         for _ in range(a_						):
             step += 1
             expected_events += ["on_step_begin", "on_step_end"]
             if step % trainer.args.logging_steps == 0:
                 expected_events.append('''on_log'''						)
             if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                 expected_events += evaluation_events.copy()
             if step % trainer.args.save_steps == 0:
                 expected_events.append('''on_save'''						)
         expected_events.append('''on_epoch_end'''						)
         if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
             expected_events += evaluation_events.copy()
     expected_events += ["on_log", "on_train_end"]
     return expected_events


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Union[str, Any]:
     a					=self.get_trainer()
     a					=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     # Callbacks passed at init are added to the default callbacks
     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]						)
     expected_callbacks.append(a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
     a					=self.get_trainer(disable_tqdm=a_						)
     a					=DEFAULT_CALLBACKS.copy() + [PrinterCallback]
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)


 def 				SCREAMING_SNAKE_CASE							(		self						) ->					str:
     a					=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
     a					=self.get_trainer()

     # We can add, pop, or remove by class name
     trainer.remove_callback(a_						)
     expected_callbacks.remove(a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     a					=self.get_trainer()
     a					=trainer.pop_callback(a_						)
     self.assertEqual(cb.__class__			,   a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     trainer.add_callback(a_						)
     expected_callbacks.insert(0			,   a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     # We can also add, pop, or remove by instance
     a					=self.get_trainer()
     a					=trainer.callback_handler.callbacks[0]
     trainer.remove_callback(a_						)
     expected_callbacks.remove(a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     a					=self.get_trainer()
     a					=trainer.callback_handler.callbacks[0]
     a					=trainer.pop_callback(a_						)
     self.assertEqual(a_			,   a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)

     trainer.add_callback(a_						)
     expected_callbacks.insert(0			,   a_						)
     self.check_callbacks_equality(trainer.callback_handler.callbacks			,   a_						)





 def 				SCREAMING_SNAKE_CASE							(		self						) ->					Union[str, Any]:
     import warnings

     # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
     warnings.simplefilter(action='''ignore'''			,   category=a_						)

     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]						)
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     # Independent log/save/eval
     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]			,   logging_steps=5						)
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]			,   save_steps=5						)
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]			,   eval_steps=5			,   evaluation_strategy='''steps'''						)
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     a					=self.get_trainer(callbacks=[MyTestTrainerCallback]			,   evaluation_strategy='''epoch'''						)
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     # A bit of everything
     a					=self.get_trainer(
         callbacks=[MyTestTrainerCallback]			,   logging_steps=3			,   save_steps=10			,   eval_steps=5			,   evaluation_strategy='''steps'''			,   )
     trainer.train()
     a					=trainer.callback_handler.callbacks[-2].events
     self.assertEqual(a_			,   self.get_expected_events(a_						)						)

     # warning should be emitted for duplicated callbacks
     with patch('''transformers.trainer_callback.logger.warning'''						) as warn_mock:
         a					=self.get_trainer(
             callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]			,   )
         assert str(a_						) in warn_mock.call_args[0][0]



"""simple docstring"""







import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser
def _A						(						lowercase		):







    """simple docstring"""
    a					={}
    a					=tokenizer(example['''content''']	,   truncation=lowercase		)['''input_ids''']
    a					=len(example['''content''']		) / len(output['''input_ids''']		)
    return output


lowerCamelCase_       :       Optional[int]      						=  HfArgumentParser(PretokenizationArguments)
lowerCamelCase_       :       Optional[Any]      						=  parser.parse_args()
if args.num_workers is None:
       lowerCamelCase_       :       Tuple      						=  multiprocessing.cpu_count()
lowerCamelCase_       :       Any      						=  AutoTokenizer.from_pretrained(args.tokenizer_dir)

lowerCamelCase_       :       Any      						=  time.time()
lowerCamelCase_       :       int      						=  load_dataset(args.dataset_name, split="""train""")
print(F'Dataset loaded in {time.time()-t_start:.2f}s')

lowerCamelCase_       :       List[str]      						=  time.time()
lowerCamelCase_       :       str      						=  ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F'Dataset tokenized in {time.time()-t_start:.2f}s')

lowerCamelCase_       :       Union[str, Any]      						=  time.time()
ds.push_to_hub(args.tokenized_data_repo)





print(F'Data pushed to the hub in {time.time()-t_start:.2f}s')