





'''simple docstring'''



import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length




def 				_lowerCAmelCase  (			lowerCamelCase_     :					List[Any]			):
			__lowercase			  =							int(lowerCamelCase_			)
			__lowercase  ,						__lowercase  ,						__lowercase			  =							t // 3_6_0_0, (t // 6_0) % 6_0, t % 6_0
			return f"{h}:{m:02d}:{s:02d}" if h != 0 else f"{m:02d}:{s:02d}"




def 				_lowerCAmelCase  (			lowerCamelCase_     :					List[str]						,    lowerCamelCase_     :					int						,    lowerCamelCase_     :					Union[str, Any]						,    lowerCamelCase_     :					Union[str, Any]						,    lowerCamelCase_     :					Union[str, Any]=3_0_0			):
			# docstyle-ignore
			return f"\n    <div>\n      {prefix}\n      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>\n      {label}\n    </div>\n    "




def 				_lowerCAmelCase  (			lowerCamelCase_     :					Optional[Any]			):
			__lowercase			  =							'''<table border="1" class="dataframe">\n'''
			html_code += """  <thead>\n <tr style="text-align: left;">\n"""
			for i in items[0]:
						html_code += f"      <th>{i}</th>\n"
			html_code += "    </tr>\n  </thead>\n  <tbody>\n"
			for line in items[1:]:
						html_code += "    <tr>\n"
						for elt in line:
									__lowercase			  =							f"{elt:.6f}" if isinstance(lowerCamelCase_						,    lowerCamelCase_			) else str(lowerCamelCase_			)
									html_code += f"      <td>{elt}</td>\n"
						html_code += "    </tr>\n"
			html_code += "  </tbody>\n</table><p>"
			return html_code






class 							__lowercase    :







	'''simple docstring'''

	a      :			Dict     =    5
	a      :			Optional[Any]     =    0.2


	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase = None       ,_lowerCamelCase = True       ,_lowerCamelCase = None       ,_lowerCamelCase = 300       ,)							->							Any:







				'''simple docstring'''





				__lowercase			  =							total
				__lowercase			  =							'''''' if prefix is None else prefix
				__lowercase			  =							leave
				__lowercase			  =							parent
				__lowercase			  =							width
				__lowercase			  =							None
				__lowercase			  =							None
				__lowercase			  =							None


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = False       ,_lowerCamelCase = None	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							value
				if comment is not None:
							__lowercase			  =							comment
				if self.last_value is None:
							__lowercase			  =							__lowercase			  =							time.time()
							__lowercase			  =							__lowercase			  =							value
							__lowercase			  =							__lowercase			  =							None
							__lowercase			  =							self.warmup
							__lowercase			  =							1
							self.update_bar(_lowerCamelCase	)
				elif value <= self.last_value and not force_update:
							return
				elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for       ,self.total	):
							if self.first_calls > 0:
										self.first_calls -= 1
							__lowercase			  =							time.time()
							__lowercase			  =							current_time - self.start_time
							# We could have value = self.start_value if the update is called twixe with the same start value.
							if value > self.start_value:
										__lowercase			  =							self.elapsed_time / (value - self.start_value)
							else:
										__lowercase			  =							None
							if value >= self.total:
										__lowercase			  =							self.total
										__lowercase			  =							None
										if not self.leave:
													self.close()
							elif self.average_time_per_item is not None:
										__lowercase			  =							self.average_time_per_item * (self.total - value)
							self.update_bar(_lowerCamelCase	)
							__lowercase			  =							value
							__lowercase			  =							current_time
							if self.average_time_per_item is None:
										__lowercase			  =							1
							else:
										__lowercase			  =							max(int(self.update_every / self.average_time_per_item	)       ,1	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None	)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							''' ''' * (len(str(self.total	)	) - len(str(_lowerCamelCase	)	)) + str(_lowerCamelCase	)
				if self.elapsed_time is None:
							__lowercase			  =							f"[{spaced_value}/{self.total} : < :"
				elif self.predicted_remaining is None:
							__lowercase			  =							f"[{spaced_value}/{self.total} {format_time(self.elapsed_time	)}"
				else:
							__lowercase			  =							(
							    f"[{spaced_value}/{self.total} {format_time(self.elapsed_time	)} <"
							    f" {format_time(self.predicted_remaining	)}"
							)
							self.label += f", {1/self.average_time_per_item:.2f} it/s"
				self.label += "]" if self.comment is None or len(self.comment	) == 0 else f", {self.comment}]"
				self.display()


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							html_progress_bar(self.value       ,self.total       ,self.prefix       ,self.label       ,self.width	)
				if self.parent is not None:
							# If this is a child bar, the parent will take care of the display.
							self.parent.display()
							return
				if self.output is None:
							__lowercase			  =							disp.display(disp.HTML(self.html_code	)       ,display_id=_lowerCamelCase	)
				else:
							self.output.update(disp.HTML(self.html_code	)	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				if self.parent is None and self.output is not None:
							self.output.update(disp.HTML(''''''	)	)






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase=None	)							->							Any:







				'''simple docstring'''





				super().__init__(_lowerCamelCase	)
				__lowercase			  =							None if column_names is None else [column_names]
				__lowercase			  =							None


	def   _UpperCAmelCase       (self	)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							html_progress_bar(self.value       ,self.total       ,self.prefix       ,self.label       ,self.width	)
				if self.inner_table is not None:
							self.html_code += text_to_html_table(self.inner_table	)
				if self.child_bar is not None:
							self.html_code += self.child_bar.html_code
				if self.output is None:
							__lowercase			  =							disp.display(disp.HTML(self.html_code	)       ,display_id=_lowerCamelCase	)
				else:
							self.output.update(disp.HTML(self.html_code	)	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							List[Any]:







				'''simple docstring'''





				if self.inner_table is None:
							__lowercase			  =							[list(values.keys()	), list(values.values()	)]
				else:
							__lowercase			  =							self.inner_table[0]
							if len(self.inner_table	) == 1:
										# We give a chance to update the column names at the first iteration
										for key in values.keys():
													if key not in columns:
																columns.append(_lowerCamelCase	)
										__lowercase			  =							columns
							self.inner_table.append([values[c] for c in columns]	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None       ,_lowerCamelCase=300	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							NotebookProgressBar(_lowerCamelCase       ,prefix=_lowerCamelCase       ,parent=self       ,width=_lowerCamelCase	)
				return self.child_bar


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				__lowercase			  =							None
				self.display()






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							None
				__lowercase			  =							None
				__lowercase			  =							False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,**_lowerCamelCase	)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							'''Epoch''' if args.evaluation_strategy == IntervalStrategy.EPOCH else '''Step'''
				__lowercase			  =							0
				__lowercase			  =							0
				__lowercase			  =							[self.first_column] + ['''Training Loss''']
				if args.evaluation_strategy != IntervalStrategy.NO:
							column_names.append('''Validation Loss'''	)
				__lowercase			  =							NotebookTrainingTracker(state.max_steps       ,_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,**_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							int(state.epoch	) if int(state.epoch	) == state.epoch else f"{state.epoch:.2f}"
				self.training_tracker.update(
				    state.global_step + 1       ,comment=f"Epoch {epoch}/{state.num_train_epochs}"       ,force_update=self._force_next_update       ,)
				__lowercase			  =							False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase=None       ,**_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if not has_length(_lowerCamelCase	):
							return
				if self.prediction_bar is None:
							if self.training_tracker is not None:
										__lowercase			  =							self.training_tracker.add_child(len(_lowerCamelCase	)	)
							else:
										__lowercase			  =							NotebookProgressBar(len(_lowerCamelCase	)	)
							self.prediction_bar.update(1	)
				else:
							self.prediction_bar.update(self.prediction_bar.value + 1	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,**_lowerCamelCase	)							->							Tuple:







				'''simple docstring'''





				if self.prediction_bar is not None:
							self.prediction_bar.close()
				__lowercase			  =							None


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase=None       ,**_lowerCamelCase	)							->							Union[str, Any]:







				'''simple docstring'''





				if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
							__lowercase			  =							{'''Training Loss''': logs['''loss''']}
							# First column is necessarily Step sine we're not in epoch eval strategy
							__lowercase			  =							state.global_step
							self.training_tracker.write_line(_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase=None       ,**_lowerCamelCase	)							->							Union[str, Any]:







				'''simple docstring'''





				if self.training_tracker is not None:
							__lowercase			  =							{'''Training Loss''': '''No log''', '''Validation Loss''': '''No log'''}
							for log in reversed(state.log_history	):
										if "loss" in log:
													__lowercase			  =							log['''loss''']
													break

							if self.first_column == "Epoch":
										__lowercase			  =							int(state.epoch	)
							else:
										__lowercase			  =							state.global_step
							__lowercase			  =							'''eval'''
							for k in metrics:
										if k.endswith('''_loss'''	):
													__lowercase			  =							re.sub(R'''\_loss$'''       ,''''''       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop('''total_flos'''       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop('''epoch'''       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop(f"{metric_key_prefix}_runtime"       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop(f"{metric_key_prefix}_samples_per_second"       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop(f"{metric_key_prefix}_steps_per_second"       ,_lowerCamelCase	)
							__lowercase			  =							metrics.pop(f"{metric_key_prefix}_jit_compilation_time"       ,_lowerCamelCase	)
							for k, v in metrics.items():
										if k == f"{metric_key_prefix}_loss":
													__lowercase			  =							v
										else:
													__lowercase			  =							k.split('''_'''	)
													__lowercase			  =							''' '''.join([part.capitalize() for part in splits[1:]]	)
													__lowercase			  =							v
							self.training_tracker.write_line(_lowerCamelCase	)
							self.training_tracker.remove_child()
							__lowercase			  =							None
							# Evaluation takes a long time so we should force the next update.
							__lowercase			  =							True


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase       ,**_lowerCamelCase	)							->							List[str]:







				'''simple docstring'''





				self.training_tracker.update(
				    state.global_step       ,comment=f"Epoch {int(state.epoch	)}/{state.num_train_epochs}"       ,force_update=_lowerCamelCase	)
				__lowercase			  =							None












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Dict     =    "open-llama"


	def __init__(self       ,_lowerCamelCase=100000       ,_lowerCamelCase=4096       ,_lowerCamelCase=11008       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase="silu"       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-6       ,_lowerCamelCase=True       ,_lowerCamelCase=0       ,_lowerCamelCase=1       ,_lowerCamelCase=2       ,_lowerCamelCase=False       ,_lowerCamelCase=True       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.1       ,_lowerCamelCase=True       ,_lowerCamelCase=True       ,_lowerCamelCase=None       ,**_lowerCamelCase       ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							intermediate_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							hidden_act
				__lowercase			  =							initializer_range
				__lowercase			  =							rms_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							kwargs.pop(
				    '''use_memorry_efficient_attention'''       ,_lowerCamelCase	)
				__lowercase			  =							hidden_dropout_prob
				__lowercase			  =							attention_dropout_prob
				__lowercase			  =							use_stable_embedding
				__lowercase			  =							shared_input_output_embedding
				__lowercase			  =							rope_scaling
				self._rope_scaling_validation()

				super().__init__(
				    pad_token_id=_lowerCamelCase       ,bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,tie_word_embeddings=_lowerCamelCase       ,**_lowerCamelCase       ,)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				if self.rope_scaling is None:
							return

				if not isinstance(self.rope_scaling       ,_lowerCamelCase	) or len(self.rope_scaling	) != 2:
							raise ValueError(
							    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
							    f"got {self.rope_scaling}"	)
				__lowercase			  =							self.rope_scaling.get('''type'''       ,_lowerCamelCase	)
				__lowercase			  =							self.rope_scaling.get('''factor'''       ,_lowerCamelCase	)
				if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
							raise ValueError(
							    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"	)
				if rope_scaling_factor is None or not isinstance(_lowerCamelCase       ,_lowerCamelCase	) or rope_scaling_factor <= 1.0:
							raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"	)





