





'''simple docstring'''



from manim import *






class 							__lowercase    ( _a  ):







	'''simple docstring'''

	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							Rectangle(height=0.5       ,width=0.5	)
				__lowercase			  =							Rectangle(height=0.4_6       ,width=0.4_6	).set_stroke(width=0	)

				__lowercase			  =							[mem.copy() for i in range(6	)]
				__lowercase			  =							[mem.copy() for i in range(6	)]
				__lowercase			  =							VGroup(*_a	).arrange(_a       ,buff=0	)
				__lowercase			  =							VGroup(*_a	).arrange(_a       ,buff=0	)
				__lowercase			  =							VGroup(_a       ,_a	).arrange(_a       ,buff=0	)
				__lowercase			  =							Text('''CPU'''       ,font_size=24	)
				__lowercase			  =							Group(_a       ,_a	).arrange(_a       ,buff=0.5       ,aligned_edge=_a	)
				cpu.move_to([-2.5, -0.5, 0]	)
				self.add(_a	)

				__lowercase			  =							[mem.copy() for i in range(4	)]
				__lowercase			  =							VGroup(*_a	).arrange(_a       ,buff=0	)
				__lowercase			  =							Text('''GPU'''       ,font_size=24	)
				__lowercase			  =							Group(_a       ,_a	).arrange(_a       ,buff=0.5       ,aligned_edge=_a	)
				gpu.move_to([-1, -1, 0]	)
				self.add(_a	)

				__lowercase			  =							[mem.copy() for i in range(6	)]
				__lowercase			  =							VGroup(*_a	).arrange(_a       ,buff=0	)

				__lowercase			  =							Text('''Model'''       ,font_size=24	)
				__lowercase			  =							Group(_a       ,_a	).arrange(_a       ,buff=0.5       ,aligned_edge=_a	)
				model.move_to([3, -1.0, 0]	)
				self.add(_a	)

				__lowercase			  =							[]
				for i, rect in enumerate(_a	):
							rect.set_stroke(_a	)
							# target = fill.copy().set_fill(YELLOW, opacity=0.7)
							# target.move_to(rect)
							# self.add(target)

							__lowercase			  =							Rectangle(height=0.4_6 / 4       ,width=0.4_6 / 3	).set_stroke(width=0.0	).set_fill(_a       ,opacity=0.7	)

							if i == 0:
										cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT	)       ,buff=0.0_2       ,direction=_a	)
										cpu_target.set_x(cpu_target.get_x() + 0.1	)
							elif i == 3:
										cpu_target.next_to(cpu_targs[0]       ,direction=_a       ,buff=0.0	)
							else:
										cpu_target.next_to(cpu_targs[i - 1]       ,direction=_a       ,buff=0.0	)
							self.add(_a	)
							cpu_targs.append(_a	)

				__lowercase			  =							[mem.copy() for i in range(6	)]
				__lowercase			  =							VGroup(*_a	).arrange(_a       ,buff=0	)

				__lowercase			  =							Text('''Loaded Checkpoint'''       ,font_size=24	)
				__lowercase			  =							Group(_a       ,_a	).arrange(_a       ,aligned_edge=_a       ,buff=0.4	)
				checkpoint.move_to([3, 0.5, 0]	)

				__lowercase			  =							Square(side_length=2.2	)
				key.move_to([-5, 2, 0]	)

				__lowercase			  =							MarkupText(
				    f"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"       ,font_size=18       ,)

				key_text.move_to([-5, 2.4, 0]	)

				self.add(_a       ,_a	)

				__lowercase			  =							MarkupText(
				    f"<span fgcolor='{BLUE}'>●</span> Checkpoint"       ,font_size=18       ,)

				blue_text.next_to(_a       ,DOWN * 2.4       ,aligned_edge=key_text.get_left()	)

				__lowercase			  =							MarkupText(
				    f"Next, a <i><span fgcolor=\"{BLUE}\">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor=\"{BLUE}\">single shard</span>."       ,font_size=24       ,)
				step_a.move_to([2, 2, 0]	)
				self.play(Write(_a	)       ,Write(_a	)	)

				self.play(Write(_a       ,run_time=1	)       ,Create(_a       ,run_time=1	)	)

				__lowercase			  =							[]
				__lowercase			  =							[]
				for i, rect in enumerate(_a	):
							__lowercase			  =							fill.copy().set_fill(_a       ,opacity=0.7	)
							target.move_to(_a	)
							first_animations.append(GrowFromCenter(_a       ,run_time=1	)	)

							__lowercase			  =							target.copy()
							cpu_target.generate_target()
							if i < 5:
										cpu_target.target.move_to(cpu_left_col_base[i + 1]	)
							else:
										cpu_target.target.move_to(cpu_right_col_base[i - 5]	)
							second_animations.append(MoveToTarget(_a       ,run_time=1.5	)	)

				self.play(*_a	)
				self.play(*_a	)
				self.wait()












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Union[str, Any]     =    "gpt_neox_japanese"


	def __init__(self       ,_lowerCamelCase=32000       ,_lowerCamelCase=2560       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase=4       ,_lowerCamelCase="gelu"       ,_lowerCamelCase=1.0_0       ,_lowerCamelCase=10000       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-5       ,_lowerCamelCase=True       ,_lowerCamelCase=31996       ,_lowerCamelCase=31999       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.0       ,**_lowerCamelCase       ,)							->							Optional[int]:







				'''simple docstring'''





				super().__init__(bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,**_lowerCamelCase	)
				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							intermediate_multiple_size
				__lowercase			  =							hidden_act
				__lowercase			  =							rotary_pct
				__lowercase			  =							rotary_emb_base
				__lowercase			  =							initializer_range
				__lowercase			  =							layer_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							attention_dropout
				__lowercase			  =							hidden_dropout





