

from manim import *






class 	UpperCAmelCase__		(	A__	):






  """simple docstring"""


  def 				lowercase_		(							self						:		Dict )		->							Union[str, Any]:
       SCREAMING_SNAKE_CASE__   =   Rectangle(height=0.5     ,       width=0.5 )
       SCREAMING_SNAKE_CASE__   =   Rectangle(height=0.25     ,       width=0.25 )
       SCREAMING_SNAKE_CASE__   =   Rectangle(height=0.46     ,       width=0.46 ).set_stroke(width=0 )

       SCREAMING_SNAKE_CASE__   =   [mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   [mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   VGroup(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   Text('''CPU'''     ,       font_size=24 )
       SCREAMING_SNAKE_CASE__   =   Group(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0.5     ,       aligned_edge=__lowerCamelCase )
       cpu.move_to([-2.5, -0.5, 0] )
       self.add(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   [mem.copy() for i in range(4 )]
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   Text('''GPU'''     ,       font_size=24 )
       SCREAMING_SNAKE_CASE__   =   Group(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0.5     ,       aligned_edge=__lowerCamelCase )
       gpu.move_to([-1, -1, 0] )
       self.add(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   [mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )

       SCREAMING_SNAKE_CASE__   =   Text('''Model'''     ,       font_size=24 )
       SCREAMING_SNAKE_CASE__   =   Group(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0.5     ,       aligned_edge=__lowerCamelCase )
       model.move_to([3, -1.0, 0] )
       self.add(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   []
       SCREAMING_SNAKE_CASE__   =   []
       SCREAMING_SNAKE_CASE__   =   []

       for i, rect in enumerate(__lowerCamelCase ):
            rect.set_stroke(__lowerCamelCase )

            SCREAMING_SNAKE_CASE__   =   Rectangle(height=0.46 / 4     ,       width=0.46 / 3 ).set_stroke(width=0.0 ).set_fill(__lowerCamelCase     ,       opacity=0.7 )

            if i == 0:
                 cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT )     ,       buff=0.02     ,       direction=__lowerCamelCase )
                 cpu_target.set_x(cpu_target.get_x() + 0.1 )
            elif i == 3:
                 cpu_target.next_to(model_cpu_arr[0]     ,       direction=__lowerCamelCase     ,       buff=0.0 )
            else:
                 cpu_target.next_to(model_cpu_arr[i - 1]     ,       direction=__lowerCamelCase     ,       buff=0.0 )
            self.add(__lowerCamelCase )
            model_cpu_arr.append(__lowerCamelCase )

       self.add(*__lowerCamelCase     ,       *__lowerCamelCase     ,       *__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   [mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )

       SCREAMING_SNAKE_CASE__   =   Text('''Loaded Checkpoint'''     ,       font_size=24 )
       SCREAMING_SNAKE_CASE__   =   Group(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0.5     ,       aligned_edge=__lowerCamelCase )
       checkpoint.move_to([3, 0.5, 0] )

       self.add(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   []
       SCREAMING_SNAKE_CASE__   =   []

       for i, rect in enumerate(__lowerCamelCase ):
            SCREAMING_SNAKE_CASE__   =   fill.copy().set_fill(__lowerCamelCase     ,       opacity=0.7 )
            target.move_to(__lowerCamelCase )
            ckpt_arr.append(__lowerCamelCase )

            SCREAMING_SNAKE_CASE__   =   target.copy()
            if i < 5:
                 cpu_target.move_to(cpu_left_col_base[i + 1] )
            else:
                 cpu_target.move_to(cpu_right_col_base[i - 5] )
            ckpt_cpu_arr.append(__lowerCamelCase )
       self.add(*__lowerCamelCase     ,       *__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   Square(side_length=2.2 )
       key.move_to([-5, 2, 0] )

       SCREAMING_SNAKE_CASE__   =   MarkupText(
           f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''     ,       font_size=18     ,       )

       key_text.move_to([-5, 2.4, 0] )

       self.add(__lowerCamelCase     ,       __lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   MarkupText(
           f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''     ,       font_size=18     ,       )

       blue_text.next_to(__lowerCamelCase     ,       DOWN * 2.4     ,       aligned_edge=key_text.get_left() )
       self.add(__lowerCamelCase )

       SCREAMING_SNAKE_CASE__   =   MarkupText(
           f'''Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'''     ,       font_size=24     ,       )
       step_a.move_to([2, 2, 0] )

       SCREAMING_SNAKE_CASE__   =   [meta_mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   [meta_mem.copy() for i in range(6 )]
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   VGroup(*__lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   VGroup(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0 )
       SCREAMING_SNAKE_CASE__   =   Text('''Disk'''     ,       font_size=24 )
       SCREAMING_SNAKE_CASE__   =   Group(__lowerCamelCase     ,       __lowerCamelCase ).arrange(__lowerCamelCase     ,       buff=0.5     ,       aligned_edge=__lowerCamelCase )
       disk.move_to([-4.0, -1.25, 0] )
       self.play(Write(__lowerCamelCase     ,       run_time=3 )     ,       Write(__lowerCamelCase     ,       run_time=1 )     ,       Create(__lowerCamelCase     ,       run_time=1 ) )

       SCREAMING_SNAKE_CASE__   =   []
       for i, rect in enumerate(__lowerCamelCase ):
            SCREAMING_SNAKE_CASE__   =   rect.copy()
            target.generate_target()
            target.target.move_to(disk_left_col_base[i] ).scale(0.5 )
            animations.append(MoveToTarget(__lowerCamelCase     ,       run_time=1.5 ) )
       self.play(*__lowerCamelCase )

       self.play(FadeOut(__lowerCamelCase ) )

       SCREAMING_SNAKE_CASE__   =   MarkupText(f'''Then, the checkpoint is removed from memory\nthrough garbage collection.'''     ,       font_size=24 )
       step_a.move_to([2, 2, 0] )

       self.play(Write(__lowerCamelCase     ,       run_time=3 ) )

       self.play(
           FadeOut(__lowerCamelCase     ,       __lowerCamelCase     ,       *__lowerCamelCase     ,       *__lowerCamelCase )     ,       )

       self.wait()





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE					:  List[str]             =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					:  Tuple             =      {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}






class 	UpperCAmelCase__		(	A__	):






  """simple docstring"""


  a									= "transfo-xl"
  a									= ["mems"]
  a									= {
      "n_token": "vocab_size",
      "hidden_size": "d_model",
      "num_attention_heads": "n_head",
      "num_hidden_layers": "n_layer",
  }






  def __init__(							self						:		Any     ,       __lowerCamelCase						:		int=26_7735     ,       __lowerCamelCase						:		Any=[2_0000, 4_0000, 20_0000]     ,       __lowerCamelCase						:		Dict=1024     ,       __lowerCamelCase						:		Optional[int]=1024     ,       __lowerCamelCase						:		List[Any]=16     ,       __lowerCamelCase						:		Union[str, Any]=64     ,       __lowerCamelCase						:		Dict=4096     ,       __lowerCamelCase						:		int=4     ,       __lowerCamelCase						:		Dict=False     ,       __lowerCamelCase						:		Tuple=18     ,       __lowerCamelCase						:		Optional[int]=1600     ,       __lowerCamelCase						:		str=1000     ,       __lowerCamelCase						:		Any=True     ,       __lowerCamelCase						:		Dict=True     ,       __lowerCamelCase						:		List[Any]=0     ,       __lowerCamelCase						:		int=-1     ,       __lowerCamelCase						:		Tuple=True     ,       __lowerCamelCase						:		List[Any]=0.1     ,       __lowerCamelCase						:		int=0.0     ,       __lowerCamelCase						:		int=True     ,       __lowerCamelCase						:		str="normal"     ,       __lowerCamelCase						:		List[str]=0.01     ,       __lowerCamelCase						:		Any=0.01     ,       __lowerCamelCase						:		List[str]=0.02     ,       __lowerCamelCase						:		List[str]=1e-5     ,       __lowerCamelCase						:		Union[str, Any]=0     ,       **__lowerCamelCase						:		int     ,       )		->							Optional[Any]:
       SCREAMING_SNAKE_CASE__   =   vocab_size
       SCREAMING_SNAKE_CASE__   =   []
       self.cutoffs.extend(__lowerCamelCase )
       if proj_share_all_but_first:
            SCREAMING_SNAKE_CASE__   =   [False] + [True] * len(self.cutoffs )
       else:
            SCREAMING_SNAKE_CASE__   =   [False] + [False] * len(self.cutoffs )
       SCREAMING_SNAKE_CASE__   =   d_model
       SCREAMING_SNAKE_CASE__   =   d_embed
       SCREAMING_SNAKE_CASE__   =   d_head
       SCREAMING_SNAKE_CASE__   =   d_inner
       SCREAMING_SNAKE_CASE__   =   div_val
       SCREAMING_SNAKE_CASE__   =   pre_lnorm
       SCREAMING_SNAKE_CASE__   =   n_layer
       SCREAMING_SNAKE_CASE__   =   n_head
       SCREAMING_SNAKE_CASE__   =   mem_len
       SCREAMING_SNAKE_CASE__   =   same_length
       SCREAMING_SNAKE_CASE__   =   attn_type
       SCREAMING_SNAKE_CASE__   =   clamp_len
       SCREAMING_SNAKE_CASE__   =   sample_softmax
       SCREAMING_SNAKE_CASE__   =   adaptive
       SCREAMING_SNAKE_CASE__   =   dropout
       SCREAMING_SNAKE_CASE__   =   dropatt
       SCREAMING_SNAKE_CASE__   =   untie_r
       SCREAMING_SNAKE_CASE__   =   init
       SCREAMING_SNAKE_CASE__   =   init_range
       SCREAMING_SNAKE_CASE__   =   proj_init_std
       SCREAMING_SNAKE_CASE__   =   init_std
       SCREAMING_SNAKE_CASE__   =   layer_norm_epsilon
       super().__init__(eos_token_id=__lowerCamelCase     ,       **__lowerCamelCase )






  @property
  def 				lowercase_		(							self						:		str )		->							Dict:
       # Message copied from Transformer-XL documentation
       logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )
       return -1







  @max_position_embeddings.setter
  def 				lowercase_		(							self						:		List[str]     ,       __lowerCamelCase						:		Any )		->							List[Any]:
       # Message copied from Transformer-XL documentation
       raise NotImplementedError(
           f'''The model {self.model_type} is one of the few models that has no sequence length limit.''' )


