




import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def       _SCREAMING_SNAKE_CASE      (							lowercase     :	int     ,							lowercase     :	Union[str, Any]=()     ,							lowercase     :	Union[str, Any]=None     ,							lowercase     :	List[str]="no"     ,							lowercase     :	List[Any]="29500"					):


      '''simple docstring'''



      lowerCamelCase_							=   False
      lowerCamelCase_							=   False
      if any(key.startswith('KAGGLE'					) for key in os.environ.keys()					):
            lowerCamelCase_							=   True
      elif "IPython" in sys.modules:
            lowerCamelCase_							=   'google.colab' in str(sys.modules['IPython'].get_ipython()					)

      try:
            lowerCamelCase_							=   PrecisionType(mixed_precision.lower()					)
      except ValueError:
            raise ValueError(
                f"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."""					)

      if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'     ,							lowercase					) is not None):
            # TPU launch
            import torch_xla.distributed.xla_multiprocessing as xmp

            if len(AcceleratorState._shared_state					) > 0:
                  raise ValueError(
                      'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
                      'your training function. Restart your notebook and make sure no cells initializes an '
                      '`Accelerator`.'					)
            if num_processes is None:
                  lowerCamelCase_							=   8

            lowerCamelCase_							=   PrepareForLaunch(lowercase     ,							distributed_type='TPU'					)
            print(f"""Launching a training on {num_processes} TPU cores."""					)
            xmp.spawn(lowercase     ,							args=lowercase     ,							nprocs=lowercase     ,							start_method='fork'					)
      elif in_colab:
            # No need for a distributed launch otherwise as it's either CPU or one GPU.
            if torch.cuda.is_available():
                  print('Launching training on one GPU.'					)
            else:
                  print('Launching training on one CPU.'					)
            function(*lowercase					)
      else:
            if num_processes is None:
                  raise ValueError(
                      'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'					)

            if num_processes > 1:
                  # Multi-GPU launch
                  from torch.multiprocessing import start_processes
                  from torch.multiprocessing.spawn import ProcessRaisedException

                  if len(AcceleratorState._shared_state					) > 0:
                        raise ValueError(
                            'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
                            'inside your training function. Restart your notebook and make sure no cells initializes an '
                            '`Accelerator`.'					)

                  if torch.cuda.is_initialized():
                        raise ValueError(
                            'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
                            'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
                            'function.'					)

                  # torch.distributed will expect a few environment variable to be here. We set the ones common to each
                  # process here (the other ones will be set be the launcher).
                  with patch_environment(
                      world_size=lowercase     ,							master_addr='127.0.01'     ,							master_port=lowercase     ,							mixed_precision=lowercase					):
                        lowerCamelCase_							=   PrepareForLaunch(lowercase     ,							distributed_type='MULTI_GPU'					)
                        print(f"""Launching training on {num_processes} GPUs."""					)
                        try:
                              start_processes(lowercase     ,							args=lowercase     ,							nprocs=lowercase     ,							start_method='fork'					)
                        except ProcessRaisedException as e:
                              if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                                    raise RuntimeError(
                                        'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                                        'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                                        'Please review your imports and test them when running the `notebook_launcher()` to identify '
                                        'which one is problematic.'					) from e

            else:
                  # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
                  if is_mps_available():
                        lowerCamelCase_							=   '1'
                        print('Launching training on MPS.'					)
                  elif torch.cuda.is_available():
                        print('Launching training on one GPU.'					)
                  else:
                        print('Launching training on CPU.'					)
                  function(*lowercase					)






def       _SCREAMING_SNAKE_CASE      (							lowercase     :	Dict     ,							lowercase     :	int=()     ,							lowercase     :	Tuple=2					):


      '''simple docstring'''



      from torch.multiprocessing import start_processes

      with tempfile.NamedTemporaryFile() as tmp_file:
            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=lowercase     ,							master_addr='127.0.01'     ,							master_port='29500'     ,							accelerate_mixed_precision='no'     ,							accelerate_debug_rdv_file=tmp_file.name     ,							accelerate_use_cpu='yes'     ,							):
                  lowerCamelCase_							=   PrepareForLaunch(lowercase     ,							debug=lowercase					)
                  start_processes(lowercase     ,							args=lowercase     ,							nprocs=lowercase     ,							start_method='fork'					)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase    :		Optional[Any]			   =       logging.get_logger(__name__)

lowerCamelCase    :		Tuple			   =       {
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						'''transfo-xl'''
 UpperCamelCase						       =						['''mems''']
 UpperCamelCase						       =						{
     '''n_token''': '''vocab_size''',
     '''hidden_size''': '''d_model''',
     '''num_attention_heads''': '''n_head''',
     '''num_hidden_layers''': '''n_layer''',
 }

 def __init__(    self :  Any						,							A_ :  Optional[Any]=267735						,							A_ :  Optional[Any]=[20000, 40000, 200000]						,							A_ :  Union[str, Any]=1024						,							A_ :  Optional[Any]=1024						,							A_ :  Optional[int]=16						,							A_ :  Any=64						,							A_ :  List[Any]=4096						,							A_ :  str=4						,							A_ :  int=False						,							A_ :  List[Any]=18						,							A_ :  Optional[int]=1600						,							A_ :  Union[str, Any]=1000						,							A_ :  Optional[Any]=True						,							A_ :  Optional[int]=True						,							A_ :  List[str]=0						,							A_ :  int=-1						,							A_ :  List[Any]=True						,							A_ :  List[Any]=0.1						,							A_ :  str=0.0						,							A_ :  Dict=True						,							A_ :  Dict="normal"						,							A_ :  Dict=0.01						,							A_ :  Optional[Any]=0.01						,							A_ :  Any=0.02						,							A_ :  int=1E-5						,							A_ :  List[str]=0						,							**A_ :  Optional[Any]						,							)       ->     Optional[int]:

       """simple docstring"""






       lowerCamelCase_							=   vocab_size
       lowerCamelCase_							=   []
       self.cutoffs.extend(A_  )
       if proj_share_all_but_first:
             lowerCamelCase_							=   [False] + [True] * len(self.cutoffs  )
       else:
             lowerCamelCase_							=   [False] + [False] * len(self.cutoffs  )
       lowerCamelCase_							=   d_model
       lowerCamelCase_							=   d_embed
       lowerCamelCase_							=   d_head
       lowerCamelCase_							=   d_inner
       lowerCamelCase_							=   div_val
       lowerCamelCase_							=   pre_lnorm
       lowerCamelCase_							=   n_layer
       lowerCamelCase_							=   n_head
       lowerCamelCase_							=   mem_len
       lowerCamelCase_							=   same_length
       lowerCamelCase_							=   attn_type
       lowerCamelCase_							=   clamp_len
       lowerCamelCase_							=   sample_softmax
       lowerCamelCase_							=   adaptive
       lowerCamelCase_							=   dropout
       lowerCamelCase_							=   dropatt
       lowerCamelCase_							=   untie_r
       lowerCamelCase_							=   init
       lowerCamelCase_							=   init_range
       lowerCamelCase_							=   proj_init_std
       lowerCamelCase_							=   init_std
       lowerCamelCase_							=   layer_norm_epsilon
       super().__init__(eos_token_id=A_						,							**A_  )

 @property
 def     a__			(    self :  Tuple  )       ->     Any:

       """simple docstring"""






       logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )
       return -1





 @max_position_embeddings.setter
 def     a__			(    self :  Dict						,							A_ :  Optional[int]  )       ->     List[Any]:

       """simple docstring"""






       raise NotImplementedError(
           f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )



