





'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__    :       List[str]              =logging.get_logger(__name__)

A__    :       int              ={
    '''tanreinama/GPTSAN-2.8B-spout_is_uniform''': (
        '''https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json'''
    ),
}




class 			UpperCAmelCase					(      snake_case_       ):
      _lowercase:      Tuple											=       '''gptsan-japanese'''
      _lowercase:      List[Any]											=       [
          '''past_key_values''',
      ]
      _lowercase:      Tuple											=       {
          '''hidden_size''': '''d_model''',
          '''num_attention_heads''': '''num_heads''',
          '''num_hidden_layers''': '''num_layers''',
      }




      def __init__(      self      :			Optional[Any]      ,      __snake_case      :			Optional[Any]=3_60_00      ,      __snake_case      :			Optional[Any]=12_80      ,      __snake_case      :			Optional[int]=10_24      ,      __snake_case      :			int=81_92      ,      __snake_case      :			Tuple=40_96      ,      __snake_case      :			Any=1_28      ,      __snake_case      :			Optional[Any]=10      ,      __snake_case      :			List[Any]=0      ,      __snake_case      :			Union[str, Any]=16      ,      __snake_case      :			str=16      ,      __snake_case      :			List[str]=1_28      ,      __snake_case      :			Optional[Any]=0.0      ,      __snake_case      :			str=1E-5      ,      __snake_case      :			Any=False      ,      __snake_case      :			Tuple=0.0      ,      __snake_case      :			Any="float32"      ,      __snake_case      :			Optional[Any]=False      ,      __snake_case      :			Tuple=False      ,      __snake_case      :			Union[str, Any]=False      ,      __snake_case      :			Any=0.0_02      ,      __snake_case      :			List[str]=False      ,      __snake_case      :			int=True      ,      __snake_case      :			int=3_59_98      ,      __snake_case      :			Any=3_59_95      ,      __snake_case      :			int=3_59_99      ,      **__snake_case      :			str      ,      )    ->				Optional[Any]:
            _lowerCAmelCase												=							vocab_size
            _lowerCAmelCase												=							max_position_embeddings
            _lowerCAmelCase												=							d_model
            _lowerCAmelCase												=							d_ff
            _lowerCAmelCase												=							d_ext
            _lowerCAmelCase												=							d_spout
            _lowerCAmelCase												=							num_switch_layers
            _lowerCAmelCase												=							num_ext_layers
            _lowerCAmelCase												=							num_switch_layers + num_ext_layers
            _lowerCAmelCase												=							num_heads
            _lowerCAmelCase												=							num_experts
            _lowerCAmelCase												=							expert_capacity
            _lowerCAmelCase												=							dropout_rate
            _lowerCAmelCase												=							layer_norm_epsilon
            _lowerCAmelCase												=							router_bias
            _lowerCAmelCase												=							router_jitter_noise
            _lowerCAmelCase												=							router_dtype
            _lowerCAmelCase												=							router_ignore_padding_tokens
            _lowerCAmelCase												=							output_hidden_states
            _lowerCAmelCase												=							output_attentions
            _lowerCAmelCase												=							initializer_factor
            _lowerCAmelCase												=							output_router_logits
            _lowerCAmelCase												=							use_cache

            super().__init__(
                separator_token_id=__snake_case      ,      pad_token_id=__snake_case      ,      eos_token_id=__snake_case      ,      **__snake_case      ,      )













'''simple docstring'''






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment




@require_huggingface_suite
class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			int  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							inspect.getfile(accelerate.test_utils  )
            _lowerCAmelCase												=							os.path.sep.join(
                mod_file.split(os.path.sep  )[:-1] + ["""scripts""", """external_deps""", """test_metrics.py"""]  )

            from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

            _lowerCAmelCase												=							test_metrics




      @require_cpu
      def     lowercase__   (      self      :			Optional[Any]  )    ->				Optional[Any]:
            debug_launcher(self.test_metrics.main      ,      num_processes=1  )




      @require_cpu
      def     lowercase__   (      self      :			Tuple  )    ->				Tuple:
            debug_launcher(self.test_metrics.main  )




      @require_single_gpu
      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            self.test_metrics.main()




      @require_multi_gpu
      def     lowercase__   (      self      :			str  )    ->				List[str]:
            print(f"Found {torch.cuda.device_count()} devices."  )
            _lowerCAmelCase												=							["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
            with patch_environment(omp_num_threads=1  ):
                  execute_subprocess_async(__snake_case      ,      env=os.environ.copy()  )






