



'''simple docstring'''
import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment






def A__   (					):
						_UpperCamelCase :		List[str]			    =       torch.nn.Linear(2  ,  4							)
						_UpperCamelCase :		Any			    =       torch.optim.AdamW(model.parameters()  ,  lr=1.0							)
						_UpperCamelCase :		List[Any]			    =       torch.optim.lr_scheduler.OneCycleLR(UpperCAmelCase_  ,  max_lr=0.01  ,  steps_per_epoch=2  ,  epochs=1							)
						_UpperCamelCase :		List[Any]			    =       DataLoader(TensorDataset(torch.tensor([1, 2, 3]							)							)							)
						_UpperCamelCase :		Dict			    =       DataLoader(TensorDataset(torch.tensor([4, 5, 6]							)							)							)

						return model, optimizer, scheduler, train_dl, valid_dl






def A__   (					UpperCAmelCase_							):
						return (model.weight.abs().sum() + model.bias.abs().sum()).item()






def A__   (					UpperCAmelCase_							):
						_UpperCamelCase :		Union[str, Any]			    =       torch.nn.Linear(*tuple(model.weight.T.shape							)							).state_dict()
						model.load_state_dict(UpperCAmelCase_							)






class 							lowercase__     (							UpperCAmelCase_       ):





	@require_cuda
	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       Accelerator()
							assert PartialState._shared_state["_cpu"] is False
							assert PartialState._shared_state["device"].type == "cuda"
							with self.assertRaises(__lowercase			):
													_UpperCamelCase :		Any			    =       Accelerator(cpu=__lowercase			)





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							_UpperCamelCase :		List[Any]			    =       Accelerator()
							_UpperCamelCase :		List[Any]			    =       GradientState()
							assert state.num_steps == 1
							_UpperCamelCase :		Any			    =       4
							assert state.num_steps == 4

							assert state.sync_gradients is True
							_UpperCamelCase :		int			    =       False
							assert state.sync_gradients is False
							GradientState._reset_state()





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       Accelerator()
							_UpperCamelCase :		Optional[Any]			    =       create_components()

							(
							    _UpperCamelCase	
							) :		int			    =       accelerator.prepare(__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)

							self.assertTrue(prepared_model in accelerator._models			)
							self.assertTrue(prepared_optimizer in accelerator._optimizers			)
							self.assertTrue(prepared_scheduler in accelerator._schedulers			)
							self.assertTrue(prepared_train_dl in accelerator._dataloaders			)
							self.assertTrue(prepared_valid_dl in accelerator._dataloaders			)





	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							_UpperCamelCase :		str			    =       Accelerator()
							_UpperCamelCase :		Optional[Any]			    =       create_components()
							accelerator.prepare(__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)
							accelerator.free_memory()

							self.assertTrue(len(accelerator._models			) == 0			)
							self.assertTrue(len(accelerator._optimizers			) == 0			)
							self.assertTrue(len(accelerator._schedulers			) == 0			)
							self.assertTrue(len(accelerator._dataloaders			) == 0			)





	def      UpperCamelCase_  (			self :    str			):







							'''simple docstring'''
							PartialState._reset_state()

							# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
							def noop(*lowerCamelCase__ :    Union[str, Any]			,**lowerCamelCase__ :    Tuple			):
													pass

							with patch('torch.cuda.set_device'			,__lowercase			), patch_environment(ACCELERATE_TORCH_DEVICE='cuda:64'			):
													_UpperCamelCase :		Optional[Any]			    =       Accelerator()
													self.assertEqual(str(accelerator.state.device			)			,'cuda:64'			)





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							_UpperCamelCase :		List[Any]			    =       Accelerator()
							_UpperCamelCase :		List[Any]			    =       create_components()
							accelerator.prepare(__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)

							_UpperCamelCase :		Tuple			    =       get_signature(__lowercase			)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__lowercase			)

													# make sure random weights don't match
													load_random_weights(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) > 1E-3			)

													# make sure loaded weights match
													accelerator.load_state(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) < 1E-3			)





	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		List[Any]			    =       Accelerator()
							_UpperCamelCase :		Any			    =       create_components()
							accelerator.prepare(__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)

							_UpperCamelCase :		Any			    =       get_signature(__lowercase			)

							# saving hook
							def save_config(lowerCamelCase__ :    List[Any]			,lowerCamelCase__ :    Optional[int]			,lowerCamelCase__ :    Optional[int]			):
													_UpperCamelCase :		Union[str, Any]			    =       {'''class_name''': models[0].__class__.__name__}

													with open(os.path.join(__lowercase			,'data.json'			)			,'w'			) as f:
																			json.dump(__lowercase			,__lowercase			)

        # loading hook
							def load_config(lowerCamelCase__ :    List[Any]			,lowerCamelCase__ :    str			):
													with open(os.path.join(__lowercase			,'data.json'			)			,'r'			) as f:
																			_UpperCamelCase :		Dict			    =       json.load(__lowercase			)

													_UpperCamelCase :		Dict			    =       config['''class_name''']

							_UpperCamelCase :		Union[str, Any]			    =       accelerator.register_save_state_pre_hook(__lowercase			)
							_UpperCamelCase :		Any			    =       accelerator.register_load_state_pre_hook(__lowercase			)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__lowercase			)

													# make sure random weights don't match with hooks
													load_random_weights(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) > 1E-3			)

													# random class name to verify correct one is loaded
													_UpperCamelCase :		int			    =       '''random'''

													# make sure loaded weights match with hooks
													accelerator.load_state(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) < 1E-3			)

													# mode.class_name is loaded from config
													self.assertTrue(model.class_name == model.__class__.__name__			)

							# remove hooks
							save_hook.remove()
							load_hook.remove()

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__lowercase			)

													# make sure random weights don't match with hooks removed
													load_random_weights(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) > 1E-3			)

													# random class name to verify correct one is loaded
													_UpperCamelCase :		Dict			    =       '''random'''

													# make sure loaded weights match with hooks removed
													accelerator.load_state(__lowercase			)
													self.assertTrue(abs(model_signature - get_signature(__lowercase			)			) < 1E-3			)

													# mode.class_name is NOT loaded from config
													self.assertTrue(model.class_name != model.__class__.__name__			)





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       Accelerator()
							_UpperCamelCase :		Union[str, Any]			    =       create_components()
							_UpperCamelCase :		Optional[Any]			    =       None

							# This should work
							_UpperCamelCase :		List[Any]			    =       accelerator.prepare(
							    __lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)
							self.assertTrue(dummy_obj is None			)





	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							_UpperCamelCase :		List[str]			    =       Accelerator()
							_UpperCamelCase :		Any			    =       create_components()
							_UpperCamelCase :		Dict			    =       [1, 2, 3]

							# This should work
							_UpperCamelCase :		Tuple			    =       accelerator.prepare(
							    __lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			,__lowercase			)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Dummy object should have `_is_accelerate_prepared` set to `True`'			,)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Model is missing `_is_accelerator_prepared` or is set to `False`'			,)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Optimizer is missing `_is_accelerator_prepared` or is set to `False`'			,)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Scheduler is missing `_is_accelerator_prepared` or is set to `False`'			,)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`'			,)
							self.assertEqual(
							    getattr(__lowercase			,'_is_accelerate_prepared'			,__lowercase			)			,__lowercase			,'Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`'			,)





	@slow
	@require_bnb
	def      UpperCamelCase_  (			self :    Dict			):







							'''simple docstring'''
							from transformers import AutoModelForCausalLM

							_UpperCamelCase :		Optional[int]			    =       AutoModelForCausalLM.from_pretrained(
							    'EleutherAI/gpt-neo-125m'			,load_in_abit=__lowercase			,device_map={'': 0}			,)
							_UpperCamelCase :		Optional[Any]			    =       Accelerator()

							# This should work
							_UpperCamelCase :		int			    =       accelerator.prepare(__lowercase			)





	@slow
	@require_bnb
	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							from transformers import AutoModelForCausalLM

							_UpperCamelCase :		str			    =       Accelerator()

							with init_empty_weights():
													_UpperCamelCase :		List[Any]			    =       AutoModelForCausalLM.from_pretrained(
													    'EleutherAI/gpt-neo-125m'			,)
													model.tie_weights()
													_UpperCamelCase :		List[str]			    =       infer_auto_device_map(__lowercase			)
													_UpperCamelCase :		str			    =       '''cpu'''

							_UpperCamelCase :		List[Any]			    =       AutoModelForCausalLM.from_pretrained(
							    'EleutherAI/gpt-neo-125m'			,device_map=__lowercase			,load_in_abit=__lowercase			,llm_inta_enable_fpaa_cpu_offload=__lowercase			)

							# This should not work and get value error
							with self.assertRaises(__lowercase			):
													_UpperCamelCase :		Union[str, Any]			    =       accelerator.prepare(__lowercase			)





	@slow
	@require_bnb
	@require_multi_gpu
	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							from transformers import AutoModelForCausalLM

							_UpperCamelCase :		int			    =       {'''distributed_type''': DistributedType.MULTI_GPU}

							with init_empty_weights():
													_UpperCamelCase :		Tuple			    =       AutoModelForCausalLM.from_pretrained(
													    'EleutherAI/gpt-neo-125m'			,)
													model.tie_weights()
													_UpperCamelCase :		int			    =       infer_auto_device_map(__lowercase			)
													_UpperCamelCase :		List[Any]			    =       1

							_UpperCamelCase :		int			    =       AutoModelForCausalLM.from_pretrained(
							    'EleutherAI/gpt-neo-125m'			,load_in_abit=__lowercase			,device_map=__lowercase			,)
							_UpperCamelCase :		Dict			    =       Accelerator()

							# This should not work and get value error
							with self.assertRaises(__lowercase			):
													_UpperCamelCase :		Any			    =       accelerator.prepare(__lowercase			)

							PartialState._reset_state()





	@slow
	@require_bnb
	@require_multi_gpu
	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							from transformers import AutoModelForCausalLM

							with init_empty_weights():
													_UpperCamelCase :		Optional[int]			    =       AutoModelForCausalLM.from_pretrained(
													    'EleutherAI/gpt-neo-125m'			,)
													_UpperCamelCase :		List[str]			    =       infer_auto_device_map(__lowercase			)
													_UpperCamelCase :		Optional[int]			    =       1

							_UpperCamelCase :		Optional[int]			    =       AutoModelForCausalLM.from_pretrained(
							    'EleutherAI/gpt-neo-125m'			,load_in_abit=__lowercase			,device_map=__lowercase			,)
							_UpperCamelCase :		int			    =       Accelerator()

							# This should work
							_UpperCamelCase :		int			    =       accelerator.prepare(__lowercase			)





	@require_cuda
	def      UpperCamelCase_  (			self :    List[str]			):







							'''simple docstring'''
							_UpperCamelCase :		Tuple			    =       torch.nn.Linear(10			,10			)
							_UpperCamelCase :		Optional[Any]			    =       torch.optim.SGD(model.parameters()			,lr=0.0_1			)
							_UpperCamelCase :		Any			    =       Accelerator(cpu=__lowercase			)
							_UpperCamelCase :		Tuple			    =       accelerator.prepare(__lowercase			)








'''simple docstring'''
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


snake_case_			:     Optional[int]  	=  logging.get_logger(__name__)

snake_case_			:     List[Any]  	=  {
    'facebook/xlm-roberta-xl': 'https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json',
    'facebook/xlm-roberta-xxl': 'https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json',
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}






class 							lowercase__     (							lowercase       ):
	lowercase__      =      """xlm-roberta-xl"""





	def __init__(			self :    Optional[int]			,lowerCamelCase__ :    Optional[Any]=250880			,lowerCamelCase__ :    Tuple=2560			,lowerCamelCase__ :    Union[str, Any]=36			,lowerCamelCase__ :    List[str]=32			,lowerCamelCase__ :    Optional[Any]=10240			,lowerCamelCase__ :    Tuple="gelu"			,lowerCamelCase__ :    int=0.1			,lowerCamelCase__ :    int=0.1			,lowerCamelCase__ :    Optional[int]=514			,lowerCamelCase__ :    List[str]=1			,lowerCamelCase__ :    Dict=0.0_2			,lowerCamelCase__ :    Any=1E-05			,lowerCamelCase__ :    Union[str, Any]=1			,lowerCamelCase__ :    str=0			,lowerCamelCase__ :    Tuple=2			,lowerCamelCase__ :    Union[str, Any]="absolute"			,lowerCamelCase__ :    Optional[Any]=True			,lowerCamelCase__ :    List[str]=None			,**lowerCamelCase__ :    Dict			,):







							'''simple docstring'''
							super().__init__(pad_token_id=lowerCamelCase__			,bos_token_id=lowerCamelCase__			,eos_token_id=lowerCamelCase__			,**lowerCamelCase__			)
							_UpperCamelCase :		Optional[int]			    =       vocab_size
							_UpperCamelCase :		Optional[Any]			    =       hidden_size
							_UpperCamelCase :		str			    =       num_hidden_layers
							_UpperCamelCase :		str			    =       num_attention_heads
							_UpperCamelCase :		Any			    =       hidden_act
							_UpperCamelCase :		Dict			    =       intermediate_size
							_UpperCamelCase :		Optional[int]			    =       hidden_dropout_prob
							_UpperCamelCase :		Any			    =       attention_probs_dropout_prob
							_UpperCamelCase :		List[Any]			    =       max_position_embeddings
							_UpperCamelCase :		str			    =       type_vocab_size
							_UpperCamelCase :		Optional[Any]			    =       initializer_range
							_UpperCamelCase :		Optional[int]			    =       layer_norm_eps
							_UpperCamelCase :		Optional[int]			    =       position_embedding_type
							_UpperCamelCase :		Optional[Any]			    =       use_cache
							_UpperCamelCase :		Optional[Any]			    =       classifier_dropout






class 							lowercase__     (							lowercase       ):





	@property
	def      UpperCamelCase_  (			self :    List[Any]			):







							'''simple docstring'''
							if self.task == "multiple-choice":
													_UpperCamelCase :		Union[str, Any]			    =       {0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
													_UpperCamelCase :		Optional[Any]			    =       {0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]			)


