


import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
		import tensorflow as tf
		from tensorflow.python.eager import context
		from tensorflow.python.framework import ops

		from transformers import GradientAccumulator, create_optimizer





@require_tf
class  _a							(    unittest.TestCase):




	def 							UpperCAmelCase__( self						:		Dict		, _SCREAMING_SNAKE_CASE						:		Dict		, _SCREAMING_SNAKE_CASE						:		Optional[int]		, _SCREAMING_SNAKE_CASE						:		Dict		)->	str:
			self.assertEqual(len(_SCREAMING_SNAKE_CASE		)		, len(_SCREAMING_SNAKE_CASE		)		)
			for a, b in zip(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		):
					self.assertAlmostEqual(_SCREAMING_SNAKE_CASE		, _SCREAMING_SNAKE_CASE		, delta=_SCREAMING_SNAKE_CASE		)




	def 							UpperCAmelCase__( self						:		Any		)->	Optional[int]:
			lowerCAmelCase__       :      int   					= GradientAccumulator()
			accumulator([tf.constant([1.0, 2.0]		)]		)
			accumulator([tf.constant([-2.0, 1.0]		)]		)
			accumulator([tf.constant([-1.0, 2.0]		)]		)
			with self.assertRaises(_SCREAMING_SNAKE_CASE		):
					accumulator([tf.constant([1.0, 1.0]		), tf.constant([2.0, 2.0]		)]		)
			self.assertEqual(accumulator.step		, 3		)
			self.assertEqual(len(accumulator.gradients		)		, 1		)
			self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()		, [-2.0, 5.0]		, tol=1E-2		)
			accumulator.reset()
			self.assertEqual(accumulator.step		, 0		)
			self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()		, [0.0, 0.0]		, tol=1E-2		)




	def 							UpperCAmelCase__( self						:		Any		)->	Union[str, Any]:
			lowerCAmelCase__       :      Tuple   					= None
			ops.enable_eager_execution_internal()
			lowerCAmelCase__       :      Tuple   					= tf.config.list_physical_devices('''CPU'''		)
			if len(_SCREAMING_SNAKE_CASE		) == 1:
					tf.config.set_logical_device_configuration(
					    physical_devices[0]		, [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]		)
			lowerCAmelCase__       :      Optional[int]   					= tf.config.list_logical_devices(device_type='''CPU'''		)
			lowerCAmelCase__       :      Any   					= tf.distribute.MirroredStrategy(devices=devices[:2]		)

			with strategy.scope():
					lowerCAmelCase__       :      Optional[int]   					= GradientAccumulator()
					lowerCAmelCase__       :      List[str]   					= tf.Variable([4.0, 3.0]		)
					lowerCAmelCase__       :      Dict   					= create_optimizer(5E-5		, 10		, 5		)
					lowerCAmelCase__       :      int   					= tf.Variable([0.0, 0.0]		, trainable=_SCREAMING_SNAKE_CASE		)

			def accumulate_on_replica(_SCREAMING_SNAKE_CASE						:		Any		):
					accumulator([gradient]		)

			def apply_on_replica():
					optimizer.apply_gradients(list(zip(accumulator.gradients		, [variable]		)		)		)

			@tf.function
			def accumulate(_SCREAMING_SNAKE_CASE						:		Any		, _SCREAMING_SNAKE_CASE						:		Optional[Any]		):
					with strategy.scope():
							lowerCAmelCase__       :      Optional[Any]   					= strategy.experimental_local_results(_SCREAMING_SNAKE_CASE		)
							local_variables[0].assign(_SCREAMING_SNAKE_CASE		)
							local_variables[1].assign(_SCREAMING_SNAKE_CASE		)
							strategy.run(_SCREAMING_SNAKE_CASE		, args=(gradient_placeholder,)		)

			@tf.function
			def apply_grad():
					with strategy.scope():
							strategy.run(_SCREAMING_SNAKE_CASE		)

			def _check_local_values(_SCREAMING_SNAKE_CASE						:		Optional[Any]		, _SCREAMING_SNAKE_CASE						:		List[Any]		):
					lowerCAmelCase__       :      Optional[Any]   					= strategy.experimental_local_results(accumulator._gradients[0]		)
					self.assertListAlmostEqual(values[0].value()		, _SCREAMING_SNAKE_CASE		, tol=1E-2		)
					self.assertListAlmostEqual(values[1].value()		, _SCREAMING_SNAKE_CASE		, tol=1E-2		)

			accumulate([1.0, 2.0]		, [-1.0, 1.0]		)
			accumulate([3.0, -1.0]		, [-1.0, -1.0]		)
			accumulate([-2.0, 2.0]		, [3.0, -2.0]		)
			self.assertEqual(accumulator.step		, 3		)
			_check_local_values([2.0, 3.0]		, [1.0, -2.0]		)
			apply_grad()
			self.assertListAlmostEqual(variable.value()		, [4.0, 3.0]		, tol=1E-2		)
			accumulator.reset()
			self.assertEqual(accumulator.step		, 0		)
			_check_local_values([0.0, 0.0]		, [0.0, 0.0]		)




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase						=		logging.get_logger(__name__)

lowerCamelCase						=		{
    '''facebook/dpr-ctx_encoder-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-question_encoder-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-reader-single-nq-base''': (
        '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-ctx_encoder-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-question_encoder-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json'''
    ),
    '''facebook/dpr-reader-multiset-base''': (
        '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json'''
    ),
}





class  _a							(    _lowercase):
	_a  :  List[Any]				   =							'''dpr'''




	def __init__( self						:		List[str]		, _SCREAMING_SNAKE_CASE						:		List[str]=3_0522		, _SCREAMING_SNAKE_CASE						:		Optional[int]=768		, _SCREAMING_SNAKE_CASE						:		List[str]=12		, _SCREAMING_SNAKE_CASE						:		Tuple=12		, _SCREAMING_SNAKE_CASE						:		str=3072		, _SCREAMING_SNAKE_CASE						:		Optional[Any]="gelu"		, _SCREAMING_SNAKE_CASE						:		Dict=0.1		, _SCREAMING_SNAKE_CASE						:		Optional[int]=0.1		, _SCREAMING_SNAKE_CASE						:		List[str]=512		, _SCREAMING_SNAKE_CASE						:		int=2		, _SCREAMING_SNAKE_CASE						:		Optional[Any]=0.02		, _SCREAMING_SNAKE_CASE						:		Tuple=1E-12		, _SCREAMING_SNAKE_CASE						:		Union[str, Any]=0		, _SCREAMING_SNAKE_CASE						:		List[str]="absolute"		, _SCREAMING_SNAKE_CASE						:		int = 0		, **_SCREAMING_SNAKE_CASE						:		Optional[Any]		, )->	Optional[int]:
			super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE		, **_SCREAMING_SNAKE_CASE		)

			lowerCAmelCase__       :      Union[str, Any]   					= vocab_size
			lowerCAmelCase__       :      Tuple   					= hidden_size
			lowerCAmelCase__       :      Union[str, Any]   					= num_hidden_layers
			lowerCAmelCase__       :      List[Any]   					= num_attention_heads
			lowerCAmelCase__       :      Optional[int]   					= hidden_act
			lowerCAmelCase__       :      Optional[Any]   					= intermediate_size
			lowerCAmelCase__       :      Optional[Any]   					= hidden_dropout_prob
			lowerCAmelCase__       :      int   					= attention_probs_dropout_prob
			lowerCAmelCase__       :      str   					= max_position_embeddings
			lowerCAmelCase__       :      List[Any]   					= type_vocab_size
			lowerCAmelCase__       :      Optional[int]   					= initializer_range
			lowerCAmelCase__       :      List[str]   					= layer_norm_eps
			lowerCAmelCase__       :      Dict   					= projection_dim
			lowerCAmelCase__       :      int   					= position_embedding_type




