




import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
 import tensorflow as tf
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops

 from transformers import GradientAccumulator, create_optimizer



@require_tf
class       a					(       unittest.TestCase      ):

     '''simple docstring'''





     def   lowerCamelCase_       (     self       :      Optional[int] ,   __snake_case       :      List[str] ,   __snake_case       :      Optional[Any] ,   __snake_case       :      Dict    ):
         self.assertEqual(len(__snake_case    ) ,   len(__snake_case    )    )
         for a, b in zip(__snake_case ,   __snake_case    ):
             self.assertAlmostEqual(__snake_case ,   __snake_case ,   delta=__snake_case    )
     def   lowerCamelCase_       (     self       :      int    ):
         UpperCAmelCase_										=					GradientAccumulator()
         accumulator([tf.constant([1.0, 2.0]    )]    )
         accumulator([tf.constant([-2.0, 1.0]    )]    )
         accumulator([tf.constant([-1.0, 2.0]    )]    )
         with self.assertRaises(__snake_case    ):
             accumulator([tf.constant([1.0, 1.0]    ), tf.constant([2.0, 2.0]    )]    )
         self.assertEqual(accumulator.step ,   3    )
         self.assertEqual(len(accumulator.gradients    ) ,   1    )
         self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist() ,   [-2.0, 5.0] ,   tol=1E-2    )
         accumulator.reset()
         self.assertEqual(accumulator.step ,   0    )
         self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist() ,   [0.0, 0.0] ,   tol=1E-2    )




     def   lowerCamelCase_       (     self       :      str    ):
         UpperCAmelCase_										=					None
         ops.enable_eager_execution_internal()
         UpperCAmelCase_										=					tf.config.list_physical_devices('''CPU'''    )
         if len(__snake_case    ) == 1:
             tf.config.set_logical_device_configuration(
                 physical_devices[0] ,   [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]    )
         UpperCAmelCase_										=					tf.config.list_logical_devices(device_type='''CPU'''    )
         UpperCAmelCase_										=					tf.distribute.MirroredStrategy(devices=devices[:2]    )

         with strategy.scope():
             UpperCAmelCase_										=					GradientAccumulator()
             UpperCAmelCase_										=					tf.Variable([4.0, 3.0]    )
             UpperCAmelCase_			,							UpperCAmelCase_										=					create_optimizer(5E-5 ,   10 ,   5    )
             UpperCAmelCase_										=					tf.Variable([0.0, 0.0] ,   trainable=__snake_case    )

         def accumulate_on_replica(__snake_case       :      Optional[Any]    ):
             accumulator([gradient]    )

         def apply_on_replica():
             optimizer.apply_gradients(list(zip(accumulator.gradients ,   [variable]    )    )    )

         @tf.function
         def accumulate(__snake_case       :      Any ,   __snake_case       :      Tuple    ):
             with strategy.scope():
                 UpperCAmelCase_										=					strategy.experimental_local_results(__snake_case    )
                 local_variables[0].assign(__snake_case    )
                 local_variables[1].assign(__snake_case    )
                 strategy.run(__snake_case ,   args=(gradient_placeholder,)    )

         @tf.function
         def apply_grad():
             with strategy.scope():
                 strategy.run(__snake_case    )

         def _check_local_values(__snake_case       :      List[Any] ,   __snake_case       :      Optional[int]    ):
             UpperCAmelCase_										=					strategy.experimental_local_results(accumulator._gradients[0]    )
             self.assertListAlmostEqual(values[0].value() ,   __snake_case ,   tol=1E-2    )
             self.assertListAlmostEqual(values[1].value() ,   __snake_case ,   tol=1E-2    )

         accumulate([1.0, 2.0] ,   [-1.0, 1.0]    )
         accumulate([3.0, -1.0] ,   [-1.0, -1.0]    )
         accumulate([-2.0, 2.0] ,   [3.0, -2.0]    )
         self.assertEqual(accumulator.step ,   3    )
         _check_local_values([2.0, 3.0] ,   [1.0, -2.0]    )
         apply_grad()
         self.assertListAlmostEqual(variable.value() ,   [4.0, 3.0] ,   tol=1E-2    )
         accumulator.reset()
         self.assertEqual(accumulator.step ,   0    )
         _check_local_values([0.0, 0.0] ,   [0.0, 0.0]    )











import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
_lowerCamelCase      			=						logging.getLogger(__name__)

if __name__ == "__main__":
 _lowerCamelCase      			=						argparse.ArgumentParser(
     description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
 )
 parser.add_argument(
     '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
 )
 parser.add_argument(
     '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
 )
 parser.add_argument('--vocab_size', default=3_05_22, type=int)
 _lowerCamelCase      			=						parser.parse_args()

 logger.info(F"Loading data from {args.data_file}")
 with open(args.data_file, 'rb') as fp:
  _lowerCamelCase      			=						pickle.load(fp)

 logger.info('Counting occurrences for MLM.')
 _lowerCamelCase      			=						Counter()
 for tk_ids in data:
  counter.update(tk_ids)
 _lowerCamelCase      			=						[0] * args.vocab_size
 for k, v in counter.items():
  _lowerCamelCase      			=						v

 logger.info(F"Dump to {args.token_counts_dump}")
 with open(args.token_counts_dump, 'wb') as handle:
  pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)





