




import os

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home


__a          =			HUGGINGFACE_HUB_CACHE


__a          =			'config.json'
__a          =			'diffusion_pytorch_model.bin'
__a          =			'diffusion_flax_model.msgpack'
__a          =			'model.onnx'
__a          =			'diffusion_pytorch_model.safetensors'
__a          =			'weights.pb'
__a          =			'https://huggingface.co'
__a          =			default_cache_path
__a          =			'diffusers_modules'
__a          =			os.getenv('HF_MODULES_CACHE', os.path.join(hf_cache_home, 'modules'))
__a          =			['fp16', 'non-ema']
__a          =			'.self_attn'











import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
__a          =			logging.getLogger(__name__)

if __name__ == "__main__":
     __a          =			argparse.ArgumentParser(
         description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
     )
     parser.add_argument(
         '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
     )
     parser.add_argument(
         '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
     )
     parser.add_argument('--vocab_size', default=30_522, type=int)
     __a          =			parser.parse_args()

     logger.info(F"""Loading data from {args.data_file}""")
     with open(args.data_file, 'rb') as fp:
          __a          =			pickle.load(fp)

     logger.info('Counting occurrences for MLM.')
     __a          =			Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __a          =			[0] * args.vocab_size
     for k, v in counter.items():
          __a          =			v

     logger.info(F"""Dump to {args.token_counts_dump}""")
     with open(args.token_counts_dump, 'wb') as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)