
"""simple docstring"""







import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
lowerCamelCase__        =		logging.getLogger(__name__)

if __name__ == "__main__":
							lowerCamelCase__        =		argparse.ArgumentParser(
							    description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
							)
							parser.add_argument(
							    """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
							)
							parser.add_argument(
							    """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
							)
							parser.add_argument("""--vocab_size""", default=30_522, type=int)
							lowerCamelCase__        =		parser.parse_args()

							logger.info(f'Loading data from {args.data_file}')
							with open(args.data_file, """rb""") as fp:
														lowerCamelCase__        =		pickle.load(fp)

							logger.info("""Counting occurrences for MLM.""")
							lowerCamelCase__        =		Counter()
							for tk_ids in data:
														counter.update(tk_ids)
							lowerCamelCase__        =		[0] * args.vocab_size
							for k, v in counter.items():
														lowerCamelCase__        =		v

							logger.info(f'Dump to {args.token_counts_dump}')
							with open(args.token_counts_dump, """wb""") as handle:
														pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

"""simple docstring"""







from typing import Optional
from urllib.parse import quote

import huggingface_hub as hfh
from packaging import version
def    __lowerCAmelCase    (_UpperCamelCase	,  _UpperCamelCase	,  _UpperCamelCase = None		):
	if version.parse(hfh.__version__		).release < version.parse('0.11.0'		).release:
		# old versions of hfh don't url-encode the file path
		__lowerCAmelCase      :						List[str]						      =				quote(_UpperCamelCase		)
	return hfh.hf_hub_url(_UpperCamelCase	,  _UpperCamelCase	,  repo_type='dataset'	,  revision=_UpperCamelCase		)