




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__a          =			{
    'configuration_xlm_roberta': [
        'XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XLMRobertaConfig',
        'XLMRobertaOnnxConfig',
    ],
}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a          =			['XLMRobertaTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a          =			['XLMRobertaTokenizerFast']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a          =			[
         'XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
         'XLMRobertaForCausalLM',
         'XLMRobertaForMaskedLM',
         'XLMRobertaForMultipleChoice',
         'XLMRobertaForQuestionAnswering',
         'XLMRobertaForSequenceClassification',
         'XLMRobertaForTokenClassification',
         'XLMRobertaModel',
         'XLMRobertaPreTrainedModel',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a          =			[
         'TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
         'TFXLMRobertaForCausalLM',
         'TFXLMRobertaForMaskedLM',
         'TFXLMRobertaForMultipleChoice',
         'TFXLMRobertaForQuestionAnswering',
         'TFXLMRobertaForSequenceClassification',
         'TFXLMRobertaForTokenClassification',
         'TFXLMRobertaModel',
         'TFXLMRobertaPreTrainedModel',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     __a          =			[
         'FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
         'FlaxXLMRobertaForMaskedLM',
         'FlaxXLMRobertaForCausalLM',
         'FlaxXLMRobertaForMultipleChoice',
         'FlaxXLMRobertaForQuestionAnswering',
         'FlaxXLMRobertaForSequenceClassification',
         'FlaxXLMRobertaForTokenClassification',
         'FlaxXLMRobertaModel',
         'FlaxXLMRobertaPreTrainedModel',
     ]

if TYPE_CHECKING:
     from .configuration_xlm_roberta import (
         XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
         XLMRobertaConfig,
         XLMRobertaOnnxConfig,
     )

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xlm_roberta import XLMRobertaTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xlm_roberta_fast import XLMRobertaTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_xlm_roberta import (
              XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
              XLMRobertaForCausalLM,
              XLMRobertaForMaskedLM,
              XLMRobertaForMultipleChoice,
              XLMRobertaForQuestionAnswering,
              XLMRobertaForSequenceClassification,
              XLMRobertaForTokenClassification,
              XLMRobertaModel,
              XLMRobertaPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_xlm_roberta import (
              TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFXLMRobertaForCausalLM,
              TFXLMRobertaForMaskedLM,
              TFXLMRobertaForMultipleChoice,
              TFXLMRobertaForQuestionAnswering,
              TFXLMRobertaForSequenceClassification,
              TFXLMRobertaForTokenClassification,
              TFXLMRobertaModel,
              TFXLMRobertaPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_xlm_roberta import (
              FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
              FlaxXLMRobertaForCausalLM,
              FlaxXLMRobertaForMaskedLM,
              FlaxXLMRobertaForMultipleChoice,
              FlaxXLMRobertaForQuestionAnswering,
              FlaxXLMRobertaForSequenceClassification,
              FlaxXLMRobertaForTokenClassification,
              FlaxXLMRobertaModel,
              FlaxXLMRobertaPreTrainedModel,
          )

else:
     import sys

     __a          =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
__a          =			logging.getLogger(__name__)

if __name__ == "__main__":
     __a          =			argparse.ArgumentParser(
         description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
     )
     parser.add_argument(
         '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
     )
     parser.add_argument(
         '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
     )
     parser.add_argument('--vocab_size', default=30_522, type=int)
     __a          =			parser.parse_args()

     logger.info(F"""Loading data from {args.data_file}""")
     with open(args.data_file, 'rb') as fp:
          __a          =			pickle.load(fp)

     logger.info('Counting occurrences for MLM.')
     __a          =			Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __a          =			[0] * args.vocab_size
     for k, v in counter.items():
          __a          =			v

     logger.info(F"""Dump to {args.token_counts_dump}""")
     with open(args.token_counts_dump, 'wb') as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)