





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_snake_case  :   List[str]						   =  logging.getLogger(__name__)

def   a_   (			):
   __lowerCAmelCase							     =     argparse.ArgumentParser(
       description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'     )
   parser.add_argument(
       '--dataset_name',     type=lowerCAmelCase_,     default='wikitext',     help='Name of the training. Explore datasets at: hf.co/datasets.',     )
   parser.add_argument(
       '--dataset_config',     type=lowerCAmelCase_,     default='wikitext-103-raw-v1',     help='Configuration name of the dataset.'     )
   parser.add_argument(
       '--tokenizer_name_or_path',     type=lowerCAmelCase_,     default='sayakpaul/unigram-tokenizer-wikitext',     help='Tokenizer identifier. Can be a local filepath or a Hub identifier.',     )
   parser.add_argument(
       '--shard_size',     type=lowerCAmelCase_,     default=1000,     help='Number of entries to go in a single shard.',     )
   parser.add_argument('--split',     type=lowerCAmelCase_,     default='train',     choices=['train', 'test', 'validation']     )
   parser.add_argument(
       '--limit',     default=lowerCAmelCase_,     type=lowerCAmelCase_,     help='Limit the number of shards (used for debugging).',     )
   parser.add_argument(
       '--max_length',     type=lowerCAmelCase_,     default=512,     help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
       ' sequence length that is a multiple of 8.',     )
   parser.add_argument(
       '--output_dir',     default='tf-tpu',     type=lowerCAmelCase_,     help='Output directory where the TFRecord shards will be saved. If the'
       ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
       ' shards will be directly saved to a Google Cloud Storage bucket.',     )

   __lowerCAmelCase							     =     parser.parse_args()
   return args

def   a_   (			lowerCAmelCase_				:      Any     ):
   def fn(lowerCAmelCase_				:      List[str]     ):
      return tokenizer(examples['text']     )

   return fn

def   a_   (			lowerCAmelCase_				:      Any     ):
   __lowerCAmelCase							     =     []
   for i in range(len(tokenized_data['input_ids']     )     ):
      __lowerCAmelCase							     =     {
          'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]     )     ),
          'attention_mask': tf.train.Feature(
              intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]     )     ),
      }
      __lowerCAmelCase							     =     tf.train.Features(feature=lowerCAmelCase_     )
      __lowerCAmelCase							     =     tf.train.Example(features=lowerCAmelCase_     )
      __lowerCAmelCase							     =     example.SerializeToString()
      records.append(lowerCAmelCase_     )
   return records

def   a_   (			lowerCAmelCase_				:      List[str]     ):
   __lowerCAmelCase							     =     datasets.load_dataset(args.dataset_name,     args.dataset_config,     split=args.split     )

   if args.limit is not None:
      __lowerCAmelCase							     =     min(len(lowerCAmelCase_     ),     args.limit     )
      __lowerCAmelCase							     =     dataset.select(range(lowerCAmelCase_     )     )
      print(F"""Limiting the dataset to {args.limit} entries."""     )

   __lowerCAmelCase							     =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

   # Handle output directory creation.
   # For serializing into a Google Cloud Storage Bucket, one needs to first
   # create a bucket.
   if "gs" not in args.output_dir:
      if not os.path.exists(args.output_dir     ):
         os.makedirs(args.output_dir     )
      __lowerCAmelCase							     =     os.path.join(args.output_dir,     args.split     )
      if not os.path.exists(lowerCAmelCase_     ):
         os.makedirs(lowerCAmelCase_     )
   else:
      __lowerCAmelCase							     =     os.path.join(args.output_dir,     args.split     )

   # Tokenize the whole dataset at once.
   __lowerCAmelCase							     =     tokenize_function(lowerCAmelCase_     )
   __lowerCAmelCase							     =     dataset.map(lowerCAmelCase_,     batched=lowerCAmelCase_,     num_proc=4,     remove_columns=['text']     )

   # We need to concatenate all our texts together, and then split the result
   # into chunks of a fixed size, which we will call block_size. To do this, we
   # will use the map method again, with the option batched=True. When we use batched=True,
   # the function we pass to map() will be passed multiple inputs at once, allowing us
   # to group them into more or fewer examples than we had in the input.
   # This allows us to create our new fixed-length samples. The advantage of this
   # method is that we don't lose a whole lot of content from the dataset compared to the
   # case where we simply tokenize with a pre-defined max_length.

   def group_texts(lowerCAmelCase_				:      Dict     ):
      # Concatenate all texts.
      __lowerCAmelCase							     =     {k: sum(examples[k],     []     ) for k in examples.keys()}
      __lowerCAmelCase							     =     len(concatenated_examples[list(examples.keys()     )[0]]     )
      # We drop the small remainder, though you could add padding instead if the model supports it
      # In this, as in all things, we advise you to follow your heart 🫀
      __lowerCAmelCase							     =     (total_length // args.max_length) * args.max_length
      # Split by chunks of max_len.
      __lowerCAmelCase							     =     {
          k: [t[i : i + args.max_length] for i in range(0,     lowerCAmelCase_,     args.max_length     )]
          for k, t in concatenated_examples.items()
      }
      return result

   __lowerCAmelCase							     =     dataset_tokenized.map(lowerCAmelCase_,     batched=lowerCAmelCase_,     batch_size=1000,     num_proc=4     )

   __lowerCAmelCase							     =     0
   __lowerCAmelCase							     =     0
   for shard in range(0,     len(lowerCAmelCase_     ),     args.shard_size     ):
      __lowerCAmelCase							     =     grouped_dataset[shard : shard + args.shard_size]
      __lowerCAmelCase							     =     len(dataset_snapshot['input_ids']     )
      __lowerCAmelCase							     =     os.path.join(lowerCAmelCase_,     F"""dataset-{shard_count}-{records_containing}.tfrecord"""     )
      __lowerCAmelCase							     =     get_serialized_examples(lowerCAmelCase_     )

      with tf.io.TFRecordWriter(lowerCAmelCase_     ) as out_file:
         for i in range(len(lowerCAmelCase_     )     ):
            __lowerCAmelCase							     =     serialized_examples[i]
            out_file.write(lowerCAmelCase_     )
         print('Wrote file {} containing {} records'.format(lowerCAmelCase_,     lowerCAmelCase_     )     )

      shard_count += 1
      total_records += records_containing

   with open(F"""split-{args.split}-records-count.txt""",     'w'     ) as f:
      print(F"""Total {args.split} records: {total_records}""",     file=lowerCAmelCase_     )


if __name__ == "__main__":
  _snake_case  :   Dict						   =  parse_args()
  main(args)








import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer

def   a_   (			lowerCAmelCase_				:      Optional[Any]     ):
   # This defines a "chinese character" as anything in the CJK Unicode block:
   #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
   #
   # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
   # despite its name. The modern Korean Hangul alphabet is a different block,
   # as is Japanese Hiragana and Katakana. Those alphabets are used to write
   # space-separated words, so they are not treated specially and handled
   # like the all of the other languages.
   if (
       (cp >= 0X4e00 and cp <= 0X9fff)
       or (cp >= 0X3400 and cp <= 0X4dbf)  #
       or (cp >= 0X20000 and cp <= 0X2a6df)  #
       or (cp >= 0X2a700 and cp <= 0X2b73f)  #
       or (cp >= 0X2b740 and cp <= 0X2b81f)  #
       or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
       or (cp >= 0Xf900 and cp <= 0Xfaff)
       or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
   ):  #
      return True

   return False

def   a_   (			lowerCAmelCase_				:      str     ):
   # word like '180' or '身高' or '神'
   for char in word:
      __lowerCAmelCase							     =     ord(lowerCAmelCase_     )
      if not _is_chinese_char(lowerCAmelCase_     ):
         return 0
   return 1

def   a_   (			lowerCAmelCase_				:      List[str]     ):
   __lowerCAmelCase							     =     set()

   for token in tokens:
      __lowerCAmelCase							     =     len(lowerCAmelCase_     ) > 1 and is_chinese(lowerCAmelCase_     )
      if chinese_word:
         word_set.add(lowerCAmelCase_     )
   __lowerCAmelCase							     =     list(lowerCAmelCase_     )
   return word_list

def   a_   (			lowerCAmelCase_				:      List[str],     lowerCAmelCase_				:      set()     ):
   if not chinese_word_set:
      return bert_tokens
   __lowerCAmelCase							     =     max([len(lowerCAmelCase_     ) for w in chinese_word_set]     )

   __lowerCAmelCase							     =     bert_tokens
   __lowerCAmelCase     ,     __lowerCAmelCase							     =     0, len(lowerCAmelCase_     )
   while start < end:
      __lowerCAmelCase							     =     True
      if is_chinese(bert_word[start]     ):
         __lowerCAmelCase							     =     min(end - start,     lowerCAmelCase_     )
         for i in range(lowerCAmelCase_,     1,     -1     ):
            __lowerCAmelCase							     =     ''.join(bert_word[start : start + i]     )
            if whole_word in chinese_word_set:
               for j in range(start + 1,     start + i     ):
                  __lowerCAmelCase							     =     '##' + bert_word[j]
               __lowerCAmelCase							     =     start + i
               __lowerCAmelCase							     =     False
               break
      if single_word:
         start += 1
   return bert_word

def   a_   (			lowerCAmelCase_				:      List[str],     lowerCAmelCase_				:      LTP,     lowerCAmelCase_				:      BertTokenizer     ):
   __lowerCAmelCase							     =     []

   for i in range(0,     len(lowerCAmelCase_     ),     100     ):
      __lowerCAmelCase							     =     ltp_tokenizer.pipeline(lines[i : i + 100],     tasks=['cws']     ).cws
      __lowerCAmelCase							     =     [get_chinese_word(lowerCAmelCase_     ) for r in res]
      ltp_res.extend(lowerCAmelCase_     )
   assert len(lowerCAmelCase_     ) == len(lowerCAmelCase_     )

   __lowerCAmelCase							     =     []
   for i in range(0,     len(lowerCAmelCase_     ),     100     ):
      __lowerCAmelCase							     =     bert_tokenizer(lines[i : i + 100],     add_special_tokens=lowerCAmelCase_,     truncation=lowerCAmelCase_,     max_length=512     )
      bert_res.extend(res['input_ids']     )
   assert len(lowerCAmelCase_     ) == len(lowerCAmelCase_     )

   __lowerCAmelCase							     =     []
   for input_ids, chinese_word in zip(lowerCAmelCase_,     lowerCAmelCase_     ):
      __lowerCAmelCase							     =     []
      for id in input_ids:
         __lowerCAmelCase							     =     bert_tokenizer._convert_id_to_token(lowerCAmelCase_     )
         input_tokens.append(lowerCAmelCase_     )
      __lowerCAmelCase							     =     add_sub_symbol(lowerCAmelCase_,     lowerCAmelCase_     )
      __lowerCAmelCase							     =     []
      # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
      for i, token in enumerate(lowerCAmelCase_     ):
         if token[:2] == "##":
            __lowerCAmelCase							     =     token[2:]
            # save chinese tokens' pos
            if len(lowerCAmelCase_     ) == 1 and _is_chinese_char(ord(lowerCAmelCase_     )     ):
               ref_id.append(lowerCAmelCase_     )
      ref_ids.append(lowerCAmelCase_     )

   assert len(lowerCAmelCase_     ) == len(lowerCAmelCase_     )

   return ref_ids

def   a_   (			lowerCAmelCase_				:      int     ):
   # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
   # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
   with open(args.file_name,     'r',     encoding='utf-8'     ) as f:
      __lowerCAmelCase							     =     f.readlines()
   __lowerCAmelCase							     =     [line.strip() for line in data if len(lowerCAmelCase_     ) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
   __lowerCAmelCase							     =     LTP(args.ltp     )  # faster in GPU device
   __lowerCAmelCase							     =     BertTokenizer.from_pretrained(args.bert     )

   __lowerCAmelCase							     =     prepare_ref(lowerCAmelCase_,     lowerCAmelCase_,     lowerCAmelCase_     )

   with open(args.save_path,     'w',     encoding='utf-8'     ) as f:
      __lowerCAmelCase							     =     [json.dumps(lowerCAmelCase_     ) + '\n' for ref in ref_ids]
      f.writelines(lowerCAmelCase_     )


if __name__ == "__main__":
     _snake_case  :   int						   =  argparse.ArgumentParser(description='prepare_chinese_ref')
     parser.add_argument(
         '--file_name',
         required=False,
         type=str,
         default='./resources/chinese-demo.txt',
         help='file need process, same as training data in lm',
     )
     parser.add_argument(
         '--ltp',
         required=False,
         type=str,
         default='./resources/ltp',
         help='resources for LTP tokenizer, usually a path',
     )
     parser.add_argument(
         '--bert',
         required=False,
         type=str,
         default='./resources/robert',
         help='resources for Bert tokenizer',
     )
     parser.add_argument(
         '--save_path',
         required=False,
         type=str,
         default='./resources/ref.txt',
         help='path to save res',
     )

     _snake_case  :   List[str]						   =  parser.parse_args()
     main(args)

