





'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A__    :       Optional[int]              =logging.getLogger(__name__)
def        UpperCamelCase__				(		):





      """simple docstring"""


      _lowerCAmelCase												=							argparse.ArgumentParser(
          description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""  )
      parser.add_argument(
          """--dataset_name"""							,     type=lowerCAmelCase							,     default="""wikitext"""							,     help="""Name of the training. Explore datasets at: hf.co/datasets."""							,     )
      parser.add_argument(
          """--dataset_config"""							,     type=lowerCAmelCase							,     default="""wikitext-103-raw-v1"""							,     help="""Configuration name of the dataset."""  )
      parser.add_argument(
          """--tokenizer_name_or_path"""							,     type=lowerCAmelCase							,     default="""sayakpaul/unigram-tokenizer-wikitext"""							,     help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""							,     )
      parser.add_argument(
          """--shard_size"""							,     type=lowerCAmelCase							,     default=10_00							,     help="""Number of entries to go in a single shard."""							,     )
      parser.add_argument("""--split"""							,     type=lowerCAmelCase							,     default="""train"""							,     choices=["""train""", """test""", """validation"""]  )
      parser.add_argument(
          """--limit"""							,     default=lowerCAmelCase							,     type=lowerCAmelCase							,     help="""Limit the number of shards (used for debugging)."""							,     )
      parser.add_argument(
          """--max_length"""							,     type=lowerCAmelCase							,     default=5_12							,     help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
          """ sequence length that is a multiple of 8."""							,     )
      parser.add_argument(
          """--output_dir"""							,     default="""tf-tpu"""							,     type=lowerCAmelCase							,     help="""Output directory where the TFRecord shards will be saved. If the"""
          """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
          """ shards will be directly saved to a Google Cloud Storage bucket."""							,     )

      _lowerCAmelCase												=							parser.parse_args()
      return args
def        UpperCamelCase__				(		lowerCAmelCase  ):





      """simple docstring"""


      def fn(lowerCAmelCase  ):
            return tokenizer(examples["""text"""]  )

      return fn
def        UpperCamelCase__				(		lowerCAmelCase  ):





      """simple docstring"""


      _lowerCAmelCase												=							[]
      for i in range(len(tokenized_data["""input_ids"""]  )  ):
            _lowerCAmelCase												=							{
                """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]  )  ),
                """attention_mask""": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]  )  ),
            }
            _lowerCAmelCase												=							tf.train.Features(feature=lowerCAmelCase  )
            _lowerCAmelCase												=							tf.train.Example(features=lowerCAmelCase  )
            _lowerCAmelCase												=							example.SerializeToString()
            records.append(lowerCAmelCase  )
      return records
def        UpperCamelCase__				(		lowerCAmelCase  ):





      """simple docstring"""


      _lowerCAmelCase												=							datasets.load_dataset(args.dataset_name							,     args.dataset_config							,     split=args.split  )

      if args.limit is not None:
            _lowerCAmelCase												=							min(len(lowerCAmelCase  )							,     args.limit  )
            _lowerCAmelCase												=							dataset.select(range(lowerCAmelCase  )  )
            print(f"Limiting the dataset to {args.limit} entries."  )

      _lowerCAmelCase												=							AutoTokenizer.from_pretrained(args.tokenizer_name_or_path  )

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir  ):
                  os.makedirs(args.output_dir  )
            _lowerCAmelCase												=							os.path.join(args.output_dir							,     args.split  )
            if not os.path.exists(lowerCAmelCase  ):
                  os.makedirs(lowerCAmelCase  )
      else:
            _lowerCAmelCase												=							os.path.join(args.output_dir							,     args.split  )

      # Tokenize the whole dataset at once.
      _lowerCAmelCase												=							tokenize_function(lowerCAmelCase  )
      _lowerCAmelCase												=							dataset.map(lowerCAmelCase							,     batched=lowerCAmelCase							,     num_proc=4							,     remove_columns=["""text"""]  )

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(lowerCAmelCase  ):
            # Concatenate all texts.
            _lowerCAmelCase												=							{k: sum(examples[k]							,     []  ) for k in examples.keys()}
            _lowerCAmelCase												=							len(concatenated_examples[list(examples.keys()  )[0]]  )
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            _lowerCAmelCase												=							(total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            _lowerCAmelCase												=							{
                k: [t[i : i + args.max_length] for i in range(0							,     lowerCAmelCase							,     args.max_length  )]
                for k, t in concatenated_examples.items()
            }
            return result

      _lowerCAmelCase												=							dataset_tokenized.map(lowerCAmelCase							,     batched=lowerCAmelCase							,     batch_size=10_00							,     num_proc=4  )

      _lowerCAmelCase												=							0
      _lowerCAmelCase												=							0
      for shard in range(0							,     len(lowerCAmelCase  )							,     args.shard_size  ):
            _lowerCAmelCase												=							grouped_dataset[shard : shard + args.shard_size]
            _lowerCAmelCase												=							len(dataset_snapshot["""input_ids"""]  )
            _lowerCAmelCase												=							os.path.join(lowerCAmelCase							,     f"dataset-{shard_count}-{records_containing}.tfrecord"  )
            _lowerCAmelCase												=							get_serialized_examples(lowerCAmelCase  )

            with tf.io.TFRecordWriter(lowerCAmelCase  ) as out_file:
                  for i in range(len(lowerCAmelCase  )  ):
                        _lowerCAmelCase												=							serialized_examples[i]
                        out_file.write(lowerCAmelCase  )
                  print("""Wrote file {} containing {} records""".format(lowerCAmelCase							,     lowerCAmelCase  )  )

            shard_count += 1
            total_records += records_containing

      with open(f"split-{args.split}-records-count.txt"							,     """w"""  ) as f:
            print(f"Total {args.split} records: {total_records}"							,     file=lowerCAmelCase  )


if __name__ == "__main__":
      A__    :       Dict              =parse_args()
      main(args)













'''simple docstring'''






def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      if len(lowerCAmelCase  ) != len(lowerCAmelCase  ):
            raise ValueError("""String lengths must match!"""  )

      _lowerCAmelCase												=							0

      for chara, chara in zip(lowerCAmelCase							,     lowerCAmelCase  ):
            if chara != chara:
                  count += 1

      return count


if __name__ == "__main__":
      import doctest

      doctest.testmod()






