






'''simple docstring'''
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a_					=	logging.getLogger(__name__)





def 	_a(					):
      SCREAMING_SNAKE_CASE__       :			str													=argparse.ArgumentParser(
          description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''							)
      parser.add_argument(
          '''--dataset_name''',		type=__lowerCAmelCase,		default='''wikitext''',		help='''Name of the training. Explore datasets at: hf.co/datasets.''',		)
      parser.add_argument(
          '''--dataset_config''',		type=__lowerCAmelCase,		default='''wikitext-103-raw-v1''',		help='''Configuration name of the dataset.'''							)
      parser.add_argument(
          '''--tokenizer_name_or_path''',		type=__lowerCAmelCase,		default='''sayakpaul/unigram-tokenizer-wikitext''',		help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.''',		)
      parser.add_argument(
          '''--shard_size''',		type=__lowerCAmelCase,		default=1_0_0_0,		help='''Number of entries to go in a single shard.''',		)
      parser.add_argument('''--split''',		type=__lowerCAmelCase,		default='''train''',		choices=['''train''', '''test''', '''validation''']							)
      parser.add_argument(
          '''--limit''',		default=__lowerCAmelCase,		type=__lowerCAmelCase,		help='''Limit the number of shards (used for debugging).''',		)
      parser.add_argument(
          '''--max_length''',		type=__lowerCAmelCase,		default=5_1_2,		help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
          ''' sequence length that is a multiple of 8.''',		)
      parser.add_argument(
          '''--output_dir''',		default='''tf-tpu''',		type=__lowerCAmelCase,		help='''Output directory where the TFRecord shards will be saved. If the'''
          ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
          ''' shards will be directly saved to a Google Cloud Storage bucket.''',		)

      SCREAMING_SNAKE_CASE__       :			List[str]													=parser.parse_args()
      return args





def 	_a(					UpperCamelCase__			:  Any							):
      def fn(UpperCamelCase__			:  Any							):
            return tokenizer(examples['''text''']							)

      return fn





def 	_a(					UpperCamelCase__			:  Union[str, Any]							):
      SCREAMING_SNAKE_CASE__       :			List[str]													=[]
      for i in range(len(tokenized_data['''input_ids''']							)							):
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													={
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]							)							),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]							)							),
            }
            SCREAMING_SNAKE_CASE__       :			int													=tf.train.Features(feature=__lowerCAmelCase							)
            SCREAMING_SNAKE_CASE__       :			List[Any]													=tf.train.Example(features=__lowerCAmelCase							)
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=example.SerializeToString()
            records.append(__lowerCAmelCase							)
      return records





def 	_a(					UpperCamelCase__			:  List[str]							):
      SCREAMING_SNAKE_CASE__       :			Dict													=datasets.load_dataset(args.dataset_name,		args.dataset_config,		split=args.split							)

      if args.limit is not None:
            SCREAMING_SNAKE_CASE__       :			Tuple													=min(len(__lowerCAmelCase							),		args.limit							)
            SCREAMING_SNAKE_CASE__       :			Optional[int]													=dataset.select(range(__lowerCAmelCase							)							)
            print(f"Limiting the dataset to {args.limit} entries."							)

      SCREAMING_SNAKE_CASE__       :			List[str]													=AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir							):
                  os.makedirs(args.output_dir							)
            SCREAMING_SNAKE_CASE__       :			Any													=os.path.join(args.output_dir,		args.split							)
            if not os.path.exists(__lowerCAmelCase							):
                  os.makedirs(__lowerCAmelCase							)
      else:
            SCREAMING_SNAKE_CASE__       :			str													=os.path.join(args.output_dir,		args.split							)

      # Tokenize the whole dataset at once.
      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=tokenize_function(__lowerCAmelCase							)
      SCREAMING_SNAKE_CASE__       :			Any													=dataset.map(__lowerCAmelCase,		batched=__lowerCAmelCase,		num_proc=4,		remove_columns=['''text''']							)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(UpperCamelCase__			:  int							):
            # Concatenate all texts.
            SCREAMING_SNAKE_CASE__       :			Optional[int]													={k: sum(examples[k],		[]							) for k in examples.keys()}
            SCREAMING_SNAKE_CASE__       :			Tuple													=len(concatenated_examples[list(examples.keys()							)[0]]							)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            SCREAMING_SNAKE_CASE__       :			Tuple													=(total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            SCREAMING_SNAKE_CASE__       :			Optional[int]													={
                k: [t[i : i + args.max_length] for i in range(0,		__lowerCAmelCase,		args.max_length							)]
                for k, t in concatenated_examples.items()
            }
            return result

      SCREAMING_SNAKE_CASE__       :			int													=dataset_tokenized.map(__lowerCAmelCase,		batched=__lowerCAmelCase,		batch_size=1_0_0_0,		num_proc=4							)

      SCREAMING_SNAKE_CASE__       :			Optional[int]													=0
      SCREAMING_SNAKE_CASE__       :			Tuple													=0
      for shard in range(0,		len(__lowerCAmelCase							),		args.shard_size							):
            SCREAMING_SNAKE_CASE__       :			List[Any]													=grouped_dataset[shard : shard + args.shard_size]
            SCREAMING_SNAKE_CASE__       :			Dict													=len(dataset_snapshot['''input_ids''']							)
            SCREAMING_SNAKE_CASE__       :			Any													=os.path.join(__lowerCAmelCase,		f"dataset-{shard_count}-{records_containing}.tfrecord"							)
            SCREAMING_SNAKE_CASE__       :			Dict													=get_serialized_examples(__lowerCAmelCase							)

            with tf.io.TFRecordWriter(__lowerCAmelCase							) as out_file:
                  for i in range(len(__lowerCAmelCase							)							):
                        SCREAMING_SNAKE_CASE__       :			int													=serialized_examples[i]
                        out_file.write(__lowerCAmelCase							)
                  print('''Wrote file {} containing {} records'''.format(__lowerCAmelCase,		__lowerCAmelCase							)							)

            shard_count += 1
            total_records += records_containing

      with open(f"split-{args.split}-records-count.txt",		'''w'''							) as f:
            print(f"Total {args.split} records: {total_records}",		file=__lowerCAmelCase							)


if __name__ == "__main__":
  a_					=	parse_args()
  main(args)







'''simple docstring'''
import math





def 	_a(					UpperCamelCase__			:  int							):




      '''simple docstring'''






      return math.sqrt(UpperCamelCase__							) * math.sqrt(UpperCamelCase__							) == num





def 	_a(					UpperCamelCase__			:  int							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			List[Any]													=0
      SCREAMING_SNAKE_CASE__       :			Any													=n
      while left <= right:
            SCREAMING_SNAKE_CASE__       :			str													=(left + right) // 2
            if mid**2 == n:
                  return True
            elif mid**2 > n:
                  SCREAMING_SNAKE_CASE__       :			Optional[Any]													=mid - 1
            else:
                  SCREAMING_SNAKE_CASE__       :			Tuple													=mid + 1
      return False


if __name__ == "__main__":
  import doctest

  doctest.testmod()