



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowercase_						=       logging.getLogger(__name__)



def __lowerCAmelCase	(       ):
			lowercase__   					=			argparse.ArgumentParser(
			    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
			parser.add_argument(
			    "--dataset_name"  ,					type=SCREAMING_SNAKE_CASE_  ,					default="wikitext"  ,					help="Name of the training. Explore datasets at: hf.co/datasets."  ,					)
			parser.add_argument(
			    "--dataset_config"  ,					type=SCREAMING_SNAKE_CASE_  ,					default="wikitext-103-raw-v1"  ,					help="Configuration name of the dataset."					)
			parser.add_argument(
			    "--tokenizer_name_or_path"  ,					type=SCREAMING_SNAKE_CASE_  ,					default="sayakpaul/unigram-tokenizer-wikitext"  ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."  ,					)
			parser.add_argument(
			    "--shard_size"  ,					type=SCREAMING_SNAKE_CASE_  ,					default=1000  ,					help="Number of entries to go in a single shard."  ,					)
			parser.add_argument("--split"  ,					type=SCREAMING_SNAKE_CASE_  ,					default="train"  ,					choices=["train", "test", "validation"]					)
			parser.add_argument(
			    "--limit"  ,					default=SCREAMING_SNAKE_CASE_  ,					type=SCREAMING_SNAKE_CASE_  ,					help="Limit the number of shards (used for debugging)."  ,					)
			parser.add_argument(
			    "--max_length"  ,					type=SCREAMING_SNAKE_CASE_  ,					default=512  ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
			    " sequence length that is a multiple of 8."  ,					)
			parser.add_argument(
			    "--output_dir"  ,					default="tf-tpu"  ,					type=SCREAMING_SNAKE_CASE_  ,					help="Output directory where the TFRecord shards will be saved. If the"
			    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
			    " shards will be directly saved to a Google Cloud Storage bucket."  ,					)

			lowercase__   					=			parser.parse_args()
			return args



def __lowerCAmelCase	(       SCREAMING_SNAKE_CASE_					):
			def fn(SCREAMING_SNAKE_CASE_					):
						return tokenizer(examples["text"]					)

			return fn



def __lowerCAmelCase	(       SCREAMING_SNAKE_CASE_					):
			lowercase__   					=			[]
			for i in range(len(tokenized_data["input_ids"]					)					):
						lowercase__   					=			{
						    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
						    "attention_mask": tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
						}
						lowercase__   					=			tf.train.Features(feature=SCREAMING_SNAKE_CASE_					)
						lowercase__   					=			tf.train.Example(features=SCREAMING_SNAKE_CASE_					)
						lowercase__   					=			example.SerializeToString()
						records.append(SCREAMING_SNAKE_CASE_					)
			return records



def __lowerCAmelCase	(       SCREAMING_SNAKE_CASE_					):
			lowercase__   					=			datasets.load_dataset(args.dataset_name  ,					args.dataset_config  ,					split=args.split					)

			if args.limit is not None:
						lowercase__   					=			min(len(SCREAMING_SNAKE_CASE_					)  ,					args.limit					)
						lowercase__   					=			dataset.select(range(SCREAMING_SNAKE_CASE_					)					)
						print(f'''Limiting the dataset to {args.limit} entries.'''					)

			lowercase__   					=			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir					):
									os.makedirs(args.output_dir					)
						lowercase__   					=			os.path.join(args.output_dir  ,					args.split					)
						if not os.path.exists(SCREAMING_SNAKE_CASE_					):
									os.makedirs(SCREAMING_SNAKE_CASE_					)
			else:
						lowercase__   					=			os.path.join(args.output_dir  ,					args.split					)

			# Tokenize the whole dataset at once.
			lowercase__   					=			tokenize_function(SCREAMING_SNAKE_CASE_					)
			lowercase__   					=			dataset.map(SCREAMING_SNAKE_CASE_  ,					batched=SCREAMING_SNAKE_CASE_  ,					num_proc=4  ,					remove_columns=["text"]					)

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(SCREAMING_SNAKE_CASE_					):
						# Concatenate all texts.
						lowercase__   					=			{k: sum(examples[k]  ,					[]					) for k in examples.keys()}
						lowercase__   					=			len(concatenated_examples[list(examples.keys()					)[0]]					)
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						lowercase__   					=			(total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						lowercase__   					=			{
						    k: [t[i : i + args.max_length] for i in range(0  ,					SCREAMING_SNAKE_CASE_  ,					args.max_length					)]
						    for k, t in concatenated_examples.items()
						}
						return result

			lowercase__   					=			dataset_tokenized.map(SCREAMING_SNAKE_CASE_  ,					batched=SCREAMING_SNAKE_CASE_  ,					batch_size=1000  ,					num_proc=4					)

			lowercase__   					=			0
			lowercase__   					=			0
			for shard in range(0  ,					len(SCREAMING_SNAKE_CASE_					)  ,					args.shard_size					):
						lowercase__   					=			grouped_dataset[shard : shard + args.shard_size]
						lowercase__   					=			len(dataset_snapshot["input_ids"]					)
						lowercase__   					=			os.path.join(SCREAMING_SNAKE_CASE_  ,					f'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
						lowercase__   					=			get_serialized_examples(SCREAMING_SNAKE_CASE_					)

						with tf.io.TFRecordWriter(SCREAMING_SNAKE_CASE_					) as out_file:
									for i in range(len(SCREAMING_SNAKE_CASE_					)					):
												lowercase__   					=			serialized_examples[i]
												out_file.write(SCREAMING_SNAKE_CASE_					)
									print("Wrote file {} containing {} records".format(SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_					)					)

						shard_count += 1
						total_records += records_containing

			with open(f'''split-{args.split}-records-count.txt'''  ,					"w"					) as f:
						print(f'''Total {args.split} records: {total_records}'''  ,					file=SCREAMING_SNAKE_CASE_					)


if __name__ == "__main__":
							lowercase_						=       parse_args()
							main(args)









import fire

from utils import calculate_rouge, save_json



def __lowerCAmelCase	(       SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_=None  ,					**SCREAMING_SNAKE_CASE_					):
			lowercase__   					=			[x.strip() for x in open(SCREAMING_SNAKE_CASE_					).readlines()]
			lowercase__   					=			[x.strip() for x in open(SCREAMING_SNAKE_CASE_					).readlines()][: len(SCREAMING_SNAKE_CASE_					)]
			lowercase__   					=			calculate_rouge(SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_  ,					**SCREAMING_SNAKE_CASE_					)
			if save_path is not None:
						save_json(SCREAMING_SNAKE_CASE_  ,					SCREAMING_SNAKE_CASE_  ,					indent=SCREAMING_SNAKE_CASE_					)
			return metrics  # these print nicely


if __name__ == "__main__":
							fire.Fire(calculate_rouge_path)




