suploader

suploader.c (6358B)

---

 /* See LICENSE file for copyright and license details.
  *
  * suploader - Simple Uploader
  *
  * Uploads URLs to web archive services (Internet Archive,
  * archive.today, Wikiwix). Reads URLs from a file or stdin.
  */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
 
 #include "config.h"
 #include "services.h"
 #include "util.h"
 
 /* Global options */
 static int verbose = 0;
 static int daemon_mode = 0;
 static int delay_secs = DEFAULT_DELAY;
 static int random_order = 0;
 static int services = SVC_ALL;
 
 static void
 usage(void)
 {
 	fprintf(stderr,
 	    "usage: suploader [-vrD] [-d secs] [-s services]"
 	    " [file | -]\n"
 	    "\n"
 	    "  -v          verbose output\n"
 	    "  -r          random URL selection order\n"
 	    "  -D          daemon mode (loop forever)\n"
 	    "  -d secs     delay between submissions"
 	    " (default: %d)\n"
 	    "  -s services comma-separated: ia,archiveph,"
 	    "wikiwix (default: all)\n"
 	    "\n"
 	    "  file        file with URLs, one per line\n"
 	    "  -           read from stdin\n",
 	    DEFAULT_DELAY);
 	exit(1);
 }
 
 /* Read all URLs from a file into an array.
  * Returns array of strings, sets *count.
  * Caller frees the array and each string. */
 static char **
 read_urls(const char *path, int *count)
 {
 	FILE *fp;
 	char line[MAX_URL_LEN];
 	char **urls;
 	int n, cap;
 	char *trimmed;
 
 	if (strcmp(path, "-") == 0)
 		fp = stdin;
 	else
 		fp = fopen(path, "r");
 
 	if (!fp)
 		die("cannot open: %s:", path);
 
 	cap = 256;
 	n = 0;
 	urls = xmalloc(cap * sizeof(char *));
 
 	while (fgets(line, sizeof(line), fp)) {
 		/* Strip newline */
 		line[strcspn(line, "\r\n")] = '\0';
 		trimmed = str_trim(line);
 
 		/* Skip empty lines and comments */
 		if (!*trimmed || *trimmed == '#')
 			continue;
 
 		/* Must look like a URL */
 		if (!str_starts_with(trimmed, "http://") &&
 		    !str_starts_with(trimmed, "https://"))
 			continue;
 
 		if (n >= cap) {
 			cap *= 2;
 			urls = xrealloc(urls,
 			    cap * sizeof(char *));
 		}
 		urls[n++] = xstrdup(trimmed);
 	}
 
 	if (fp != stdin)
 		fclose(fp);
 
 	*count = n;
 	return urls;
 }
 
 /* Write remaining URLs back to the file (for daemon mode) */
 static void
 write_urls(const char *path, char **urls, int count)
 {
 	FILE *fp;
 	int i;
 
 	if (strcmp(path, "-") == 0)
 		return;
 
 	fp = fopen(path, "w");
 	if (!fp) {
 		warn("cannot write: %s", path);
 		return;
 	}
 
 	for (i = 0; i < count; i++) {
 		if (urls[i])
 			fprintf(fp, "%s\n", urls[i]);
 	}
 
 	fclose(fp);
 }
 
 /* Remove an entry from the URL array by setting it to NULL */
 static void
 remove_url(char **urls, int idx)
 {
 	free(urls[idx]);
 	urls[idx] = NULL;
 }
 
 /* Count non-NULL entries */
 static int
 count_remaining(char **urls, int count)
 {
 	int i, n;
 
 	n = 0;
 	for (i = 0; i < count; i++) {
 		if (urls[i])
 			n++;
 	}
 	return n;
 }
 
 /* Pick a random non-NULL index */
 static int
 pick_random(char **urls, int count)
 {
 	int remaining, target, i;
 
 	remaining = count_remaining(urls, count);
 	if (remaining == 0)
 		return -1;
 
 	target = rand() % remaining;
 	for (i = 0; i < count; i++) {
 		if (urls[i]) {
 			if (target == 0)
 				return i;
 			target--;
 		}
 	}
 	return -1;
 }
 
 /* Pick next non-NULL index sequentially */
 static int
 pick_next(char **urls, int count)
 {
 	int i;
 
 	for (i = 0; i < count; i++) {
 		if (urls[i])
 			return i;
 	}
 	return -1;
 }
 
 /* Parse the -s services flag */
 static int
 parse_services(const char *arg)
 {
 	int svc;
 	char *copy, *tok, *saveptr;
 
 	svc = 0;
 	copy = xstrdup(arg);
 
 	for (tok = strtok_r(copy, ",", &saveptr); tok;
 	     tok = strtok_r(NULL, ",", &saveptr)) {
 		tok = str_trim(tok);
 		if (strcmp(tok, "ia") == 0)
 			svc |= SVC_IA;
 		else if (strcmp(tok, "archiveph") == 0)
 			svc |= SVC_ARCHIVEPH;
 		else if (strcmp(tok, "wikiwix") == 0)
 			svc |= SVC_WIKIWIX;
 		else if (strcmp(tok, "all") == 0)
 			svc = SVC_ALL;
 		else
 			die("unknown service: %s", tok);
 	}
 
 	free(copy);
 	return svc ? svc : SVC_ALL;
 }
 
 int
 main(int argc, char *argv[])
 {
 	const char *file;
 	char **urls;
 	int count, idx, ok, processed, opt;
 	time_t start;
 
 	while ((opt = getopt(argc, argv, "vrDd:s:h")) != -1) {
 		switch (opt) {
 		case 'v':
 			verbose = 1;
 			break;
 		case 'r':
 			random_order = 1;
 			break;
 		case 'D':
 			daemon_mode = 1;
 			break;
 		case 'd':
 			delay_secs = atoi(optarg);
 			if (delay_secs < 1)
 				delay_secs = 1;
 			break;
 		case 's':
 			services = parse_services(optarg);
 			break;
 		case 'h': /* fallthrough */
 		default:
 			usage();
 		}
 	}
 
 	if (optind >= argc)
 		usage();
 
 	file = argv[optind];
 	srand(time(NULL));
 
 	fprintf(stderr, "%s %s\n", PROG_NAME, PROG_VERSION);
 	if (services & SVC_IA)
 		fprintf(stderr, "  service: Internet Archive\n");
 	if (services & SVC_ARCHIVEPH)
 		fprintf(stderr, "  service: archive.ph\n");
 	if (services & SVC_WIKIWIX)
 		fprintf(stderr, "  service: Wikiwix\n");
 	fprintf(stderr, "  delay: %ds\n", delay_secs);
 	if (daemon_mode)
 		fprintf(stderr, "  mode: daemon\n");
 	fprintf(stderr, "\n");
 
 	svc_init();
 
 	processed = 0;
 	start = time(NULL);
 
 	do {
 		urls = read_urls(file, &count);
 
 		if (count == 0) {
 			if (daemon_mode) {
 				if (verbose)
 					fprintf(stderr,
 					    "queue empty, waiting..."
 					    "\n");
 				sleep(delay_secs);
 				free(urls);
 				continue;
 			}
 			fprintf(stderr, "no URLs to process\n");
 			free(urls);
 			break;
 		}
 
 		fprintf(stderr, "loaded %d URL(s)\n", count);
 
 		while (count_remaining(urls, count) > 0) {
 			if (random_order)
 				idx = pick_random(urls, count);
 			else
 				idx = pick_next(urls, count);
 
 			if (idx < 0)
 				break;
 
 			fprintf(stderr, "[%d] %s\n",
 			    processed + 1, urls[idx]);
 
 			ok = svc_submit(urls[idx], services,
 			    verbose);
 			fprintf(stderr, "  %d service(s) OK\n", ok);
 
 			remove_url(urls, idx);
 			processed++;
 
 			/* Write back remaining URLs */
 			if (daemon_mode)
 				write_urls(file, urls, count);
 
 			/* Rate limit */
 			if (count_remaining(urls, count) > 0 ||
 			    daemon_mode)
 				sleep(delay_secs);
 
 			/* In non-daemon mode, process all URLs */
 			if (!daemon_mode)
 				continue;
 
 			/* In daemon mode, re-read file each cycle
 			 * in case new URLs were appended */
 			break;
 		}
 
 		/* Free URL array */
 		{
 			int i;
 
 			for (i = 0; i < count; i++)
 				free(urls[i]);
 			free(urls);
 		}
 
 	} while (daemon_mode);
 
 	fprintf(stderr,
 	    "\nDone: %d URLs processed in %lds\n",
 	    processed, (long)(time(NULL) - start));
 
 	svc_cleanup();
 	return 0;
 }