sparser

sparser.c (6009B)

---

 /* See LICENSE file for copyright and license details.
  *
  * sparser - Simple Parser
  *
  * Extracts external URLs from text files.
  * Supports HTML, Markdown, MDX, plain text.
  * Can recursively walk directories.
  */
 
 #include <dirent.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include "config.h"
 #include "extract.h"
 #include "util.h"
 
 /* Hash table for URL deduplication */
 #define DEDUP_SIZE 65521
 
 typedef struct DeNode {
 	char *url;
 	struct DeNode *next;
 } DeNode;
 
 /* Global options */
 static int verbose = 0;
 static int recurse = 0;
 static int dedup = 0;
 static DeNode *dedup_table[DEDUP_SIZE];
 
 static void
 usage(void)
 {
 	fprintf(stderr,
 	    "usage: sparser [-vuR] [path | -]\n"
 	    "\n"
 	    "  -v    verbose (print filenames to stderr)\n"
 	    "  -u    deduplicate URLs\n"
 	    "  -R    recursive directory scan\n"
 	    "\n"
 	    "  path  file or directory to scan\n"
 	    "  -     read from stdin\n");
 	exit(1);
 }
 
 /* FNV-1a hash */
 static unsigned long
 fnv1a(const char *s)
 {
 	unsigned long h = 2166136261UL;
 
 	for (; *s; s++) {
 		h ^= (unsigned char)*s;
 		h *= 16777619UL;
 	}
 	return h;
 }
 
 static int
 dedup_seen(const char *url)
 {
 	unsigned long h;
 	DeNode *n;
 
 	h = fnv1a(url) % DEDUP_SIZE;
 	for (n = dedup_table[h]; n; n = n->next) {
 		if (strcmp(n->url, url) == 0)
 			return 1;
 	}
 	return 0;
 }
 
 static void
 dedup_add(const char *url)
 {
 	unsigned long h;
 	DeNode *n;
 
 	h = fnv1a(url) % DEDUP_SIZE;
 	n = xmalloc(sizeof(DeNode));
 	n->url = xstrdup(url);
 	n->next = dedup_table[h];
 	dedup_table[h] = n;
 }
 
 static void
 dedup_free(void)
 {
 	size_t i;
 	DeNode *n, *next;
 
 	for (i = 0; i < DEDUP_SIZE; i++) {
 		for (n = dedup_table[i]; n; n = next) {
 			next = n->next;
 			free(n->url);
 			free(n);
 		}
 	}
 }
 
 /* Callback for each extracted URL */
 static void
 url_found(const char *url, void *ctx)
 {
 	(void)ctx;
 
 	if (dedup) {
 		if (dedup_seen(url))
 			return;
 		dedup_add(url);
 	}
 
 	puts(url);
 }
 
 /* Check if a filename has a text-like extension */
 static int
 is_text_ext(const char *name)
 {
 	/* Common text extensions we want to process */
 	static const char *exts[] = {
 		".html", ".htm", ".xhtml",
 		".md", ".mdx", ".markdown",
 		".txt", ".text", ".rst",
 		".xml", ".rss", ".atom",
 		".json", ".yaml", ".yml",
 		".css", ".js", ".jsx", ".ts", ".tsx",
 		".org", ".adoc", ".tex", ".bib",
 		".csv", ".tsv",
 		".cfg", ".conf", ".ini",
 		".sh", ".bash", ".zsh", ".fish",
 		".py", ".rb", ".pl", ".c", ".h",
 		".go", ".rs", ".java", ".hs",
 		NULL
 	};
 	int i;
 
 	for (i = 0; exts[i]; i++) {
 		if (str_ends_with(name, exts[i]))
 			return 1;
 	}
 
 	/* Files without extension (README, LICENSE, etc.) */
 	if (!strchr(name, '.'))
 		return 1;
 
 	return 0;
 }
 
 /* Read entire file into memory. Returns NULL on error. */
 static char *
 read_file(const char *path, size_t *out_len)
 {
 	FILE *fp;
 	char *data;
 	long fsize;
 
 	if (strcmp(path, "-") == 0) {
 		/* Read stdin into buffer */
 		size_t cap, len, n;
 
 		cap = 4096;
 		len = 0;
 		data = xmalloc(cap);
 
 		while ((n = fread(data + len, 1, cap - len,
 		       stdin)) > 0) {
 			len += n;
 			if (len >= cap) {
 				cap *= 2;
 				if (cap > MAX_FILE_SIZE)
 					break;
 				data = xrealloc(data, cap);
 			}
 		}
 
 		data[len] = '\0';
 		*out_len = len;
 		return data;
 	}
 
 	fp = fopen(path, "rb");
 	if (!fp)
 		return NULL;
 
 	if (fseek(fp, 0, SEEK_END) != 0) {
 		fclose(fp);
 		return NULL;
 	}
 
 	fsize = ftell(fp);
 	if (fsize < 0 || fsize > MAX_FILE_SIZE) {
 		fclose(fp);
 		return NULL;
 	}
 
 	rewind(fp);
 
 	data = xmalloc(fsize + 1);
 	if (fread(data, 1, fsize, fp) != (size_t)fsize) {
 		free(data);
 		fclose(fp);
 		return NULL;
 	}
 
 	data[fsize] = '\0';
 	fclose(fp);
 
 	*out_len = fsize;
 	return data;
 }
 
 /* Process a single file */
 static void
 process_file(const char *path)
 {
 	char *data;
 	size_t len;
 
 	if (verbose)
 		fprintf(stderr, "%s\n", path);
 
 	data = read_file(path, &len);
 	if (!data) {
 		if (verbose)
 			warn("cannot read: %s", path);
 		return;
 	}
 
 	if (len == 0) {
 		free(data);
 		return;
 	}
 
 	/* Skip binary files */
 	if (is_binary(data, len)) {
 		if (verbose)
 			fprintf(stderr, "  skip binary: %s\n", path);
 		free(data);
 		return;
 	}
 
 	extract_urls(data, len, url_found, NULL);
 	free(data);
 }
 
 /* Recursively walk a directory */
 static void
 walk_dir(const char *dirpath)
 {
 	DIR *d;
 	struct dirent *ent;
 	struct stat st;
 	char path[4096];
 
 	d = opendir(dirpath);
 	if (!d) {
 		warn("cannot open directory: %s", dirpath);
 		return;
 	}
 
 	while ((ent = readdir(d)) != NULL) {
 		/* Skip hidden files and . / .. */
 		if (ent->d_name[0] == '.')
 			continue;
 
 		/* Skip common non-content directories */
 		if (strcmp(ent->d_name, "node_modules") == 0 ||
 		    strcmp(ent->d_name, ".git") == 0 ||
 		    strcmp(ent->d_name, "__pycache__") == 0 ||
 		    strcmp(ent->d_name, "vendor") == 0 ||
 		    strcmp(ent->d_name, ".next") == 0 ||
 		    strcmp(ent->d_name, "dist") == 0 ||
 		    strcmp(ent->d_name, "build") == 0)
 			continue;
 
 		snprintf(path, sizeof(path), "%s/%s",
 		    dirpath, ent->d_name);
 
 		if (stat(path, &st) != 0)
 			continue;
 
 		if (S_ISDIR(st.st_mode)) {
 			walk_dir(path);
 		} else if (S_ISREG(st.st_mode)) {
 			if (is_text_ext(ent->d_name))
 				process_file(path);
 		}
 	}
 
 	closedir(d);
 }
 
 int
 main(int argc, char *argv[])
 {
 	const char *path;
 	struct stat st;
 	int opt;
 
 	while ((opt = getopt(argc, argv, "vuRh")) != -1) {
 		switch (opt) {
 		case 'v':
 			verbose = 1;
 			break;
 		case 'u':
 			dedup = 1;
 			break;
 		case 'R':
 			recurse = 1;
 			break;
 		case 'h': /* fallthrough */
 		default:
 			usage();
 		}
 	}
 
 	if (optind >= argc)
 		usage();
 
 	path = argv[optind];
 
 	/* Reading from stdin */
 	if (strcmp(path, "-") == 0) {
 		process_file("-");
 		goto done;
 	}
 
 	if (stat(path, &st) != 0)
 		die("cannot stat: %s:", path);
 
 	if (S_ISDIR(st.st_mode)) {
 		if (!recurse)
 			die("use -R to scan directories");
 		walk_dir(path);
 	} else if (S_ISREG(st.st_mode)) {
 		process_file(path);
 	} else {
 		die("not a regular file or directory: %s", path);
 	}
 
 done:
 	if (dedup)
 		dedup_free();
 	return 0;
 }