dyndep: add initial dyndep support

Author: Duncaen

Makefile

@@ -6,6 +6,7 @@ ALL_CFLAGS=$(CFLAGS) -std=c99 -Wall -Wextra -Wpedantic
 OBJ=\
 	build.o\
 	deps.o\
+	dyndep.o\
 	env.o\
 	graph.o\
 	htab.o\

build.c

@@ -13,6 +13,7 @@
 #include <unistd.h>
 #include "build.h"
 #include "deps.h"
+#include "dyndep.h"
 #include "env.h"
 #include "graph.h"
 #include "log.h"
@@ -241,6 +242,8 @@ buildadd(struct node *n)
 			++ntotal;
 	}
 	e->flags &= ~FLAG_CYCLE;
+	if (e->dyndep)
+		dyndepload(e->dyndep, false);
 }
 
 static size_t
@@ -406,6 +409,10 @@ nodedone(struct node *n, bool prune)
 	/* mark node clean for computedirty of edges with dyndeps */
 	n->dirty = false;
 
+	/* if this node is a dyndep it can be loaded now */
+	if (n->dyndep)
+		dyndepload(n->dyndep, false);
+
 	for (i = 0; i < n->nuse; ++i) {
 		e = n->use[i];
 		/* skip edges not used in this build */

build.ninja

@@ -10,6 +10,7 @@ rule link
 
 build build.o: cc build.c
 build deps.o: cc deps.c
+build dyndep.o: cc dyndep.c
 build env.o: cc env.c
 build graph.o: cc graph.c
 build htab.o: cc htab.c
@@ -20,6 +21,6 @@ build scan.o: cc scan.c
 build tool.o: cc tool.c
 build tree.o: cc tree.c
 build util.o: cc util.c
-build samu: link build.o deps.o env.o graph.o htab.o log.o parse.o samu.o scan.o tool.o tree.o util.o
+build samu: link build.o deps.o dyndep.o env.o graph.o htab.o log.o parse.o samu.o scan.o tool.o tree.o util.o
 
 default samu

dyndep.c

@@ -0,0 +1,272 @@
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include "build.h"
+#include "dyndep.h"
+#include "env.h"
+#include "graph.h"
+#include "htab.h"
+#include "scan.h"
+#include "util.h"
+
+struct nodearray {
+	struct node **node;
+	size_t len;
+};
+
+const char *dyndepversion = "1.0";
+
+static struct dyndep *alldyndeps;
+
+void
+dyndepinit(void)
+{
+	struct dyndep *d;
+
+	/* delete old dyndeps in case we rebuilt the manifest */
+	while (alldyndeps) {
+		d = alldyndeps;
+		alldyndeps = d->allnext;
+		free(d->use);
+		free(d);
+	}
+}
+
+struct dyndep *
+mkdyndep(struct node *n)
+{
+	struct dyndep *d;
+
+	if (n->dyndep)
+		return n->dyndep;
+
+	d = xmalloc(sizeof(*d));
+	d->node = n;
+	d->use = NULL;
+	d->nuse = 0;
+	d->update = NULL;
+	d->nupdate = 0;
+	d->done = false;
+	d->allnext = alldyndeps;
+	alldyndeps = d;
+	n->dyndep = d;
+
+	return d;
+}
+
+void
+dyndepuse(struct dyndep *d, struct edge *e)
+{
+	e->dyndep = d;
+
+	/* allocate in powers of two */
+	if (!(d->nuse & (d->nuse - 1)))
+		d->use = xreallocarray(d->use, d->nuse ? d->nuse * 2 : 1, sizeof(e));
+	d->use[d->nuse++] = e;
+}
+
+static void
+dyndepupdate(struct dyndep *d, struct node *n)
+{
+	/* allocate in powers of two */
+	if (!(d->nupdate & (d->nupdate - 1)))
+		d->update = xreallocarray(d->update, d->nupdate ? d->nupdate * 2 : 1, sizeof(n));
+	d->update[d->nupdate++] = n;
+}
+
+static void
+parselet(struct scanner *s, struct evalstring **val)
+{
+	scanchar(s, '=');
+	*val = scanstring(s, false);
+	scannewline(s);
+}
+
+static void
+checkversion(const char *ver)
+{
+	if (strcmp(ver, dyndepversion) > 0)
+		fatal("ninja_dyndep_version %s is newer than %s", ver, dyndepversion);
+}
+
+static void
+dyndepparseedge(struct scanner *s, struct environment *env)
+{
+	struct dyndep *d;
+	struct edge *e;
+	struct node *n;
+	static struct nodearray nodes;
+	static size_t nodescap;
+	struct evalstring *str;
+	char *name;
+	struct string *val;
+
+	str = scanstring(s, true);
+	if (!str)
+		scanerror(s, "expected explicit output");
+
+	val = enveval(env, str);
+	n = nodeget(val->s, val->n);
+	if (!n)
+		fatal("unknown target '%s'", val->s);
+	e = n->gen;
+	if (!e)
+		fatal("no action to generate '%s' in dyndep", val->s);
+	free(val);
+	d = e->dyndep;
+	if (!d)
+		fatal("target '%s' does not mention this dyndep", n->path->s);
+
+	dyndepupdate(d, n);
+
+	nodes.len = 0;
+	if (scanpipe(s, 1)) {
+		for (; (str = scanstring(s, true)); ) {
+			val = enveval(e->env, str);
+			canonpath(val);
+			if (nodes.len == nodescap) {
+				nodescap = nodes.node ? nodescap * 2 : 32;
+				nodes.node = xreallocarray(nodes.node, nodescap, sizeof(nodes.node[0]));
+			}
+			nodes.node[nodes.len++] = mknode(val);
+		}
+	}
+	if (nodes.len)
+		edgeadddynouts(e, nodes.node, nodes.len);
+
+	scanchar(s, ':');
+	name = scanname(s);
+	if (strcmp(name, "dyndep") != 0)
+		fatal("unexpected rule '%s' in dyndep", name);
+	free(name);
+
+	nodes.len = 0;
+	if (scanpipe(s, 1)) {
+		for (; (str = scanstring(s, true)); ) {
+			val = enveval(e->env, str);
+			canonpath(val);
+			if (nodes.len == nodescap) {
+				nodescap = nodes.node ? nodescap * 2 : 32;
+				nodes.node = xreallocarray(nodes.node, nodescap, sizeof(nodes.node[0]));
+			}
+			nodes.node[nodes.len++] = mknode(val);
+		}
+	}
+	if (nodes.len)
+		edgeadddyndeps(e, nodes.node, nodes.len);
+
+	scannewline(s);
+	while (scanindent(s)) {
+		name = scanname(s);
+		if (strcmp(name, "restat") != 0)
+			fatal("unexpected variable '%s' in dyndep", name);
+		parselet(s, &str);
+		envaddvar(e->env, "restat", enveval(env, str));
+	}
+
+	e->flags |= FLAG_DYNDEP;
+}
+
+static void
+dyndepparse(const char *name)
+{
+	struct scanner s;
+	struct environment *env;
+	struct evalstring *str;
+	struct string *val;
+	bool version = false;
+	char *var;
+
+	scaninit(&s, name);
+
+	env = mkenv(NULL);
+	for (;;) {
+		switch (scankeyword(&s, &var)) {
+		case BUILD:
+			if (!version)
+				goto version;
+			dyndepparseedge(&s, env);
+			break;
+		case VARIABLE:
+			if (version)
+				scanerror(&s, "unexpected variable '%s'", var);
+			parselet(&s, &str);
+			val = enveval(env, str);
+			if (strcmp(var, "ninja_dyndep_version") == 0)
+				checkversion(val->s);
+			envaddvar(env, var, val);
+			version = true;
+			break;
+		case EOF:
+			if (!version)
+				goto version;
+			scanclose(&s);
+			return;
+		default:
+			scanerror(&s, "unexpected keyword");
+		}
+	}
+version:
+	scanerror(&s, "expected 'ninja_dyndep_version'");
+}
+
+bool
+dyndepload(struct dyndep *d, bool prune)
+{
+	const char *name;
+	struct edge *e;
+	struct node *n;
+	size_t i;
+
+	if (d->done)
+		return true;
+
+	name = d->node->path->s;
+
+	if (!d->node->gen)
+		fatal("dyndep not created by any action: '%s'", name);
+
+	if (!prune) {
+		/* only load dyndep file if its clean and nothing is blocking it */
+		if (d->node->gen->flags & FLAG_DIRTY || d->node->gen->nblock > 0)
+			return false;
+	} else {
+		nodestat(d->node);
+		if (d->node->mtime == MTIME_MISSING)
+			return false;
+	}
+
+	if (buildopts.explain)
+		warn("loading dyndep file: '%s'", name);
+
+	dyndepparse(name);
+
+	d->done = true;
+
+	if (prune)
+		return true;
+
+	/* verify that all nodes with this dyndep are loaded */
+	for (i = 0; i < d->nuse; i++) {
+		e = d->use[i];
+		if (!(e->flags & FLAG_DYNDEP))
+			fatal("target '%s' not mentioned in dyndep file '%s'",
+			    e->out[0]->path->s, d->node->path->s);
+	}
+	/* (re)-schedule the nodes updated by this dyndep */
+	for (i = 0; i < d->nupdate; i++) {
+		n = d->update[i];
+		/* only update the node if it was previously added to the build */
+		if (n->gen->flags & FLAG_WORK)
+			buildupdate(n);
+		else
+			buildadd(n);
+		n->gen->flags &= ~FLAG_DYNDEP;
+	}
+
+	return true;
+}

dyndep.h

@@ -0,0 +1,29 @@
+struct node;
+
+struct dyndep {
+	/* the node building the dyndep file */
+	struct node *node;
+
+	/* edges using this dyndep */
+	struct edge **use;
+	size_t nuse;
+
+	/* nodes this dyndep updated */
+	struct node **update;
+	size_t nupdate;
+
+	/* is this dyndep file already loaded */
+	_Bool done;
+
+	/* used for alldyndeps linked list */
+	struct dyndep *allnext;
+};
+
+void dyndepinit(void);
+
+/* create a new dyndep or return existing dyndep */
+struct dyndep *mkdyndep(struct node *);
+/* load the dyndep */
+bool dyndepload(struct dyndep *, bool);
+/* record the usage of a dyndep by an edge */
+void dyndepuse(struct dyndep *, struct edge *);