fw: a firewall, at a card, between two networks, or in front of a namespace

One program with three modes, sharing one rule engine and one ndb rule
language.  Which mode it is depends on what you point it at, and it says
so at startup rather than choosing silently.

  fw -e /net/ether0 rules.ndb     a card: every packet in or out
  fw rules.ndb <side> <side>      two networks: everything crossing
  fw rules.ndb                    one namespace: what programs ask for

The first two filter packets on a wire, using the pkt medium: the stack
gives up its card and gets a synthetic one with fw on the other end, so
nothing reaches it that fw did not pass.  Since the stack no longer has
ethernet, fw answers ARP for the address it stands in for.

The third serves a filtered /net and matches connect and announce before
they reach the kernel, so a refusal comes back out of dial(2) with a
reason.  That is only a boundary if the program also loses #I, which
/dev/drivers does and cannot be undone; fw.rc does it in the right order.

Rules are ndb, matched top to bottom, first match wins, no match denies.
Connections are tracked, so permitting traffic one way permits the
replies.  A rule change drops connections the new rules forbid rather
than letting them finish: a block blocks.  Logging is per rule, to
/sys/log/fw.

Tested on the init-test VM in all three modes: a page fetched through a
real card, a TCP handshake across two networks, request filtering with
the escape routes closed, live rule changes killing established
connections, and one rule file working unchanged at both altitudes.

doc/todo.md has what is not done.  Item 1 is the one that matters: a fw
that dies takes the card's address with it, so the machine loses its
network and fw cannot restart unaided.  That also blocks svc supervision.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 1391 insertions, 0 deletions

diff --git a/fw/src/fw.c b/fw/src/fw.c
new file mode 100644
index 0000000..4f94090
--- /dev/null
+++ b/fw/src/fw.c
@@ -0,0 +1,1391 @@
+/*
+ * fw - a firewall between two networks.
+ *
+ * Each side is an interface bound to the pkt medium, whose data file is
+ * a raw IP wire: reading gives packets that stack wants to transmit,
+ * writing injects packets as if they had arrived.  fw sits between
+ * two of them and copies packets across, or doesn't.
+ *
+ *	LAN --ether-- stack --pkt-- fw --pkt-- stack --ether-- WAN
+ *
+ * Because nothing crosses without passing through here, a dropped
+ * packet is dropped before the far stack allocates anything or answers
+ * anything: this is filtering at the gate, in both directions, which is
+ * what the request-filtering half cannot do.
+ *
+ * pktmedium is unbindonclose, so the ctl fd of each interface has to be
+ * held for as long as the interface should exist.  That is why the
+ * interfaces are created here rather than by a script.
+ */
+#include <u.h>
+#include <libc.h>
+#include <bio.h>
+#include <ndb.h>
+#include <ip.h>
+#include <fcall.h>
+#include <thread.h>
+#include <9p.h>
+#include "rules.h"
+
+typedef struct Wire Wire;
+struct Wire
+{
+	char	*net;			/* mountpoint of this side's stack */
+	char	*addr;
+	char	*mask;
+	char	*side;			/* "outside" or "inside", for messages */
+	int	ifc;
+	int	cfd;			/* held open: unbindonclose */
+	int	dfd;			/* the raw IP wire */
+};
+
+typedef struct Pkt Pkt;
+struct Pkt
+{
+	int	ok;
+	int	proto;
+	uchar	src[IPaddrlen];
+	uchar	dst[IPaddrlen];
+	int	sport;
+	int	dport;
+	int	verb;
+};
+
+static int debug;
+static int stateless;
+static int nallow, ndeny;
+
+/*
+ * Connection tracking.
+ *
+ * Without this a rule set is unwritable: permitting a connection out
+ * would mean separately permitting every reply back in, which in
+ * practice means opening every ephemeral port and calling it strict.
+ * So a packet the rules permit creates a flow, and anything belonging
+ * to that flow - in either direction - passes without consulting the
+ * rules again.  This is what "keep state" means in pf and what every
+ * sysadmin already assumes is happening.
+ *
+ * ICMP has no ports, so its flows are keyed on the addresses alone;
+ * that is enough for ping to work.  ICMP errors *about* a flow are not
+ * yet recognised as related to it.
+ */
+typedef struct Flow Flow;
+struct Flow
+{
+	int	proto;
+	uchar	src[IPaddrlen];
+	uchar	dst[IPaddrlen];
+	int	sport;
+	int	dport;
+	int	verb;			/* direction that created it */
+	long	last;
+	Flow	*next;
+};
+
+enum { Nflow = 257, Maxpkt = 16*1024 };
+
+static Flow	*flowtab[Nflow];
+static Lock	flowlock;
+static int	sweep;
+
+static int
+flowtimeout(int proto)
+{
+	switch(proto){
+	case 6:				/* tcp */
+		return 300;
+	case 17:			/* udp */
+		return 60;
+	default:
+		return 30;
+	}
+}
+
+static uint
+flowhash(int proto, uchar *src, uchar *dst, int sport, int dport)
+{
+	uint h;
+	int i;
+
+	h = proto*31 + sport*7 + dport;
+	for(i = 0; i < IPaddrlen; i++)
+		h = h*33 + src[i]*3 + dst[i];
+	return h % Nflow;
+}
+
+static int
+flowis(Flow *f, int proto, uchar *src, uchar *dst, int sport, int dport)
+{
+	return f->proto == proto && f->sport == sport && f->dport == dport
+		&& ipcmp(f->src, src) == 0 && ipcmp(f->dst, dst) == 0;
+}
+
+static int
+flowlook(int proto, uchar *src, uchar *dst, int sport, int dport, long now)
+{
+	Flow *f;
+	uint h;
+
+	h = flowhash(proto, src, dst, sport, dport);
+	for(f = flowtab[h]; f != nil; f = f->next)
+		if(flowis(f, proto, src, dst, sport, dport)){
+			f->last = now;
+			return 1;
+		}
+	return 0;
+}
+
+/* does this packet belong to a flow we already permitted, either way round? */
+static int
+flowseen(Pkt *p)
+{
+	long now;
+	int r;
+
+	if(stateless)
+		return 0;
+	now = time(0);
+	lock(&flowlock);
+	r = flowlook(p->proto, p->src, p->dst, p->sport, p->dport, now)
+	 || flowlook(p->proto, p->dst, p->src, p->dport, p->sport, now);
+	unlock(&flowlock);
+	return r;
+}
+
+static void
+reapbucket(int i, long now)
+{
+	Flow *f, **pp;
+
+	for(pp = &flowtab[i]; (f = *pp) != nil; ){
+		if(now - f->last > flowtimeout(f->proto)){
+			*pp = f->next;
+			free(f);
+		}else
+			pp = &f->next;
+	}
+}
+
+/*
+ * Re-check every live flow against the current rules and drop the ones
+ * the rules no longer permit.  Called whenever the rule set changes.
+ *
+ * pf and iptables do not do this: there, blocking a port stops new
+ * connections and leaves established ones running until they time out.
+ * That is a wart everybody has to learn.  A block should block, so a
+ * rule change kills the traffic it now forbids.
+ */
+int
+revalidate(void)
+{
+	Flow *f, **pp;
+	uchar *peer;
+	int i, port, lport, n;
+
+	n = 0;
+	lock(&flowlock);
+	for(i = 0; i < Nflow; i++)
+		for(pp = &flowtab[i]; (f = *pp) != nil; ){
+			if(f->verb == Vout){
+				peer = f->dst;
+				port = f->dport;
+				lport = f->sport;
+			}else{
+				peer = f->src;
+				port = f->sport;
+				lport = f->dport;
+			}
+			if(matchrule(f->verb, protonum2name(f->proto), peer, 0, port, lport, nil) != nil){
+				*pp = f->next;
+				free(f);
+				n++;
+			}else
+				pp = &f->next;
+		}
+	unlock(&flowlock);
+	return n;
+}
+
+static void
+flowadd(Pkt *p)
+{
+	Flow *f;
+	uint h;
+	long now;
+
+	if(stateless)
+		return;
+	now = time(0);
+	h = flowhash(p->proto, p->src, p->dst, p->sport, p->dport);
+	lock(&flowlock);
+	reapbucket(h, now);
+	/* and sweep one other bucket, so idle flows do not accumulate */
+	sweep = (sweep + 1) % Nflow;
+	reapbucket(sweep, now);
+
+	f = emalloc(sizeof *f);
+	f->proto = p->proto;
+	ipmove(f->src, p->src);
+	ipmove(f->dst, p->dst);
+	f->sport = p->sport;
+	f->dport = p->dport;
+	f->verb = p->verb;
+	f->last = now;
+	f->next = flowtab[h];
+	flowtab[h] = f;
+	unlock(&flowlock);
+}
+
+void	servenet(char*, char*, char*);
+int	etheropen(char*, uchar*);
+void	ethersetaddr(uchar*, uchar*, int);
+int	etherisarp(uchar*, int);
+int	etherisip(uchar*, int);
+int	etherwriteip(uchar*, int, uchar*);
+
+enum { Ehdrlen = 14 };
+
+static int efd = -1;
+int etherdebug;
+static uchar ethermask[IPaddrlen];
+
+static void
+usage(void)
+{
+	fprint(2, "usage: fw [-dSW] [-m ctl] [-s srv] [-n net]\n");
+	fprint(2, "          [-e ether -a addr/mask [-g gateway]]\n");
+	fprint(2, "          rules.ndb [outside inside]\n");
+	fprint(2, "  -e:        filter packets on a card, for this machine\n");
+	fprint(2, "  two sides: filter packets between them (a gateway)\n");
+	fprint(2, "  neither:   filter requests on /net (one namespace)\n");
+	fprint(2, "  a side is net!addr!mask, eg /net.alt!10.9.9.1!255.255.255.0\n");
+	exits("usage");
+}
+
+static Wire*
+parseside(char *spec, char *side)
+{
+	char *f[4];
+	Wire *w;
+
+	w = emalloc(sizeof *w);
+	w->side = side;
+	if(getfields(estrdup(spec), f, nelem(f), 0, "!") != 3)
+		sysfatal("%s: want net!addr!mask", spec);
+	w->net = f[0];
+	w->addr = f[1];
+	w->mask = f[2];
+	return w;
+}
+
+static void
+wireup(Wire *w)
+{
+	char path[128], buf[64];
+	int n;
+
+	snprint(path, sizeof path, "%s/ipifc/clone", w->net);
+	if((w->cfd = open(path, ORDWR)) < 0)
+		sysfatal("open %s: %r", path);
+	if((n = read(w->cfd, buf, sizeof buf - 1)) <= 0)
+		sysfatal("read %s: %r", path);
+	buf[n] = '\0';
+	w->ifc = atoi(buf);
+
+	if(fprint(w->cfd, "bind pkt") < 0)
+		sysfatal("%s: bind pkt: %r", w->net);
+	if(fprint(w->cfd, "add %s %s", w->addr, w->mask) < 0)
+		sysfatal("%s: add %s %s: %r", w->net, w->addr, w->mask);
+
+	snprint(path, sizeof path, "%s/ipifc/%d/data", w->net, w->ifc);
+	if((w->dfd = open(path, ORDWR)) < 0)
+		sysfatal("open %s: %r", path);
+
+	fprint(2, "fw: %s %s/ipifc/%d addr %s %s\n",
+		w->side, w->net, w->ifc, w->addr, w->mask);
+}
+
+/*
+ * Enough of an IP header to make a decision.  IPv6 extension headers
+ * are not walked: a packet carrying them is reported as its next-header
+ * protocol and matched on addresses only, which is conservative given
+ * default deny but is a real gap worth closing.
+ */
+static void
+parsepkt(uchar *b, int n, Pkt *p)
+{
+	uchar *t;
+	int hl;
+
+	memset(p, 0, sizeof *p);
+	p->sport = p->dport = -1;
+	if(n < 1)
+		return;
+
+	switch(b[0] >> 4){
+	case 4:
+		if(n < 20)
+			return;
+		hl = (b[0] & 0xF) * 4;
+		if(hl < 20 || n < hl)
+			return;
+		p->proto = b[9];
+		v4tov6(p->src, b + 12);
+		v4tov6(p->dst, b + 16);
+		t = b + hl;
+		n -= hl;
+		break;
+	case 6:
+		if(n < 40)
+			return;
+		p->proto = b[6];
+		ipmove(p->src, b + 8);
+		ipmove(p->dst, b + 24);
+		t = b + 40;
+		n -= 40;
+		break;
+	default:
+		return;
+	}
+
+	if((p->proto == 6 || p->proto == 17) && n >= 4){
+		p->sport = nhgets(t);
+		p->dport = nhgets(t + 2);
+	}
+	p->ok = 1;
+}
+
+/*
+ * One direction.  "verb" says which way this is, and the rule's ip and
+ * port always refer to the peer - the far end - so a rule reads the
+ * same whichever direction it governs.
+ */
+int revalidate(void);
+
+static void
+relay(Wire *from, Wire *to, int verb)
+{
+	uchar *buf;
+	char *e;
+	uchar *peer;
+	Rule *rule;
+	Pkt p;
+	int n, port, lport;
+
+	/*
+	 * On the heap, not the stack: these run as libthread procs with a
+	 * small stack, and pktmedium's maxtu is 4k anyway.
+	 */
+	buf = emalloc(Maxpkt);
+	for(;;){
+		if((n = read(from->dfd, buf, Maxpkt)) <= 0){
+			fprint(2, "fw: %s: read failed: %r\n", from->side);
+			syslog(0, "fw", "%s: read failed: %r", from->side);
+			return;
+		}
+
+		parsepkt(buf, n, &p);
+		if(!p.ok){
+			ndeny++;
+			if(debug)
+				fprint(2, "drop %s: unparseable, %d bytes\n", from->side, n);
+			continue;
+		}
+
+		p.verb = verb;
+		if(verb == Vout){
+			peer = p.dst;
+			port = p.dport;
+			lport = p.sport;
+		}else{
+			peer = p.src;
+			port = p.sport;
+			lport = p.dport;
+		}
+
+		if(flowseen(&p)){
+			nallow++;
+			if(debug)
+				fprint(2, "pass %s %s %I!%d -> %I!%d (state)\n",
+					verb == Vout ? "out" : "in",
+					protonum2name(p.proto),
+					p.src, p.sport, p.dst, p.dport);
+			if(write(to->dfd, buf, n) != n)
+				fprint(2, "fw: write %s wire: %r\n", to->side);
+			continue;
+		}
+
+		e = matchrule(verb, protonum2name(p.proto), peer, 0, port, lport, &rule);
+		if(e != nil){
+			if(rule != nil && rule->log)
+				syslog(0, "fw", "drop %s %s %I!%d -> %I!%d: %s",
+					verb == Vout ? "out" : "in",
+					protonum2name(p.proto),
+					p.src, p.sport, p.dst, p.dport, e);
+			ndeny++;
+			if(debug)
+				fprint(2, "drop %s %s %I!%d -> %I!%d: %s\n",
+					verb == Vout ? "out" : "in",
+					protonum2name(p.proto),
+					p.src, p.sport, p.dst, p.dport, e);
+			continue;
+		}
+
+		nallow++;
+		if(rule != nil && rule->log)
+			syslog(0, "fw", "pass %s %s %I!%d -> %I!%d",
+				verb == Vout ? "out" : "in",
+				protonum2name(p.proto),
+				p.src, p.sport, p.dst, p.dport);
+		flowadd(&p);
+		if(debug)
+			fprint(2, "pass %s %s %I!%d -> %I!%d (new)\n",
+				verb == Vout ? "out" : "in",
+				protonum2name(p.proto),
+				p.src, p.sport, p.dst, p.dport);
+		if(write(to->dfd, buf, n) != n)
+			fprint(2, "fw: write %s wire: %r\n", to->side);
+	}
+}
+
+enum
+{
+	Qctl,
+	Qrules,
+	Qflows,
+	Qstats,
+};
+
+static char *rulepath;		/* the file we were started with */
+
+static char *ctltext =
+	"prepend <rule>   insert a rule at the top, where it wins\n"
+	"append <rule>    add a rule at the bottom\n"
+	"delete <n>       remove rule n\n"
+	"flush            remove every rule; then nothing is permitted\n"
+	"reload [file]    re-read the rule file; no argument means the\n"
+	"                 one it was started with\n"
+	"save [file]      write the current rules back out\n"
+	"flushflows       forget tracked connections\n"
+	"\n"
+	"a rule is ndb, eg: deny=out proto=tcp port=80\n"
+	"changes take effect at once, and live connections that the\n"
+	"new rules forbid are dropped rather than left running.\n";
+
+enum { Rulebuf = 64*1024 };
+
+/*
+ * ctl edits are done by writing the rule set back out as ndb, editing
+ * the text, and parsing the whole thing again.  It is not the quickest
+ * way, but rule changes are rare, and it means a rule typed at ctl and
+ * a rule in the file go through exactly one parser - they cannot come
+ * to disagree about what a rule means.
+ */
+static Rule*
+rulesfromtext(char *text, char **err)
+{
+	char tmp[64];
+	Rule *r;
+	long n;
+	int fd;
+
+	snprint(tmp, sizeof tmp, "/tmp/fw.%d", getpid());
+	if((fd = create(tmp, OWRITE, 0600)) < 0){
+		*err = "cannot create a temporary file";
+		return nil;
+	}
+	n = strlen(text);
+	if(write(fd, text, n) != n){
+		close(fd);
+		remove(tmp);
+		*err = "cannot write a temporary file";
+		return nil;
+	}
+	close(fd);
+	r = parserules(tmp, err);
+	remove(tmp);
+	return r;
+}
+
+static char*
+replacerules(char *text)
+{
+	Rule *new;
+	char *err;
+
+	new = rulesfromtext(text, &err);
+	if(err != nil)
+		return err;
+	installrules(new);
+	return nil;
+}
+
+static char*
+editrules(char *add, int atfront, int delete)
+{
+	char *cur, *all, *err, *p, *nl;
+	int i;
+
+	if((cur = mallocz(Rulebuf, 1)) == nil)
+		return "out of memory";
+	cur[fmtrules(cur, Rulebuf-1)] = '\0';
+
+	if(delete > 0){
+		/* fmtrules writes one line per rule, so line n is rule n */
+		p = cur;
+		for(i = 1; i < delete && p != nil; i++)
+			if((p = strchr(p, '\n')) != nil)
+				p++;
+		if(p == nil || *p == '\0'){
+			free(cur);
+			return "no such rule";
+		}
+		if((nl = strchr(p, '\n')) != nil)
+			memmove(p, nl+1, strlen(nl+1)+1);
+		else
+			*p = '\0';
+		all = strdup(cur);
+	}else if(atfront)
+		all = smprint("%s\n%s", add, cur);
+	else
+		all = smprint("%s%s\n", cur, add);
+	free(cur);
+	if(all == nil)
+		return "out of memory";
+
+	err = replacerules(all);
+	free(all);
+	return err;
+}
+
+static char*
+reloadrules(char *file)
+{
+	Rule *new;
+	char *err;
+
+	if(file == nil)
+		return "no rule file to reload";
+	if((new = parserules(file, &err)) == nil && err != nil)
+		return err;
+	installrules(new);
+	return nil;
+}
+
+/* the other direction: keep what was typed at ctl across a restart */
+static char*
+saverules(char *file)
+{
+	char *buf;
+	long n;
+	int fd;
+
+	if(file == nil)
+		return "no rule file to save to";
+	if((buf = mallocz(Rulebuf, 1)) == nil)
+		return "out of memory";
+	n = fmtrules(buf, Rulebuf-1);
+	if((fd = create(file, OWRITE, 0644)) < 0){
+		free(buf);
+		return "cannot create the rule file";
+	}
+	if(write(fd, buf, n) != n){
+		close(fd);
+		free(buf);
+		return "cannot write the rule file";
+	}
+	close(fd);
+	free(buf);
+	return nil;
+}
+
+static char*
+flushflows(void)
+{
+	Flow *f, *next;
+	int i;
+
+	lock(&flowlock);
+	for(i = 0; i < Nflow; i++){
+		for(f = flowtab[i]; f != nil; f = next){
+			next = f->next;
+			free(f);
+		}
+		flowtab[i] = nil;
+	}
+	unlock(&flowlock);
+	return nil;
+}
+
+static char*
+flowtext(void)
+{
+	char *buf, *p, *e;
+	Flow *f;
+	long now;
+	int i;
+
+	if((buf = mallocz(Rulebuf, 1)) == nil)
+		return nil;
+	p = buf;
+	e = buf + Rulebuf;
+	now = time(0);
+	lock(&flowlock);
+	for(i = 0; i < Nflow; i++)
+		for(f = flowtab[i]; f != nil; f = f->next)
+			p = seprint(p, e, "%s %s %I!%d -> %I!%d idle %ld\n",
+				f->verb == Vout ? "out" : "in",
+				protonum2name(f->proto),
+				f->src, f->sport, f->dst, f->dport,
+				now - f->last);
+	unlock(&flowlock);
+	USED(p);
+	return buf;
+}
+
+/*
+ * A rule set written to "rules" may not arrive in one 9P message, and
+ * treating each write as a complete set would install fragments.  So
+ * writes accumulate on the fid and the set is replaced when the fid is
+ * clunked, which is the transaction boundary the protocol already has.
+ *
+ * A clunk cannot fail, so a set that will not parse is reported to the
+ * log and the running rules are kept.  Use ctl, whose writes do report
+ * errors, when you want to be told.
+ */
+typedef struct Wbuf Wbuf;
+struct Wbuf
+{
+	char	*b;
+	long	n;
+	long	max;
+};
+
+static void
+wbufput(Wbuf *w, char *p, long n, vlong off)
+{
+	long need;
+
+	need = off + n;
+	if(need + 1 > w->max){
+		w->max = need + 8192;
+		if((w->b = realloc(w->b, w->max)) == nil)
+			sysfatal("out of memory");
+	}
+	if(off > w->n)
+		memset(w->b + w->n, 0, off - w->n);
+	memmove(w->b + off, p, n);
+	if(need > w->n)
+		w->n = need;
+}
+
+static void
+fsdestroyfid(Fid *fid)
+{
+	Wbuf *w;
+	char *err;
+
+	if((w = fid->aux) == nil)
+		return;
+	fid->aux = nil;
+	if(w->n > 0){
+		w->b[w->n] = '\0';
+		if((err = replacerules(w->b)) != nil)
+			syslog(0, "fw", "rules rejected, keeping the old ones: %s", err);
+		else
+			syslog(0, "fw", "rule set replaced");
+	}
+	free(w->b);
+	free(w);
+}
+
+static void
+fsread(Req *r)
+{
+	char buf[256], *s;
+	long n;
+
+	switch((int)(uintptr)r->fid->file->aux){
+	case Qctl:
+		readstr(r, ctltext);
+		break;
+	case Qrules:
+		if((s = mallocz(Rulebuf, 1)) == nil){
+			respond(r, "out of memory");
+			return;
+		}
+		n = fmtrules(s, Rulebuf-1);
+		s[n] = '\0';
+		readstr(r, s);
+		free(s);
+		break;
+	case Qflows:
+		if((s = flowtext()) == nil){
+			respond(r, "out of memory");
+			return;
+		}
+		readstr(r, s);
+		free(s);
+		break;
+	case Qstats:
+		if((s = mallocz(Rulebuf, 1)) == nil){
+			respond(r, "out of memory");
+			return;
+		}
+		n = snprint(s, Rulebuf-1, "passed %d\ndropped %d\n\n", nallow, ndeny);
+		fmthits(s+n, Rulebuf-1-n);
+		readstr(r, s);
+		free(s);
+		break;
+	default:
+		respond(r, "not a readable file");
+		return;
+	}
+	respond(r, nil);
+}
+
+static void
+fswrite(Req *r)
+{
+	char *buf, *err, *arg, *p;
+	int type;
+	long n;
+
+	type = (int)(uintptr)r->fid->file->aux;
+	if(type != Qctl && type != Qrules){
+		respond(r, "not a writable file");
+		return;
+	}
+	if((buf = mallocz(r->ifcall.count+1, 1)) == nil){
+		respond(r, "out of memory");
+		return;
+	}
+	memmove(buf, r->ifcall.data, r->ifcall.count);
+	buf[r->ifcall.count] = '\0';
+
+	if(type == Qrules){
+		Wbuf *w;
+
+		if((w = r->fid->aux) == nil){
+			w = emalloc(sizeof *w);
+			r->fid->aux = w;
+		}
+		wbufput(w, r->ifcall.data, r->ifcall.count, r->ifcall.offset);
+		err = nil;
+		goto Done;
+	}
+
+	/*
+	 * A write may or may not carry a trailing newline, and the
+	 * argument of prepend is a whole ndb rule with spaces and tabs
+	 * in it, so split off the first word by hand rather than
+	 * tokenizing the lot.
+	 */
+	for(n = strlen(buf); n > 0; n--){
+		if(buf[n-1] != '\n' && buf[n-1] != '\r'
+		&& buf[n-1] != ' ' && buf[n-1] != '\t')
+			break;
+		buf[n-1] = '\0';
+	}
+	if(*buf == '\0'){
+		err = "no command";
+		goto Done;
+	}
+	arg = nil;
+	for(p = buf; *p != '\0' && *p != ' ' && *p != '\t'; p++)
+		;
+	if(*p != '\0'){
+		*p++ = '\0';
+		while(*p == ' ' || *p == '\t')
+			p++;
+		if(*p != '\0')
+			arg = p;
+	}
+
+	if(strcmp(buf, "flush") == 0)
+		err = replacerules("");
+	else if(strcmp(buf, "flushflows") == 0)
+		err = flushflows();
+	else if(strcmp(buf, "reload") == 0)
+		err = reloadrules(arg != nil ? arg : rulepath);
+	else if(strcmp(buf, "save") == 0)
+		err = saverules(arg != nil ? arg : rulepath);
+	else if(arg == nil)
+		err = "usage: prepend, append or delete <rule>, "
+			"flush, reload, save or flushflows";
+	else if(strcmp(buf, "prepend") == 0)
+		err = editrules(arg, 1, 0);
+	else if(strcmp(buf, "append") == 0)
+		err = editrules(arg, 0, 0);
+	else if(strcmp(buf, "delete") == 0)
+		err = editrules(nil, 0, atoi(arg));
+	else
+		err = "unknown command; read ctl for the list";
+
+Done:
+	free(buf);
+	if(err != nil){
+		respond(r, err);
+		return;
+	}
+	r->ofcall.count = r->ifcall.count;
+	respond(r, nil);
+}
+
+static Srv fs =
+{
+.read=		fsread,
+.write=		fswrite,
+.destroyfid=	fsdestroyfid,
+};
+
+/* a rule change must not leave traffic running that the rules now forbid */
+static void
+rulesdidchange(void)
+{
+	int n;
+
+	if((n = revalidate()) > 0)
+		syslog(0, "fw", "rules changed; dropped %d flow%s they forbid",
+			n, n == 1 ? "" : "s");
+}
+
+static void
+servectl(char *mtpt, char *srvname)
+{
+	File *root;
+
+	fs.tree = alloctree("fw", "fw", DMDIR|0555, nil);
+	root = fs.tree->root;
+	closefile(createfile(root, "ctl", "fw", 0666, (void*)Qctl));
+	closefile(createfile(root, "rules", "fw", 0666, (void*)Qrules));
+	closefile(createfile(root, "flows", "fw", 0444, (void*)Qflows));
+	closefile(createfile(root, "stats", "fw", 0444, (void*)Qstats));
+	threadpostmountsrv(&fs, srvname, mtpt, MREPL);
+}
+
+static void
+relayproc(void *a)
+{
+	Wire **w;
+
+	w = a;
+	relay(w[0], w[1], (int)(uintptr)w[2]);
+}
+
+/*
+ * Decide on one packet.  Shared by every relay: the wire it came from
+ * only changes which way "in" and "out" mean.
+ */
+static int
+permitted(uchar *buf, int n, int verb, Pkt *p)
+{
+	Rule *rule;
+	uchar *peer;
+	char *e;
+	int port, lport;
+
+	parsepkt(buf, n, p);
+	if(!p->ok){
+		ndeny++;
+		if(debug)
+			fprint(2, "drop %s: unparseable, %d bytes\n",
+				verb == Vout ? "out" : "in", n);
+		return 0;
+	}
+	p->verb = verb;
+	if(verb == Vout){
+		peer = p->dst;
+		port = p->dport;
+		lport = p->sport;
+	}else{
+		peer = p->src;
+		port = p->sport;
+		lport = p->dport;
+	}
+	if(flowseen(p)){
+		nallow++;
+		return 1;
+	}
+	e = matchrule(verb, protonum2name(p->proto), peer, 0, port, lport, &rule);
+	if(e != nil){
+		ndeny++;
+		if(rule != nil && rule->log)
+			syslog(0, "fw", "drop %s %s %I!%d -> %I!%d: %s",
+				verb == Vout ? "out" : "in", protonum2name(p->proto),
+				p->src, p->sport, p->dst, p->dport, e);
+		if(debug)
+			fprint(2, "drop %s %s %I!%d -> %I!%d: %s\n",
+				verb == Vout ? "out" : "in", protonum2name(p->proto),
+				p->src, p->sport, p->dst, p->dport, e);
+		return 0;
+	}
+	nallow++;
+	if(rule != nil && rule->log)
+		syslog(0, "fw", "pass %s %s %I!%d -> %I!%d",
+			verb == Vout ? "out" : "in", protonum2name(p->proto),
+			p->src, p->sport, p->dst, p->dport);
+	if(debug)
+		fprint(2, "pass %s %s %I!%d -> %I!%d\n",
+			verb == Vout ? "out" : "in", protonum2name(p->proto),
+			p->src, p->sport, p->dst, p->dport);
+	flowadd(p);
+	return 1;
+}
+
+/* the wire -> the protected stack */
+static void
+etherin(void *a)
+{
+	uchar *buf;
+	Wire *w;
+	Pkt p;
+	int n;
+
+	w = a;
+	buf = emalloc(Maxpkt);
+	for(;;){
+		if((n = read(efd, buf, Maxpkt)) <= 0){
+			fprint(2, "fw: %s: read the card: %r\n", w->side);
+			syslog(0, "fw", "stopped reading the card: %r");
+			return;
+		}
+		if(debug > 1)
+			fprint(2, "wire: %d bytes type %.4ux\n", n, (buf[12]<<8)|buf[13]);
+		if(etherisarp(buf, n))
+			continue;
+		if(!etherisip(buf, n))
+			continue;
+		if(permitted(buf+Ehdrlen, n-Ehdrlen, Vin, &p))
+			write(w->dfd, buf+Ehdrlen, n-Ehdrlen);
+	}
+}
+
+/* the protected stack -> the wire */
+static void
+etherout(void *a)
+{
+	uchar *buf;
+	Wire *w;
+	Pkt p;
+	int n;
+
+	w = a;
+	buf = emalloc(Maxpkt);
+	for(;;){
+		if((n = read(w->dfd, buf, Maxpkt)) <= 0){
+			fprint(2, "fw: %s: read the stack: %r\n", w->side);
+			syslog(0, "fw", "stopped reading the stack: %r");
+			return;
+		}
+		if(permitted(buf, n, Vout, &p))
+			etherwriteip(buf, n, ethermask);
+	}
+}
+
+/*
+ * The address, mask and gateway the stack is already using.  Asking the
+ * administrator to repeat what ipconfig(8) was told is a way of getting
+ * the two out of step, so read them instead: the interface status names
+ * the address and mask, and the routing table names the gateway.
+ */
+static int
+learnaddr(char *net, char *dev, char *addr, int naddr, char *gw, int ngw)
+{
+	char path[128], buf[1024], *lines[8], *f[8], *p;
+	int i, fd, n, nl, nf, found;
+
+	found = 0;
+	for(i = 0; i < 16 && !found; i++){
+		snprint(path, sizeof path, "%s/ipifc/%d/status", net, i);
+		if((fd = open(path, OREAD)) < 0)
+			continue;
+		n = read(fd, buf, sizeof buf - 1);
+		close(fd);
+		if(n <= 0)
+			continue;
+		buf[n] = '\0';
+		if((p = strstr(buf, dev)) == nil)
+			continue;
+		USED(p);
+		nl = getfields(buf, lines, nelem(lines), 0, "\n");
+		if(nl < 2)
+			continue;
+		/* the address line: address, then the mask as a prefix */
+		nf = tokenize(lines[1], f, nelem(f));
+		if(nf < 2)
+			continue;
+		snprint(addr, naddr, "%s%s", f[0], f[1]);
+		found = 1;
+	}
+	if(!found)
+		return -1;
+
+	*gw = '\0';
+	snprint(path, sizeof path, "%s/iproute", net);
+	if((fd = open(path, OREAD)) >= 0){
+		n = read(fd, buf, sizeof buf - 1);
+		close(fd);
+		if(n > 0){
+			buf[n] = '\0';
+			nl = getfields(buf, lines, nelem(lines), 0, "\n");
+			for(i = 0; i < nl; i++){
+				nf = tokenize(lines[i], f, nelem(f));
+				if(nf >= 3 && strcmp(f[0], "0.0.0.0") == 0
+				&& strcmp(f[1], "/96") == 0){
+					snprint(gw, ngw, "%s", f[2]);
+					break;
+				}
+			}
+		}
+	}
+	return 0;
+}
+
+/*
+ * Undo a previous fw that died.
+ *
+ * An IP stack outlives the program that configured it, so a fw that is
+ * killed leaves its pkt interface behind, holding the address, with
+ * nothing on the other end of it.  The machine has no network until
+ * someone unpicks that by hand, and the next fw to start makes a second
+ * interface with the same address and routes that could go to either.
+ *
+ * So before taking anything, throw away any pkt interface already
+ * carrying the address we are about to use.  Nothing else can have made
+ * it: a live fw would still be holding the card we are about to take.
+ */
+/*
+ * Put the card back.
+ *
+ * Taking a card is destructive: the stack loses it, and the pkt
+ * interface that replaced it is unbindonclose, so when fw stops the
+ * address goes with it and the machine is left with a card bound to
+ * nothing.  Restore it on the way out, so that everything short of an
+ * uncatchable kill leaves the machine as we found it.
+ */
+static char *backdev, *backaddr, *backmask, *backnet, *backgw;
+
+static void
+putback(void)
+{
+	char path[128], buf[64];
+	int cfd, n, ifc;
+
+	if(backdev == nil)
+		return;
+	snprint(path, sizeof path, "%s/ipifc/clone", backnet);
+	if((cfd = open(path, ORDWR)) < 0)
+		return;
+	if((n = read(cfd, buf, sizeof buf - 1)) <= 0){
+		close(cfd);
+		return;
+	}
+	buf[n] = '\0';
+	ifc = atoi(buf);
+	if(fprint(cfd, "bind ether %s", backdev) > 0
+	&& fprint(cfd, "add %s %s", backaddr, backmask) > 0){
+		if(backgw != nil){
+			snprint(path, sizeof path, "%s/iproute", backnet);
+			if((n = open(path, OWRITE)) >= 0){
+				fprint(n, "add 0.0.0.0 0.0.0.0 %s", backgw);
+				close(n);
+			}
+		}
+		fprint(2, "fw: put %s back on %s/ipifc/%d\n", backdev, backnet, ifc);
+	}
+	/* the ctl fd must stay open for the binding to last */
+	backdev = nil;
+}
+
+/* clean up, then let the note do what it was going to do */
+static int
+notehandler(void*, char*)
+{
+	putback();
+	return 0;
+}
+
+static void
+reclaim(char *net, char *addr)
+{
+	char path[128], buf[1024], *lines[8], *f[8];
+	int i, fd, n, nl, nf;
+
+	for(i = 0; i < 16; i++){
+		snprint(path, sizeof path, "%s/ipifc/%d/status", net, i);
+		if((fd = open(path, OREAD)) < 0)
+			continue;
+		n = read(fd, buf, sizeof buf - 1);
+		close(fd);
+		if(n <= 0)
+			continue;
+		buf[n] = '\0';
+		nl = getfields(buf, lines, nelem(lines), 0, "\n");
+		if(nl < 2)
+			continue;
+		nf = tokenize(lines[0], f, nelem(f));
+		if(nf < 2 || strncmp(f[1], "pkt", 3) != 0)
+			continue;
+		nf = tokenize(lines[1], f, nelem(f));
+		if(nf < 1 || strcmp(f[0], addr) != 0)
+			continue;
+
+		snprint(path, sizeof path, "%s/ipifc/%d/ctl", net, i);
+		if((fd = open(path, OWRITE)) < 0)
+			continue;
+		if(fprint(fd, "unbind") > 0){
+			fprint(2, "fw: cleared %s/ipifc/%d, left behind by an earlier fw\n",
+				net, i);
+			syslog(0, "fw", "cleared a stale interface holding %s", addr);
+		}
+		close(fd);
+	}
+}
+
+/*
+ * The card cannot stay attached to the stack we are protecting, or
+ * packets reach it whatever the rules say.  Find whichever interface
+ * has it and take it away.
+ */
+static void
+takecard(char *net, char *dev)
+{
+	char path[128], buf[512], *p;
+	int i, fd, n, found;
+
+	found = 0;
+	for(i = 0; i < 16; i++){
+		snprint(path, sizeof path, "%s/ipifc/%d/status", net, i);
+		if((fd = open(path, OREAD)) < 0)
+			continue;
+		n = read(fd, buf, sizeof buf - 1);
+		close(fd);
+		if(n <= 0)
+			continue;
+		buf[n] = '\0';
+		if((p = strchr(buf, ' ')) == nil)
+			continue;
+		*p = '\0';
+		if(strcmp(buf, "device") != 0 && strstr(buf, dev) == nil)
+			continue;
+		if(strstr(p+1, dev) == nil && strstr(buf, dev) == nil)
+			continue;
+		snprint(path, sizeof path, "%s/ipifc/%d/ctl", net, i);
+		if((fd = open(path, OWRITE)) < 0)
+			continue;
+		if(fprint(fd, "unbind") > 0){
+			fprint(2, "fw: took %s away from %s/ipifc/%d\n", dev, net, i);
+			found = 1;
+		}
+		close(fd);
+	}
+	if(!found)
+		fprint(2, "fw: warning: %s was not bound to %s; "
+			"is something else still using it?\n", dev, net);
+}
+
+void
+threadmain(int argc, char **argv)
+{
+	static Wire *outargs[3], *inargs[3];
+	Wire *out, *in, *prot;
+	char *mtpt, *srvname, *netmtpt, *etherdev, *etheraddr, *ethergw;
+	uchar ip[IPaddrlen], gw[IPaddrlen];
+	int wireonly;
+
+	mtpt = "/mnt/fw";
+	netmtpt = "/net";
+	srvname = nil;
+	etherdev = nil;
+	etheraddr = nil;
+	ethergw = nil;
+	wireonly = 0;
+	ARGBEGIN{
+	case 'd':
+		debug++;
+		break;
+	case 'S':
+		stateless++;
+		break;
+	case 'm':
+		mtpt = EARGF(usage());
+		break;
+	case 's':
+		srvname = EARGF(usage());
+		break;
+	case 'n':
+		netmtpt = EARGF(usage());
+		break;
+	case 'W':
+		wireonly++;
+		break;
+	case 'e':
+		/*
+		 * One card per fw.  Taking two would mean two pkt
+		 * interfaces, two ARP caches and one rule set that could
+		 * not say which card it meant; run one fw per card until
+		 * the rules can name one.
+		 */
+		if(etherdev != nil)
+			sysfatal("one -e at a time; run one fw per card");
+		etherdev = EARGF(usage());
+		break;
+	case 'a':
+		etheraddr = EARGF(usage());
+		break;
+	case 'g':
+		ethergw = EARGF(usage());
+		break;
+	default:
+		usage();
+	}ARGEND
+
+	if(argc != 1 && argc != 3)
+		usage();
+
+	fmtinstall('I', eipfmt);
+	fmtinstall('V', eipfmt);
+	fmtinstall('E', eipfmt);
+
+	rulepath = argv[0];
+	readrules(rulepath);
+	checklogging();
+	if(debug)
+		dumprules();
+
+	/*
+	 * Naming two sides means there are wires to sit between, so we
+	 * filter packets.  Naming none means there is one namespace to
+	 * protect, so we filter the requests it makes.  Same rules
+	 * either way; the mode is what you pointed it at, and it is
+	 * never chosen silently.
+	 */
+	if(etherdev != nil){
+		char abuf[64], rpath[128], *m;
+		uchar mac[6];
+		int haveg, k;
+
+		if(etheraddr == nil){
+			static char abuf2[64], gbuf[64];
+
+			if(learnaddr(netmtpt, etherdev, abuf2, sizeof abuf2,
+			    gbuf, sizeof gbuf) < 0)
+				sysfatal("%s has no address on %s; give -a addr/mask",
+					etherdev, netmtpt);
+			etheraddr = abuf2;
+			if(ethergw == nil && *gbuf != '\0')
+				ethergw = gbuf;
+			fprint(2, "fw: %s has %s%s%s\n", etherdev, etheraddr,
+				ethergw != nil ? ", gateway " : "",
+				ethergw != nil ? ethergw : "");
+		}
+		m = nil;
+		if((m = strchr(etheraddr, '/')) != nil){
+			k = m - etheraddr;
+			if(k >= sizeof abuf)
+				sysfatal("%s: address too long", etheraddr);
+			memmove(abuf, etheraddr, k);
+			abuf[k] = '\0';
+		}else
+			sysfatal("-a wants addr/mask, eg 10.0.2.15/24");
+		if(parseipandmask(ip, ethermask, abuf, m) == -1)
+			sysfatal("%s: unparseable address or mask", etheraddr);
+		haveg = 0;
+		if(ethergw != nil){
+			if(parseip(gw, ethergw) == -1)
+				sysfatal("%s: unparseable gateway", ethergw);
+			haveg = 1;
+		}
+
+		/*
+		 * Check we can mount before touching the card.  Taking it
+		 * and then failing leaves the machine with an interface
+		 * that has no card behind it and no network at all, which
+		 * is a bad way to discover a typo in -m.
+		 */
+		if(access(mtpt, AEXIST) < 0)
+			sysfatal("%s: %r; not touching %s until it exists",
+				mtpt, etherdev);
+
+		fprint(2, "fw: filtering packets on %s\n", etherdev);
+		syslog(0, "fw", "started, filtering %s for %s", etherdev, etheraddr);
+
+		etherdebug = debug;
+		efd = etheropen(etherdev, mac);
+		ethersetaddr(ip, gw, haveg);
+		reclaim(netmtpt, abuf);
+
+		backdev = etherdev;
+		backaddr = abuf;
+		backmask = m;
+		backnet = netmtpt;
+		backgw = ethergw;
+		atexit(putback);
+		threadnotify(notehandler, 1);
+
+		takecard(netmtpt, etherdev);
+
+		prot = emalloc(sizeof *prot);
+		prot->net = netmtpt;
+		prot->addr = abuf;
+		prot->mask = m;
+		prot->side = "protected";
+		wireup(prot);
+
+		/*
+		 * Taking the card away took the routes with it.  The stack
+		 * still needs to know how to leave its own network, and we
+		 * are the only thing that knows where the gateway is.
+		 */
+		if(haveg){
+			snprint(rpath, sizeof rpath, "%s/iproute", netmtpt);
+			if((k = open(rpath, OWRITE)) < 0)
+				fprint(2, "fw: cannot open %s: %r\n", rpath);
+			else{
+				if(fprint(k, "add 0.0.0.0 0.0.0.0 %s", ethergw) < 0)
+					fprint(2, "fw: cannot add default route: %r\n");
+				else
+					fprint(2, "fw: default route via %s\n", ethergw);
+				close(k);
+			}
+		}
+
+		rulechanged = rulesdidchange;
+		proccreate(etherin, prot, 32*1024);
+		proccreate(etherout, prot, 32*1024);
+		servectl(mtpt, srvname);
+		threadexits(nil);
+	}
+
+	if(argc == 1){
+		if(wireonly)
+			sysfatal("-W given, but no interfaces to filter between");
+		servenet(netmtpt, srvname, netmtpt);
+		threadexits(nil);
+	}
+
+	fprint(2, "fw: filtering packets\n");
+	syslog(0, "fw", "started, filtering packets, rules %s", rulepath);
+	out = parseside(argv[1], "outside");
+	in = parseside(argv[2], "inside");
+	wireup(out);
+	wireup(in);
+
+	rulechanged = rulesdidchange;
+
+	/* inside -> outside is "out"; outside -> inside is "in" */
+	outargs[0] = in;
+	outargs[1] = out;
+	outargs[2] = (Wire*)Vout;
+	proccreate(relayproc, outargs, 32*1024);
+
+	inargs[0] = out;
+	inargs[1] = in;
+	inargs[2] = (Wire*)Vin;
+	proccreate(relayproc, inargs, 32*1024);
+
+	servectl(mtpt, srvname);
+	threadexits(nil);
+}