Initial commit: svc supervisor, design docs, session transcript

Existing work moved from /storage/vms/9front/svc, previously unversioned.
Object files and linked binaries excluded via .gitignore.

1 files changed, 851 insertions, 0 deletions

diff --git a/svc/src/init.c b/svc/src/init.c
new file mode 100644
index 0000000..7babe28
--- /dev/null
+++ b/svc/src/init.c
@@ -0,0 +1,851 @@
+/*
+ * init - run the services described in a directory of ndb files.
+ *
+ * Stage 3: load, start in dependency order, wait for readiness,
+ * supervise, log, and serve svcfs.  No namespaces, no identity
+ * switching, no halt/reboot.  Runs as an ordinary process; being
+ * pid 1 is not assumed anywhere.
+ */
+#include <u.h>
+#include <libc.h>
+#include <bio.h>
+#include <ndb.h>
+#include "dat.h"
+
+char *statename[] =
+{
+[Sstopped]	"stopped",
+[Swaiting]	"waiting",
+[Sstarting]	"starting",
+[Srunning]	"running",
+[Sfailed]	"failed",
+[Sdone]		"done",
+};
+
+Svc *svcs;
+char *svcdir = "/lib/svc";
+/*
+ * Not /log: the root is mounted without create permission, so a new
+ * top level directory cannot be made from a running system.  /sys/log
+ * already exists and is where Plan 9 keeps this.
+ */
+char *logdir = "/sys/log";
+int verbose;
+QLock statelock;
+
+static int tickpid;
+static int fsrunning;
+
+typedef struct Cmd Cmd;
+struct Cmd
+{
+	char	verb[16];
+	char	name[64];
+	Cmd	*next;
+};
+static Cmd *cmdq;
+
+char *known[] =
+{
+	"svc", "exec", "args", "env", "restart", "enable", "ready", "needs",
+	"ns", "user", "stop", "adopt",
+	nil
+};
+
+char *unimpl[] = { "ns", "user", "stop", "adopt", nil };
+
+void*
+emalloc(ulong n)
+{
+	void *p;
+
+	p = mallocz(n, 1);
+	if(p == nil)
+		sysfatal("out of memory");
+	setmalloctag(p, getcallerpc(&n));
+	return p;
+}
+
+char*
+estrdup(char *s)
+{
+	char *p;
+
+	p = strdup(s);
+	if(p == nil)
+		sysfatal("out of memory");
+	return p;
+}
+
+int
+inlist(char **list, char *s)
+{
+	int i;
+
+	for(i = 0; list[i] != nil; i++)
+		if(strcmp(list[i], s) == 0)
+			return 1;
+	return 0;
+}
+
+Svc*
+findname(char *name)
+{
+	Svc *s;
+
+	for(s = svcs; s != nil; s = s->next)
+		if(strcmp(s->name, name) == 0)
+			return s;
+	return nil;
+}
+
+static Svc*
+findpid(int pid)
+{
+	Svc *s;
+
+	for(s = svcs; s != nil; s = s->next)
+		if(s->pid == pid)
+			return s;
+	return nil;
+}
+
+static void
+failsvc(Svc *s, char *why)
+{
+	s->state = Sfailed;
+	s->wanted = 0;
+	free(s->exits);
+	s->exits = estrdup(why);
+	fprint(2, "init: %s: %s\n", s->name, why);
+}
+
+static int
+parseready(Svc *s, char *v)
+{
+	if(strcmp(v, "exec") == 0)
+		s->kind = Kexec;
+	else if(strcmp(v, "exit") == 0)
+		s->kind = Kexit;
+	else if(strncmp(v, "srv:", 4) == 0){
+		s->kind = Ksrv;
+		s->readyarg = estrdup(v+4);
+	}else if(strncmp(v, "dial:", 5) == 0){
+		s->kind = Kdial;
+		s->readyarg = estrdup(v+5);
+	}else
+		return -1;
+	return 0;
+}
+
+static Svc*
+loadone(char *path, char *name)
+{
+	Ndb *db;
+	Ndbtuple *tu, *t;
+	Svc *s;
+
+	db = ndbopen(path);
+	if(db == nil){
+		fprint(2, "init: %s: %r\n", path);
+		return nil;
+	}
+	tu = ndbparse(db);
+	if(tu == nil){
+		fprint(2, "init: %s: no service defined\n", path);
+		ndbclose(db);
+		return nil;
+	}
+
+	s = emalloc(sizeof *s);
+	s->restart = "onfail";
+	s->state = Sstopped;
+	s->kind = Kexec;
+
+	for(t = tu; t != nil; t = t->entry){
+		if(!inlist(known, t->attr)){
+			fprint(2, "init: %s: unknown attribute %q\n", path, t->attr);
+			goto Bad;
+		}
+		if(inlist(unimpl, t->attr)){
+			fprint(2, "init: %s: %s not implemented yet, ignored\n",
+				path, t->attr);
+			continue;
+		}
+		if(strcmp(t->attr, "svc") == 0)
+			s->name = estrdup(t->val);
+		else if(strcmp(t->attr, "exec") == 0)
+			s->exec = estrdup(t->val);
+		else if(strcmp(t->attr, "args") == 0){
+			if(s->argc < Maxargs-2)
+				s->argv[++s->argc] = estrdup(t->val);
+		}else if(strcmp(t->attr, "env") == 0){
+			if(s->envc < Maxargs-1)
+				s->env[s->envc++] = estrdup(t->val);
+		}else if(strcmp(t->attr, "needs") == 0){
+			if(s->nneeds < Maxneeds)
+				s->needs[s->nneeds++] = estrdup(t->val);
+		}else if(strcmp(t->attr, "ready") == 0){
+			if(parseready(s, t->val) < 0){
+				fprint(2, "init: %s: ready=%q is not exec, exit, "
+					"srv:name or dial:addr\n", path, t->val);
+				goto Bad;
+			}
+		}else if(strcmp(t->attr, "restart") == 0)
+			s->restart = estrdup(t->val);
+		else if(strcmp(t->attr, "enable") == 0)
+			s->enable = strcmp(t->val, "yes") == 0;
+	}
+	ndbfree(tu);
+	ndbclose(db);
+
+	if(s->name == nil){
+		fprint(2, "init: %s: no svc= attribute\n", path);
+		goto Bad2;
+	}
+	if(strcmp(s->name, name) != 0){
+		fprint(2, "init: %s: svc=%s does not match file name\n", path, s->name);
+		goto Bad2;
+	}
+	if(s->exec == nil){
+		fprint(2, "init: %s: no exec= attribute\n", path);
+		goto Bad2;
+	}
+	if(strcmp(s->restart, "never") != 0 && strcmp(s->restart, "onfail") != 0
+	&& strcmp(s->restart, "always") != 0){
+		fprint(2, "init: %s: restart=%q is not never, onfail or always\n",
+			path, s->restart);
+		goto Bad2;
+	}
+	if(s->kind == Kexit && strcmp(s->restart, "never") != 0){
+		fprint(2, "init: %s: restart=%s with ready=exit; a oneshot is "
+			"never restarted\n", path, s->restart);
+		goto Bad2;
+	}
+
+	s->argv[0] = strrchr(s->exec, '/');
+	if(s->argv[0] == nil)
+		s->argv[0] = s->exec;
+	else
+		s->argv[0]++;
+	s->argv[s->argc+1] = nil;
+
+	return s;
+
+Bad:
+	ndbfree(tu);
+	ndbclose(db);
+Bad2:
+	free(s);
+	return nil;
+}
+
+static int
+cyclic(Svc *s)
+{
+	Svc *d;
+	int i;
+
+	if(s->mark == 1)
+		return 1;
+	if(s->mark == 2)
+		return 0;
+	s->mark = 1;
+	for(i = 0; i < s->nneeds; i++){
+		d = findname(s->needs[i]);
+		if(d != nil && cyclic(d)){
+			fprint(2, "init: %s: needs= cycle through %s\n",
+				s->name, d->name);
+			return 1;
+		}
+	}
+	s->mark = 2;
+	return 0;
+}
+
+static void
+dropsvc(Svc *bad)
+{
+	Svc **p;
+
+	for(p = &svcs; *p != nil; p = &(*p)->next)
+		if(*p == bad){
+			*p = bad->next;
+			free(bad);
+			return;
+		}
+}
+
+static void
+checkdeps(void)
+{
+	Svc *s, *next;
+	int i;
+
+	for(s = svcs; s != nil; s = next){
+		next = s->next;
+		for(i = 0; i < s->nneeds; i++)
+			if(findname(s->needs[i]) == nil){
+				fprint(2, "init: %s: needs=%s, which is not a service\n",
+					s->name, s->needs[i]);
+				dropsvc(s);
+				break;
+			}
+	}
+	for(s = svcs; s != nil; s = next){
+		next = s->next;
+		if(cyclic(s))
+			dropsvc(s);
+	}
+}
+
+static void
+loadsvcs(void)
+{
+	Dir *d;
+	int fd, i, n;
+	char path[512];
+	Svc *s, **tail;
+
+	fd = open(svcdir, OREAD);
+	if(fd < 0)
+		sysfatal("open %s: %r", svcdir);
+	n = dirreadall(fd, &d);
+	close(fd);
+	if(n < 0)
+		sysfatal("read %s: %r", svcdir);
+
+	tail = &svcs;
+	for(i = 0; i < n; i++){
+		if(d[i].qid.type & QTDIR)
+			continue;
+		snprint(path, sizeof path, "%s/%s", svcdir, d[i].name);
+		s = loadone(path, d[i].name);
+		if(s == nil)
+			continue;
+		if(findname(s->name) != nil){
+			fprint(2, "init: %s: duplicate service name\n", path);
+			free(s);
+			continue;
+		}
+		*tail = s;
+		tail = &s->next;
+	}
+	free(d);
+	checkdeps();
+}
+
+static int
+toofast(Svc *s)
+{
+	long now, old;
+
+	now = time(nil);
+	old = s->rtimes[s->rnext];
+	s->rtimes[s->rnext] = now;
+	s->rnext = (s->rnext + 1) % Maxrestarts;
+	return old != 0 && now - old < Restartwindow;
+}
+
+static int
+srvexists(char *name)
+{
+	char path[256];
+
+	snprint(path, sizeof path, "/srv/%s", name);
+	return access(path, AEXIST) == 0;
+}
+
+static int
+dialok(char *addr)
+{
+	int fd;
+
+	fd = dial(addr, nil, nil, nil);
+	if(fd < 0)
+		return 0;
+	close(fd);
+	return 1;
+}
+
+static int
+isready(Svc *s)
+{
+	switch(s->kind){
+	case Ksrv:
+		return srvexists(s->readyarg);
+	case Kdial:
+		return dialok(s->readyarg);
+	}
+	return 1;
+}
+
+static int
+depsok(Svc *s)
+{
+	Svc *d;
+	int i;
+
+	for(i = 0; i < s->nneeds; i++){
+		d = findname(s->needs[i]);
+		if(d == nil)
+			return 0;
+		if(d->state != Srunning && d->state != Sdone)
+			return 0;
+	}
+	return 1;
+}
+
+void
+startsvc(Svc *s)
+{
+	int pid, fd, i;
+	char path[512];
+
+	if(s->pid != 0)
+		return;
+
+	switch(pid = rfork(RFPROC|RFFDG|RFNOTEG|RFENVG)){
+	case -1:
+		failsvc(s, "rfork failed");
+		return;
+	case 0:
+		snprint(path, sizeof path, "%s/%s", logdir, s->name);
+		fd = open(path, OWRITE);
+		if(fd < 0)
+			fd = create(path, OWRITE, 0644);
+		if(fd >= 0){
+			seek(fd, 0, 2);
+			dup(fd, 1);
+			dup(fd, 2);
+			if(fd > 2)
+				close(fd);
+		}
+		for(i = 0; i < s->envc; i++){
+			char *p;
+
+			p = strchr(s->env[i], '=');
+			if(p == nil)
+				continue;
+			*p = '\0';
+			putenv(s->env[i], p+1);
+		}
+		exec(s->exec, s->argv);
+		fprint(2, "init: exec %s: %r\n", s->exec);
+		exits("exec");
+	}
+	s->pid = pid;
+	s->wanted = 1;
+	s->tstart = time(nil);
+	free(s->exits);
+	s->exits = nil;
+	s->state = s->kind == Kexec ? Srunning : Sstarting;
+	if(verbose)
+		fprint(2, "init: started %s pid %d (%s)\n", s->name, pid,
+			statename[s->state]);
+}
+
+void
+stopsvc(Svc *s)
+{
+	s->wanted = 0;
+	s->tretry = 0;
+	if(s->pid == 0){
+		if(s->state != Sdone && s->state != Sfailed)
+			s->state = Sstopped;
+		return;
+	}
+	s->tstop = time(nil);
+	s->stopping = 1;
+	if(postnote(PNGROUP, s->pid, "hangup") < 0)
+		postnote(PNPROC, s->pid, "hangup");
+	if(verbose)
+		fprint(2, "init: stopping %s pid %d\n", s->name, s->pid);
+}
+
+/*
+ * The filesystem runs in another proc and cannot fork services itself,
+ * so it leaves them here.  Errors are constant strings: several procs
+ * may be in here at once.
+ */
+char*
+queuecmd(char *verb, char *name)
+{
+	Cmd *c;
+	Svc *s;
+
+	if(strcmp(verb, "start") != 0 && strcmp(verb, "stop") != 0
+	&& strcmp(verb, "restart") != 0 && strcmp(verb, "reset") != 0)
+		return "unknown command; try start, stop, restart or reset";
+
+	qlock(&statelock);
+	if(name != nil){
+		s = findname(name);
+		if(s == nil){
+			qunlock(&statelock);
+			return "no such service";
+		}
+	}
+	c = mallocz(sizeof *c, 1);
+	if(c == nil){
+		qunlock(&statelock);
+		return "out of memory";
+	}
+	strecpy(c->verb, c->verb+sizeof c->verb, verb);
+	if(name != nil)
+		strecpy(c->name, c->name+sizeof c->name, name);
+	c->next = cmdq;
+	cmdq = c;
+	qunlock(&statelock);
+	return nil;
+}
+
+int
+cmdspending(void)
+{
+	return cmdq != nil;
+}
+
+/* called with statelock held */
+void
+draincmds(void)
+{
+	Cmd *c, *next;
+	Svc *s;
+
+	c = cmdq;
+	cmdq = nil;
+	for(; c != nil; c = next){
+		next = c->next;
+		if(strcmp(c->verb, "reset") == 0){
+			for(s = svcs; s != nil; s = s->next)
+				if(s->pid != 0 || s->state == Srunning)
+					stopsvc(s);
+			for(s = svcs; s != nil; s = s->next)
+				if(s->enable){
+					s->wanted = 1;
+					s->state = Swaiting;
+				}
+			free(c);
+			continue;
+		}
+		s = findname(c->name);
+		if(s == nil){
+			free(c);
+			continue;
+		}
+		if(strcmp(c->verb, "start") == 0){
+			if(s->pid == 0 && s->state != Srunning){
+				s->wanted = 1;
+				s->tretry = 0;
+				s->state = Swaiting;
+			}
+		}else if(strcmp(c->verb, "stop") == 0)
+			stopsvc(s);
+		else if(strcmp(c->verb, "restart") == 0){
+			stopsvc(s);
+			s->wanted = 1;
+			s->state = Swaiting;
+		}
+		free(c);
+	}
+}
+
+static void
+startall(void)
+{
+	Svc *s;
+
+	for(s = svcs; s != nil; s = s->next)
+		if(s->enable){
+			s->wanted = 1;
+			s->state = Swaiting;
+		}
+}
+
+static int
+needtick(void)
+{
+	Svc *s;
+
+	if(fsrunning)			/* commands may arrive at any time */
+		return 1;
+	for(s = svcs; s != nil; s = s->next){
+		if(s->state == Swaiting || s->state == Sstarting)
+			return 1;
+		if(s->state == Srunning && s->kind == Ksrv)
+			return 1;
+		if(s->pid != 0 && s->wanted == 0)
+			return 1;	/* waiting for it to go */
+	}
+	return 0;
+}
+
+static void
+armtick(void)
+{
+	int pid;
+
+	if(tickpid != 0 || !needtick())
+		return;
+	switch(pid = rfork(RFPROC|RFNOTEG)){
+	case -1:
+		return;
+	case 0:
+		sleep(Tickms);
+		_exits(nil);
+	}
+	tickpid = pid;
+}
+
+/* called with statelock held */
+static void
+ontick(void)
+{
+	Svc *s;
+	long now;
+
+	now = time(nil);
+	for(s = svcs; s != nil; s = s->next){
+		/* a service we asked to stop that will not go */
+		if(s->pid != 0 && s->wanted == 0 && now - s->tstop >= Stopwait){
+			if(verbose)
+				fprint(2, "init: %s did not stop, killing\n", s->name);
+			postnote(PNGROUP, s->pid, "kill");
+			postnote(PNPROC, s->pid, "kill");
+			s->tstop = now;
+		}
+		switch(s->state){
+		case Swaiting:
+			if(s->wanted && now >= s->tretry && depsok(s) && s->pid == 0)
+				startsvc(s);
+			break;
+		case Sstarting:
+			if(isready(s)){
+				s->state = Srunning;
+				if(verbose)
+					fprint(2, "init: %s ready\n", s->name);
+			}else if(now - s->tstart >= Readywait)
+				failsvc(s, "not ready in time");
+			break;
+		case Srunning:
+			if(s->kind == Ksrv && s->wanted && !srvexists(s->readyarg)){
+				if(verbose)
+					fprint(2, "init: %s: /srv/%s went away\n",
+						s->name, s->readyarg);
+				s->pid = 0;
+				if(strcmp(s->restart, "never") == 0){
+					s->state = Sstopped;
+					s->wanted = 0;
+					break;
+				}
+				if(toofast(s)){
+					failsvc(s, "restarting too fast");
+					break;
+				}
+				s->restarts++;
+				s->state = Swaiting;
+				s->tretry = now + Restartwait;
+			}
+			break;
+		}
+	}
+}
+
+/* called with statelock held */
+static void
+reap(Waitmsg *w)
+{
+	Svc *s;
+	int failed;
+
+	s = findpid(w->pid);
+	if(s == nil)
+		return;
+	s->pid = 0;
+	failed = w->msg[0] != '\0';
+	free(s->exits);
+	s->exits = failed ? estrdup(w->msg) : nil;
+
+	if(verbose || failed)
+		fprint(2, "init: %s exited%s%s\n", s->name,
+			failed ? ": " : " cleanly", failed ? w->msg : "");
+
+	if(s->kind == Kexit){
+		s->wanted = 0;
+		s->state = failed ? Sfailed : Sdone;
+		return;
+	}
+	if(s->kind == Ksrv){
+		/*
+		 * Its process exiting is normal: the service is whatever is
+		 * behind /srv, and the tick watches that.
+		 */
+		if(failed && s->state == Sstarting)
+			failsvc(s, w->msg);
+		return;
+	}
+	/*
+	 * If we asked it to go, this is not a crash: do not count it
+	 * against the restart rate, or restarting a service by hand a
+	 * few times would mark it failed.
+	 */
+	if(s->stopping){
+		s->stopping = 0;
+		s->state = s->wanted ? Swaiting : Sstopped;
+		return;
+	}
+	if(!s->wanted){
+		s->state = Sstopped;
+		return;
+	}
+	if(strcmp(s->restart, "always") == 0
+	|| (strcmp(s->restart, "onfail") == 0 && failed)){
+		if(toofast(s)){
+			failsvc(s, "restarting too fast");
+			return;
+		}
+		s->restarts++;
+		s->state = Swaiting;
+		s->tretry = time(nil) + Restartwait;
+		return;
+	}
+	s->wanted = 0;
+	s->state = failed ? Sfailed : Sstopped;
+}
+
+/* called with statelock held */
+char*
+statusline(Svc *s)
+{
+	char buf[512], *p, *e;
+	Svc *d;
+	int i;
+
+	p = buf;
+	e = buf + sizeof buf;
+	p = seprint(p, e, "svc=%s state=%s", s->name, statename[s->state]);
+	if(s->pid != 0)
+		p = seprint(p, e, " pid=%d", s->pid);
+	if(s->restarts != 0)
+		p = seprint(p, e, " restarts=%d", s->restarts);
+	if(s->state == Swaiting)
+		for(i = 0; i < s->nneeds; i++){
+			d = findname(s->needs[i]);
+			if(d != nil && d->state != Srunning && d->state != Sdone)
+				p = seprint(p, e, " needs=%s", s->needs[i]);
+		}
+	if(s->enable)
+		p = seprint(p, e, " enable=yes");
+	if(s->exits != nil)
+		p = seprint(p, e, " exit=%q", s->exits);
+	seprint(p, e, "\n");
+	return strdup(buf);
+}
+
+/* called with statelock held */
+char*
+allstatus(void)
+{
+	Svc *s;
+	char *all, *one, *t;
+
+	all = strdup("");
+	if(all == nil)
+		return nil;
+	for(s = svcs; s != nil; s = s->next){
+		one = statusline(s);
+		if(one == nil)
+			break;
+		t = smprint("%s%s", all, one);
+		free(one);
+		if(t == nil)
+			break;
+		free(all);
+		all = t;
+	}
+	return all;
+}
+
+static void
+usage(void)
+{
+	fprint(2, "usage: init [-v] [-d svcdir] [-l logdir] "
+		"[-s srvname] [-m mtpt]\n");
+	exits("usage");
+}
+
+void
+main(int argc, char **argv)
+{
+	Waitmsg *w;
+	char *srvname, *mtpt;
+
+	quotefmtinstall();
+	srvname = nil;
+	mtpt = nil;
+
+	ARGBEGIN{
+	case 'd':
+		svcdir = EARGF(usage());
+		break;
+	case 'l':
+		logdir = EARGF(usage());
+		break;
+	case 's':
+		srvname = EARGF(usage());
+		break;
+	case 'm':
+		mtpt = EARGF(usage());
+		break;
+	case 'v':
+		verbose++;
+		break;
+	default:
+		usage();
+	}ARGEND;
+
+	if(argc != 0)
+		usage();
+
+	loadsvcs();
+	if(svcs == nil)
+		sysfatal("no services loaded from %s", svcdir);
+
+	qlock(&statelock);
+	startall();
+	ontick();
+	qunlock(&statelock);
+
+	/*
+	 * The control plane is optional: if it will not come up we say so
+	 * and carry on supervising.  A machine must not fail to boot over
+	 * a filesystem nobody may ever mount.
+	 */
+	if(srvname != nil || mtpt != nil){
+		startfs(srvname != nil ? srvname : "svc", mtpt);
+		fsrunning = 1;
+	}
+
+	for(;;){
+		armtick();
+		w = wait();
+		if(w == nil){
+			fprint(2, "init: wait: %r\n");
+			sleep(1000);
+			continue;
+		}
+		if(w->pid == tickpid){
+			tickpid = 0;
+			qlock(&statelock);
+			draincmds();
+			ontick();
+			qunlock(&statelock);
+		}else{
+			qlock(&statelock);
+			reap(w);
+			qunlock(&statelock);
+		}
+		free(w);
+	}
+}