test: stop poisoning the next run, and stop passing for the wrong reason

The suite passed the first time and failed the second, always on "a
permitted connection crosses, and is tracked".  A comment blamed timing
and told the reader that a failure of that check alone was not evidence
of a fault.  It was.

Each run left six fw processes alive with their pkt interfaces still
bound.  After four runs #I22 looked like this:

	0: device pkt0 maxtu 1500 ... pktout 1826 | 10.9.9.1 /120
	1: device pkt1 maxtu 1500 ... pktout 0    | 10.9.9.1 /120
	2: device pkt2 maxtu 1500 ... pktout 0    | 10.9.9.1 /120
	3: device pkt3 maxtu 1500 ... pktout 0    | 10.9.9.1 /120

Four interfaces, one address, and the stack routes out the first, so the
current fw sees nothing and its flows file is empty.  pktout 1826 into a
wire whose far end died two runs ago is the trap design.md already
records costing an afternoon.  Measured, not guessed: kill every fw,
run once, 22 passed; run again immediately, 21 passed with that check
failing.

fw cannot be stopped by pid -- it daemonizes, so the shell's $apid is
gone before the server exists, and ps shows it no arguments -- and "kill
fw" would be wrong on a machine running a real one.  So stopfw takes the
interfaces away instead and fw follows: the relay's read fails and
threadexitsall takes the rest down.  That doubles as a live test of the
fail-closed path, since a relay that goes back to dying quietly now
shows up in the two new checks at the end, which count fw processes and
bound interfaces and would have caught this on the day.

Two other ways a check could pass without meaning anything.  A "refused"
result was returned for any failure at all, so a check for a hole went
green on a kernel that never had the hole -- gre raw is refused says
nothing if there is no /net/gre.  Each such check is now paired with one
asking, outside the sandbox, whether the thing being refused exists.
And the diagnostic could not be read: a failed > is reported by rc
itself and escapes any >[2] around it, so wr does the same create(2)
with cp, whose error lands on its own standard error and is printed
when a check fails.

Finally the port is derived from the pid.  A devip Conv is never freed,
so a fixed port made one run's leftovers into the next run's "address in
use".

29 checks, and 29 pass twice in a row with no cleanup between.  With the
stopfw calls disabled the two new ones report 6 processes and 4
interfaces left behind.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 101 insertions, 21 deletions

diff --git a/fw/test/fwtest.rc b/fw/test/fwtest.rc
index 0e947f3..0b923e9 100755
--- a/fw/test/fwtest.rc
+++ b/fw/test/fwtest.rc
@@ -8,6 +8,13 @@
 #
 # Card mode is not covered: it takes the machine's card away, and a
 # test that can leave you with no network is a test nobody runs.
+#
+# Run it twice in a row.  It used to pass the first time and fail the
+# second, because it left its firewalls running and their interfaces
+# bound, and the next run's stack routed out the dead one; the failure
+# was blamed on timing in a comment that told the next person to
+# disbelieve a real result.  The last two checks exist so that cannot
+# happen again quietly.
 rfork ne
 
 fw=$1
@@ -26,12 +33,23 @@ mkdir -p $nA $nB
 mtpt=$tmp/ctl
 mkdir -p $mtpt
 
+# The stacks the packet checks build for themselves.  Fixed numbers are
+# fine because they are unbound again at the end of each block, and the
+# last check proves it.
+stacks=(20 21 22 23)
+
+# A port of our own.  Conversations outlive the run that made them - a
+# devip Conv is never freed - so a fixed port makes one run's leftovers
+# into the next run's "address in use".
+port=`{echo $pid | awk '{print 17000 + ($1 % 900)}'}
+
 # Results go to a file, not to variables: every check below runs inside
 # an @{} that needs its own namespace, and an assignment in there never
 # reaches the parent.  Counting in variables silently reported one pass
 # out of seventeen.
 res=$tmp/results
 >$res
+>$tmp/why
 
 # check <name> <expected> <got>
 fn check {
@@ -44,26 +62,70 @@ fn check {
 		echo '	FAIL ' $1
 		echo '	      want: '$2
 		echo '	      got:  '$3
+		if(test -s $tmp/why)
+			echo '	      said: '`{cat $tmp/why}
 	}
+	>$tmp/why
 }
 
-# Did that write succeed?  Not what it said: rc reports a failed
-# redirect from the outer shell, so the message cannot be captured from
-# in here.  Refused or not is the thing being tested anyway.
+# Did that write succeed?
+#
+# Not with rc's own redirection: a failed > is reported by the shell
+# itself and escapes any >[2] put around it, so there is no way to see
+# what went wrong.  cp does the same create(2) and reports it on its
+# own standard error, where a failing check can print it.
 fn wr {
-	if(@{ echo -n $2 > $1 } >[2]/dev/null)
+	echo -n $2 > $tmp/in
+	>$tmp/why
+	if(@{ cp $tmp/in $1 } >[2] $tmp/why)
 		echo ok
 	if not
 		echo refused
 }
 
 fn rd {
-	if(@{ cat $1 >/dev/null } >[2]/dev/null)
+	>$tmp/why
+	if(@{ cat $1 >/dev/null } >[2] $tmp/why)
 		echo ok
 	if not
 		echo refused
 }
 
+# Is it there at all, on the real /net?
+#
+# Every "refused" check below is also passed by a path that does not
+# exist, which is how a check for a hole can go green on a kernel that
+# never had the hole to begin with.  So each one is paired with this,
+# asked outside the sandbox: the thing being refused has to be a thing.
+fn have {
+	if(test -e $1)
+		echo yes
+	if not
+		echo no
+}
+
+# Stop the firewalls a packet check started.
+#
+# Take the interfaces away and fw follows: the relay's read fails and
+# threadexitsall takes the rest of it down.  Nothing else can find
+# these processes - fw daemonizes, so the shell's $apid is gone before
+# the server exists, and ps shows fw no arguments to match on - and
+# "kill fw" would be wrong on a machine running a real one.  Doing it
+# this way also exercises the fail-closed path: if one relay ever goes
+# back to dying quietly, the leak checks at the end catch it.
+fn stopfw {
+	for(d in $*)
+		for(i in `{ls -p $d/ipifc | grep '^[0-9]'})
+			@{ echo -n unbind > $d/ipifc/$i/ctl } >[2]/dev/null
+	sleep 2
+}
+
+# fw processes belonging to anyone, counted the same way twice
+fn nfw {
+	ps | awk '$NF == "fw"' | wc -l
+}
+fwbefore=`{nfw}
+
 echo '== rules: parsing'
 
 cat > $tmp/bad.ndb <<'!'
@@ -76,6 +138,12 @@ cat > $tmp/empty.ndb <<'!'
 #  nothing
 !
 
+echo '== namespace mode: what is refused is there to refuse'
+for(p in /net/udp/clone /net/gre/clone /net/ndb /net/log /net/ipifc/0/data){
+	r=`{have $p}
+	check 'the real /net has '^$p yes $"r
+}
+
 echo '== namespace mode'
 @{
 	rfork n
@@ -105,13 +173,11 @@ echo '== namespace mode'
 	check 'but its status still is' ok $"r
 }
 
-cat > $tmp/lport.ndb <<'!'
-allow=in	proto=tcp	lport=17099
-!
 @{
 	rfork n
+	echo 'allow=in	proto=tcp	lport='^$port > $tmp/lport.ndb
 	$fw $tmp/lport.ndb >[2]/dev/null
-	r=`{wr /net/tcp/clone 'announce 17099'}
+	r=`{wr /net/tcp/clone 'announce '^$port}
 	check 'announce matches lport, not port' ok $"r
 	r=`{wr /net/tcp/clone 'announce 17098'}
 	check 'a different port is denied' refused $"r
@@ -152,14 +218,14 @@ deny=*		log=yes
 
 	r=`{wr $mtpt/ctl 'delete 0'}
 	check 'delete 0 is refused' refused $"r
+
+	stopfw $nA $nB
 }
 
 echo '== packets, between two stacks'
-cat > $tmp/wire.ndb <<'!'
-allow=in	proto=tcp	lport=17099
-!
 @{
 	rfork n
+	echo 'allow=in	proto=tcp	lport='^$port > $tmp/wire.ndb
 	bind -a '#I22' $nA
 	bind -a '#I23' $nB
 	$fw -m $mtpt $tmp/wire.ndb $nA^'!'^10.9.9.1^'!'^/24 $nB^'!'^10.9.9.2^'!'^/24 >[2]/dev/null &
@@ -168,28 +234,42 @@ allow=in	proto=tcp	lport=17099
 	r=`{cat $nA/ipifc/0/status | sed 1q | awk '{print $4}'}
 	check 'the pkt interface does not claim a 4096 mtu' 1500 $"r
 
-	@{ echo -n 'announce 17099'; sleep 25 } > $nB/tcp/clone &
+	@{ echo -n 'announce '^$port; sleep 15 } > $nB/tcp/clone &
 	sleep 3
-	@{ echo -n 'connect 10.9.9.2!17099'; sleep 20 } > $nA/tcp/clone &
+	@{ echo -n 'connect 10.9.9.2!'^$port; sleep 12 } > $nA/tcp/clone &
 	sleep 6
 
-	# Timing-sensitive: the handshake has to complete through fw
-	# before this looks.  It passes when run on its own and fails
-	# here intermittently, so a failure of this one check alone is
-	# not evidence of a fault - check it by hand before believing it.
-	r=`{grep -c 17099 $mtpt/flows}
+	r=`{grep -c $port $mtpt/flows}
 	check 'a permitted connection crosses, and is tracked' 1 $"r
 
 	# one rule, both directions: state, not a second rule
 	r=`{grep -c . $mtpt/rules}
 	check 'it took one rule to do that' 1 $"r
 
-	echo -n 'prepend deny=in proto=tcp lport=17099' > $mtpt/ctl
+	echo -n 'prepend deny=in proto=tcp lport='^$port > $mtpt/ctl
 	sleep 1
-	r=`{grep -c 17099 $mtpt/flows}
+	r=`{grep -c $port $mtpt/flows}
 	check 'blocking a port drops the live connection' 0 $"r
+
+	stopfw $nA $nB
 }
 
+echo '== the test cleans up after itself'
+r=`{nfw}
+check 'no fw processes are left running' $"fwbefore $"r
+
+>$tmp/ifcs
+for(s in $stacks)
+	@{
+		rfork n
+		mkdir -p $tmp/z
+		bind -a '#I'^$s $tmp/z
+		for(i in `{ls -p $tmp/z/ipifc | grep '^[0-9]'})
+			sed 1q $tmp/z/ipifc/$i/status >> $tmp/ifcs
+	}
+r=`{grep -c 'device pkt' $tmp/ifcs}
+check 'no interfaces are left bound, so the next run is clean' 0 $"r
+
 echo
 npass=`{grep -c '^ok' $res}
 nfail=`{grep -c '^FAIL' $res}