From d1fc2a8047d722f90d289fafe41be0029c6e39ff Mon Sep 17 00:00:00 2001 From: Calvin Morrison Date: Wed, 19 Aug 2026 12:31:10 -0400 Subject: test: count what was skipped, and say how to stop skipping it The suite reported 73 passed, 0 failed on a machine where it had been reporting 87. Nothing had regressed: the VM had been booted without a second card, so the fourteen card-mode checks skipped. A skip printed one line among ninety and the total moved, and a total that moves is not a signal anybody reads -- 73 passed looks exactly as good as 87 unless you happen to know which to expect. So skips are counted with the rest: 87 passed, 0 failed, 0 skipped and the one that matters says what to do about it rather than only what is missing: boot with run.sh -gw and the spare card is there. The header says the same, since that is where someone looks first. Also finishes the proof the last commit could not. /sys/log/fw now has a line for every port the suite can pick, so the collision that made the old check fail by chance is certain: 3600 poison lines, and the delta version passes 87 while the total version fails both logging checks with 4 and 3 where 1 is right. Co-Authored-By: Claude Opus 5 --- fw/test/fwtest.rc | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'fw') diff --git a/fw/test/fwtest.rc b/fw/test/fwtest.rc index a7267ed..dbd7754 100755 --- a/fw/test/fwtest.rc +++ b/fw/test/fwtest.rc @@ -6,8 +6,13 @@ # machine's, because the packet checks run between two IP stacks it # makes for itself. # -# Card mode is not covered: it takes the machine's card away, and a -# test that can leave you with no network is a test nobody runs. +# The machine's own card is never touched: a test that can leave you +# with no network is a test nobody runs. Card mode is covered on a +# second card instead, so boot with a spare one - +# +# ./run.sh -headless -gw init-test.qcow2 +# +# and fourteen checks that otherwise say "skip" will run. # # Run it twice in a row. It used to pass the first time and fail the # second, because it left its firewalls running and their interfaces @@ -61,6 +66,16 @@ res=$tmp/results >$res >$tmp/why +# A check that could not be made. Counted and printed at the end, +# because the number of checks is how you notice one going missing: the +# card section quietly stopped running when its VM was booted without a +# second card, and 73 passed looks exactly like 87 passed unless you +# knew which to expect. +fn skip { + echo skip >> $res + echo ' skip ' $1 +} + # check fn check { if(~ $2 $3){ @@ -164,7 +179,7 @@ if(test -d $src){ check 'and the compiler has nothing to say about it' 0 $"r } if not - echo ' skip no '^$src^', not checking the build' + skip 'no '^$src^', not checking the build' echo '== rules: parsing' @@ -274,7 +289,7 @@ echo '== a peer is checked when there is one to check' # own address stands in for a peer, since it has no loopback. myip=`{awk '/4u$/ {print $1}' /net/ipselftab | sed 1q} if(~ $#myip 0) - echo ' skip no IPv4 address on this machine to call' + skip 'no IPv4 address on this machine to call' if not { { echo 'deny=in proto=tcp ip='^$myip^' log=yes' @@ -527,7 +542,7 @@ echo '== a card' if(~ `{grep -c ether1 /net/ipifc/*/status >[2]/dev/null | awk '{n += $1} END {print n+0}'} 0) spare=yes if(~ $spare no) - echo ' skip no spare card to take' + skip 'no spare card: boot with run.sh -gw for a second one' if not { mkdir -p $tmp/nC bind -a '#I26' $tmp/nC @@ -683,7 +698,7 @@ if(test -f $lib/fwstart) check 'and nothing it started kept our descriptors' 0 $"r } if not - echo ' skip no '^$lib^'/fwstart' + skip 'no '^$lib^'/fwstart' echo '== the test cleans up after itself' r=`{nfw} @@ -704,7 +719,8 @@ check 'no interfaces are left bound, so the next run is clean' 0 $"r echo npass=`{grep -c '^ok' $res} nfail=`{grep -c '^FAIL' $res} -echo $"npass' passed, '$"nfail' failed' +nskip=`{grep -c '^skip' $res} +echo $"npass' passed, '$"nfail' failed, '$"nskip' skipped' rm -rf $tmp if(! ~ $"nfail 0) exit failed -- cgit v1.2.3