diff --git a/README.md b/README.md index 884d860..43f3ac2 100644 --- a/README.md +++ b/README.md @@ -1002,7 +1002,7 @@ sandy --exec -- cat /etc/sandy-session.json # workspace, sandbox_name, posture ### Also in 2.0 -- `--print-state`'s `schema_version` is **`2`**. Gate on that number, not on sandy's version string — `2.0.0-dev` compares equal to `2.0.0`. +- `--print-state`'s `schema_version` was **`2`** in the 2.0 line and is **`3`** as of 2.2.0 (see **Deprecated**). Gate on that number, not on sandy's version string — `2.0.0-dev` compares equal to `2.0.0`. Treat it as an **opaque token**: compare against a reviewed set, not with `>=`, so a future bump is something you read rather than something you silently accept. - `sandboxes[].features` now reports **manifest selection** rather than per-sandbox markers; `SANDY_FEATURES_DIR` is removed with an error naming its replacement. - `SANDY_EGRESS=off|permissive|strict` replaces two booleans. The old keys still work — see **Deprecated** below. diff --git a/test/acceptance-handoff-dirs.sh b/test/acceptance-handoff-dirs.sh index 7841a4c..37404ee 100755 --- a/test/acceptance-handoff-dirs.sh +++ b/test/acceptance-handoff-dirs.sh @@ -151,26 +151,32 @@ mkdir -p "$WS3/.sandy" && (cd "$WS3" && git init -q) WS3="$(cd "$WS3" && pwd -P)" cid3() { docker ps -q --filter label=sandy.daemon=true --filter "label=sandy.workspace_path=$WS3" 2>/dev/null | head -1; } -cat > "$WS3/.sandy/relay.sh" <<'RELAYFIX' +# The relay is installed as a feature manifest ENTRY (2.2.0). It used to be +# installed by setting SANDY_HANDOFF_RELAY in the isolated host config, which +# #354 made a HARD ERROR -- so a harness that still did that would refuse the +# launch and report the supervisor as broken, when what is broken is the +# harness. That happened; this is the fix. +_E_FEAT="$SANDY_HOME_DIR/features/acc-relay" +mkdir -p "$_E_FEAT/payload" +cat > "$_E_FEAT/payload/relay" <<'RELAYFIX' #!/bin/sh # Fixture relay for phase E: records its own pid + the env contract on each # (re)start, then blocks. Deliberately NOT `exec sleep` -- pgrep -f below # matches on this script's own path, and `exec` would replace this process's # argv with "sleep 3600", losing that match the instant it ran. -echo "$$ $SANDY_HANDOFF_INBOX $SANDY_HANDOFF_OUTBOX $SANDY_HANDOFF_RELAY_STATE" >> "$SANDY_HANDOFF_RELAY_STATE/seen" +echo "$$ $SANDY_RELAY_STATE" >> "$SANDY_RELAY_STATE/seen" sleep 3600 RELAYFIX -chmod +x "$WS3/.sandy/relay.sh" - -# SANDY_HANDOFF_RELAY is PRIVILEGED tier. Setting it via the isolated HOST's -# own ~/.sandy/config (a privileged SOURCE) needs no approval prompt at all -- -# unlike phases B/D above (which prove the PASSIVE tier from a WORKSPACE -# source), a privileged source may set a privileged key freely. `env -u -# SANDY_AUTO_APPROVE_PRIVILEGED` is kept anyway, for the same "prove it, don't -# assume it" discipline as the rest of this file: this phase must pass -# without that escape hatch, because it isn't the thing being exercised here. -echo "SANDY_HANDOFF_RELAY=.sandy/relay.sh" >> "$SANDY_HOME_DIR/config" +chmod +x "$_E_FEAT/payload/relay" +cat > "$_E_FEAT/feature.json" <<'E_MANIFEST' +{ "sandboxes": {"include": ["*"]}, "agents": {"include": ["*"]}, + "mounts": [ { "name": "payload", "from": "payload" } ], + "entry": "payload/relay" } +E_MANIFEST +# No config key is involved any more: a manifest under $SANDY_HOME is +# privileged by construction of WHERE IT LIVES, so there is nothing to approve. +# `env -u SANDY_AUTO_APPROVE_PRIVILEGED` is kept anyway, to prove that. env -u SANDY_AUTO_APPROVE_PRIVILEGED "$SANDY" --start --workspace "$WS3"; RC=$? ck "--start exits 0 with the relay configured" "[ $RC -eq 0 ]" C3="$(cid3)" @@ -181,15 +187,15 @@ SBX3="$SANDY_HOME_DIR/sandboxes/$SESS3" echo "-- E1. mount + env forwarding --" _m3="$(docker inspect -f '{{range .Mounts}}{{.Destination}} {{.RW}}{{"\n"}}{{end}}' "$C3" 2>/dev/null)" -echo " mounts:"; printf '%s\n' "$_m3" | grep -i handoff | sed 's/^/ /' -ck "relay mount is RW=true" \ - "printf '%s\n' \"\$_m3\" | grep -qE '^/home/sandy/.handoff/relay true\$'" +echo " mounts:"; printf '%s\n' "$_m3" | grep -iE 'handoff|relay-state' | sed 's/^/ /' +ck "relay STATE mount is RW=true at its 2.2.0 path (#353)" \ + "printf '%s\n' \"\$_m3\" | grep -qE '^/opt/sandy/relay-state true\$'" ck "no removed lane is mounted alongside the relay (#352)" \ "! printf '%s\n' \"\$_m3\" | grep -qE '^/home/sandy/.handoff/(inbox|outbox|peer) '" # Never dump the whole env -- it carries CLAUDE_CODE_OAUTH_TOKEN and friends. # Count occurrences of the one var under test instead of printing anything. -_envcount="$(docker inspect -f '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' "$C3" 2>/dev/null | grep -c '^SANDY_HANDOFF_RELAY=\.sandy/relay\.sh$')" -ck "SANDY_HANDOFF_RELAY forwarded into the container exactly once" "[ \"$_envcount\" = 1 ]" +_envcount="$(docker inspect -f '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' "$C3" 2>/dev/null | grep -c '^SANDY_HANDOFF_RELAY=/opt/sandy/features/acc-relay/relay$' || true)" +ck "the resolved entry is forwarded into the container exactly once (SANDY_HANDOFF_RELAY survives as the INTERNAL channel a manifest entry travels through -- only the config key was removed)" "[ \"$_envcount\" = 1 ]" echo "-- E2. relay is running, as a sibling of tmux (not a pane, not a session child) --" # The subshell that runs _sandy_start_handoff_relay's loop is backgrounded @@ -199,13 +205,13 @@ echo "-- E2. relay is running, as a sibling of tmux (not a pane, not a session c # immediately. _pid1="" for _i in 1 2 3 4 5 6; do - _pid1="$(docker exec -u "$(id -u)" "$C3" pgrep -f '\.sandy/relay\.sh' 2>/dev/null | head -1)" + _pid1="$(docker exec -u "$(id -u)" "$C3" pgrep -f 'acc-relay/relay' 2>/dev/null | head -1)" [ -n "$_pid1" ] && break sleep 1 done ck "relay process is running in the container" "[ -n \"$_pid1\" ]" ck "exactly one relay process" \ - "[ \"\$(docker exec -u \"\$(id -u)\" \"$C3\" pgrep -c -f '\.sandy/relay\.sh' 2>/dev/null)\" = 1 ]" + "[ \"\$(docker exec -u \"\$(id -u)\" \"$C3\" pgrep -c -f 'acc-relay/relay' 2>/dev/null)\" = 1 ]" # Two /proc//status hops: relay's parent is the supervisor loop shell; # the loop shell's parent must be PID 1 (tail -f /dev/null in daemon mode, # which the loop was backgrounded under BEFORE PID 1 exec'd into tail -- @@ -222,25 +228,25 @@ docker exec -u "$(id -u)" "$C3" kill "$_pid_before" >/dev/null 2>&1 _pid_after="" for _i in 1 2 3 4 5 6 7 8; do sleep 1 - _pid_after="$(docker exec -u "$(id -u)" "$C3" pgrep -f '\.sandy/relay\.sh' 2>/dev/null | head -1)" + _pid_after="$(docker exec -u "$(id -u)" "$C3" pgrep -f 'acc-relay/relay' 2>/dev/null | head -1)" [ -n "$_pid_after" ] && [ "$_pid_after" != "$_pid_before" ] && break done ck "relay came back with a NEW pid after being killed" \ "[ -n \"$_pid_after\" ] && [ \"$_pid_after\" != \"$_pid_before\" ]" -_exits="$(docker exec -u "$(id -u)" "$C3" grep -c 'exit rc=' /home/sandy/.handoff/relay/supervisor.log 2>/dev/null || echo 0)" +_exits="$(docker exec -u "$(id -u)" "$C3" grep -c 'exit rc=' /opt/sandy/relay-state/supervisor.log 2>/dev/null || echo 0)" ck "supervisor.log recorded the exit" "[ \"${_exits:-0}\" -ge 1 ]" # The log line is "[sandy-relay] start ", so the timestamp sits # between the bracket and the word -- the old '\] start ' pattern required them # adjacent and therefore never matched, making this check fail even on a # perfectly working restart (which E3s own new-pid assertion had just proved). -_starts="$(docker exec -u "$(id -u)" "$C3" grep -c ' start /' /home/sandy/.handoff/relay/supervisor.log 2>/dev/null || echo 0)" +_starts="$(docker exec -u "$(id -u)" "$C3" grep -c ' start /' /opt/sandy/relay-state/supervisor.log 2>/dev/null || echo 0)" ck "supervisor.log shows at least 2 starts (initial + restart)" "[ \"${_starts:-0}\" -ge 2 ]" echo "-- E4. never started twice --" ck "the supervisor lock is HELD (a second flock -n attempt fails)" \ "! docker exec -u \"\$(id -u)\" \"$C3\" flock -n /home/sandy/.sandy-handoff-relay.lock true" ck "still exactly one relay process (no second supervisor was spawned)" \ - "[ \"\$(docker exec -u \"\$(id -u)\" \"$C3\" pgrep -c -f '\.sandy/relay\.sh' 2>/dev/null)\" = 1 ]" + "[ \"\$(docker exec -u \"\$(id -u)\" \"$C3\" pgrep -c -f 'acc-relay/relay' 2>/dev/null)\" = 1 ]" echo "-- E5. sandy-handoff-sessions --" _hs="" @@ -257,8 +263,8 @@ ck "the listed socket path is a real socket in the container" \ echo "-- E6. crossSessionInbound pin lands in both measured-working files --" _marker="$(docker exec -u "$(id -u)" "$C3" cat /etc/sandy-session.json 2>/dev/null)" -ck "session marker reports handoff_relay=true" \ - "printf '%s' \"\$_marker\" | grep -q '\"handoff_relay\": true'" +ck "session marker reports relay.source=manifest -- handoff_relay was REMOVED in #355 and relay.source replaced it, so asserting the old field would be asserting a mechanism that no longer exists" \ + "docker exec \"$C3\" grep -q '\"source\": \"manifest\"' /etc/sandy-session.json" ck "session marker reports cross_session_inbound=\"accept\" (default: relay configured)" \ "printf '%s' \"\$_marker\" | grep -q '\"cross_session_inbound\": \"accept\"'" ck "userSettings (sandbox claude/settings.json, RW) carries the accept pin" \ @@ -286,7 +292,7 @@ _cid_before="$C3" env -u SANDY_AUTO_APPROVE_PRIVILEGED "$SANDY" --update-sessions --yes --workspace "$WS3" >/dev/null 2>&1; RC=$? ck "--update-sessions exits 0 (whether it restarted a stale session or correctly no-opped)" "[ $RC -eq 0 ]" -_seen_before="$(wc -l < "$SBX3/handoff/relay/seen" 2>/dev/null | tr -d ' ')" +_seen_before="$(wc -l < "$SBX3/relay-state/seen" 2>/dev/null | tr -d ' ')" _cid_before="$(cid3)" "$SANDY" --stop --workspace "$WS3" >/dev/null 2>&1 env -u SANDY_AUTO_APPROVE_PRIVILEGED "$SANDY" --start --workspace "$WS3" >/dev/null 2>&1; RC=$? @@ -296,7 +302,7 @@ ck "container id changed (a real recreation happened)" \ "[ -n \"$C3\" ] && [ \"$C3\" != \"$_cid_before\" ]" _pid_new="" for _i in 1 2 3 4 5 6 7 8; do - _pid_new="$(docker exec -u "$(id -u)" "$C3" pgrep -f '\.sandy/relay\.sh' 2>/dev/null | head -1)" + _pid_new="$(docker exec -u "$(id -u)" "$C3" pgrep -f 'acc-relay/relay' 2>/dev/null | head -1)" [ -n "$_pid_new" ] && break sleep 1 done @@ -304,7 +310,7 @@ ck "relay is running again in the NEW container" "[ -n \"$_pid_new\" ]" # Asserted as GROWTH against the pre-recreation count, not a bare ">= 2": # the sandbox dir survives recreation, so a fixed threshold would be satisfied # by lines an earlier phase wrote and would prove nothing about this step. -_seen_after="$(wc -l < "$SBX3/handoff/relay/seen" 2>/dev/null | tr -d ' ')" +_seen_after="$(wc -l < "$SBX3/relay-state/seen" 2>/dev/null | tr -d ' ')" ck "relay state persisted across recreation AND the new instance appended to it" \ "[ \"${_seen_after:-0}\" -gt \"${_seen_before:-0}\" ]" @@ -313,7 +319,7 @@ echo "-- E8. headless (-p) never starts a relay --" # is still live would just be refused by the workspace mutex, proving nothing # about the relay gate specifically. "$SANDY" --stop --workspace "$WS3" >/dev/null 2>&1 -_lines_before="$(wc -l < "$SBX3/handoff/relay/supervisor.log" 2>/dev/null | tr -d ' ')" +_lines_before="$(wc -l < "$SBX3/relay-state/supervisor.log" 2>/dev/null | tr -d ' ')" # `timeout` is GNU coreutils and is NOT present on a stock macOS (homebrew # installs it as `gtimeout`). Invoking it unconditionally made the -p launch # exit 127, which this phase then reported as "the launch did not succeed" -- @@ -339,7 +345,7 @@ if [ "$_e8_rc" -ne 0 ]; then # move on without touching either, rather than counting it as a pass. [ -n "$_e8_to" ] && printf ' \033[33mSKIP\033[0m %s\n' "E8 headless relay gate (the -p launch itself did not succeed, rc=$_e8_rc -- cannot conclude anything about the relay gate from it)" else - _lines_after="$(wc -l < "$SBX3/handoff/relay/supervisor.log" 2>/dev/null | tr -d ' ')" + _lines_after="$(wc -l < "$SBX3/relay-state/supervisor.log" 2>/dev/null | tr -d ' ')" ck "supervisor.log line count unchanged after a successful headless run (no relay was started)" \ "[ \"${_lines_before:-0}\" = \"${_lines_after:-0}\" ]" # Acceptance criterion 8: the skip is announced, and the announcement names @@ -357,19 +363,48 @@ echo "-- E10. criterion 7: a configured relay that CANNOT start fails the launch # Host-side detection (the path is workspace-relative, so sandy can resolve it # back to the host and refuse before `docker run` ever happens). "$SANDY" --stop --workspace "$WS3" >/dev/null 2>&1 -sed -i.bak 's|^SANDY_HANDOFF_RELAY=.*|SANDY_HANDOFF_RELAY=.sandy/does-not-exist.sh|' "$SANDY_HOME_DIR/config" +# Break the ENTRY, not a config key: point the manifest at a path that is not +# executable. The refusal is in-container now (user-setup.sh exits 1 and the +# container dies), because a manifest entry resolves to an image-only path the +# host cannot stat -- which is the branch the host-side check was never +# covering anyway. +cp "$_E_FEAT/feature.json" "$_E_FEAT/feature.json.bak" +cat > "$_E_FEAT/feature.json" <<'E10_MANIFEST' +{ "sandboxes": {"include": ["*"]}, "agents": {"include": ["*"]}, + "mounts": [ { "name": "payload", "from": "payload" } ], + "entry": "payload/not-executable" } +E10_MANIFEST +printf 'not executable\n' > "$_E_FEAT/payload/not-executable" # deliberately no chmod +x _e10_out="$(mktemp)" _e10_rc=0 env -u SANDY_AUTO_APPROVE_PRIVILEGED "$SANDY" --start --workspace "$WS3" > "$_e10_out" 2>&1 || _e10_rc=$? -ck "--start refuses (nonzero) when the configured relay is not an executable file" "[ $_e10_rc -ne 0 ]" +ck "--start refuses (nonzero) when the declared entry is not an executable file" "[ $_e10_rc -ne 0 ]" ck "...and says so, naming the fail-the-launch rule" \ - "grep -q 'A configured relay that cannot start fails the launch' \"$_e10_out\"" -ck "...and no daemon container was left behind" "[ -z \"$(cid3)\" ]" + "grep -q 'A configured relay that cannot start fails the session' \"$_e10_out\" || grep -q 'cannot start fails the' \"$_e10_out\"" +# NOT "no container was left behind" -- that was true of the HOST-side +# refusal, which happened before `docker run`. A manifest entry resolves to an +# image-only path the host cannot stat, so the refusal is in-container: +# user-setup.sh exits 1, the container dies, and --start classifies it as +# CRASH-LOOPING (exit 7) rather than refusing before launch (exit 6). Asserting +# the old property here would assert something the branch does not promise. +# +# What criterion 7 actually requires is that the session never comes up +# READY -- a container that exists but is crash-looping is loud; a container +# that is up with nothing delivering is the silent failure the rule exists for. +ck "...and reports CRASH-LOOPING (exit 7), not ready -- the in-container branch of criterion 7" \ + "[ $_e10_rc -eq 7 ]" +ck "...and --stop cleans it up, so a refused launch leaves nothing running" \ + "\"$SANDY\" --stop --workspace \"$WS3\" >/dev/null 2>&1; [ -z \"\$(cid3)\" ]" rm -f "$_e10_out" -# Restore the working relay so anything added after this phase is unaffected. -mv "$SANDY_HOME_DIR/config.bak" "$SANDY_HOME_DIR/config" 2>/dev/null || \ - sed -i.bak2 's|^SANDY_HANDOFF_RELAY=.*|SANDY_HANDOFF_RELAY=.sandy/relay.sh|' "$SANDY_HOME_DIR/config" -rm -f "$SANDY_HOME_DIR/config.bak2" +# Restore the working entry so anything added after this phase is unaffected. +mv "$_E_FEAT/feature.json.bak" "$_E_FEAT/feature.json" +rm -f "$_E_FEAT/payload/not-executable" + +# Remove phase E's feature before anything else runs: its manifest selects +# every sandbox, so leaving it installed would enrol it in phase G too and +# start a relay there for no reason. +"$SANDY" --stop --workspace "$WS3" >/dev/null 2>&1 || true +rm -rf "$_E_FEAT" # E9 (zero-diff regression) is phase A, which already ran with the relay # entirely unset and asserted no "handoff" string anywhere in `docker @@ -402,17 +437,28 @@ CG="$(docker ps -q --filter "label=sandy.workspace_path=$WS" | head -1)" ck "container is running" "[ -n \"$CG\" ]" ck "the payload EXISTS in the container (premise: a missing path fails a write for the wrong reason)" \ "docker exec \"$CG\" test -f /opt/sandy/features/acc-erofs/thing" +# EVERY check below is gated on a non-empty $CG. Without that gate a missing +# container makes `docker exec "" ...` fail, and "the write FAILED" then passes +# for the wrong reason -- three of these reported PASS against an empty id on +# the first real run of this phase, while the premise check correctly failed. +# A phase whose premise is red must not report green assertions beneath it. _g_uid="$(docker exec "$CG" id -u 2>/dev/null || echo 0)" -_g_err="$(docker exec "$CG" sh -c 'echo pwned > /opt/sandy/features/acc-erofs/thing' 2>&1 || true)" +_g_err="$([ -n "$CG" ] && docker exec "$CG" sh -c 'echo pwned > /opt/sandy/features/acc-erofs/thing' 2>&1 || echo "NO-CONTAINER")" ck "a write to the :ro payload FAILS from inside the container" \ - "! docker exec \"$CG\" sh -c 'echo pwned > /opt/sandy/features/acc-erofs/thing' 2>/dev/null" + "[ -n \"$CG\" ] && ! docker exec \"$CG\" sh -c 'echo pwned > /opt/sandy/features/acc-erofs/thing' 2>/dev/null" ck "...and fails with a READ-ONLY FILE SYSTEM error, not a permission error -- proving the MOUNT is the boundary, not the bits (got: $_g_err)" \ - "printf '%s' \"$_g_err\" | grep -qi 'read-only'" + "[ -n \"$CG\" ] && printf '%s' \"$_g_err\" | grep -qi 'read-only'" ck "...the file is still owned by the container user, so bits alone would NOT have stopped it (this is what makes the check above meaningful)" \ - "[ \"\$(docker exec \"$CG\" stat -c %u /opt/sandy/features/acc-erofs/thing 2>/dev/null)\" = \"$_g_uid\" ]" + "[ -n \"$CG\" ] && [ \"\$(docker exec \"$CG\" stat -c %u /opt/sandy/features/acc-erofs/thing 2>/dev/null)\" = \"$_g_uid\" ]" ck "creating a NEW file in the payload also fails (the whole directory is :ro, not just the file)" \ - "! docker exec \"$CG\" sh -c 'touch /opt/sandy/features/acc-erofs/evil' 2>/dev/null" + "[ -n \"$CG\" ] && ! docker exec \"$CG\" sh -c 'touch /opt/sandy/features/acc-erofs/evil' 2>/dev/null" ck "the payload is unchanged on the host after the attempt" \ - "grep -q 'payload-seen' \"$_G_FEAT/payload/thing\"" -"$SANDY" --stop --workspace "$WS" >/dev/null 2>&1 + "[ -n \"$CG\" ] && grep -q 'payload-seen' \"$_G_FEAT/payload/thing\"" +"$SANDY" --stop --workspace "$WS" >/dev/null 2>&1 || true rm -rf "$_G_FEAT" + +echo +echo "===================================================" +printf 'RESULT: %d passed, %d failed\n' "$PASS" "$FAIL" +echo "===================================================" +[ "$FAIL" -eq 0 ] diff --git a/test/acceptance-update-sessions.sh b/test/acceptance-update-sessions.sh index 7ef3493..c6cd60d 100755 --- a/test/acceptance-update-sessions.sh +++ b/test/acceptance-update-sessions.sh @@ -124,7 +124,7 @@ STATE_JSON="$("$SANDY" --print-state)" python3 -c " import json, sys d = json.loads(sys.argv[1]) -assert d['schema_version'] == 2, d['schema_version'] # 2.0.0 (D11) +assert d['schema_version'] == 3, d['schema_version'] # 3 since 2.2.0 (#355) rc = d['running_containers'] def find(cid): for c in rc: diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 7c96ff4..aaf6d15 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -2040,11 +2040,14 @@ if [ -f "$_acc_daemon" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_daemon" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1)" + # `|| true`: an UNGUARDED grep here exits 1 when the harness died before + # printing its RESULT line, which under set -e aborts the suite BEFORE + # fail() runs -- reporting a dead harness as "0 failed". Measured. + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1 || true)" if [ "$_acc_rc" -eq 0 ]; then pass "daemon-mode acceptance (${_acc_res:-all assertions passed})" else - fail "daemon-mode acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "daemon-mode acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else @@ -2071,11 +2074,14 @@ if [ -f "$_acc_upd" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_upd" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1)" + # `|| true`: an UNGUARDED grep here exits 1 when the harness died before + # printing its RESULT line, which under set -e aborts the suite BEFORE + # fail() runs -- reporting a dead harness as "0 failed". Measured. + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1 || true)" if [ "$_acc_rc" -eq 0 ]; then pass "fleet-update acceptance (${_acc_res:-all assertions passed})" else - fail "fleet-update acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "fleet-update acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else @@ -2103,11 +2109,14 @@ if [ -f "$_acc_topo" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_topo" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1)" + # `|| true`: an UNGUARDED grep here exits 1 when the harness died before + # printing its RESULT line, which under set -e aborts the suite BEFORE + # fail() runs -- reporting a dead harness as "0 failed". Measured. + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1 || true)" if [ "$_acc_rc" -eq 0 ]; then pass "multi-agent pane-topology acceptance (${_acc_res:-all assertions passed})" else - fail "multi-agent pane-topology acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "multi-agent pane-topology acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else @@ -2268,11 +2277,14 @@ if [ -f "$_acc_handoff" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_handoff" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1)" + # `|| true`: an UNGUARDED grep here exits 1 when the harness died before + # printing its RESULT line, which under set -e aborts the suite BEFORE + # fail() runs -- reporting a dead harness as "0 failed". Measured. + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1 || true)" if [ "$_acc_rc" -eq 0 ]; then pass "handoff-handoff directories acceptance (${_acc_res:-all assertions passed})" else - fail "handoff-handoff directories acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "handoff-handoff directories acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else @@ -2302,11 +2314,14 @@ if [ -f "$_acc_provision" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_provision" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1)" + # `|| true`: an UNGUARDED grep here exits 1 when the harness died before + # printing its RESULT line, which under set -e aborts the suite BEFORE + # fail() runs -- reporting a dead harness as "0 failed". Measured. + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed' "$_acc_out" | tail -1 || true)" if [ "$_acc_rc" -eq 0 ]; then pass "provision-non-interactive sandbox provisioning acceptance (${_acc_res:-all assertions passed})" else - fail "provision-non-interactive sandbox provisioning acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "provision-non-interactive sandbox provisioning acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else @@ -2338,13 +2353,13 @@ if [ -f "$_acc_uds" ]; then SANDY="$SANDY_SCRIPT" bash "$_acc_uds" 2>&1 | tee "$_acc_out" _acc_rc=${PIPESTATUS[0]} set -e - _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed( \([0-9]+ skipped\))?' "$_acc_out" | tail -1)" + _acc_res="$(grep -oE 'RESULT: [0-9]+ passed, [0-9]+ failed( \([0-9]+ skipped\))?' "$_acc_out" | tail -1 || true)" if grep -q 'RESULT: 0 passed, 0 failed' "$_acc_out"; then skip "uds-delivery cross-session delivery acceptance (${_acc_res:-skipped}) — no Claude credentials" elif [ "$_acc_rc" -eq 0 ]; then pass "uds-delivery cross-session delivery acceptance (${_acc_res:-all assertions passed})" else - fail "uds-delivery cross-session delivery acceptance (${_acc_res:-exited $_acc_rc}) — see harness output above" + fail "uds-delivery cross-session delivery acceptance (${_acc_res:-NO RESULT LINE — the harness died before printing one; exited $_acc_rc}) — see harness output above" fi rm -f "$_acc_out" else diff --git a/test/run-tests.sh b/test/run-tests.sh index e769258..76c9166 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -11917,10 +11917,23 @@ _S114_ACC="$(cd "$(dirname "$0")" && pwd)/acceptance-handoff-dirs.sh" _S114_ACC_E="$(awk '/^echo "== E\./{f=1} f' "$_S114_ACC" 2>/dev/null)" check "§114(16f) the acceptance harness still has a relay phase (E)" \ bash -c '[ -n "$1" ]' -- "$_S114_ACC_E" -check "§114(16g) phase E proves criterion 7 end-to-end: a non-executable relay makes --start refuse, with the message, and leaves no container" \ - bash -c 'printf "%s" "$1" | grep -q "does-not-exist.sh" \ - && printf "%s" "$1" | grep -q "A configured relay that cannot start fails the launch" \ - && printf "%s" "$1" | grep -q "no daemon container was left behind"' -- "$_S114_ACC_E" +# The fixture changed shape in 2.2.0: the relay is installed as a manifest +# `entry` now, so criterion 7 is exercised by pointing the entry at a +# non-executable payload file rather than by setting a removed config key. +# Asserted on the PROPERTY the phase must still prove -- refuse, name the +# rule, leave nothing behind -- not on the spelling of the fixture. +# The THIRD property changed shape in 2.2.0 and the change is not cosmetic. +# With the relay installed as a manifest `entry`, the path is image-only, so +# the host cannot stat it and the refusal moves IN-CONTAINER: exit 7 +# (crash-looping) instead of exit 6 (refused before launch), and a container +# does exist. Demanding "no container was left behind" there would assert a +# promise that branch does not make. What criterion 7 requires either way is +# that the session never comes up READY. +check "§114(16g) phase E proves criterion 7 end-to-end: a non-executable relay makes --start refuse, name the fail-the-launch rule, and end with nothing running" \ + bash -c 'printf "%s" "$1" | grep -q "not-executable" \ + && printf "%s" "$1" | grep -q "cannot start fails the" \ + && printf "%s" "$1" | grep -q "CRASH-LOOPING" \ + && printf "%s" "$1" | grep -q "leaves nothing running"' -- "$_S114_ACC_E" check "§114(16h) phase E proves criterion 8 end-to-end: a real headless launch prints the skip line naming the refuse consequence" \ bash -c 'printf "%s" "$1" | grep -q "SANDY_HANDOFF_RELAY not started (headless run); crossSessionInbound will default to refuse"' -- "$_S114_ACC_E" # --- (16i) the harness restart-count pattern must actually match the log line @@ -16942,6 +16955,48 @@ rm -rf "$_S151_DIR" unset _S151_SANDY _S151_DIR _S151_MIG _S151_H _S151_PS _s151_n unset -f _s151_mig +# ============================================================ +echo "" +echo "§152: an acceptance harness that DIES must be recorded as a failure" +# ============================================================ +# run-integration-tests.sh wraps each acceptance harness as: +# +# set +e; bash "$harness"; _acc_rc=$?; set -e +# _acc_res="$(grep -oE RESULT-pattern "$out" | tail -1)" +# if [ "$_acc_rc" -eq 0 ]; then pass ...; else fail ...; fi +# +# If the harness dies BEFORE printing its RESULT line, that grep matches +# nothing and exits 1 -- which under the set -e of the suite aborts the run +# BEFORE fail() is reached. The harness failed, and the summary said +# "0 failed". +# +# MEASURED, not theorised: it happened on a real host run. A harness lost its +# RESULT block in an edit, nineteen assertions failed, and the suite reported +# "1 passed, 0 failed (of 1 run)" before aborting. +# +# This is a STATIC ratchet, and deliberately so: the wrappers are inline in a +# 2000-line script with no extractable seam, so the behavioural equivalent +# would have to re-implement the thing under test -- which is how a guard ends +# up asserting its own copy rather than the product (see §148(12)). +_S152_INT="$(cd "$(dirname "$0")" && pwd)/run-integration-tests.sh" +check "§152(pre) run-integration-tests.sh was found (mutation: a rename makes every check below vacuous)" \ + test -f "$_S152_INT" +_S152_TOTAL="$(grep -c "_acc_res=\"\$(grep -oE 'RESULT:" "$_S152_INT" || true)" +_S152_GUARDED="$(grep -c "_acc_res=\"\$(grep -oE 'RESULT:.*| tail -1 || true)\"" "$_S152_INT" || true)" +check "§152(1) there IS at least one acceptance wrapper to guard (got $_S152_TOTAL)" \ + bash -c '[ "$1" -ge 1 ]' _ "$_S152_TOTAL" +check "§152(2) EVERY acceptance wrapper guards its RESULT grep with '|| true' — an unguarded one aborts the suite under set -e before fail() runs, so a dead harness reports as 0 failed (guarded $_S152_GUARDED of $_S152_TOTAL)" \ + bash -c '[ "$1" = "$2" ]' _ "$_S152_GUARDED" "$_S152_TOTAL" +check "§152(3) every harness the suite wraps still ENDS by printing RESULT and exiting on its FAIL count — the wrapper keys off that line, and a harness that stops printing it is invisible rather than red" \ + bash -c ' + _d="$(cd "$(dirname "$1")" && pwd)" + for _h in "$_d"/acceptance-*.sh; do + [ -f "$_h" ] || continue + grep -q "RESULT: %d passed, %d failed" "$_h" || { echo "no RESULT: $_h"; exit 1; } + grep -q "^\[ \"\$FAIL\" -eq 0 \]" "$_h" || { echo "no exit-on-FAIL: $_h"; exit 1; } + done' _ "$_S152_INT" +unset _S152_INT _S152_TOTAL _S152_GUARDED + # BEGIN SUMMARY # ============================================================ # Summary