berth updates

berth/ctrl/host.sh

@@ -21,6 +21,9 @@
 # may not be.
 
 set -euo pipefail
+# Kept before the cd: a snapshot path given on the command line is relative to
+# where the user stood, not to ctrl/.
+CALLER_PWD="$PWD"
 cd "$(dirname "$0")"
 
 source ./lib/config.sh
@@ -79,6 +82,12 @@ snapshot() {
     assert_ignored "$dir/raw"          # before anything is written, not after
     mkdir -p "$dir"
 
+    # Interrupted (Ctrl-C on a slow box), keep what arrived: split it and mark
+    # it partial. Unsplit, it was invisible to check — and newer than the last
+    # good snapshot, so it hid that one too.
+    trap 'echo "  ! interrupted — keeping the partial snapshot" >&2;
+          echo "interrupted" > "$dir/PARTIAL"; split_sections "$dir/raw" "$dir";
+          [ -f "$dir/raw-admin" ] && split_sections "$dir/raw-admin" "$dir"; exit 130' INT
     echo "ssh $HOST bash -s < ctrl/remote/probe.sh"
     if ! ssh -o BatchMode=yes "$HOST" bash -s < ./remote/probe.sh > "$dir/raw" 2> "$dir/stderr"; then
         echo "  ! ssh $HOST exited non-zero — output so far is kept in $dir" >&2
@@ -93,6 +102,7 @@ snapshot() {
             echo "  ! ssh $HOST_ADMIN exited non-zero — see $dir/stderr" >&2
         split_sections "$dir/raw-admin" "$dir"
     fi
+    trap - INT
     echo "saved: $dir"
     echo
     check "$dir"
@@ -121,12 +131,13 @@ check() {
     if [ -z "$f" ]; then
         # `|| true`: with no snapshots, ls fails, pipefail fails the pipeline and
         # set -e exits silently — on exactly the path that must explain itself.
-        f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/ 2>/dev/null | sort | tail -1) || true
+        f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/meta 2>/dev/null | sort | tail -1) || true
-        f=${f%/}
+        f=${f%/meta}
     fi
     # Exit 1 here, unlike every other report in berth: with no snapshot there is
     # no report, and a check that prints nothing alarming because it looked at
     # nothing is the vacuous pass selftest exists to catch.
+    case "$f" in /*|"") ;; *) [ -d "$f" ] || f="$CALLER_PWD/$f" ;; esac
     if [ -z "$f" ] || [ ! -f "$f/meta" ]; then
         echo "no snapshot for estate '$ESTATE'. Take one — read-only, printed before it runs:"
         echo "  make host snapshot"
@@ -138,6 +149,7 @@ check() {
     age_min=$(( ( $(date -u +%s) - $(date -u -d "$taken" +%s 2>/dev/null || date -u +%s) ) / 60 ))
     echo "snapshot  $f"
     echo "          $box, taken $taken"
+    [ -f "$f/PARTIAL" ] && echo "  ! PARTIAL — interrupted; sections after the cut are missing, not empty"
     if [ "$age_min" -ge 60 ]; then
         echo "  ! $((age_min / 60))h old — this is how the box WAS. Re-snapshot before acting on it."
     fi
@@ -241,6 +253,13 @@ check() {
         echo
         if [ "$deficit" -le 0 ]; then
             printf "    %s is under 80%% — no deficit.\n" "$dmnt"
+        elif [ "$safe" -lt "$deficit" ] && [ -z "$logs_mb" ]; then
+            # Container logs are unsized without --admin. Counting that as 0 once
+            # printed "Pruning will NOT fix this" on a 15 MB margin — a verdict
+            # resting entirely on a number nobody measured.
+            printf "    %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable, NOT counting container logs.\n" "$dmnt" "$deficit" "$safe"
+            echo "    Undetermined. Logs were not sized, and they are the one lever that can"
+            echo "    flip this verdict. Size them first: bash ctrl/host.sh snapshot --run --admin"
         elif [ "$safe" -ge "$deficit" ]; then
             printf "    %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe"
             echo "    Pruning is enough. A bigger disk would not be needed yet —"
@@ -252,10 +271,32 @@ check() {
         fi
     fi
 
+    # What was measured, against what is used. The gap is not a finding about
+    # WHAT fills it — only that the unprivileged probe could not see it.
+    # Measured against used. Two estimates were wrong before this (2026-09-16):
+    # "used − docker − \$HOME" forgot the OS, and called 3 GB of /usr invisible.
+    # du / already walks the OS; only root-only trees are missing from it.
+    if [ -n "$dmnt" ] && [ -z "$logs_mb" ]; then
+        local seen gap
+        seen=$(awk -F'\t' '$2 == "/" { print $1 + 0 }' "$f/du_root" 2>/dev/null)
+        if [ -n "$seen" ]; then
+            gap=$(( (dtotal - davail) - seen ))
+            if [ "$gap" -gt 512 ]; then
+                echo
+                printf "    ≈%d MB used on %s is root-only — du / could not read it.\n" "$gap" "$dmnt"
+                echo "    Mostly /var/lib/docker, container logs included. --admin measures it."
+            fi
+        fi
+    fi
+
     # ── where the space is ──────────────────────────────────────────────────
     echo
     echo "biggest directories  (MB; unprivileged, so root-only trees are absent)"
     awk -F'\t' 'NR <= 8 { printf "    %7d  %s\n", $1, $2 }' "$f/du_root" 2>/dev/null
+    if [ -f "$f/admin_docker_dirs" ]; then
+        echo "    — /var/lib/docker (root):"
+        awk -F'\t' 'NR <= 4 && $1 ~ /^[0-9]+$/ { printf "    %7d  %s\n", $1, $2 }' "$f/admin_docker_dirs"
+    fi
     echo "    — under \$HOME:"
     awk -F'\t' 'NR >= 2 && NR <= 8 { printf "    %7d  %s\n", $1, $2 }' "$f/du_home" 2>/dev/null
 

berth/ctrl/remote/probe-admin.sh

@@ -8,10 +8,10 @@ set -u
 section() { printf '\n### %s\n' "$1"; }
 
 section admin_docker_dirs
-sudo -n du -xm --max-depth=1 /var/lib/docker 2>&1 | sort -rn | head -20
+sudo -n timeout 60 du -xm --max-depth=1 /var/lib/docker 2>&1 | sort -rn | head -20
 
 section admin_container_logs
-sudo -n sh -c 'du -m /var/lib/docker/containers/*/*-json.log 2>/dev/null' | sort -rn | head -20
+sudo -n timeout 60 sh -c 'du -m /var/lib/docker/containers/*/*-json.log 2>/dev/null' | sort -rn | head -20
 
 section admin_volumes
-sudo -n du -xm --max-depth=1 /var/lib/docker/volumes 2>&1 | sort -rn | head -20
+sudo -n timeout 60 du -xm --max-depth=1 /var/lib/docker/volumes 2>&1 | sort -rn | head -20

berth/ctrl/remote/probe.sh

@@ -12,6 +12,11 @@
 # No sudo. What needs root is probe-admin.sh, a separately announced batch.
 set -u
 section() { printf '\n### %s\n' "$1"; }
+# Every step that walks a tree is bounded. `du /` takes ~20s on this box and
+# prints nothing until it finishes (its output goes through sort), so a slow or
+# sick disk looks exactly like a hang. A step that times out says so and the
+# probe moves on: a partial answer beats none.
+bounded() { local t="$1"; shift; timeout "$t" "$@"; rc=$?; [ $rc = 124 ] && echo "(timed out after ${t}s: $*)"; return 0; }
 REAL='-x tmpfs -x devtmpfs -x overlay -x squashfs -x efivarfs'
 
 section meta
@@ -40,7 +45,7 @@ section docker_df
 docker system df 2>&1
 
 section docker_df_verbose
-docker system df -v 2>&1
+bounded 30 docker system df -v 2>&1
 
 section daemon_json
 cat /etc/docker/daemon.json 2>/dev/null || echo '(absent)'
@@ -49,10 +54,12 @@ section journal
 journalctl --disk-usage 2>&1
 
 section du_home
-du -xm --max-depth=3 "$HOME" 2>/dev/null | sort -rn | head -40
+bounded 30 du -xm --max-depth=3 "$HOME" 2>/dev/null | sort -rn | head -40
 
 section du_root
-du -xm --max-depth=2 / 2>/dev/null | sort -rn | head -40
+# Unprivileged, so /var/lib/docker is absent — probe-admin.sh sizes that. What it
+# does see matters anyway: the OS under /usr, and /var/log (779 MB, 2026-09-16).
+bounded 90 du -xm --max-depth=2 / 2>/dev/null | sort -rn | head -40
 
 section registry_repos
 # The registry answers /v2/ anonymously (curl -> 200, 2026-09-14), so what is
@@ -60,7 +67,7 @@ section registry_repos
 wd=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' registry 2>/dev/null)
 r="${wd:-$HOME/ppl/gateway}/registry-data/docker/registry/v2/repositories"
 if [ -d "$r" ]; then
-    find "$r" -type d -name _manifests -prune -print 2>/dev/null | while read -r m; do
+    timeout 30 find "$r" -type d -name _manifests -prune -print 2>/dev/null | while read -r m; do
         repo=${m%/_manifests}; repo=${repo#"$r"/}
         newest=$(find "$m" -type f -printf '%TY-%Tm-%Td\n' 2>/dev/null | sort | tail -1)
         tags=$(ls "$m/tags" 2>/dev/null | wc -l)

berth/ctrl/selftest.sh

@@ -231,7 +231,16 @@ check "a crash-looping container is flagged"   "1" "$(grep -c '! mongo .*restart
 check "an unbounded json log is flagged"       "1" "$(grep -c 'NO max-size' <<< "$hout")"
 check "reclaimable ≥ deficit → pruning is enough" "1" "$(grep -c 'Pruning is enough' <<< "$hout")"
 printf 'TYPE TOTAL ACTIVE SIZE RECLAIMABLE\nImages 3 3 5GB 0B (0%%)\n' > "$hfix/docker_df"
-check "reclaimable < deficit → pruning will NOT fix it" "1" \
+# Unknown is not zero. Without --admin, container logs are unsized, and they can
+# flip the verdict — 2026-09-16 printed "will NOT fix" on a 15 MB margin that
+# rested on counting them as 0. Unsized → undetermined, never a verdict.
+check "unsized logs + a shortfall → undetermined, not a verdict" "1 0" \
+    "$(o="$(bash ./host.sh check "$hfix" 2>&1)"; echo "$(grep -c Undetermined <<< "$o") $(grep -c 'will NOT fix' <<< "$o")")"
+printf '5000\t/var/lib/docker/containers/aaaaaaaaaaaa/aaaaaaaaaaaa-json.log\n' > "$hfix/admin_container_logs"
+check "sized logs map back to container names" "1" \
+    "$(bash ./host.sh check "$hfix" 2>&1 | grep -cE '^ +5000  mongo$')"
+: > "$hfix/admin_container_logs"
+check "measured and still short → pruning will NOT fix it" "1" \
     "$(bash ./host.sh check "$hfix" 2>&1 | grep -c 'Pruning will NOT fix')"
 rm -rf "$hfix"
 check "no snapshot is exit 1, not a silent pass" "1" \