diff --git a/berth/ctrl/host.sh b/berth/ctrl/host.sh index 5f452a1..c05683f 100644 --- a/berth/ctrl/host.sh +++ b/berth/ctrl/host.sh @@ -21,6 +21,9 @@ # may not be. set -euo pipefail +# Kept before the cd: a snapshot path given on the command line is relative to +# where the user stood, not to ctrl/. +CALLER_PWD="$PWD" cd "$(dirname "$0")" source ./lib/config.sh @@ -79,6 +82,12 @@ snapshot() { assert_ignored "$dir/raw" # before anything is written, not after mkdir -p "$dir" + # Interrupted (Ctrl-C on a slow box), keep what arrived: split it and mark + # it partial. Unsplit, it was invisible to check — and newer than the last + # good snapshot, so it hid that one too. + trap 'echo " ! interrupted — keeping the partial snapshot" >&2; + echo "interrupted" > "$dir/PARTIAL"; split_sections "$dir/raw" "$dir"; + [ -f "$dir/raw-admin" ] && split_sections "$dir/raw-admin" "$dir"; exit 130' INT echo "ssh $HOST bash -s < ctrl/remote/probe.sh" if ! ssh -o BatchMode=yes "$HOST" bash -s < ./remote/probe.sh > "$dir/raw" 2> "$dir/stderr"; then echo " ! ssh $HOST exited non-zero — output so far is kept in $dir" >&2 @@ -93,6 +102,7 @@ snapshot() { echo " ! ssh $HOST_ADMIN exited non-zero — see $dir/stderr" >&2 split_sections "$dir/raw-admin" "$dir" fi + trap - INT echo "saved: $dir" echo check "$dir" @@ -121,12 +131,13 @@ check() { if [ -z "$f" ]; then # `|| true`: with no snapshots, ls fails, pipefail fails the pipeline and # set -e exits silently — on exactly the path that must explain itself. - f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/ 2>/dev/null | sort | tail -1) || true - f=${f%/} + f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/meta 2>/dev/null | sort | tail -1) || true + f=${f%/meta} fi # Exit 1 here, unlike every other report in berth: with no snapshot there is # no report, and a check that prints nothing alarming because it looked at # nothing is the vacuous pass selftest exists to catch. + case "$f" in /*|"") ;; *) [ -d "$f" ] || f="$CALLER_PWD/$f" ;; esac if [ -z "$f" ] || [ ! -f "$f/meta" ]; then echo "no snapshot for estate '$ESTATE'. Take one — read-only, printed before it runs:" echo " make host snapshot" @@ -138,6 +149,7 @@ check() { age_min=$(( ( $(date -u +%s) - $(date -u -d "$taken" +%s 2>/dev/null || date -u +%s) ) / 60 )) echo "snapshot $f" echo " $box, taken $taken" + [ -f "$f/PARTIAL" ] && echo " ! PARTIAL — interrupted; sections after the cut are missing, not empty" if [ "$age_min" -ge 60 ]; then echo " ! $((age_min / 60))h old — this is how the box WAS. Re-snapshot before acting on it." fi @@ -241,6 +253,13 @@ check() { echo if [ "$deficit" -le 0 ]; then printf " %s is under 80%% — no deficit.\n" "$dmnt" + elif [ "$safe" -lt "$deficit" ] && [ -z "$logs_mb" ]; then + # Container logs are unsized without --admin. Counting that as 0 once + # printed "Pruning will NOT fix this" on a 15 MB margin — a verdict + # resting entirely on a number nobody measured. + printf " %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable, NOT counting container logs.\n" "$dmnt" "$deficit" "$safe" + echo " Undetermined. Logs were not sized, and they are the one lever that can" + echo " flip this verdict. Size them first: bash ctrl/host.sh snapshot --run --admin" elif [ "$safe" -ge "$deficit" ]; then printf " %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe" echo " Pruning is enough. A bigger disk would not be needed yet —" @@ -252,10 +271,32 @@ check() { fi fi + # What was measured, against what is used. The gap is not a finding about + # WHAT fills it — only that the unprivileged probe could not see it. + # Measured against used. Two estimates were wrong before this (2026-09-16): + # "used − docker − \$HOME" forgot the OS, and called 3 GB of /usr invisible. + # du / already walks the OS; only root-only trees are missing from it. + if [ -n "$dmnt" ] && [ -z "$logs_mb" ]; then + local seen gap + seen=$(awk -F'\t' '$2 == "/" { print $1 + 0 }' "$f/du_root" 2>/dev/null) + if [ -n "$seen" ]; then + gap=$(( (dtotal - davail) - seen )) + if [ "$gap" -gt 512 ]; then + echo + printf " ≈%d MB used on %s is root-only — du / could not read it.\n" "$gap" "$dmnt" + echo " Mostly /var/lib/docker, container logs included. --admin measures it." + fi + fi + fi + # ── where the space is ────────────────────────────────────────────────── echo echo "biggest directories (MB; unprivileged, so root-only trees are absent)" awk -F'\t' 'NR <= 8 { printf " %7d %s\n", $1, $2 }' "$f/du_root" 2>/dev/null + if [ -f "$f/admin_docker_dirs" ]; then + echo " — /var/lib/docker (root):" + awk -F'\t' 'NR <= 4 && $1 ~ /^[0-9]+$/ { printf " %7d %s\n", $1, $2 }' "$f/admin_docker_dirs" + fi echo " — under \$HOME:" awk -F'\t' 'NR >= 2 && NR <= 8 { printf " %7d %s\n", $1, $2 }' "$f/du_home" 2>/dev/null diff --git a/berth/ctrl/remote/probe-admin.sh b/berth/ctrl/remote/probe-admin.sh index 0c522b4..46ff710 100644 --- a/berth/ctrl/remote/probe-admin.sh +++ b/berth/ctrl/remote/probe-admin.sh @@ -8,10 +8,10 @@ set -u section() { printf '\n### %s\n' "$1"; } section admin_docker_dirs -sudo -n du -xm --max-depth=1 /var/lib/docker 2>&1 | sort -rn | head -20 +sudo -n timeout 60 du -xm --max-depth=1 /var/lib/docker 2>&1 | sort -rn | head -20 section admin_container_logs -sudo -n sh -c 'du -m /var/lib/docker/containers/*/*-json.log 2>/dev/null' | sort -rn | head -20 +sudo -n timeout 60 sh -c 'du -m /var/lib/docker/containers/*/*-json.log 2>/dev/null' | sort -rn | head -20 section admin_volumes -sudo -n du -xm --max-depth=1 /var/lib/docker/volumes 2>&1 | sort -rn | head -20 +sudo -n timeout 60 du -xm --max-depth=1 /var/lib/docker/volumes 2>&1 | sort -rn | head -20 diff --git a/berth/ctrl/remote/probe.sh b/berth/ctrl/remote/probe.sh index 3ef791b..e6f7a12 100644 --- a/berth/ctrl/remote/probe.sh +++ b/berth/ctrl/remote/probe.sh @@ -12,6 +12,11 @@ # No sudo. What needs root is probe-admin.sh, a separately announced batch. set -u section() { printf '\n### %s\n' "$1"; } +# Every step that walks a tree is bounded. `du /` takes ~20s on this box and +# prints nothing until it finishes (its output goes through sort), so a slow or +# sick disk looks exactly like a hang. A step that times out says so and the +# probe moves on: a partial answer beats none. +bounded() { local t="$1"; shift; timeout "$t" "$@"; rc=$?; [ $rc = 124 ] && echo "(timed out after ${t}s: $*)"; return 0; } REAL='-x tmpfs -x devtmpfs -x overlay -x squashfs -x efivarfs' section meta @@ -40,7 +45,7 @@ section docker_df docker system df 2>&1 section docker_df_verbose -docker system df -v 2>&1 +bounded 30 docker system df -v 2>&1 section daemon_json cat /etc/docker/daemon.json 2>/dev/null || echo '(absent)' @@ -49,10 +54,12 @@ section journal journalctl --disk-usage 2>&1 section du_home -du -xm --max-depth=3 "$HOME" 2>/dev/null | sort -rn | head -40 +bounded 30 du -xm --max-depth=3 "$HOME" 2>/dev/null | sort -rn | head -40 section du_root -du -xm --max-depth=2 / 2>/dev/null | sort -rn | head -40 +# Unprivileged, so /var/lib/docker is absent — probe-admin.sh sizes that. What it +# does see matters anyway: the OS under /usr, and /var/log (779 MB, 2026-09-16). +bounded 90 du -xm --max-depth=2 / 2>/dev/null | sort -rn | head -40 section registry_repos # The registry answers /v2/ anonymously (curl -> 200, 2026-09-14), so what is @@ -60,7 +67,7 @@ section registry_repos wd=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' registry 2>/dev/null) r="${wd:-$HOME/ppl/gateway}/registry-data/docker/registry/v2/repositories" if [ -d "$r" ]; then - find "$r" -type d -name _manifests -prune -print 2>/dev/null | while read -r m; do + timeout 30 find "$r" -type d -name _manifests -prune -print 2>/dev/null | while read -r m; do repo=${m%/_manifests}; repo=${repo#"$r"/} newest=$(find "$m" -type f -printf '%TY-%Tm-%Td\n' 2>/dev/null | sort | tail -1) tags=$(ls "$m/tags" 2>/dev/null | wc -l) diff --git a/berth/ctrl/selftest.sh b/berth/ctrl/selftest.sh index a05f4a0..9d4016d 100644 --- a/berth/ctrl/selftest.sh +++ b/berth/ctrl/selftest.sh @@ -231,7 +231,16 @@ check "a crash-looping container is flagged" "1" "$(grep -c '! mongo .*restart check "an unbounded json log is flagged" "1" "$(grep -c 'NO max-size' <<< "$hout")" check "reclaimable ≥ deficit → pruning is enough" "1" "$(grep -c 'Pruning is enough' <<< "$hout")" printf 'TYPE TOTAL ACTIVE SIZE RECLAIMABLE\nImages 3 3 5GB 0B (0%%)\n' > "$hfix/docker_df" -check "reclaimable < deficit → pruning will NOT fix it" "1" \ +# Unknown is not zero. Without --admin, container logs are unsized, and they can +# flip the verdict — 2026-09-16 printed "will NOT fix" on a 15 MB margin that +# rested on counting them as 0. Unsized → undetermined, never a verdict. +check "unsized logs + a shortfall → undetermined, not a verdict" "1 0" \ + "$(o="$(bash ./host.sh check "$hfix" 2>&1)"; echo "$(grep -c Undetermined <<< "$o") $(grep -c 'will NOT fix' <<< "$o")")" +printf '5000\t/var/lib/docker/containers/aaaaaaaaaaaa/aaaaaaaaaaaa-json.log\n' > "$hfix/admin_container_logs" +check "sized logs map back to container names" "1" \ + "$(bash ./host.sh check "$hfix" 2>&1 | grep -cE '^ +5000 mongo$')" +: > "$hfix/admin_container_logs" +check "measured and still short → pruning will NOT fix it" "1" \ "$(bash ./host.sh check "$hfix" 2>&1 | grep -c 'Pruning will NOT fix')" rm -rf "$hfix" check "no snapshot is exit 1, not a silent pass" "1" \