From 907b616d1babc4a58b6ea64dac5033c78fafdda8 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Wed, 16 Sep 2026 19:22:16 -0300 Subject: [PATCH] berth updates --- berth/.gitignore | 5 + berth/Makefile | 2 +- berth/ctrl/host.sh | 253 ++++++++++++++++++++++++++++++- berth/ctrl/lib/estate.sh | 13 ++ berth/ctrl/remote/probe-admin.sh | 17 +++ berth/ctrl/remote/probe.sh | 72 +++++++++ berth/ctrl/selftest.sh | 42 +++++ berth/ctrl/vpn.sh | 12 +- 8 files changed, 399 insertions(+), 17 deletions(-) create mode 100644 berth/ctrl/remote/probe-admin.sh create mode 100644 berth/ctrl/remote/probe.sh diff --git a/berth/.gitignore b/berth/.gitignore index 4736aeb..8e2045e 100644 --- a/berth/.gitignore +++ b/berth/.gitignore @@ -19,3 +19,8 @@ def/ # slash resolves against this file's own directory. `git check-ignore -v` is the # only way to confirm it, and ctrl/vpn.sh refuses to write a key until it does. ctrl/.secrets/ + +# Snapshots of a remote box's state (`host.sh snapshot --run`). Not secrets — +# the probe refuses to read environments, compose files or logs — but machine +# state from production, and never source. Anchored to ctrl/ for the reason above. +ctrl/.snapshots/ diff --git a/berth/Makefile b/berth/Makefile index fb1d9f3..d092cc1 100644 --- a/berth/Makefile +++ b/berth/Makefile @@ -50,7 +50,7 @@ certs: ## TLS [status|verify|renew|push] (default stat # ── the box ──────────────────────────────────────────────────────────────── -host: ## the remote box [status|ports|services] (default status) +host: ## the remote box [status|ports|services|snapshot|check] (default status) bash ctrl/host.sh $(or $(ARGS),status) registry: ## the image registry [status] (default status) diff --git a/berth/ctrl/host.sh b/berth/ctrl/host.sh index 52172aa..5f452a1 100644 --- a/berth/ctrl/host.sh +++ b/berth/ctrl/host.sh @@ -2,11 +2,23 @@ # The remote box — the other half of the inventory. # # Usage: -# ./host.sh status|ports|services +# ./host.sh status|ports|services announce a batch; run nothing +# ./host.sh snapshot [--admin] print the probe that would run +# ./host.sh snapshot --run [--admin] run it, save it, then check it +# ./host.sh check [] is the box healthy? offline, never ssh # -# Announces what it would run over `ssh $HOST` and does not run it. Always the -# ssh alias, never a hostname: there is no `Host mcrn.ar` block, so a bare -# hostname offers every agent key and trips MaxAuthTries. +# Always the ssh alias, never a hostname: there is no `Host mcrn.ar` block, so +# a bare hostname offers every agent key and trips MaxAuthTries. +# +# snapshot + check is rig's `make check` asked of a REMOTE machine. rig's check +# reports and instructs and never fixes; so does this. rig's check already reads +# /proc/meminfo through an override path so its branches can be exercised +# "against another machine's real numbers" — this is that idea taken the rest +# of the way: collect the other machine's numbers ONCE, over one announced ssh +# session, then answer every question from the saved copy. The analysis can +# then be re-run, tested against fixtures, and compared across days, and it +# never needs the box to be reachable — which, on a box that is failing, it +# may not be. set -euo pipefail cd "$(dirname "$0")" @@ -31,6 +43,235 @@ announce_batch() { echo "learned rather than scrolled past." } +# Overridable so selftest can point check at an empty root. snapshot still +# refuses to write anywhere git does not confirm is ignored — a root outside +# the repo is refused, not honoured. +SNAP_ROOT="${SNAP_ROOT:-./.snapshots}" + +snapshot() { + local run="" admin="" + for a in "$@"; do + case "$a" in + --run) run=1 ;; + --admin) admin=1 ;; + *) echo "snapshot: unknown argument '$a'" >&2; exit 1 ;; + esac + done + + if [ -z "$run" ]; then + echo "would send, over 'ssh $HOST bash -s', exactly this file — ctrl/remote/probe.sh:" + echo + sed 's/^/ | /' ./remote/probe.sh + if [ -n "$admin" ]; then + echo + echo "and, over 'ssh $HOST_ADMIN bash -s', exactly this — ctrl/remote/probe-admin.sh:" + echo + sed 's/^/ | /' ./remote/probe-admin.sh + fi + echo + echo "read-only, and NOT run. What you read above is the file that is sent;" + echo "there is no second copy. To run it:" + echo " bash ctrl/host.sh snapshot --run${admin:+ --admin}" + return 0 + fi + + local dir; dir="$SNAP_ROOT/$ESTATE/$(date -u +%Y%m%dT%H%M%SZ)" + assert_ignored "$dir/raw" # before anything is written, not after + mkdir -p "$dir" + + echo "ssh $HOST bash -s < ctrl/remote/probe.sh" + if ! ssh -o BatchMode=yes "$HOST" bash -s < ./remote/probe.sh > "$dir/raw" 2> "$dir/stderr"; then + echo " ! ssh $HOST exited non-zero — output so far is kept in $dir" >&2 + sed 's/^/ /' "$dir/stderr" >&2 + fi + split_sections "$dir/raw" "$dir" + + if [ -n "$admin" ]; then + echo "ssh $HOST_ADMIN bash -s < ctrl/remote/probe-admin.sh" + ssh -o BatchMode=yes "$HOST_ADMIN" bash -s < ./remote/probe-admin.sh \ + > "$dir/raw-admin" 2>> "$dir/stderr" || \ + echo " ! ssh $HOST_ADMIN exited non-zero — see $dir/stderr" >&2 + split_sections "$dir/raw-admin" "$dir" + fi + echo "saved: $dir" + echo + check "$dir" +} + +# One file per `### name` section. An empty section still gets its file, so +# "the probe ran and found nothing" is distinguishable from "it never ran". +split_sections() { + awk -v d="$2" '/^### /{ f = d "/" $2; printf "" > f; next } f { print > f }' "$1" +} + +# docker prints decimal units (kB, MB, GB). Approximate on purpose: these are +# for choosing between levers, not for accounting. +AWK_MB='function mb(s, v, u) { + v = s; u = s; sub(/[A-Za-z]+$/, "", v); sub(/^[0-9.]+/, "", u) + if (u == "B") return v / 1e6 + if (u == "kB" || u == "KB") return v / 1e3 + if (u == "MB") return v + 0 + if (u == "GB") return v * 1e3 + if (u == "TB") return v * 1e6 + return 0 +}' + +check() { + local f="${1:-}" + if [ -z "$f" ]; then + # `|| true`: with no snapshots, ls fails, pipefail fails the pipeline and + # set -e exits silently — on exactly the path that must explain itself. + f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/ 2>/dev/null | sort | tail -1) || true + f=${f%/} + fi + # Exit 1 here, unlike every other report in berth: with no snapshot there is + # no report, and a check that prints nothing alarming because it looked at + # nothing is the vacuous pass selftest exists to catch. + if [ -z "$f" ] || [ ! -f "$f/meta" ]; then + echo "no snapshot for estate '$ESTATE'. Take one — read-only, printed before it runs:" + echo " make host snapshot" + exit 1 + fi + + local taken box age_min + taken=$(sed -n 1p "$f/meta"); box=$(sed -n 2p "$f/meta") + age_min=$(( ( $(date -u +%s) - $(date -u -d "$taken" +%s 2>/dev/null || date -u +%s) ) / 60 )) + echo "snapshot $f" + echo " $box, taken $taken" + if [ "$age_min" -ge 60 ]; then + echo " ! $((age_min / 60))h old — this is how the box WAS. Re-snapshot before acting on it." + fi + + # ── disk ──────────────────────────────────────────────────────────────── + echo + echo "disk" + local rows; rows=$(awk 'NR > 1 && NF >= 6 { p = $5; sub(/%/, "", p); printf "%s\t%d\t%d\t%d\n", $6, p, $4, $2 }' "$f/df") + local full=0 mnt pct avail total + while IFS=$'\t' read -r mnt pct avail total; do + [ -z "$mnt" ] && continue + local mark=" "; [ "$pct" -ge 85 ] && mark=" !" + printf "%s %-22s %3d%% %7d MB free of %7d MB\n" "$mark" "$mnt" "$pct" "$avail" "$total" + [ "$pct" -ge 95 ] && full=1 + done <<< "$rows" + awk 'NR > 1 && NF >= 6 { p = $5; sub(/%/, "", p); if (p + 0 >= 85) printf " ! inodes %s at %s%% — space can be free and writes still fail\n", $6, $5 }' "$f/df_inodes" + + # The filesystem docker writes to: the longest mount point that prefixes + # /var/lib/docker. Usually /, but a separate /var would otherwise be missed. + local dmnt dpct davail dtotal + IFS=$'\t' read -r dmnt dpct davail dtotal < <( + awk -F'\t' '{ m = $1; if (index("/var/lib/docker", m) == 1 && (m == "/" || substr("/var/lib/docker", length(m) + 1, 1) == "/") && length(m) > best) { best = length(m); row = $0 } } END { print row }' <<< "$rows") + + if [ "$full" = 1 ]; then + echo + echo " A full disk fails WRITES, not reads — so it rarely presents as 'disk':" + echo " pages render logged-out but 500 logged-in (sessions, DB writes)" + echo " databases crash-loop on start (they must write to start)" + echo " pushes and uploads die partway through" + fi + + # ── memory — rig's check.sh mb_of, recomputed here (✖ B8), pointed at the box's /proc/meminfo ── + echo + echo "memory" + mb_of() { + awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 } END { if (!found) printf "0" }' "$f/meminfo" + } + local mtotal mavail swap_used + mtotal=$(mb_of MemTotal); mavail=$(mb_of MemAvailable) + swap_used=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) )) + printf " available %d MB of %d MB\n" "$mavail" "$mtotal" + [ "$mtotal" -gt 0 ] && [ $(( mavail * 100 / mtotal )) -lt 10 ] && \ + echo " ! under 10% available — expect the OOM killer before anything says 'memory'" + [ "$swap_used" -gt 0 ] && printf " ! %d MB in swap\n" "$swap_used" + + # ── containers ────────────────────────────────────────────────────────── + echo + echo "containers" + awk -F'\t' 'NF >= 3 { n++; if ($2 == "running") r++; else printf " ! %-26s %-11s %s\n", $1, $2, $3 } + END { printf " %d of %d running\n", r, n }' "$f/containers" + awk -F'\t' 'NF >= 3 && $3 + 0 > 0 { n = $2; sub(/^\//, "", n); printf "%d\t%s\n", $3, n }' "$f/container_config" \ + | sort -rn | head -5 | awk -F'\t' '{ printf " restarted %4d× %s\n", $1, $2 }' + local unbounded + unbounded=$(awk -F'\t' 'NF >= 6 && $5 == "json-file" && ($6 == "" || $6 == "") { n = $2; sub(/^\//, "", n); printf "%s ", n }' "$f/container_config") + if [ -n "$unbounded" ]; then + echo " ! json-file logs with NO max-size — each grows until the disk is full:" + echo "$unbounded" | fold -s -w 70 | sed 's/^/ /' + fi + + # ── what would free space, cheapest first ─────────────────────────────── + echo + echo "space — what would free it, and what it costs (≈MB; printed, never run)" + local dfrows; dfrows=$(awk "$AWK_MB"' + NR > 1 && NF >= 5 { + line = $0; gsub(/\([0-9]+%\)/, "", line); n = split(line, a, " ") + t = a[1]; for (i = 2; i <= n - 4; i++) t = t " " a[i] + printf "%s\t%d\t%d\n", t, mb(a[n - 1]), mb(a[n]) + }' "$f/docker_df") + rec() { awk -F'\t' -v t="$1" '$1 == t { print $3 }' <<< "$dfrows"; } + local r_cache r_img r_ctr r_vol j_mb logs_mb="" + r_cache=$(rec "Build Cache"); r_img=$(rec "Images"); r_ctr=$(rec "Containers"); r_vol=$(rec "Local Volumes") + j_mb=$(awk '/take up/ { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9.]+[KMGT]$/) { v = $i; u = substr(v, length(v)); sub(/[KMGT]$/, "", v); m = (u == "K" ? v / 1024 : u == "M" ? v : u == "G" ? v * 1024 : v * 1048576); printf "%d", m } }' "$f/journal" 2>/dev/null) + + printf " %6s build cache docker builder prune free — nothing running uses it\n" "${r_cache:-?}" + printf " %6s unused images docker image prune -a next deploy re-pulls or rebuilds\n" "${r_img:-?}" + printf " %6s stopped ctrs docker container prune they cannot be 'start'ed again\n" "${r_ctr:-?}" + printf " %6s host journal sudo journalctl --vacuum-size=200M older host logs are gone\n" "${j_mb:-?}" + if [ -f "$f/admin_container_logs" ]; then + logs_mb=$(awk '$1 ~ /^[0-9]+$/ { s += $1 } END { print s + 0 }' "$f/admin_container_logs") + printf " %6s container logs sudo truncate -s 0 RELIEF, not cure: they refill\n" "$logs_mb" + awk -F'\t' 'NR == FNR { id[substr($1, 1, 12)] = $2; next } + $1 ~ /^[0-9]+$/ && $1 >= 50 { n = $2; sub(/.*containers\//, "", n); k = substr(n, 1, 12); nm = (k in id ? id[k] : k); sub(/^\//, "", nm); printf " %6d %s\n", $1, nm }' \ + "$f/container_config" "$f/admin_container_logs" + else + echo " ? container logs not sized: /var/lib/docker is root-only" + echo " — 'make host snapshot' with --admin sizes them" + fi + printf " %6s unused VOLUMES (no command) DATA. Unreferenced is not unwanted —\n" "${r_vol:-?}" + echo " a dangling volume can be the only copy. Decide per volume." + if [ -s "$f/registry_repos" ] && ! grep -q '^(not found' "$f/registry_repos"; then + local blobs; blobs=$(awk -F'\t' '$1 == "blobs_mb" { print $2 }' "$f/registry_repos") + printf " %6s registry blobs registry garbage-collect a DECISION — see registry below\n" "${blobs:-?}" + fi + + # rig's check tells apart two failures with opposite fixes ("stopping that + # is what helps — a bigger VM would not"). The disk has the same pair. + if [ -n "$dmnt" ] && [ "${dtotal:-0}" -gt 0 ]; then + local deficit safe + deficit=$(( (dtotal - davail) - dtotal * 80 / 100 )) + safe=$(( ${r_cache:-0} + ${r_img:-0} + ${r_ctr:-0} + ${j_mb:-0} + ${logs_mb:-0} )) + echo + if [ "$deficit" -le 0 ]; then + printf " %s is under 80%% — no deficit.\n" "$dmnt" + elif [ "$safe" -ge "$deficit" ]; then + printf " %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe" + echo " Pruning is enough. A bigger disk would not be needed yet —" + echo " and if it refills, the cause is unbounded logs or builds, not size." + else + printf " %s needs ≈%d MB to reach 80%%; only ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe" + echo " Pruning will NOT fix this. The rest is data — see the biggest directories." + echo " Grow the volume (EBS resizes online, then growpart + resize2fs), or move data off." + fi + fi + + # ── where the space is ────────────────────────────────────────────────── + echo + echo "biggest directories (MB; unprivileged, so root-only trees are absent)" + awk -F'\t' 'NR <= 8 { printf " %7d %s\n", $1, $2 }' "$f/du_root" 2>/dev/null + echo " — under \$HOME:" + awk -F'\t' 'NR >= 2 && NR <= 8 { printf " %7d %s\n", $1, $2 }' "$f/du_home" 2>/dev/null + + # ── registry ──────────────────────────────────────────────────────────── + if [ -s "$f/registry_repos" ] && ! grep -q '^(not found' "$f/registry_repos"; then + echo + echo "registry (answers /v2/ anonymously — so read this list before collecting garbage)" + awk -F'\t' '$1 != "blobs_mb" { printf " %s %3s tag(s) %s\n", $1, $2, $3 }' "$f/registry_repos" + echo " A repository nobody recognises here is a finding, not clutter: with" + echo " anonymous push, it may be someone else's image in your deploy path." + fi + + echo + echo "reports, never fixes. Every command above is printed; none was run." +} + guard_alias case "${1:-status}" in status) @@ -59,5 +300,7 @@ for r in json.load(sys.stdin): echo "stacks joining the shared 'gateway' network — intended design, but" echo "nothing in the tree lists it. The inventory produces that list." ;; - *) echo "usage: $0 [status|ports|services]" >&2; exit 1 ;; + snapshot) shift; snapshot "$@" ;; + check) shift; check "$@" ;; + *) echo "usage: $0 [status|ports|services|snapshot [--run] [--admin]|check []]" >&2; exit 1 ;; esac diff --git a/berth/ctrl/lib/estate.sh b/berth/ctrl/lib/estate.sh index b0c71bc..d4c397f 100644 --- a/berth/ctrl/lib/estate.sh +++ b/berth/ctrl/lib/estate.sh @@ -145,3 +145,16 @@ service_upstream() { *) printf '%s' "$up" ;; esac } + +# A .gitignore pattern containing a slash anchors to its own directory, so the +# only way to know a path is ignored is to ask git. Used before writing key +# material (vpn.sh) and machine snapshots (host.sh) — refuse, don't hope. +assert_ignored() { + local path="$1" + if ! git check-ignore -q "$path" 2>/dev/null; then + echo "REFUSING: '$path' is not gitignored." >&2 + echo " Writing there would stage it on the next 'git add'." >&2 + echo " Verify with: git check-ignore -v $path" >&2 + exit 1 + fi +} diff --git a/berth/ctrl/remote/probe-admin.sh b/berth/ctrl/remote/probe-admin.sh new file mode 100644 index 0000000..0c522b4 --- /dev/null +++ b/berth/ctrl/remote/probe-admin.sh @@ -0,0 +1,17 @@ +# Read-only probe of what only root can size (`ssh $HOST_ADMIN`, sudo). +# +# Separate from probe.sh on purpose: INDEX §7 — the admin alias only where sudo +# is genuinely required. It is here: /var/lib/docker is root-only, so without +# this a container's json log — the classic way a small disk fills — has no size. +# Sizes only. Nothing is read out of any file. +set -u +section() { printf '\n### %s\n' "$1"; } + +section admin_docker_dirs +sudo -n du -xm --max-depth=1 /var/lib/docker 2>&1 | sort -rn | head -20 + +section admin_container_logs +sudo -n sh -c 'du -m /var/lib/docker/containers/*/*-json.log 2>/dev/null' | sort -rn | head -20 + +section admin_volumes +sudo -n du -xm --max-depth=1 /var/lib/docker/volumes 2>&1 | sort -rn | head -20 diff --git a/berth/ctrl/remote/probe.sh b/berth/ctrl/remote/probe.sh new file mode 100644 index 0000000..3ef791b --- /dev/null +++ b/berth/ctrl/remote/probe.sh @@ -0,0 +1,72 @@ +# Read-only probe of the box, as the unprivileged user (`ssh $HOST`). +# +# host.sh sends THIS FILE, verbatim, over `ssh $HOST bash -s`, and prints this +# file, verbatim, when asked what it would run. The commands you read are the +# commands that run — there is no second copy to drift from. +# +# Never read here, because a snapshot is written to disk on another machine: +# - container environments (bare `docker inspect` prints Env — every +# inspect below names the fields it prints) +# - compose files and .env (inline secrets) +# - container logs (tokens and credentials leak into logs) +# No sudo. What needs root is probe-admin.sh, a separately announced batch. +set -u +section() { printf '\n### %s\n' "$1"; } +REAL='-x tmpfs -x devtmpfs -x overlay -x squashfs -x efivarfs' + +section meta +date -u +%Y-%m-%dT%H:%M:%SZ +hostname +uname -r +uptime + +section df +df -P -B1M $REAL 2>/dev/null + +section df_inodes +df -P -i $REAL 2>/dev/null + +section meminfo +cat /proc/meminfo + +section containers +docker ps -a --format '{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Image}}' 2>&1 + +section container_config +ids=$(docker ps -aq 2>/dev/null) +[ -n "$ids" ] && docker inspect --format '{{.Id}} {{.Name}} {{.RestartCount}} {{.HostConfig.RestartPolicy.Name}} {{.HostConfig.LogConfig.Type}} {{index .HostConfig.LogConfig.Config "max-size"}} {{index .Config.Labels "com.docker.compose.project.working_dir"}}' $ids 2>&1 + +section docker_df +docker system df 2>&1 + +section docker_df_verbose +docker system df -v 2>&1 + +section daemon_json +cat /etc/docker/daemon.json 2>/dev/null || echo '(absent)' + +section journal +journalctl --disk-usage 2>&1 + +section du_home +du -xm --max-depth=3 "$HOME" 2>/dev/null | sort -rn | head -40 + +section du_root +du -xm --max-depth=2 / 2>/dev/null | sort -rn | head -40 + +section registry_repos +# The registry answers /v2/ anonymously (curl -> 200, 2026-09-14), so what is +# stored in it is a question in its own right, not bookkeeping. +wd=$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' registry 2>/dev/null) +r="${wd:-$HOME/ppl/gateway}/registry-data/docker/registry/v2/repositories" +if [ -d "$r" ]; then + find "$r" -type d -name _manifests -prune -print 2>/dev/null | while read -r m; do + repo=${m%/_manifests}; repo=${repo#"$r"/} + newest=$(find "$m" -type f -printf '%TY-%Tm-%Td\n' 2>/dev/null | sort | tail -1) + tags=$(ls "$m/tags" 2>/dev/null | wc -l) + printf '%s\t%s\t%s\n' "${newest:-?}" "$tags" "$repo" + done | sort + printf 'blobs_mb\t%s\n' "$(du -sm "${r%/repositories}/blobs" 2>/dev/null | cut -f1)" +else + echo "(not found at $r)" +fi diff --git a/berth/ctrl/selftest.sh b/berth/ctrl/selftest.sh index 9311aac..a05f4a0 100644 --- a/berth/ctrl/selftest.sh +++ b/berth/ctrl/selftest.sh @@ -195,6 +195,48 @@ check "a roaming peer's port is not the overlay's port" "51820" \ check "the capture warning clears once keys are in" "0" \ "$(bash ./check.sh 2>/dev/null | grep -c 'public keys not captured')" +note "host snapshot + check — rig's check, asked of a remote box" + +# Announce-first made mechanical: without --run nothing reaches ssh or disk. +before="$(ls -1 .snapshots 2>/dev/null | wc -l)" +out="$(bash ./host.sh snapshot 2>&1)" +check "snapshot without --run says it ran nothing" "1" "$(grep -c 'and NOT run' <<< "$out")" +check "snapshot without --run writes nothing" "$before" "$(ls -1 .snapshots 2>/dev/null | wc -l)" +# The printed probe IS the sent probe — no second copy to drift from. +check "what is announced is byte-for-byte what is sent" "same" \ + "$(diff -q <(sed -n 's/^ | //p; s/^ |$//p' <<< "$out" | sed -n "1,$(wc -l < remote/probe.sh)p") remote/probe.sh >/dev/null && echo same || echo differs)" +check "snapshots are gitignored" "yes" \ + "$(cd .. && git check-ignore -q ctrl/.snapshots/x/raw && echo yes || echo no)" + +# A snapshot is written to another machine's disk. Environments, compose files +# and logs carry secrets, so the probes must never read them. Comments excluded: +# probe.sh's header names exactly these things in order to forbid them. +check "the probes never read env, compose, or logs" "0" \ + "$(grep -hvE '^\s*#' remote/probe.sh remote/probe-admin.sh \ + | grep -cE 'docker logs|docker inspect [^-]|docker inspect \$|\.env\b|compose\.ya?ml|printenv|/environ')" + +hfix="$(mktemp -d)" +T="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +printf '%s\nfixture\n' "$T" > "$hfix/meta" +printf 'Filesystem 1048576-blocks Used Available Capacity Mounted\n/dev/x 15731 15714 0 100%% /\n' > "$hfix/df" +printf 'Filesystem Inodes IUsed IFree IUse%% Mounted\n/dev/x 100 10 90 10%% /\n' > "$hfix/df_inodes" +printf 'MemTotal: 2000000 kB\nMemAvailable: 1000000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n' > "$hfix/meminfo" +printf 'mongo\trestarting\tRestarting (1) 2s ago\tmongo:6\n' > "$hfix/containers" +printf 'aaaaaaaaaaaa\t/mongo\t99\talways\tjson-file\t\t/x\n' > "$hfix/container_config" +printf 'TYPE TOTAL ACTIVE SIZE RECLAIMABLE\nImages 3 1 5GB 4GB (80%%)\nBuild Cache 1 0 1GB 1GB\n' > "$hfix/docker_df" +: > "$hfix/journal"; : > "$hfix/du_root"; : > "$hfix/du_home" +hout="$(bash ./host.sh check "$hfix" 2>&1)" +check "a full disk is flagged" "1" "$(grep -c '^ ! / ' <<< "$hout")" +check "a crash-looping container is flagged" "1" "$(grep -c '! mongo .*restarting' <<< "$hout")" +check "an unbounded json log is flagged" "1" "$(grep -c 'NO max-size' <<< "$hout")" +check "reclaimable ≥ deficit → pruning is enough" "1" "$(grep -c 'Pruning is enough' <<< "$hout")" +printf 'TYPE TOTAL ACTIVE SIZE RECLAIMABLE\nImages 3 3 5GB 0B (0%%)\n' > "$hfix/docker_df" +check "reclaimable < deficit → pruning will NOT fix it" "1" \ + "$(bash ./host.sh check "$hfix" 2>&1 | grep -c 'Pruning will NOT fix')" +rm -rf "$hfix" +check "no snapshot is exit 1, not a silent pass" "1" \ + "$(empty="$(mktemp -d)"; SNAP_ROOT="$empty" bash ./host.sh check >/dev/null 2>&1; echo $?; rmdir "$empty")" + note "every STALE entry has a check here" # Not "$0": line 15 cd's into this script's directory, so a relative $0 no diff --git a/berth/ctrl/vpn.sh b/berth/ctrl/vpn.sh index c0ee2b6..9554d56 100644 --- a/berth/ctrl/vpn.sh +++ b/berth/ctrl/vpn.sh @@ -196,17 +196,7 @@ PY return 0 } -# A .gitignore pattern containing a slash anchors to its own directory, so the -# only way to know a path is ignored is to ask git. -assert_ignored() { - local path="$1" - if ! git check-ignore -q "$path" 2>/dev/null; then - echo "REFUSING: '$path' is not gitignored." >&2 - echo " Writing key material there would stage it on the next 'git add'." >&2 - echo " Verify with: git check-ignore -v $path" >&2 - exit 1 - fi -} +# assert_ignored lives in lib/estate.sh — host.sh's snapshots need it too. keygen() { local peer="${1:-}"