#!/usr/bin/env bash
# The remote box — the other half of the inventory.
#
# Usage:
#   ./host.sh status|ports|services         announce a batch; run nothing
#   ./host.sh snapshot [--admin]            print the probe that would run
#   ./host.sh snapshot --run [--admin]      run it, save it, then check it
#   ./host.sh check [<snapshot-dir>]        is the box healthy? offline, never ssh
#
# Always the ssh alias, never a hostname: there is no `Host mcrn.ar` block, so
# a bare hostname offers every agent key and trips MaxAuthTries.
#
# snapshot + check is rig's `make check` asked of a REMOTE machine. rig's check
# reports and instructs and never fixes; so does this. rig's check already reads
# /proc/meminfo through an override path so its branches can be exercised
# "against another machine's real numbers" — this is that idea taken the rest
# of the way: collect the other machine's numbers ONCE, over one announced ssh
# session, then answer every question from the saved copy. The analysis can
# then be re-run, tested against fixtures, and compared across days, and it
# never needs the box to be reachable — which, on a box that is failing, it
# may not be.

set -euo pipefail
# Kept before the cd: a snapshot path given on the command line is relative to
# where the user stood, not to ctrl/.
CALLER_PWD="$PWD"
cd "$(dirname "$0")"

source ./lib/config.sh
source ./lib/estate.sh
load_config

guard_alias() {
    if [[ "$HOST" == *.* ]]; then
        echo "HOST='$HOST' is a hostname, not a ~/.ssh/config alias. Refusing." >&2
        exit 1
    fi
}

announce_batch() {
    echo "would run, over 'ssh $HOST':"
    printf '  %s\n' "$@"
    echo
    echo "read-only, and NOT run. Announce-first applies to EACH batch, not"
    echo "once per session — so the commands can be read and"
    echo "learned rather than scrolled past."
}

# Overridable so selftest can point check at an empty root. snapshot still
# refuses to write anywhere git does not confirm is ignored — a root outside
# the repo is refused, not honoured.
SNAP_ROOT="${SNAP_ROOT:-./.snapshots}"

snapshot() {
    local run="" admin=""
    for a in "$@"; do
        case "$a" in
            --run)   run=1 ;;
            --admin) admin=1 ;;
            *) echo "snapshot: unknown argument '$a'" >&2; exit 1 ;;
        esac
    done

    if [ -z "$run" ]; then
        echo "would send, over 'ssh $HOST bash -s', exactly this file — ctrl/remote/probe.sh:"
        echo
        sed 's/^/  | /' ./remote/probe.sh
        if [ -n "$admin" ]; then
            echo
            echo "and, over 'ssh $HOST_ADMIN bash -s', exactly this — ctrl/remote/probe-admin.sh:"
            echo
            sed 's/^/  | /' ./remote/probe-admin.sh
        fi
        echo
        echo "read-only, and NOT run. What you read above is the file that is sent;"
        echo "there is no second copy. To run it:"
        echo "  bash ctrl/host.sh snapshot --run${admin:+ --admin}"
        return 0
    fi

    local dir; dir="$SNAP_ROOT/$ESTATE/$(date -u +%Y%m%dT%H%M%SZ)"
    assert_ignored "$dir/raw"          # before anything is written, not after
    mkdir -p "$dir"

    # Interrupted (Ctrl-C on a slow box), keep what arrived: split it and mark
    # it partial. Unsplit, it was invisible to check — and newer than the last
    # good snapshot, so it hid that one too.
    trap 'echo "  ! interrupted — keeping the partial snapshot" >&2;
          echo "interrupted" > "$dir/PARTIAL"; split_sections "$dir/raw" "$dir";
          [ -f "$dir/raw-admin" ] && split_sections "$dir/raw-admin" "$dir"; exit 130' INT
    echo "ssh $HOST bash -s < ctrl/remote/probe.sh"
    if ! ssh -o BatchMode=yes "$HOST" bash -s < ./remote/probe.sh > "$dir/raw" 2> "$dir/stderr"; then
        echo "  ! ssh $HOST exited non-zero — output so far is kept in $dir" >&2
        sed 's/^/    /' "$dir/stderr" >&2
    fi
    split_sections "$dir/raw" "$dir"

    if [ -n "$admin" ]; then
        echo "ssh $HOST_ADMIN bash -s < ctrl/remote/probe-admin.sh"
        ssh -o BatchMode=yes "$HOST_ADMIN" bash -s < ./remote/probe-admin.sh \
            > "$dir/raw-admin" 2>> "$dir/stderr" || \
            echo "  ! ssh $HOST_ADMIN exited non-zero — see $dir/stderr" >&2
        split_sections "$dir/raw-admin" "$dir"
    fi
    trap - INT
    echo "saved: $dir"
    echo
    check "$dir"
}

# One file per `### name` section. An empty section still gets its file, so
# "the probe ran and found nothing" is distinguishable from "it never ran".
split_sections() {
    awk -v d="$2" '/^### /{ f = d "/" $2; printf "" > f; next } f { print > f }' "$1"
}

# docker prints decimal units (kB, MB, GB). Approximate on purpose: these are
# for choosing between levers, not for accounting.
AWK_MB='function mb(s,   v, u) {
    v = s; u = s; sub(/[A-Za-z]+$/, "", v); sub(/^[0-9.]+/, "", u)
    if (u == "B")               return v / 1e6
    if (u == "kB" || u == "KB") return v / 1e3
    if (u == "MB")              return v + 0
    if (u == "GB")              return v * 1e3
    if (u == "TB")              return v * 1e6
    return 0
}'

check() {
    local f="${1:-}"
    if [ -z "$f" ]; then
        # `|| true`: with no snapshots, ls fails, pipefail fails the pipeline and
        # set -e exits silently — on exactly the path that must explain itself.
        f=$(ls -1d "$SNAP_ROOT/$ESTATE"/*/meta 2>/dev/null | sort | tail -1) || true
        f=${f%/meta}
    fi
    # Exit 1 here, unlike every other report in berth: with no snapshot there is
    # no report, and a check that prints nothing alarming because it looked at
    # nothing is the vacuous pass selftest exists to catch.
    case "$f" in /*|"") ;; *) [ -d "$f" ] || f="$CALLER_PWD/$f" ;; esac
    if [ -z "$f" ] || [ ! -f "$f/meta" ]; then
        echo "no snapshot for estate '$ESTATE'. Take one — read-only, printed before it runs:"
        echo "  make host snapshot"
        exit 1
    fi

    local taken box age_min
    taken=$(sed -n 1p "$f/meta"); box=$(sed -n 2p "$f/meta")
    age_min=$(( ( $(date -u +%s) - $(date -u -d "$taken" +%s 2>/dev/null || date -u +%s) ) / 60 ))
    echo "snapshot  $f"
    echo "          $box, taken $taken"
    [ -f "$f/PARTIAL" ] && echo "  ! PARTIAL — interrupted; sections after the cut are missing, not empty"
    if [ "$age_min" -ge 60 ]; then
        echo "  ! $((age_min / 60))h old — this is how the box WAS. Re-snapshot before acting on it."
    fi

    # ── disk ────────────────────────────────────────────────────────────────
    echo
    echo "disk"
    local rows; rows=$(awk 'NR > 1 && NF >= 6 { p = $5; sub(/%/, "", p); printf "%s\t%d\t%d\t%d\n", $6, p, $4, $2 }' "$f/df")
    local full=0 mnt pct avail total
    while IFS=$'\t' read -r mnt pct avail total; do
        [ -z "$mnt" ] && continue
        local mark="   "; [ "$pct" -ge 85 ] && mark="  !"
        printf "%s %-22s %3d%%   %7d MB free of %7d MB\n" "$mark" "$mnt" "$pct" "$avail" "$total"
        [ "$pct" -ge 95 ] && full=1
    done <<< "$rows"
    awk 'NR > 1 && NF >= 6 { p = $5; sub(/%/, "", p); if (p + 0 >= 85) printf "  ! inodes %s at %s%% — space can be free and writes still fail\n", $6, $5 }' "$f/df_inodes"

    # The filesystem docker writes to: the longest mount point that prefixes
    # /var/lib/docker. Usually /, but a separate /var would otherwise be missed.
    local dmnt dpct davail dtotal
    IFS=$'\t' read -r dmnt dpct davail dtotal < <(
        awk -F'\t' '{ m = $1; if (index("/var/lib/docker", m) == 1 && (m == "/" || substr("/var/lib/docker", length(m) + 1, 1) == "/") && length(m) > best) { best = length(m); row = $0 } } END { print row }' <<< "$rows")

    if [ "$full" = 1 ]; then
        echo
        echo "  A full disk fails WRITES, not reads — so it rarely presents as 'disk':"
        echo "    pages render logged-out but 500 logged-in    (sessions, DB writes)"
        echo "    databases crash-loop on start                (they must write to start)"
        echo "    pushes and uploads die partway through"
    fi

    # ── memory — rig's check.sh mb_of, recomputed here (✖ B8), pointed at the box's /proc/meminfo ──
    echo
    echo "memory"
    mb_of() {
        awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 } END { if (!found) printf "0" }' "$f/meminfo"
    }
    local mtotal mavail swap_used
    mtotal=$(mb_of MemTotal); mavail=$(mb_of MemAvailable)
    swap_used=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
    printf "    available %d MB of %d MB\n" "$mavail" "$mtotal"
    [ "$mtotal" -gt 0 ] && [ $(( mavail * 100 / mtotal )) -lt 10 ] && \
        echo "  ! under 10% available — expect the OOM killer before anything says 'memory'"
    [ "$swap_used" -gt 0 ] && printf "  ! %d MB in swap\n" "$swap_used"

    # ── containers ──────────────────────────────────────────────────────────
    echo
    echo "containers"
    awk -F'\t' 'NF >= 3 { n++; if ($2 == "running") r++; else printf "  ! %-26s %-11s %s\n", $1, $2, $3 }
                END { printf "    %d of %d running\n", r, n }' "$f/containers"
    awk -F'\t' 'NF >= 3 && $3 + 0 > 0 { n = $2; sub(/^\//, "", n); printf "%d\t%s\n", $3, n }' "$f/container_config" \
        | sort -rn | head -5 | awk -F'\t' '{ printf "    restarted %4d×  %s\n", $1, $2 }'
    local unbounded
    unbounded=$(awk -F'\t' 'NF >= 6 && $5 == "json-file" && ($6 == "" || $6 == "<no value>") { n = $2; sub(/^\//, "", n); printf "%s ", n }' "$f/container_config")
    if [ -n "$unbounded" ]; then
        echo "  ! json-file logs with NO max-size — each grows until the disk is full:"
        echo "$unbounded" | fold -s -w 70 | sed 's/^/      /'
    fi

    # ── what would free space, cheapest first ───────────────────────────────
    echo
    echo "space — what would free it, and what it costs  (≈MB; printed, never run)"
    local dfrows; dfrows=$(awk "$AWK_MB"'
        NR > 1 && NF >= 5 {
            line = $0; gsub(/\([0-9]+%\)/, "", line); n = split(line, a, " ")
            t = a[1]; for (i = 2; i <= n - 4; i++) t = t " " a[i]
            printf "%s\t%d\t%d\n", t, mb(a[n - 1]), mb(a[n])
        }' "$f/docker_df")
    rec() { awk -F'\t' -v t="$1" '$1 == t { print $3 }' <<< "$dfrows"; }
    local r_cache r_img r_ctr r_vol j_mb logs_mb=""
    r_cache=$(rec "Build Cache"); r_img=$(rec "Images"); r_ctr=$(rec "Containers"); r_vol=$(rec "Local Volumes")
    j_mb=$(awk '/take up/ { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9.]+[KMGT]$/) { v = $i; u = substr(v, length(v)); sub(/[KMGT]$/, "", v); m = (u == "K" ? v / 1024 : u == "M" ? v : u == "G" ? v * 1024 : v * 1048576); printf "%d", m } }' "$f/journal" 2>/dev/null)

    printf "    %6s  build cache      docker builder prune                 free — nothing running uses it\n" "${r_cache:-?}"
    printf "    %6s  unused images    docker image prune -a                next deploy re-pulls or rebuilds\n" "${r_img:-?}"
    printf "    %6s  stopped ctrs     docker container prune               they cannot be 'start'ed again\n" "${r_ctr:-?}"
    printf "    %6s  host journal     sudo journalctl --vacuum-size=200M   older host logs are gone\n" "${j_mb:-?}"
    if [ -f "$f/admin_container_logs" ]; then
        logs_mb=$(awk '$1 ~ /^[0-9]+$/ { s += $1 } END { print s + 0 }' "$f/admin_container_logs")
        printf "    %6s  container logs   sudo truncate -s 0 <log>             RELIEF, not cure: they refill\n" "$logs_mb"
        awk -F'\t' 'NR == FNR { id[substr($1, 1, 12)] = $2; next }
                    $1 ~ /^[0-9]+$/ && $1 >= 50 { n = $2; sub(/.*containers\//, "", n); k = substr(n, 1, 12); nm = (k in id ? id[k] : k); sub(/^\//, "", nm); printf "              %6d  %s\n", $1, nm }' \
            "$f/container_config" "$f/admin_container_logs"
    else
        echo "         ?  container logs   not sized: /var/lib/docker is root-only"
        echo "                             — 'make host snapshot' with --admin sizes them"
    fi
    printf "    %6s  unused VOLUMES   (no command)                         DATA. Unreferenced is not unwanted —\n" "${r_vol:-?}"
    echo "                             a dangling volume can be the only copy. Decide per volume."
    if [ -s "$f/registry_repos" ] && ! grep -q '^(not found' "$f/registry_repos"; then
        local blobs; blobs=$(awk -F'\t' '$1 == "blobs_mb" { print $2 }' "$f/registry_repos")
        printf "    %6s  registry blobs   registry garbage-collect             a DECISION — see registry below\n" "${blobs:-?}"
    fi

    # rig's check tells apart two failures with opposite fixes ("stopping that
    # is what helps — a bigger VM would not"). The disk has the same pair.
    if [ -n "$dmnt" ] && [ "${dtotal:-0}" -gt 0 ]; then
        local deficit safe
        deficit=$(( (dtotal - davail) - dtotal * 80 / 100 ))
        safe=$(( ${r_cache:-0} + ${r_img:-0} + ${r_ctr:-0} + ${j_mb:-0} + ${logs_mb:-0} ))
        echo
        if [ "$deficit" -le 0 ]; then
            printf "    %s is under 80%% — no deficit.\n" "$dmnt"
        elif [ "$safe" -lt "$deficit" ] && [ -z "$logs_mb" ]; then
            # Container logs are unsized without --admin. Counting that as 0 once
            # printed "Pruning will NOT fix this" on a 15 MB margin — a verdict
            # resting entirely on a number nobody measured.
            printf "    %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable, NOT counting container logs.\n" "$dmnt" "$deficit" "$safe"
            echo "    Undetermined. Logs were not sized, and they are the one lever that can"
            echo "    flip this verdict. Size them first: bash ctrl/host.sh snapshot --run --admin"
        elif [ "$safe" -ge "$deficit" ]; then
            printf "    %s needs ≈%d MB to reach 80%%; ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe"
            echo "    Pruning is enough. A bigger disk would not be needed yet —"
            echo "    and if it refills, the cause is unbounded logs or builds, not size."
        else
            printf "    %s needs ≈%d MB to reach 80%%; only ≈%d MB is reclaimable without touching data.\n" "$dmnt" "$deficit" "$safe"
            echo "    Pruning will NOT fix this. The rest is data — see the biggest directories."
            echo "    Grow the volume (EBS resizes online, then growpart + resize2fs), or move data off."
        fi
    fi

    # What was measured, against what is used. The gap is not a finding about
    # WHAT fills it — only that the unprivileged probe could not see it.
    # Measured against used. Two estimates were wrong before this (2026-09-16):
    # "used − docker − \$HOME" forgot the OS, and called 3 GB of /usr invisible.
    # du / already walks the OS; only root-only trees are missing from it.
    if [ -n "$dmnt" ] && [ -z "$logs_mb" ]; then
        local seen gap
        seen=$(awk -F'\t' '$2 == "/" { print $1 + 0 }' "$f/du_root" 2>/dev/null)
        if [ -n "$seen" ]; then
            gap=$(( (dtotal - davail) - seen ))
            if [ "$gap" -gt 512 ]; then
                echo
                printf "    ≈%d MB used on %s is root-only — du / could not read it.\n" "$gap" "$dmnt"
                echo "    Mostly /var/lib/docker, container logs included. --admin measures it."
            fi
        fi
    fi

    # ── where the space is ──────────────────────────────────────────────────
    echo
    echo "biggest directories  (MB; unprivileged, so root-only trees are absent)"
    awk -F'\t' 'NR <= 8 { printf "    %7d  %s\n", $1, $2 }' "$f/du_root" 2>/dev/null
    if [ -f "$f/admin_docker_dirs" ]; then
        echo "    — /var/lib/docker (root):"
        awk -F'\t' 'NR <= 4 && $1 ~ /^[0-9]+$/ { printf "    %7d  %s\n", $1, $2 }' "$f/admin_docker_dirs"
    fi
    echo "    — under \$HOME:"
    awk -F'\t' 'NR >= 2 && NR <= 8 { printf "    %7d  %s\n", $1, $2 }' "$f/du_home" 2>/dev/null

    # ── registry ────────────────────────────────────────────────────────────
    if [ -s "$f/registry_repos" ] && ! grep -q '^(not found' "$f/registry_repos"; then
        echo
        echo "registry  (answers /v2/ anonymously — so read this list before collecting garbage)"
        awk -F'\t' '$1 != "blobs_mb" { printf "    %s  %3s tag(s)  %s\n", $1, $2, $3 }' "$f/registry_repos"
        echo "    A repository nobody recognises here is a finding, not clutter: with"
        echo "    anonymous push, it may be someone else's image in your deploy path."
    fi

    echo
    echo "reports, never fixes. Every command above is printed; none was run."
}

guard_alias
case "${1:-status}" in
    status)
        announce_batch \
            "uname -a; uptime; df -h /" \
            "docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'" \
            "docker network inspect gateway --format '{{range .Containers}}{{.Name}} {{end}}'" \
            "systemctl list-units --type=service --state=running --no-pager" \
            "systemctl list-timers --no-pager"
        echo
        echo "sudo-only, over 'ssh $HOST_ADMIN' and only where genuinely needed:"
        echo "  wg show           # the WireGuard peers that exist nowhere in the tree"
        ;;
    ports)
        announce_batch "ss -ltnp"
        echo "the estate declares these firewall rules:"
        estate_get "firewall" | python3 -c '
import json,sys
for r in json.load(sys.stdin):
    print("  %-6s %-5s %s" % (r["port"], r.get("proto","tcp"), r.get("desc","")))'
        ;;
    services)
        announce_batch "docker compose -f ~/ppl/gateway/docker-compose.yml ps"
        echo "the estate declares $(estate_services "$TARGET" | grep -c . ) service(s) for target '$TARGET'."
        echo "the gateway compose declares 8. The difference is sibling repos'"
        echo "stacks joining the shared 'gateway' network — intended design, but"
        echo "nothing in the tree lists it. The inventory produces that list."
        ;;
    snapshot) shift; snapshot "$@" ;;
    check)    shift; check "$@" ;;
    *) echo "usage: $0 [status|ports|services|snapshot [--run] [--admin]|check [<dir>]]" >&2; exit 1 ;;
esac