#!/usr/bin/env bash # Readiness check: is this machine ready to run rig? # # Reports and instructs; never silently fixes anything. Everything it finds is # either already fine, or something a human has to decide on. # # Runs ctrl/deps.sh host detection in a container when Docker is the only thing # installed, or directly when the toolchain is already present. Then adds the # checks that need this repo's config: profile sanity, CA trust, port clashes. set -euo pipefail cd "$(dirname "$0")" DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}" # Host detection. Prefer running it bare — it needs no dependencies beyond # coreutils — and fall back to the container only if this shell can't. bash ./deps.sh detect # ── repo-level checks ────────────────────────────────────────────────────── source ./lib/config.sh load_config echo echo "config" echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})" echo " cluster ${CLUSTER} (context ${KUBECONTEXT})" echo " registry ${REGISTRY_MODE}" echo " ingress ${INGRESS_MODE}" if [ ! -f ./.env ]; then echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env" fi # ── memory ───────────────────────────────────────────────────────────────── # # A profile on a box that is already full is the most common first failure, and # it presents as pods stuck Pending rather than anything that says "memory". # Warns; never blocks. Whether to try anyway is the user's call. # A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist # only so the tight and does-not-fit branches can be exercised against another # machine's real numbers; in normal use they are the kernel's own files. mb_of() { awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 } END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}" } # What one node costs, measured rather than guessed. On 2026-09-11 a minimal # control-plane node ran at 620 MiB idle and ~728 MiB with a small mock, plus # 16 MiB for the local registry — ~745 MiB of working set. 800 rounds that up, # and agrees with the 800 MB observed independently on a larger rig. Worker # nodes carry no etcd or apiserver and are lighter, so for a multi-node shape # this errs high. It is the cluster alone: whatever you deploy comes on top. NODE_MB=800 # Every running container's working set in MB, tagged with the kind cluster it # belongs to ('-' when it is not kind). docker stats reports usage minus page # cache, which is what actually competes — cache is handed back under pressure. # Counting only kind would hide the usual culprit on a managed workspace, where # the memory is held by other containers entirely. container_mb() { docker info >/dev/null 2>&1 || return 0 awk -F'\t' ' FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next } { grp = ($1 in cl ? cl[$1] : "-") # A kind cluster'"'"'s local registry is a plain container with no kind # label, named -registry, so on its own it would read as a # stranger. It belongs to its cluster — but only if that cluster exists: # a registry whose cluster is gone is a genuine stray, and says so. if (grp == "-" && $1 ~ /-registry$/) { base = $1; sub(/-registry$/, "", base) if (base in isc) grp = base } split($2, u, " "); v = u[1]; mb = 0 if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 } else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v } else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 } else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 } printf "%d\t%s\t%s\n", mb, grp, $1 } ' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \ <(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null) } total_mb=$(mb_of MemTotal) avail_mb=$(mb_of MemAvailable) swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) )) overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?') need_mb=$(( NODES * NODE_MB )) rows=$(container_mb) # Once this environment's own cluster is running, its real footprint is already # out of MemAvailable and the per-node estimate stops being relevant. Subtracting # the measurement from the estimate would count the same memory twice, and a # running cluster that happens to sit under 800 MB would still "need" the gap. ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows") still_mb=$(( ours_mb > 0 ? 0 : need_mb )) echo echo "memory" printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \ "$need_mb" "$NODES" "$NODE_MB" if [ "$ours_mb" -gt 0 ]; then printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER" fi printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb" # The biggest things holding memory right now, other than this cluster: kind # clusters summed per cluster, everything else by container name. others=$(awk -F'\t' -v c="$CLUSTER" ' $2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 } $2 == "-" { k["container \x27" $3 "\x27"] += $1 } END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn) if [ -n "$others" ]; then echo " held elsewhere:" head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }' n_others=$(wc -l <<< "$others") if [ "$n_others" -gt 6 ]; then echo " ... and $((n_others - 6)) more" fi fi headroom=$(( avail_mb - still_mb )) if [ "$still_mb" -eq 0 ]; then if [ "$headroom" -ge 512 ]; then printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom" else printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom" fi elif [ "$headroom" -ge 512 ]; then printf " fits — %d MB headroom for what you deploy\n" "$headroom" elif [ "$headroom" -ge 0 ]; then printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom" else printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb" # Two failures with opposite fixes, and telling them apart is the point. if [ "$still_mb" -le "$total_mb" ]; then echo " The machine is big enough; something else is holding memory (above)." echo " Stopping that is what helps — a bigger VM would not." if grep -q 'kind cluster' <<< "$others"; then echo " 'make cluster free' stops the other kind clusters. It stops, never deletes." fi else echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total." fi fi if [ "$swap_used_mb" -gt 0 ]; then printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb" echo " cluster here to be slow well before it fails" fi if [ "$overcommit" = "1" ]; then echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling." echo " A cluster that starts cleanly can still lose processes to the OOM killer." fi # The CA reaches three places and only one of them is ours. Report the other two. if [ -n "${REGISTRY_CA_FILE:-}" ]; then echo echo "registry CA" if [ ! -r "$REGISTRY_CA_FILE" ]; then echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE" else echo " file $REGISTRY_CA_FILE" host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}" if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then echo " ! the HOST docker daemon does not trust it yet:" echo " sudo mkdir -p /etc/docker/certs.d/${host}" echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt" echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)" fi fi fi # Host ports this environment will try to bind. Checked before cluster creation # because docker reports a clash halfway through, as an opaque # "failed to bind host port ...: address already in use". echo echo "ports (block derived from the directory name — see 'make ports')" port_busy() { if command -v ss >/dev/null 2>&1; then ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1 fi # iproute2 is absent from a minimal Debian, so fall back to procfs rather # than silently reporting everything as free. local hex; hex=$(printf ':%04X' "$1") grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null } # A port held by THIS environment's own cluster is not a clash — it is the thing # working. Reporting it as a problem every time the cluster is up would train # people to ignore this section, which is the opposite of the point. # Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the # character RANGE ':' to '>', which does not contain '-', so the trailing dash # survives and nothing ever matches. ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \ --format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \ | grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true) clash=0 for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \ "TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do name="${entry%%:*}"; p="${entry#*:}" [ -n "$p" ] || continue if ! port_busy "$p"; then printf " %-9s %-6s free\n" "$name" "$p" elif echo "$ours" | grep -qx "$p"; then printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p" else printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p" clash=1 fi done if [ "$clash" -eq 1 ]; then echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080" echo " (or rename this directory — the whole block follows the name)" fi