#!/usr/bin/env bash # Readiness check: is this machine ready to run rig? Reports and instructs; never fixes. # Usage: check.sh [all | mem [status|push|all|backup|restore]] (all = every detail) # Notes: docs/notes/check.md set -euo pipefail cd "$(dirname "$0")" # `check mem` goes deeper on memory than the summary below: how far allocation # really climbs, and the WSL .wslconfig backup/restore. if [ "${1:-}" = mem ]; then shift exec bash ./mem.sh "${@:-status}" fi # Compact by default: facts only with `all`; problems (!) always print. VERBOSE="" if [ "${1:-}" = all ]; then VERBOSE=1; fi fact() { if [ -n "$VERBOSE" ]; then echo "$@"; fi; } bash ./deps.sh detect ${VERBOSE:+all} source ./lib/config.sh load_config # A /proc/meminfo field in MB, 0 if absent. MEMINFO/OVERCOMMIT_FILE override for testing. mb_of() { awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 } END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}" } # NODE_MB (cost of one node) comes from load_config in lib/config.sh; do not copy it here. # Every running container's working set in MB, tagged with its kind cluster ('-' if none). container_mb() { docker info >/dev/null 2>&1 || return 0 awk -F'\t' ' FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next } { grp = ($1 in cl ? cl[$1] : "-") # A kind cluster'"'"'s local registry is a plain container with no kind # label, named -registry, so on its own it would read as a # stranger. It belongs to its cluster — but only if that cluster exists: # a registry whose cluster is gone is a genuine stray, and says so. if (grp == "-" && $1 ~ /-registry$/) { base = $1; sub(/-registry$/, "", base) if (base in isc) grp = base } split($2, u, " "); v = u[1]; mb = 0 if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 } else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v } else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 } else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 } printf "%d\t%s\t%s\n", mb, grp, $1 } ' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \ <(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null) } port_busy() { if command -v ss >/dev/null 2>&1; then ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1 fi # iproute2 is absent from a minimal Debian, so fall back to procfs rather # than silently reporting everything as free. local hex; hex=$(printf ':%04X' "$1") grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null } echo echo "rig" echo " cluster ${CLUSTER} (${KUBECONTEXT}) profile ${PROFILE_NAME}, ${NODES} node(s), registry ${REGISTRY_MODE}" fact " ingress ${INGRESS_MODE}" if [ ! -f ./.env ]; then fact " .env none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)" fi # ── memory: does this cluster fit right now? Warns; never blocks. ────────── total_mb=$(mb_of MemTotal) avail_mb=$(mb_of MemAvailable) swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) )) overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?') need_mb=$(( NODES * NODE_MB )) rows=$(container_mb) # If our cluster is already up, its memory is already out of MemAvailable: need nothing more. ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows") still_mb=$(( ours_mb > 0 ? 0 : need_mb )) headroom=$(( avail_mb - still_mb )) # The biggest things holding memory, other than this cluster: kind clusters summed, the rest by name. others=$(awk -F'\t' -v c="$CLUSTER" ' $2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 } $2 == "-" { k["container \x27" $3 "\x27"] += $1 } END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn) if [ "$still_mb" -eq 0 ] && [ "$headroom" -ge 512 ]; then printf " memory up, holding %d MB — %d MB headroom for what you deploy\n" "$ours_mb" "$headroom" elif [ "$still_mb" -eq 0 ]; then printf " ! memory up, but only %d MB headroom for anything you deploy\n" "$headroom" elif [ "$headroom" -ge 512 ]; then printf " memory fits — ~%d MB for %s node(s), %d MB headroom\n" "$need_mb" "$NODES" "$headroom" elif [ "$headroom" -ge 0 ]; then printf " ! memory fits, but only %d MB headroom (~%d MB for %s node(s))\n" "$headroom" "$need_mb" "$NODES" else printf " ! memory does not fit: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb" # Two failures with opposite fixes, and telling them apart is the point. if [ "$still_mb" -le "$total_mb" ]; then echo " something else holds it (below) — stopping that helps, a bigger VM would not." if grep -q 'kind cluster' <<< "$others"; then echo " 'make cluster free' stops the other kind clusters. It stops, never deletes." fi else echo " the machine itself is too small: ${total_mb} MB total." fi fi if [ -n "$others" ] && { [ -n "$VERBOSE" ] || [ "$headroom" -lt 512 ]; }; then echo " held elsewhere:" head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }' n_others=$(wc -l <<< "$others") if [ "$n_others" -gt 6 ]; then echo " ... and $((n_others - 6)) more" fi fi if [ "$swap_used_mb" -gt 0 ]; then fact " ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing" fi if [ "$overcommit" = "1" ]; then fact " overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)" fi # ── ports: checked before creation; docker reports a clash only halfway through. ── # Ports held by our own cluster are not clashes. Second grep, not `tr -d ':->'` (a tr range). ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \ --format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \ | grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true) clash=0 list="" mine=0 for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \ "TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do name="${entry%%:*}"; p="${entry#*:}" [ -n "$p" ] || continue list+="$p " if ! port_busy "$p"; then fact "$(printf " %-9s %-6s free" "$name" "$p")" elif echo "$ours" | grep -qx "$p"; then mine=1 fact "$(printf " %-9s %-6s in use by this environment's cluster" "$name" "$p")" else printf " ! ports %s %s IN USE by something else\n" "$name" "$p" clash=1 fi done if [ "$clash" -eq 1 ]; then echo " override it in ctrl/.env (e.g. HTTP_PORT=21080), or rename this directory" elif [ "$mine" -eq 1 ]; then echo " ports ${list% } held by this cluster" else echo " ports ${list% } free" fi fact " derived from the directory name; pin them: bash ctrl/ports.sh persist" # ── what `make cluster up` wires in beside the cluster ───────────────────── REG_NAME="${CLUSTER}-registry" if state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then echo " registry localhost:${REGISTRY_PORT} ($state)" else fact " registry no container yet — 'make cluster up' starts it" fi echo " addons ${ADDONS:-none}" fact " available: $(ls addons/*.sh 2>/dev/null | xargs -n1 basename | sed 's/\.sh$//' | tr '\n' ' ')" # The CA reaches three places and only one of them is ours. Report the other two. if [ -n "${REGISTRY_CA_FILE:-}" ]; then if [ ! -r "$REGISTRY_CA_FILE" ]; then echo " ! CA REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE" else fact " CA $REGISTRY_CA_FILE" host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}" if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then echo " ! CA the HOST docker daemon does not trust it yet:" echo " sudo mkdir -p /etc/docker/certs.d/${host}" echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt" echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)" fi fi fi