#!/usr/bin/env bash # Readiness check: is this machine ready to run rig? Reports and instructs; never fixes. # Usage: check.sh [all | mem [status|push|all|backup|restore]] (all = every detail) # Prints the memory budget and what to do next first, then the configuration as it is. # Notes: docs/notes/check.md set -euo pipefail cd "$(dirname "$0")" # `check mem` goes deeper on memory than the summary below: how far allocation # really climbs, and the WSL .wslconfig backup/restore. if [ "${1:-}" = mem ]; then shift exec bash ./mem.sh "${@:-status}" fi # Facts only with `all`; problems (!) always print. VERBOSE="" if [ "${1:-}" = all ]; then VERBOSE=1; fi fact() { if [ -n "$VERBOSE" ]; then echo "$@"; fi; } source ./lib/config.sh load_config # A /proc/meminfo field in MB, 0 if absent. MEMINFO/OVERCOMMIT_FILE override for testing. mb_of() { awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 } END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}" } # NODE_MB (cost of one node) comes from load_config in lib/config.sh; do not copy it here. # Every running container's working set in MB, tagged with its kind cluster ('-' if none). container_mb() { docker info >/dev/null 2>&1 || return 0 awk -F'\t' ' FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next } { grp = ($1 in cl ? cl[$1] : "-") # A kind cluster'"'"'s local registry is a plain container with no kind # label, named -registry, so on its own it would read as a # stranger. It belongs to its cluster — but only if that cluster exists: # a registry whose cluster is gone is a genuine stray, and says so. if (grp == "-" && $1 ~ /-registry$/) { base = $1; sub(/-registry$/, "", base) if (base in isc) grp = base } split($2, u, " "); v = u[1]; mb = 0 if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 } else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v } else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 } else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 } printf "%d\t%s\t%s\n", mb, grp, $1 } ' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \ <(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null) } port_busy() { if command -v ss >/dev/null 2>&1; then ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1 fi # iproute2 is absent from a minimal Debian, so fall back to procfs rather # than silently reporting everything as free. local hex; hex=$(printf ':%04X' "$1") grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null } # Who listens on a port, as "pid name", when ss can tell (only for our own processes). port_owner() { command -v ss >/dev/null 2>&1 || return 0 ss -ltnpH "sport = :$1" 2>/dev/null \ | sed -n 's/.*users:(("\([^"]*\)",pid=\([0-9]*\).*/\2 \1/p' | head -1 } # This environment's Tilt: a tilt started from this ctrl/ (the Makefile runs it there). tilt_pid="" if tilt_owner=$(port_owner "$TILT_PORT") && [ -n "$tilt_owner" ]; then pid=${tilt_owner%% *} if [ "${tilt_owner#* }" = tilt ] && [ "$(readlink "/proc/$pid/cwd" 2>/dev/null)" = "$(pwd -P)" ]; then tilt_pid=$pid fi fi # none | stopped | up | unknown (no docker) cluster_state() { docker info >/dev/null 2>&1 || { echo unknown; return; } local f="label=io.x-k8s.kind.cluster=${CLUSTER}" if [ -n "$(docker ps -q --filter "$f")" ]; then echo up elif [ -n "$(docker ps -aq --filter "$f")" ]; then echo stopped else echo none; fi } state=$(cluster_state) # ── memory budget: what this environment needs, against what the machine has ── # The workload's share comes from the overlay: MEMORY_BUDGET="name=MB ..." in its rig.env, # and MEMORY_BUDGET_TIP, optionally, says how to switch its largest part off. # rig only knows what a node costs. Warns; never blocks. total_mb=$(mb_of MemTotal) avail_mb=$(mb_of MemAvailable) swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) )) overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?') node_mb=$(( NODES * NODE_MB )) budget="" work_mb=0 big_mb=0 big="" for item in ${MEMORY_BUDGET:-}; do name=${item%=*} mb=${item##*=} if [ "$name" = "$item" ] || ! [[ "$mb" =~ ^[0-9]+$ ]]; then echo " ! budget '$item' in MEMORY_BUDGET is not name=MB" continue fi [ "$mb" -gt 0 ] || continue budget+=$(printf ' %-18s %6d MB' "$name" "$mb")$'\n' work_mb=$(( work_mb + mb )) if [ "$mb" -gt "$big_mb" ]; then big_mb=$mb big=$name; fi done need_mb=$(( work_mb + node_mb )) rows=$(container_mb) # If our cluster is already up, its memory is already out of MemAvailable. ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows") if [ "$state" = up ]; then still_mb=$(( need_mb > ours_mb ? need_mb - ours_mb : 0 )) else still_mb=$need_mb fi headroom=$(( avail_mb - still_mb )) # The biggest things holding memory, other than this cluster: kind clusters summed, the rest by name. others=$(awk -F'\t' -v c="$CLUSTER" ' $2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 } $2 == "-" { k["container \x27" $3 "\x27"] += $1 } END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn) echo echo "memory budget (${CLUSTER})" printf '%s' "$budget" if [ -z "$budget" ]; then echo " workload ? (the overlay lists none: MEMORY_BUDGET in its rig.env)" fi printf ' %-18s %6d MB (kind control plane, kube-system; %s node(s))\n' "node overhead" "$node_mb" "$NODES" echo " ---------" printf ' %-18s %6d MB\n' "required" "$need_mb" if [ "$state" = up ]; then printf ' %-18s %6d MB (%s is up)\n' "held by it now" "$ours_mb" "$CLUSTER" fi printf ' %-18s %6d MB\n' "this machine has" "$total_mb" printf ' %-18s %6d MB (right now)\n' "available" "$avail_mb" echo if [ "$headroom" -ge 512 ]; then if [ "$state" = up ]; then echo "fits: ${CLUSTER} holds ${ours_mb} MB of its ${need_mb} MB; ${avail_mb} MB still available — ${headroom} MB headroom." else echo "fits: ${need_mb} MB needed, ${avail_mb} MB available — ${headroom} MB headroom." fi elif [ "$headroom" -ge 0 ]; then echo "! tight: ${still_mb} MB still needed, ${avail_mb} MB available — only ${headroom} MB headroom." else echo "! does not fit: ${still_mb} MB still needed, ${avail_mb} MB available." # Two failures with opposite fixes, and telling them apart is the point. if [ "$need_mb" -le "$total_mb" ]; then echo " Something else holds the memory (below): stopping it helps, a bigger machine would not." if grep -q 'kind cluster' <<< "$others"; then echo " 'make cluster free' stops the other kind clusters. It stops, never deletes." fi else echo " The machine itself is too small: ${total_mb} MB in all. Give it more memory" echo " ('make check mem' has the WSL steps), or switch parts of the workload off." fi fi if [ -n "$others" ] && { [ -n "$VERBOSE" ] || [ "$headroom" -lt 512 ]; }; then echo " held elsewhere:" head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }' n_others=$(wc -l <<< "$others") if [ "$n_others" -gt 6 ]; then echo " ... and $((n_others - 6)) more" fi fi # One part that outweighs the rest is the first thing to switch off on a smaller machine. if [ "$work_mb" -gt 0 ] && [ $(( big_mb * 2 )) -gt "$work_mb" ] && [ "$big_mb" -ge 1024 ]; then echo echo "${big} is ${big_mb} of the ${work_mb} MB the workload needs — more than the rest" echo "combined. If you are not working on it, ${MEMORY_BUDGET_TIP:-switching it off in the overlay} gets that back." fi if [ "$swap_used_mb" -gt 0 ]; then fact " ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing" fi if [ "$overcommit" = "1" ]; then fact " overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)" fi # ── where things stand, and the next step ────────────────────────────────── echo case "$state" in none) echo "${CLUSTER} does not exist yet. 'make cluster up' creates it${ADDONS:+ (with ${ADDONS})}," echo "then 'make tilt' starts the dev loop." ;; stopped) echo "${CLUSTER} exists but is stopped (nothing is lost). Start it again with:" echo " docker start \$(docker ps -aq -f label=io.x-k8s.kind.cluster=${CLUSTER})" ;; up) if [ -n "$tilt_pid" ]; then echo "${CLUSTER} is up, and its Tilt is running (pid ${tilt_pid}): http://localhost:${TILT_PORT}" else echo "${CLUSTER} is up. 'make tilt' starts the dev loop, at http://localhost:${TILT_PORT}." fi echo "Through the gateway: http://localhost:${HTTP_PORT}" ;; *) echo "docker is not reachable, so the cluster's state is unknown (the machine block below says why)." ;; esac # ── the configuration, as it is ──────────────────────────────────────────── echo bash ./deps.sh detect ${VERBOSE:+all} echo echo "config" echo " cluster ${CLUSTER} ($([ "$state" = none ] && echo 'not created' || echo "$state"))" echo " context ${KUBECONTEXT}" if [ -n "${OVERLAY:-}" ]; then echo " overlay $(_abs_from_ctrl "$OVERLAY_DIR")" elif [ -n "$OVERLAY_DIR" ]; then echo " overlay none named — rig's own ${OVERLAY_DIR}" fi if [ -n "$VERBOSE" ] && [ -n "$OVERLAY_DIR" ]; then ov=$(_from_ctrl "$OVERLAY_DIR") pieces="" for piece in rig.env k8s/overlays/dev kind-config.yaml.tpl addons Tiltfile; do [ -e "$ov/$piece" ] && pieces+="$piece " done echo " provides: ${pieces:-nothing rig reads}" fi echo " profile ${PROFILE_NAME}" echo " nodes ${NODES}" echo " kind config ${KIND_CONFIG}" echo " manifests ${MANIFESTS_DIR:-none}" echo " ingress ${INGRESS_MODE}" if [ ! -f ./.env ]; then fact " .env none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)" fi if [ -n "${STALE_MANIFESTS_DIR:-}" ]; then echo " ! .env MANIFESTS_DIR=${STALE_MANIFESTS_DIR} is the old default; rig's examples moved" echo " to examples/ — delete that line from ctrl/.env (ignored until then)" fi # registry.sh points containerd at certs.d, which only works if the kind config says so, # and a kind config is fixed at creation: a project's own file that drops it fails silently. if [ "$REGISTRY_MODE" != none ] && ! grep -q 'config_path *= *"/etc/containerd/certs.d"' "$KIND_CONFIG"; then echo " ! kind ${KIND_CONFIG} lacks the containerd config_path patch that registry mode" echo " '${REGISTRY_MODE}' needs — copy it from ctrl/k8s/kind-config.yaml.tpl" fi # ── ports: checked before creation; docker reports a clash only halfway through. ── # Ports held by our own cluster are not clashes: its node containers, and its registry, # which is a container of its own. Second grep, not `tr -d ':->'` (a tr range). ours=$({ docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" --format '{{.Ports}}' docker ps --filter "name=^${CLUSTER}-registry\$" --format '{{.Ports}}'; } 2>/dev/null \ | tr ',' '\n' | grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true) clash=0 list="" for entry in "http:${HTTP_PORT}" "https:${HTTPS_PORT}" \ "tilt:${TILT_PORT}" "registry:${REGISTRY_PORT}"; do name="${entry%%:*}"; p="${entry#*:}" [ -n "$p" ] || continue list+="$name $p " if ! port_busy "$p"; then fact "$(printf " %-9s %-6s free" "$name" "$p")" elif echo "$ours" | grep -qx "$p"; then fact "$(printf " %-9s %-6s in use by this environment's cluster" "$name" "$p")" elif [ "$name" = tilt ] && [ -n "$tilt_pid" ]; then fact "$(printf " %-9s %-6s in use by this environment's Tilt" "$name" "$p")" else owner=$(port_owner "$p") if [ "$name" = tilt ] && [ "${owner#* }" = tilt ]; then printf " ! ports tilt %s is held by another Tilt (pid %s, in %s)\n" "$p" "${owner%% *}" \ "$(readlink "/proc/${owner%% *}/cwd" 2>/dev/null || echo '?')" echo " stop it (kill ${owner%% *}) before 'make tilt'" else printf " ! ports %s %s IN USE by %s\n" "$name" "$p" "${owner:+pid ${owner}}${owner:-something else}" clash=1 fi fi done echo " ports ${list% }" if [ "$clash" -eq 1 ]; then echo " override it in ctrl/.env (e.g. HTTP_PORT=21080), or rename this directory" fi # ── what `make cluster up` wires in beside the cluster ───────────────────── REG_NAME="${CLUSTER}-registry" if reg_state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then echo " registry ${REGISTRY_MODE} localhost:${REGISTRY_PORT} (${reg_state})" else echo " registry ${REGISTRY_MODE} (no container yet)" fi echo " addons ${ADDONS:-none}" if [ -n "$VERBOSE" ]; then bash ./addons.sh list | sed -n '3,$p' | sed 's/^/ /' fi # The CA reaches three places and only one of them is ours. Report the other two. if [ -n "${REGISTRY_CA_FILE:-}" ]; then if [ ! -r "$REGISTRY_CA_FILE" ]; then echo " ! CA REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE" else fact " CA $REGISTRY_CA_FILE" host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}" if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then echo " ! CA the HOST docker daemon does not trust it yet:" echo " sudo mkdir -p /etc/docker/certs.d/${host}" echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt" echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)" fi fi fi