187 lines
8.5 KiB
Bash
Executable File
187 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Readiness check: is this machine ready to run rig? Reports and instructs; never fixes.
|
|
# Usage: check.sh [all | mem [status|push|all|backup|restore]] (all = every detail)
|
|
# Notes: docs/notes/check.md
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
# `check mem` goes deeper on memory than the summary below: how far allocation
|
|
# really climbs, and the WSL .wslconfig backup/restore.
|
|
if [ "${1:-}" = mem ]; then
|
|
shift
|
|
exec bash ./mem.sh "${@:-status}"
|
|
fi
|
|
|
|
# Compact by default: facts only with `all`; problems (!) always print.
|
|
VERBOSE=""
|
|
if [ "${1:-}" = all ]; then VERBOSE=1; fi
|
|
fact() { if [ -n "$VERBOSE" ]; then echo "$@"; fi; }
|
|
|
|
bash ./deps.sh detect ${VERBOSE:+all}
|
|
|
|
source ./lib/config.sh
|
|
load_config
|
|
|
|
# A /proc/meminfo field in MB, 0 if absent. MEMINFO/OVERCOMMIT_FILE override for testing.
|
|
mb_of() {
|
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
|
}
|
|
|
|
# NODE_MB (cost of one node) comes from load_config in lib/config.sh; do not copy it here.
|
|
|
|
# Every running container's working set in MB, tagged with its kind cluster ('-' if none).
|
|
container_mb() {
|
|
docker info >/dev/null 2>&1 || return 0
|
|
awk -F'\t' '
|
|
FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next }
|
|
{
|
|
grp = ($1 in cl ? cl[$1] : "-")
|
|
# A kind cluster'"'"'s local registry is a plain container with no kind
|
|
# label, named <cluster>-registry, so on its own it would read as a
|
|
# stranger. It belongs to its cluster — but only if that cluster exists:
|
|
# a registry whose cluster is gone is a genuine stray, and says so.
|
|
if (grp == "-" && $1 ~ /-registry$/) {
|
|
base = $1; sub(/-registry$/, "", base)
|
|
if (base in isc) grp = base
|
|
}
|
|
split($2, u, " "); v = u[1]; mb = 0
|
|
if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 }
|
|
else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v }
|
|
else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 }
|
|
else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 }
|
|
printf "%d\t%s\t%s\n", mb, grp, $1
|
|
}
|
|
' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \
|
|
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
|
}
|
|
|
|
port_busy() {
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
|
fi
|
|
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
|
# than silently reporting everything as free.
|
|
local hex; hex=$(printf ':%04X' "$1")
|
|
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
|
}
|
|
|
|
echo
|
|
echo "rig"
|
|
echo " cluster ${CLUSTER} (${KUBECONTEXT}) profile ${PROFILE_NAME}, ${NODES} node(s), registry ${REGISTRY_MODE}"
|
|
fact " ingress ${INGRESS_MODE}"
|
|
if [ ! -f ./.env ]; then
|
|
fact " .env none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)"
|
|
fi
|
|
|
|
# ── memory: does this cluster fit right now? Warns; never blocks. ──────────
|
|
total_mb=$(mb_of MemTotal)
|
|
avail_mb=$(mb_of MemAvailable)
|
|
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
|
overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
|
need_mb=$(( NODES * NODE_MB ))
|
|
|
|
rows=$(container_mb)
|
|
# If our cluster is already up, its memory is already out of MemAvailable: need nothing more.
|
|
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
|
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
|
headroom=$(( avail_mb - still_mb ))
|
|
|
|
# The biggest things holding memory, other than this cluster: kind clusters summed, the rest by name.
|
|
others=$(awk -F'\t' -v c="$CLUSTER" '
|
|
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
|
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
|
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
|
|
|
if [ "$still_mb" -eq 0 ] && [ "$headroom" -ge 512 ]; then
|
|
printf " memory up, holding %d MB — %d MB headroom for what you deploy\n" "$ours_mb" "$headroom"
|
|
elif [ "$still_mb" -eq 0 ]; then
|
|
printf " ! memory up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
|
elif [ "$headroom" -ge 512 ]; then
|
|
printf " memory fits — ~%d MB for %s node(s), %d MB headroom\n" "$need_mb" "$NODES" "$headroom"
|
|
elif [ "$headroom" -ge 0 ]; then
|
|
printf " ! memory fits, but only %d MB headroom (~%d MB for %s node(s))\n" "$headroom" "$need_mb" "$NODES"
|
|
else
|
|
printf " ! memory does not fit: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
|
# Two failures with opposite fixes, and telling them apart is the point.
|
|
if [ "$still_mb" -le "$total_mb" ]; then
|
|
echo " something else holds it (below) — stopping that helps, a bigger VM would not."
|
|
if grep -q 'kind cluster' <<< "$others"; then
|
|
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
|
fi
|
|
else
|
|
echo " the machine itself is too small: ${total_mb} MB total."
|
|
fi
|
|
fi
|
|
if [ -n "$others" ] && { [ -n "$VERBOSE" ] || [ "$headroom" -lt 512 ]; }; then
|
|
echo " held elsewhere:"
|
|
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
|
n_others=$(wc -l <<< "$others")
|
|
if [ "$n_others" -gt 6 ]; then
|
|
echo " ... and $((n_others - 6)) more"
|
|
fi
|
|
fi
|
|
if [ "$swap_used_mb" -gt 0 ]; then
|
|
fact " ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing"
|
|
fi
|
|
if [ "$overcommit" = "1" ]; then
|
|
fact " overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)"
|
|
fi
|
|
|
|
# ── ports: checked before creation; docker reports a clash only halfway through. ──
|
|
# Ports held by our own cluster are not clashes. Second grep, not `tr -d ':->'` (a tr range).
|
|
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
|
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
|
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
|
|
|
clash=0 list="" mine=0
|
|
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
|
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
|
name="${entry%%:*}"; p="${entry#*:}"
|
|
[ -n "$p" ] || continue
|
|
list+="$p "
|
|
if ! port_busy "$p"; then
|
|
fact "$(printf " %-9s %-6s free" "$name" "$p")"
|
|
elif echo "$ours" | grep -qx "$p"; then
|
|
mine=1
|
|
fact "$(printf " %-9s %-6s in use by this environment's cluster" "$name" "$p")"
|
|
else
|
|
printf " ! ports %s %s IN USE by something else\n" "$name" "$p"
|
|
clash=1
|
|
fi
|
|
done
|
|
if [ "$clash" -eq 1 ]; then
|
|
echo " override it in ctrl/.env (e.g. HTTP_PORT=21080), or rename this directory"
|
|
elif [ "$mine" -eq 1 ]; then
|
|
echo " ports ${list% } held by this cluster"
|
|
else
|
|
echo " ports ${list% } free"
|
|
fi
|
|
fact " derived from the directory name; pin them: bash ctrl/ports.sh persist"
|
|
|
|
# ── what `make cluster up` wires in beside the cluster ─────────────────────
|
|
REG_NAME="${CLUSTER}-registry"
|
|
if state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then
|
|
echo " registry localhost:${REGISTRY_PORT} ($state)"
|
|
else
|
|
fact " registry no container yet — 'make cluster up' starts it"
|
|
fi
|
|
echo " addons ${ADDONS:-none}"
|
|
fact " available: $(ls addons/*.sh 2>/dev/null | xargs -n1 basename | sed 's/\.sh$//' | tr '\n' ' ')"
|
|
|
|
# The CA reaches three places and only one of them is ours. Report the other two.
|
|
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
|
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
|
echo " ! CA REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
|
else
|
|
fact " CA $REGISTRY_CA_FILE"
|
|
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
|
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
|
echo " ! CA the HOST docker daemon does not trust it yet:"
|
|
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
|
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
|
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
|
fi
|
|
fi
|
|
fi
|