219 lines
10 KiB
Bash
Executable File
219 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Readiness check: is this machine ready to run rig?
|
|
#
|
|
# Reports and instructs; never silently fixes anything. Everything it finds is
|
|
# either already fine, or something a human has to decide on.
|
|
#
|
|
# Runs ctrl/deps.sh host detection in a container when Docker is the only thing
|
|
# installed, or directly when the toolchain is already present. Then adds the
|
|
# checks that need this repo's config: profile sanity, CA trust, port clashes.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}"
|
|
|
|
# Host detection. Prefer running it bare — it needs no dependencies beyond
|
|
# coreutils — and fall back to the container only if this shell can't.
|
|
bash ./deps.sh detect
|
|
|
|
# ── repo-level checks ──────────────────────────────────────────────────────
|
|
|
|
source ./lib/config.sh
|
|
load_config
|
|
|
|
echo
|
|
echo "config"
|
|
echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})"
|
|
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
|
|
echo " registry ${REGISTRY_MODE}"
|
|
echo " ingress ${INGRESS_MODE}"
|
|
|
|
if [ ! -f ./.env ]; then
|
|
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
|
fi
|
|
|
|
# ── memory ─────────────────────────────────────────────────────────────────
|
|
#
|
|
# A profile on a box that is already full is the most common first failure, and
|
|
# it presents as pods stuck Pending rather than anything that says "memory".
|
|
# Warns; never blocks. Whether to try anyway is the user's call.
|
|
|
|
# A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist
|
|
# only so the tight and does-not-fit branches can be exercised against another
|
|
# machine's real numbers; in normal use they are the kernel's own files.
|
|
mb_of() {
|
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
|
}
|
|
|
|
# NODE_MB — what one node costs — comes from load_config (lib/config.sh), where
|
|
# its measurement is recorded. It lives there, not here, because the memory tool
|
|
# and every standalone kit need the same number: a copy of it is how rigmini.sh
|
|
# came to say 2 GB per node long after rig had measured 800 MB.
|
|
|
|
# Every running container's working set in MB, tagged with the kind cluster it
|
|
# belongs to ('-' when it is not kind). docker stats reports usage minus page
|
|
# cache, which is what actually competes — cache is handed back under pressure.
|
|
# Counting only kind would hide the usual culprit on a managed workspace, where
|
|
# the memory is held by other containers entirely.
|
|
container_mb() {
|
|
docker info >/dev/null 2>&1 || return 0
|
|
awk -F'\t' '
|
|
FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next }
|
|
{
|
|
grp = ($1 in cl ? cl[$1] : "-")
|
|
# A kind cluster'"'"'s local registry is a plain container with no kind
|
|
# label, named <cluster>-registry, so on its own it would read as a
|
|
# stranger. It belongs to its cluster — but only if that cluster exists:
|
|
# a registry whose cluster is gone is a genuine stray, and says so.
|
|
if (grp == "-" && $1 ~ /-registry$/) {
|
|
base = $1; sub(/-registry$/, "", base)
|
|
if (base in isc) grp = base
|
|
}
|
|
split($2, u, " "); v = u[1]; mb = 0
|
|
if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 }
|
|
else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v }
|
|
else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 }
|
|
else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 }
|
|
printf "%d\t%s\t%s\n", mb, grp, $1
|
|
}
|
|
' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \
|
|
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
|
}
|
|
|
|
total_mb=$(mb_of MemTotal)
|
|
avail_mb=$(mb_of MemAvailable)
|
|
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
|
overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
|
need_mb=$(( NODES * NODE_MB ))
|
|
|
|
rows=$(container_mb)
|
|
# Once this environment's own cluster is running, its real footprint is already
|
|
# out of MemAvailable and the per-node estimate stops being relevant. Subtracting
|
|
# the measurement from the estimate would count the same memory twice, and a
|
|
# running cluster that happens to sit under 800 MB would still "need" the gap.
|
|
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
|
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
|
|
|
echo
|
|
echo "memory"
|
|
printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \
|
|
"$need_mb" "$NODES" "$NODE_MB"
|
|
if [ "$ours_mb" -gt 0 ]; then
|
|
printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER"
|
|
fi
|
|
printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb"
|
|
|
|
# The biggest things holding memory right now, other than this cluster: kind
|
|
# clusters summed per cluster, everything else by container name.
|
|
others=$(awk -F'\t' -v c="$CLUSTER" '
|
|
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
|
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
|
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
|
if [ -n "$others" ]; then
|
|
echo " held elsewhere:"
|
|
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
|
n_others=$(wc -l <<< "$others")
|
|
if [ "$n_others" -gt 6 ]; then
|
|
echo " ... and $((n_others - 6)) more"
|
|
fi
|
|
fi
|
|
|
|
headroom=$(( avail_mb - still_mb ))
|
|
if [ "$still_mb" -eq 0 ]; then
|
|
if [ "$headroom" -ge 512 ]; then
|
|
printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom"
|
|
else
|
|
printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
|
fi
|
|
elif [ "$headroom" -ge 512 ]; then
|
|
printf " fits — %d MB headroom for what you deploy\n" "$headroom"
|
|
elif [ "$headroom" -ge 0 ]; then
|
|
printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom"
|
|
else
|
|
printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
|
# Two failures with opposite fixes, and telling them apart is the point.
|
|
if [ "$still_mb" -le "$total_mb" ]; then
|
|
echo " The machine is big enough; something else is holding memory (above)."
|
|
echo " Stopping that is what helps — a bigger VM would not."
|
|
if grep -q 'kind cluster' <<< "$others"; then
|
|
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
|
fi
|
|
else
|
|
echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total."
|
|
fi
|
|
fi
|
|
|
|
if [ "$swap_used_mb" -gt 0 ]; then
|
|
printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb"
|
|
echo " cluster here to be slow well before it fails"
|
|
fi
|
|
if [ "$overcommit" = "1" ]; then
|
|
echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling."
|
|
echo " A cluster that starts cleanly can still lose processes to the OOM killer."
|
|
fi
|
|
|
|
# The CA reaches three places and only one of them is ours. Report the other two.
|
|
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
|
echo
|
|
echo "registry CA"
|
|
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
|
echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
|
else
|
|
echo " file $REGISTRY_CA_FILE"
|
|
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
|
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
|
echo " ! the HOST docker daemon does not trust it yet:"
|
|
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
|
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
|
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Host ports this environment will try to bind. Checked before cluster creation
|
|
# because docker reports a clash halfway through, as an opaque
|
|
# "failed to bind host port ...: address already in use".
|
|
echo
|
|
echo "ports (block derived from the directory name — see 'make ports')"
|
|
|
|
port_busy() {
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
|
fi
|
|
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
|
# than silently reporting everything as free.
|
|
local hex; hex=$(printf ':%04X' "$1")
|
|
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
|
}
|
|
|
|
# A port held by THIS environment's own cluster is not a clash — it is the thing
|
|
# working. Reporting it as a problem every time the cluster is up would train
|
|
# people to ignore this section, which is the opposite of the point.
|
|
# Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the
|
|
# character RANGE ':' to '>', which does not contain '-', so the trailing dash
|
|
# survives and nothing ever matches.
|
|
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
|
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
|
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
|
|
|
clash=0
|
|
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
|
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
|
name="${entry%%:*}"; p="${entry#*:}"
|
|
[ -n "$p" ] || continue
|
|
if ! port_busy "$p"; then
|
|
printf " %-9s %-6s free\n" "$name" "$p"
|
|
elif echo "$ours" | grep -qx "$p"; then
|
|
printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p"
|
|
else
|
|
printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p"
|
|
clash=1
|
|
fi
|
|
done
|
|
|
|
if [ "$clash" -eq 1 ]; then
|
|
echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080"
|
|
echo " (or rename this directory — the whole block follows the name)"
|
|
fi
|