simpler check and deps messages
This commit is contained in:
@@ -1,12 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Readiness check: is this machine ready to run rig?
|
||||
#
|
||||
# Reports and instructs; never silently fixes anything. Everything it finds is
|
||||
# either already fine, or something a human has to decide on.
|
||||
#
|
||||
# Runs ctrl/deps.sh host detection in a container when Docker is the only thing
|
||||
# installed, or directly when the toolchain is already present. Then adds the
|
||||
# checks that need this repo's config: profile sanity, CA trust, port clashes.
|
||||
# Readiness check: is this machine ready to run rig? Reports and instructs; never fixes.
|
||||
# Usage: check.sh [all | mem [status|push|all|backup|restore]] (all = every detail)
|
||||
# Notes: docs/notes/check.md
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
@@ -17,51 +12,25 @@ if [ "${1:-}" = mem ]; then
|
||||
exec bash ./mem.sh "${@:-status}"
|
||||
fi
|
||||
|
||||
DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}"
|
||||
# Compact by default: facts only with `all`; problems (!) always print.
|
||||
VERBOSE=""
|
||||
if [ "${1:-}" = all ]; then VERBOSE=1; fi
|
||||
fact() { if [ -n "$VERBOSE" ]; then echo "$@"; fi; }
|
||||
|
||||
# Host detection. Prefer running it bare — it needs no dependencies beyond
|
||||
# coreutils — and fall back to the container only if this shell can't.
|
||||
bash ./deps.sh detect
|
||||
|
||||
# ── repo-level checks ──────────────────────────────────────────────────────
|
||||
bash ./deps.sh detect ${VERBOSE:+all}
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
echo
|
||||
echo "config"
|
||||
echo " profile ${PROFILE_NAME} (nodes=${NODES})"
|
||||
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
|
||||
echo " ingress ${INGRESS_MODE}"
|
||||
|
||||
if [ ! -f ./.env ]; then
|
||||
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
||||
fi
|
||||
|
||||
# ── memory ─────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# A profile on a box that is already full is the most common first failure, and
|
||||
# it presents as pods stuck Pending rather than anything that says "memory".
|
||||
# Warns; never blocks. Whether to try anyway is the user's call.
|
||||
|
||||
# A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist
|
||||
# only so the tight and does-not-fit branches can be exercised against another
|
||||
# machine's real numbers; in normal use they are the kernel's own files.
|
||||
# A /proc/meminfo field in MB, 0 if absent. MEMINFO/OVERCOMMIT_FILE override for testing.
|
||||
mb_of() {
|
||||
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||
}
|
||||
|
||||
# NODE_MB — what one node costs — comes from load_config (lib/config.sh), where
|
||||
# its measurement is recorded. It lives there, not here, because the memory tool
|
||||
# and every standalone kit need the same number: a copy of it is how rigmini.sh
|
||||
# came to say 2 GB per node long after rig had measured 800 MB.
|
||||
# NODE_MB (cost of one node) comes from load_config in lib/config.sh; do not copy it here.
|
||||
|
||||
# Every running container's working set in MB, tagged with the kind cluster it
|
||||
# belongs to ('-' when it is not kind). docker stats reports usage minus page
|
||||
# cache, which is what actually competes — cache is handed back under pressure.
|
||||
# Counting only kind would hide the usual culprit on a managed workspace, where
|
||||
# the memory is held by other containers entirely.
|
||||
# Every running container's working set in MB, tagged with its kind cluster ('-' if none).
|
||||
container_mb() {
|
||||
docker info >/dev/null 2>&1 || return 0
|
||||
awk -F'\t' '
|
||||
@@ -87,6 +56,25 @@ container_mb() {
|
||||
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
||||
}
|
||||
|
||||
port_busy() {
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
||||
fi
|
||||
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
||||
# than silently reporting everything as free.
|
||||
local hex; hex=$(printf ':%04X' "$1")
|
||||
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
||||
}
|
||||
|
||||
echo
|
||||
echo "rig"
|
||||
echo " cluster ${CLUSTER} (${KUBECONTEXT}) profile ${PROFILE_NAME}, ${NODES} node(s), registry ${REGISTRY_MODE}"
|
||||
fact " ingress ${INGRESS_MODE}"
|
||||
if [ ! -f ./.env ]; then
|
||||
fact " .env none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)"
|
||||
fi
|
||||
|
||||
# ── memory: does this cluster fit right now? Warns; never blocks. ──────────
|
||||
total_mb=$(mb_of MemTotal)
|
||||
avail_mb=$(mb_of MemAvailable)
|
||||
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
||||
@@ -94,141 +82,105 @@ overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/nul
|
||||
need_mb=$(( NODES * NODE_MB ))
|
||||
|
||||
rows=$(container_mb)
|
||||
# Once this environment's own cluster is running, its real footprint is already
|
||||
# out of MemAvailable and the per-node estimate stops being relevant. Subtracting
|
||||
# the measurement from the estimate would count the same memory twice, and a
|
||||
# running cluster that happens to sit under 800 MB would still "need" the gap.
|
||||
# If our cluster is already up, its memory is already out of MemAvailable: need nothing more.
|
||||
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
||||
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
||||
headroom=$(( avail_mb - still_mb ))
|
||||
|
||||
echo
|
||||
echo "memory"
|
||||
printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \
|
||||
"$need_mb" "$NODES" "$NODE_MB"
|
||||
if [ "$ours_mb" -gt 0 ]; then
|
||||
printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER"
|
||||
fi
|
||||
printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb"
|
||||
|
||||
# The biggest things holding memory right now, other than this cluster: kind
|
||||
# clusters summed per cluster, everything else by container name.
|
||||
# The biggest things holding memory, other than this cluster: kind clusters summed, the rest by name.
|
||||
others=$(awk -F'\t' -v c="$CLUSTER" '
|
||||
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
||||
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
||||
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
||||
if [ -n "$others" ]; then
|
||||
echo " held elsewhere:"
|
||||
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
||||
n_others=$(wc -l <<< "$others")
|
||||
if [ "$n_others" -gt 6 ]; then
|
||||
echo " ... and $((n_others - 6)) more"
|
||||
fi
|
||||
fi
|
||||
|
||||
headroom=$(( avail_mb - still_mb ))
|
||||
if [ "$still_mb" -eq 0 ]; then
|
||||
if [ "$headroom" -ge 512 ]; then
|
||||
printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom"
|
||||
else
|
||||
printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||
fi
|
||||
if [ "$still_mb" -eq 0 ] && [ "$headroom" -ge 512 ]; then
|
||||
printf " memory up, holding %d MB — %d MB headroom for what you deploy\n" "$ours_mb" "$headroom"
|
||||
elif [ "$still_mb" -eq 0 ]; then
|
||||
printf " ! memory up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||
elif [ "$headroom" -ge 512 ]; then
|
||||
printf " fits — %d MB headroom for what you deploy\n" "$headroom"
|
||||
printf " memory fits — ~%d MB for %s node(s), %d MB headroom\n" "$need_mb" "$NODES" "$headroom"
|
||||
elif [ "$headroom" -ge 0 ]; then
|
||||
printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||
printf " ! memory fits, but only %d MB headroom (~%d MB for %s node(s))\n" "$headroom" "$need_mb" "$NODES"
|
||||
else
|
||||
printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
||||
printf " ! memory does not fit: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
||||
# Two failures with opposite fixes, and telling them apart is the point.
|
||||
if [ "$still_mb" -le "$total_mb" ]; then
|
||||
echo " The machine is big enough; something else is holding memory (above)."
|
||||
echo " Stopping that is what helps — a bigger VM would not."
|
||||
echo " something else holds it (below) — stopping that helps, a bigger VM would not."
|
||||
if grep -q 'kind cluster' <<< "$others"; then
|
||||
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
||||
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
||||
fi
|
||||
else
|
||||
echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total."
|
||||
echo " the machine itself is too small: ${total_mb} MB total."
|
||||
fi
|
||||
fi
|
||||
if [ -n "$others" ] && { [ -n "$VERBOSE" ] || [ "$headroom" -lt 512 ]; }; then
|
||||
echo " held elsewhere:"
|
||||
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
||||
n_others=$(wc -l <<< "$others")
|
||||
if [ "$n_others" -gt 6 ]; then
|
||||
echo " ... and $((n_others - 6)) more"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$swap_used_mb" -gt 0 ]; then
|
||||
printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb"
|
||||
echo " cluster here to be slow well before it fails"
|
||||
fact " ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing"
|
||||
fi
|
||||
if [ "$overcommit" = "1" ]; then
|
||||
echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling."
|
||||
echo " A cluster that starts cleanly can still lose processes to the OOM killer."
|
||||
fact " overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)"
|
||||
fi
|
||||
|
||||
# The CA reaches three places and only one of them is ours. Report the other two.
|
||||
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
||||
echo
|
||||
echo "registry CA"
|
||||
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
||||
echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
||||
else
|
||||
echo " file $REGISTRY_CA_FILE"
|
||||
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
||||
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
||||
echo " ! the HOST docker daemon does not trust it yet:"
|
||||
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
||||
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
||||
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Host ports this environment will try to bind. Checked before cluster creation
|
||||
# because docker reports a clash halfway through, as an opaque
|
||||
# "failed to bind host port ...: address already in use".
|
||||
echo
|
||||
echo "ports (block derived from the directory name; pin it: bash ctrl/ports.sh persist)"
|
||||
|
||||
port_busy() {
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
||||
fi
|
||||
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
||||
# than silently reporting everything as free.
|
||||
local hex; hex=$(printf ':%04X' "$1")
|
||||
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
||||
}
|
||||
|
||||
# A port held by THIS environment's own cluster is not a clash — it is the thing
|
||||
# working. Reporting it as a problem every time the cluster is up would train
|
||||
# people to ignore this section, which is the opposite of the point.
|
||||
# Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the
|
||||
# character RANGE ':' to '>', which does not contain '-', so the trailing dash
|
||||
# survives and nothing ever matches.
|
||||
# ── ports: checked before creation; docker reports a clash only halfway through. ──
|
||||
# Ports held by our own cluster are not clashes. Second grep, not `tr -d ':->'` (a tr range).
|
||||
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
||||
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
||||
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
||||
|
||||
clash=0
|
||||
clash=0 list="" mine=0
|
||||
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
||||
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
||||
name="${entry%%:*}"; p="${entry#*:}"
|
||||
[ -n "$p" ] || continue
|
||||
list+="$p "
|
||||
if ! port_busy "$p"; then
|
||||
printf " %-9s %-6s free\n" "$name" "$p"
|
||||
fact "$(printf " %-9s %-6s free" "$name" "$p")"
|
||||
elif echo "$ours" | grep -qx "$p"; then
|
||||
printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p"
|
||||
mine=1
|
||||
fact "$(printf " %-9s %-6s in use by this environment's cluster" "$name" "$p")"
|
||||
else
|
||||
printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p"
|
||||
printf " ! ports %s %s IN USE by something else\n" "$name" "$p"
|
||||
clash=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$clash" -eq 1 ]; then
|
||||
echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080"
|
||||
echo " (or rename this directory — the whole block follows the name)"
|
||||
echo " override it in ctrl/.env (e.g. HTTP_PORT=21080), or rename this directory"
|
||||
elif [ "$mine" -eq 1 ]; then
|
||||
echo " ports ${list% } held by this cluster"
|
||||
else
|
||||
echo " ports ${list% } free"
|
||||
fi
|
||||
fact " derived from the directory name; pin them: bash ctrl/ports.sh persist"
|
||||
|
||||
# What `make cluster up` wires in beside the cluster. Both are set up by it —
|
||||
# listed here only so there is nothing to run just to look.
|
||||
echo
|
||||
echo "registry"
|
||||
bash ./registry.sh status | sed 's/^/ /'
|
||||
# ── what `make cluster up` wires in beside the cluster ─────────────────────
|
||||
REG_NAME="${CLUSTER}-registry"
|
||||
if state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then
|
||||
echo " registry localhost:${REGISTRY_PORT} ($state)"
|
||||
else
|
||||
fact " registry no container yet — 'make cluster up' starts it"
|
||||
fi
|
||||
echo " addons ${ADDONS:-none}"
|
||||
fact " available: $(ls addons/*.sh 2>/dev/null | xargs -n1 basename | sed 's/\.sh$//' | tr '\n' ' ')"
|
||||
|
||||
echo
|
||||
echo "addons"
|
||||
bash ./addons.sh list | sed 's/^/ /'
|
||||
# The CA reaches three places and only one of them is ours. Report the other two.
|
||||
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
||||
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
||||
echo " ! CA REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
||||
else
|
||||
fact " CA $REGISTRY_CA_FILE"
|
||||
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
||||
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
||||
echo " ! CA the HOST docker daemon does not trust it yet:"
|
||||
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
||||
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
||||
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user