#!/usr/bin/env bash # What rig has settled, written down as assertions: one decision per check. # No cluster, docker or network; exits 1 on failure (unlike `make check`). # Usage: make selftest (or: bash ctrl/selftest.sh) # Notes: docs/notes/selftest.md set -uo pipefail # NOT -e: one failing check must not abort the rest cd "$(dirname "$0")" source ./lib/config.sh rc=0 passed=0 check() { # name, expected, actual if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1" passed=$((passed + 1)) else printf ' FAIL %s\n expected: %s\n got: %s\n' "$1" "$2" "$3" rc=1 fi } note() { printf '\n%s\n' "$1"; } # A scratch copy of rig for a check to change freely. local/ (overlays, possibly # someone else's) and def/ (scratch) never ride along, and neither do this # machine's PROFILE/OVERLAY/CLUSTER choices: a check sets what it tests. copy_rig() { # dest-dir mkdir -p "$1" tar -C .. --exclude=./local --exclude=./def -cf - . | tar -C "$1" -xf - if [ -f "$1/ctrl/.env" ]; then sed -i '/^PROFILE=/d; /^OVERLAY=/d; /^CLUSTER=/d; /^MANIFESTS_DIR=/d' "$1/ctrl/.env" fi } # Resolve one key the way every rig script does, in a clean shell so the # caller's exported value is the only thing in play. resolved() { bash -c 'source ./lib/config.sh; load_config >/dev/null 2>&1; printf "%s" "${!1}"' _ "$1" } note "rig needs no profile" # No env.d/ must still resolve and generate a kit; an unknown profile stays an error. NP="$(mktemp -d)" copy_rig "$NP/rig"; rm -rf "$NP/rig/ctrl/env.d" check "no env.d: config resolves" "default" \ "$(cd "$NP/rig/ctrl" && bash -c 'source ./lib/config.sh; load_config >/dev/null && echo "$PROFILE_NAME"' 2>&1)" check "no env.d: the k8s version comes from the pins" "yes" \ "$(cd "$NP/rig/ctrl" && bash -c 'source ./lib/config.sh; load_config >/dev/null && [ -n "$NODE_IMAGE" ] && echo yes' 2>&1)" check "no env.d: ports.sh active works" "8" \ "$(cd "$NP/rig/ctrl" && bash ports.sh active 2>/dev/null | wc -w)" check "no env.d: a kit is generated for the defaults" "yes" \ "$( (cd "$NP/rig/ctrl" && rm -rf ../standalone/*/ && bash standalone.sh write >/dev/null 2>&1) && [ -f "$NP/rig/standalone/default/rigdeps.sh" ] && echo yes || echo no)" check "a profile that does not exist is still an error" "yes" \ "$( (cd "$NP/rig/ctrl" && PROFILE=no-such-profile bash -c 'source ./lib/config.sh; load_config' >/dev/null 2>&1) && echo no || echo yes)" rm -rf "$NP" note "the ports.sh active contract" # ports.sh active is read positionally by the Makefile and Tiltfile: pin field count and order. FACTS="$(bash ports.sh active)" check "active: exactly 8 fields" "8" "$(printf '%s' "$FACTS" | wc -w)" read -r F_CLUSTER F_CTX F_HTTP F_HTTPS F_TILT F_REG F_MANIFESTS F_OVERLAY <<< "$FACTS" check "active: field 2 is kind-" "kind-$F_CLUSTER" "$F_CTX" check "active: fields 3-6 are numeric" "yes" \ "$([[ "$F_HTTP$F_HTTPS$F_TILT$F_REG" =~ ^[0-9]+$ ]] && echo yes || echo no)" # Absolute, or - when there is none: an empty field would shift every later one. check "active: fields 7-8 are absolute paths or -" "yes" \ "$(for f in "$F_MANIFESTS" "$F_OVERLAY"; do case "$f" in -|/*) ;; *) echo no; exit; esac; done; echo yes)" # derive answers a different question and must keep its own shape: it reports # what the directory name implies, ignoring ctrl/.env, so nothing should # configure itself from it. check "derive: still 4 fields, not 7" "4" "$(bash ports.sh derive | wc -w)" note "the caller's env beats the files" # Every key in CONFIG_OVERRIDABLE must lose to the caller's env; the loop follows the list. test_value() { case "$1" in # Picked from what exists, never named: rig must not need any particular # profile, template or pinned version to be present for this to run. PROFILE) config_profiles | head -1 ;; K8S_VERSION) (set -a; source ./versions.env; compgen -v NODE_IMAGE_v | sort -V | head -1 | sed 's/^NODE_IMAGE_//') ;; # An absolute path, as a project passing its own file does. Never equal # to the default, so the check cannot pass by accident. KIND_CONFIG) echo "$PWD/k8s/kind-config.yaml.tpl" ;; *_PORT) echo "19999" ;; CLUSTER) echo "selftest-name" ;; # Named folders must exist, and must not be the default. MANIFESTS_DIR) echo "examples/starter/k8s/base" ;; OVERLAY) echo "examples/data" ;; ADDONS) echo "metallb" ;; *) echo "selftest-sentinel" ;; esac } for key in $CONFIG_OVERRIDABLE; do [ -n "$key" ] || continue want="$(test_value "$key")" if [ -z "$want" ]; then check "precedence: $key has a test value" "yes" "no — add one to test_value()" continue fi got="$(export "$key=$want"; resolved "$key")" check "precedence: caller's $key wins" "$want" "$got" done note "one derivation, not three" # The Makefile must take context/port from ports.sh active, checked on real `make -n` output. # --no-print-directory + grep, not tail -1: under `make selftest` this is a recursive make. MK="$(cd .. && make --no-print-directory -n tilt 2>/dev/null | grep -m1 'tilt ')" check "Makefile: --context comes from active" "$F_CTX" \ "$(printf '%s' "$MK" | sed -n 's/.*--context \([^ ]*\).*/\1/p')" check "Makefile: --port comes from active" "$F_TILT" \ "$(printf '%s' "$MK" | sed -n 's/.*--port \([^ ]*\).*/\1/p')" note "identity follows the folder, safely" # The cluster name is the folder name made a DNS label, derived only in lib/config.sh. TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT mkdir -p "$TMP/My_Proj" cp -r . "$TMP/My_Proj/ctrl" # A pinned CLUSTER in .env would be an override, not a derivation, and this # check is about the derivation. (An OVERLAY would be another derivation.) sed -i '/^CLUSTER=/d; /^OVERLAY=/d' "$TMP/My_Proj/ctrl/.env" 2>/dev/null COPY="$(cd "$TMP/My_Proj/ctrl" && bash ports.sh active)" check "a dir named My_Proj derives a DNS label" "my-proj" "$(awk '{print $1}' <<< "$COPY")" check "and a context to match" "kind-my-proj" "$(awk '{print $2}' <<< "$COPY")" check "a renamed copy gets a DIFFERENT block" "different" \ "$([ "$(awk '{print $3}' <<< "$COPY")" != "$F_HTTP" ] && echo different || echo COLLIDES)" note "ports are stable across versions" # Ports are derived, never stored: a changed derivation moves every existing env's ports. check "derive_port_base rig" "20310" "$(derive_port_base rig)" check "derive_port_base foo" "21690" "$(derive_port_base foo)" check "derive_port_base my-proj" "21030" "$(derive_port_base my-proj)" note "rig stays standalone" # rig must be copyable out of its host project: no references to the host. # The pattern is assembled from fragments so this file does not match itself. # The host project's word for a backing service counts too: rig described its # workload addons with it until they left. local/ holds overlays, which may say anything. HOST_PAT="$(printf '%s' 'sole' 'print' '|\b' 'sp' 'r\b' '|' 'cab' 'inet')" check "no host-project references" "0" \ "$(cd .. && grep -rIl -iE "$HOST_PAT" . --exclude-dir=def --exclude-dir=local 2>/dev/null | wc -l)" note "what runs is an overlay; rig only reads it" # docs/notes/overlay.md. Every check runs in a scratch copy with its own overlay. OV="$TMP/overlay-proof"; copy_rig "$OV/rig" OVR="$OV/rig" mkdir -p "$OVR/local/My_Env/addons" "$OVR/local/My_Env/k8s/prod" "$OVR/ctrl/env.d" printf 'ADDONS="from-profile"\nDATA_NAMESPACE=from-profile\n' > "$OVR/ctrl/env.d/selftest.env" cat > "$OVR/local/My_Env/rig.env" <<'EOF' ADDONS="metallb" DATA_NAMESPACE=from-overlay MANIFESTS_DIR=k8s/prod SELFTEST_SENTINEL=selftest-overlay-sentinel EOF printf 'resources: []\n' > "$OVR/local/My_Env/k8s/prod/kustomization.yaml" printf '#!/usr/bin/env bash\necho "overlay-metallb from $PWD with ${RIG_CTRL:-no RIG_CTRL}"\n' \ > "$OVR/local/My_Env/addons/metallb.sh" in_ov() { (cd "$OVR/ctrl" && "$@"); } ov_key() { # key [env assignments...] local k="$1"; shift in_ov env "$@" bash -c 'source ./lib/config.sh; load_config >/dev/null 2>&1; printf "%s" "${!1}"' _ "$k" } # With nothing named, rig behaves as it did before overlays: same name, ports, addons, nodes. check "no overlay: the cluster, ports and addons of before" "rig kind-rig 20310 20311 20312 20313" \ "$(in_ov bash ports.sh active | awk '{print $1, $2, $3, $4, $5, $6}')" check "no overlay: no addons, one node, rig's own kind config" "|1|./k8s/kind-config.yaml.tpl" \ "$(ov_key ADDONS)|$(ov_key NODES)|$(ov_key KIND_CONFIG)" # The overlay's rig.env sits between the profile and ctrl/.env; the caller beats all. check "rig.env beats the profile" "from-overlay" \ "$(ov_key DATA_NAMESPACE PROFILE=selftest OVERLAY=local/My_Env)" echo 'DATA_NAMESPACE=from-dotenv' >> "$OVR/ctrl/.env" check "ctrl/.env beats rig.env" "from-dotenv" \ "$(ov_key DATA_NAMESPACE PROFILE=selftest OVERLAY=local/My_Env)" sed -i '/^DATA_NAMESPACE=from-dotenv$/d' "$OVR/ctrl/.env" check "the caller beats rig.env" "from-caller" \ "$(ov_key ADDONS OVERLAY=local/My_Env ADDONS=from-caller)" # Identity follows the overlay's folder, so one rig serves several without collisions. check "identity follows the overlay's folder" "my-env kind-my-env" \ "$(in_ov env OVERLAY=local/My_Env bash ports.sh active | awk '{print $1, $2}')" check "paths in rig.env are relative to the overlay" "$OVR/local/My_Env/k8s/prod" \ "$(in_ov env OVERLAY=local/My_Env bash ports.sh active | awk '{print $7}')" check "a named overlay that does not exist is an error" "yes" \ "$(in_ov env OVERLAY=local/nope bash ports.sh active >/dev/null 2>&1 && echo no || echo yes)" printf 'PROFILE=x\n' > "$OV/bad-rig.env"; mkdir -p "$OVR/local/bad"; cp "$OV/bad-rig.env" "$OVR/local/bad/rig.env" check "rig.env may not choose the profile or the overlay" "yes" \ "$(in_ov env OVERLAY=local/bad bash ports.sh active >/dev/null 2>&1 && echo no || echo yes)" # Addons: the overlay's is found before rig's own, and runs from rig's ctrl/. check "an overlay's addon comes before rig's of the same name" \ "overlay-metallb from $OVR/ctrl with $OVR/ctrl" \ "$(in_ov env OVERLAY=local/My_Env ADDONS=metallb bash addons.sh install 2>&1 | grep '^overlay-metallb')" # ctrl/.env is this rig's: a pinned block would follow every overlay. check "ports.sh persist refuses while an overlay is set" "yes" \ "$(in_ov env OVERLAY=local/My_Env bash ports.sh persist >/dev/null 2>&1 && echo no || echo yes)" # make's $(shell) must see an OVERLAY given as a make argument (make < 4.4 does not pass it). check "make -n tilt OVERLAY=... asks for the overlay's context" "kind-data" \ "$(cd .. && make --no-print-directory -n tilt OVERLAY=examples/data 2>/dev/null | grep -m1 'tilt ' | sed -n 's/.*--context \([^ ]*\).*/\1/p')" # rig reads an overlay and never writes into it; its values never reach a committed kit. sum_ov() { (cd "$OVR/local/My_Env" && find . -type f | sort | xargs sha256sum | sha256sum); } before=$(sum_ov) echo 'OVERLAY=local/My_Env' >> "$OVR/ctrl/.env" in_ov bash ports.sh active >/dev/null 2>&1 in_ov bash addons.sh list >/dev/null 2>&1 in_ov bash -c 'source ./lib/config.sh; load_config >/dev/null; render_kind_config >/dev/null' 2>/dev/null in_ov bash standalone.sh write >/dev/null 2>&1 in_ov bash standalone.sh export "$OV/export" >/dev/null 2>&1 check "rig writes nothing into an overlay" "$before" "$(sum_ov)" check "an overlay's values never reach a committed kit" "0" \ "$(grep -rlE 'selftest-overlay-sentinel|local/My_Env' "$OVR/standalone" 2>/dev/null | wc -l)" check "a committed kit holds no path of this machine" "0" \ "$(grep -rlF "$OVR" "$OVR/standalone" 2>/dev/null | wc -l)" note "the Tiltfile hardcodes nothing" # The Tiltfile asks ports.sh for its context; a literal kind- would undo that. check "no literal kind-" "0" "$(grep -cE "['\"]kind-[a-z0-9]" Tiltfile)" check "guards on the variable" "1" "$(grep -c 'allow_k8s_contexts(CTX)' Tiltfile)" check "asks ports.sh for facts" "1" "$(grep -c "local('bash ports.sh active'" Tiltfile)" check "hands over to the overlay's Tiltfile" "1" "$(grep -c "include(OVERLAY + '/Tiltfile')" Tiltfile)" note "standalone kits are generated, current, and call only real verbs" # A kit left stale by a change to rig fails here, not on another machine. check "every kit matches what rig generates now" "yes" \ "$(bash standalone.sh check >/dev/null 2>&1 && echo yes || echo "no — run make standalone")" # Every kit Makefile target must call a verb its script's own dispatch accepts. verbs_of() { sed -n '/^case "\$cmd" in/,/^esac/p' "$1" | grep -oE '^ [a-z]+\)' | tr -d ' )' } kits=0 for mk in ../standalone/*/Makefile; do [ -f "$mk" ] || continue kit=$(dirname "$mk"); kits=$((kits + 1)) for target in $(grep -oE '^[a-z][a-z-]*:' "$mk" | tr -d ':' | grep -vx help); do line="$(make --no-print-directory -s -n -f "$mk" "$target" 2>/dev/null | head -1)" script=$(basename "$(printf '%s' "$line" | awk '{print $2}')") verb=$(printf '%s' "$line" | awk '{print $NF}') check "$(basename "$kit"): make $target -> $script $verb, a verb it accepts" "yes" \ "$(verbs_of "$kit/$script" | grep -qx "$verb" && echo yes || echo "no: '$verb'")" done check "$(basename "$kit"): no \`mini\` target, which already means minimal footprint" "0" \ "$(grep -cE '^mini:' "$mk")" done check "there is a kit for every profile" "$(config_profiles | wc -l)" "$kits" # An export carries this machine's choices but never its credentials; committed kits carry neither. # Proven with sentinel values in a scratch copy, since the real ctrl/.env may leave them empty. SX="$TMP/export-proof"; copy_rig "$SX/rig" mkdir -p "$SX/selftest-sentinel-choice/overlays/dev" # a named MANIFESTS_DIR must exist cat >> "$SX/rig/ctrl/.env" <<'EOF' REGISTRY_USER=selftest-sentinel-user REGISTRY_PASSWORD=selftest-sentinel-password MANIFESTS_DIR=../selftest-sentinel-choice/overlays/dev EOF ( cd "$SX/rig/ctrl" && bash standalone.sh export "$SX/out" >/dev/null 2>&1 ) count_in() { grep -rcF -- "$1" "$2" 2>/dev/null | awk -F: '{s+=$2} END{print s+0}'; } check "export: carries this machine's choices" "yes" \ "$([ "$(count_in selftest-sentinel-choice "$SX/out")" -gt 0 ] && echo yes || echo no)" check "export: carries no credential" "0" \ "$(( $(count_in selftest-sentinel-user "$SX/out") + $(count_in selftest-sentinel-password "$SX/out") ))" check "per-profile kits: carry neither, whatever this machine has" "0" \ "$( (cd "$SX/rig/ctrl" && source ./lib/config.sh && for p in $(config_profiles); do config_snapshot "$p"; done) \ | grep -cE 'selftest-sentinel-(choice|user|password)')" check "export: refuses to write inside the repository" "yes" \ "$( (bash standalone.sh export ../standalone/selftest-mine >/dev/null 2>&1) && echo no || echo yes)" note "rig's addons apply verified files, never URLs" # The offline profile must need no network for manifests: each addon asks deps.sh for a # pinned manifest, verified on disk (versions.md). Their images still need preloading. check "no rig addon applies a URL" "0" \ "$(cat addons/*.sh | grep -cE 'apply -f "?https?://')" check "every manifest an addon asks for is pinned with a sum" "" \ "$(for n in $(grep -ohE 'deps\.sh manifest [A-Z_]+' addons/*.sh | awk '{print $3}' | sort -u); do grep -q "^${n}_MANIFEST_URL=" versions.env && grep -q "^${n}_MANIFEST_SHA256=[0-9a-f]\{64\}$" versions.env \ || printf '%s ' "$n"; done)" note "withdrawn stays withdrawn (STALE.md)" # One check per entry; the reasoning is in STALE.md, not here. check "✖ S1 rig's Tiltfile has no Images section of its own" "0" "$(grep -c '^# ── Images' Tiltfile)" check "✖ S2 local/ is where overlays live, and ignored" "yes" \ "$(grep -qx '/local/' ../.gitignore && echo yes || echo no)" # Patterns assembled from fragments so this file does not match itself. COPIES_PAT="$(printf '%s' 'ac' 'me-rig|ac' 'mebank')" HOUSE_PAT="$(printf '%s' 'semes' 'ter|local' '\.ar\b')" check "✖ S2 no example environment name from the copies era" "0" \ "$(cd .. && grep -rIlE "$COPIES_PAT" . --exclude-dir=def --exclude-dir=local --exclude=STALE.md 2>/dev/null | wc -l)" check "✖ S3 ctrl/addons makes the cluster work, nothing more" "cert-manager metallb metrics-server" \ "$(ls addons/ | sed 's/\.sh$//' | sort | xargs)" check "✖ S3 versions.env pins no workload image" "0" \ "$(grep -cE '^(POSTGRES|REDIS|AIRFLOW)_IMAGE=' versions.env)" check "✖ S4 no namespace named after the cluster" "0" "$(grep -c "CLUSTER + ':namespace'" Tiltfile)" check "✖ S5 rig's examples left ctrl/k8s" "no" "$([ -d k8s/overlays ] && echo yes || echo no)" check "✖ S5 .env.example does not pin MANIFESTS_DIR" "0" "$(grep -c '^MANIFESTS_DIR=' .env.example)" check "✖ S6 no client or data example profile" "0" \ "$(ls env.d/ | grep -cE '^(client|data)\.')" check "✖ S7 no house path or host name in rig" "0" \ "$(cd .. && grep -rIlE "$HOUSE_PAT" . --exclude-dir=def --exclude-dir=local --exclude=STALE.md 2>/dev/null | wc -l)" note "the dev loop parses — needs tilt and kubectl, not a cluster" # A throwaway kubeconfig with kind-named entries (Tilt trusts kind contexts) and a # kubectl that swallows `apply`: the Tiltfile evaluates for real, nothing is contacted. # The second run is a copy under another name, the case that once failed at load. if ! command -v tilt >/dev/null || ! command -v kubectl >/dev/null; then printf ' skip tilt or kubectl is not installed\n' else FK="$TMP/fake-kube"; mkdir -p "$FK" real_kubectl=$(command -v kubectl) printf '#!/usr/bin/env bash\nfor a in "$@"; do [ "$a" = apply ] && { cat >/dev/null; exit 0; }; done\nexec %q "$@"\n' \ "$real_kubectl" > "$FK/kubectl" chmod +x "$FK/kubectl" parses() { # cluster-name [env...] -> the manifests Tilt would deploy, or the error local name="$1"; shift cat > "$FK/kubeconfig" < "$FK/out.json" 2> "$FK/err" \ && grep -o '"Name": *"[^"]*"' "$FK/out.json" | sed 's/.*"\([^"]*\)"$/\1/' | sort -u | xargs \ || grep -m1 -iE 'error|no object' "$FK/err" } check "the starter overlay parses" "example-service infra uncategorized" "$(parses rig)" check "and under another name" "example-service infra uncategorized" \ "$(parses selftest-copy CLUSTER=selftest-copy)" check "the data overlay parses" "items-api uncategorized" "$(parses data OVERLAY=examples/data)" fi note "the examples are overlays that work as shipped" # They are what a real overlay is copied from, so they must at least parse. bad="" for f in ../examples/*/addons/*.sh; do [ -e "$f" ] && { bash -n "$f" 2>/dev/null || bad+="$f "; }; done check "every example addon parses" "" "$bad" if command -v python3 >/dev/null; then bad="" for f in ../examples/*/dags/*.py; do [ -e "$f" ] && { python3 -c 'import ast, sys; ast.parse(open(sys.argv[1]).read())' "$f" 2>/dev/null || bad+="$f "; } done check "every example DAG parses" "" "$bad" else printf ' skip python3 is not installed\n' fi printf '\n' if [ "$rc" -eq 0 ]; then printf '%d checks passed — rig still does what it says\n' "$passed" else printf 'FAILED — a decision above has drifted; read the comment next to it\n' >&2 fi exit "$rc"