bring secrets per repo

rig/VERSION

@@ -1 +0,0 @@
-0.1.0

rig/ctrl/addons/metallb.sh

@@ -3,7 +3,6 @@
 # The pool is derived from the kind Docker network at install time.
 # Notes: docs/notes/addons.md
 set -euo pipefail
-ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:-$(dirname "$0")/..}"
 
 source ./lib/config.sh
@@ -62,7 +61,27 @@ $K rollout status daemonset/speaker -n metallb-system --timeout=240s
 # on a race that resolves itself in seconds.
 echo "  configuring the address pool"
 for attempt in 1 2 3 4 5 6 7 8 9 10; do
-    if render_manifest "$ADDON_DIR/metallb.yaml" | $K apply -f - >/dev/null 2>&1
+    if $K apply -f - >/dev/null 2>&1 <<YAML
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  addresses:
+    - ${pool_start}-${pool_end}
+---
+# Layer 2 mode: one node answers ARP for each address. No BGP peer needed, which
+# is what makes this work on a laptop.
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+    - default
+YAML
     then
         echo "  pool ready: ${pool_start}-${pool_end}"
         exit 0

rig/ctrl/addons/metallb.yaml

@@ -1,19 +0,0 @@
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: default
-  namespace: metallb-system
-spec:
-  addresses:
-    - ${pool_start}-${pool_end}
----
-# Layer 2 mode: one node answers ARP for each address. No BGP peer needed, which
-# is what makes this work on a laptop.
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: default
-  namespace: metallb-system
-spec:
-  ipAddressPools:
-    - default

rig/ctrl/check.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 # Readiness check: is this machine ready to run rig? Reports and instructs; never fixes.
 # Usage: check.sh [all | mem [status|push|all|backup|restore]]   (all = every detail)
-# Prints the memory budget and what to do next first, then the configuration as it is.
 # Notes: docs/notes/check.md
 set -euo pipefail
 cd "$(dirname "$0")"
@@ -13,11 +12,13 @@ if [ "${1:-}" = mem ]; then
     exec bash ./mem.sh "${@:-status}"
 fi
 
-# Facts only with `all`; problems (!) always print.
+# Compact by default: facts only with `all`; problems (!) always print.
 VERBOSE=""
 if [ "${1:-}" = all ]; then VERBOSE=1; fi
 fact() { if [ -n "$VERBOSE" ]; then echo "$@"; fi; }
 
+bash ./deps.sh detect ${VERBOSE:+all}
+
 source ./lib/config.sh
 load_config
 
@@ -65,64 +66,49 @@ port_busy() {
     grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
 }
 
-# Who listens on a port, as "pid name", when ss can tell (only for our own processes).
+echo
-port_owner() {
+echo "rig"
-    command -v ss >/dev/null 2>&1 || return 0
+echo "  cluster     ${CLUSTER} (${KUBECONTEXT})  profile ${PROFILE_NAME}, ${NODES} node(s), registry ${REGISTRY_MODE}"
-    ss -ltnpH "sport = :$1" 2>/dev/null \
+if [ -n "${OVERLAY:-}" ]; then
-        | sed -n 's/.*users:(("\([^"]*\)",pid=\([0-9]*\).*/\2 \1/p' | head -1
+    echo "  overlay     $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")  ($(_abs_from_ctrl "$OVERLAY_DIR"))"
-}
+elif [ -n "$OVERLAY_DIR" ]; then
-
+    fact "  overlay     none named — rig's own ${OVERLAY_DIR}"
-# This environment's Tilt: a tilt started from this ctrl/ (the Makefile runs it there).
+fi
-tilt_pid=""
+if [ -n "$VERBOSE" ] && [ -n "$OVERLAY_DIR" ]; then
-if tilt_owner=$(port_owner "$TILT_PORT") && [ -n "$tilt_owner" ]; then
+    ov=$(_from_ctrl "$OVERLAY_DIR") pieces=""
-    pid=${tilt_owner%% *}
+    for piece in rig.env k8s/overlays/dev kind-config.yaml.tpl addons Tiltfile; do
-    if [ "${tilt_owner#* }" = tilt ] && [ "$(readlink "/proc/$pid/cwd" 2>/dev/null)" = "$(pwd -P)" ]; then
+        [ -e "$ov/$piece" ] && pieces+="$piece "
-        tilt_pid=$pid
+    done
-    fi
+    echo "              provides: ${pieces:-nothing rig reads}"
+fi
+fact "  manifests   ${MANIFESTS_DIR:-none}"
+fact "  kind config ${KIND_CONFIG}"
+fact "  ingress     ${INGRESS_MODE}"
+if [ ! -f ./.env ]; then
+    fact "  .env        none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)"
+fi
+if [ -n "${STALE_MANIFESTS_DIR:-}" ]; then
+    echo "  ! .env      MANIFESTS_DIR=${STALE_MANIFESTS_DIR} is the old default; rig's examples moved"
+    echo "              to examples/ — delete that line from ctrl/.env (ignored until then)"
+fi
+# registry.sh points containerd at certs.d, which only works if the kind config says so,
+# and a kind config is fixed at creation: a project's own file that drops it fails silently.
+if [ "$REGISTRY_MODE" != none ] && ! grep -q 'config_path *= *"/etc/containerd/certs.d"' "$KIND_CONFIG"; then
+    echo "  ! kind      ${KIND_CONFIG} lacks the containerd config_path patch that registry mode"
+    echo "              '${REGISTRY_MODE}' needs — copy it from ctrl/k8s/kind-config.yaml.tpl"
 fi
 
-# none | stopped | up | unknown (no docker)
+# ── memory: does this cluster fit right now? Warns; never blocks. ──────────
-cluster_state() {
-    docker info >/dev/null 2>&1 || { echo unknown; return; }
-    local f="label=io.x-k8s.kind.cluster=${CLUSTER}"
-    if [ -n "$(docker ps -q --filter "$f")" ]; then echo up
-    elif [ -n "$(docker ps -aq --filter "$f")" ]; then echo stopped
-    else echo none; fi
-}
-state=$(cluster_state)
-
-# ── memory budget: what this environment needs, against what the machine has ──
-# The workload's share comes from the overlay: MEMORY_BUDGET="name=MB ..." in its rig.env,
-# and MEMORY_BUDGET_TIP, optionally, says how to switch its largest part off.
-# rig only knows what a node costs. Warns; never blocks.
 total_mb=$(mb_of MemTotal)
 avail_mb=$(mb_of MemAvailable)
 swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
 overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
-node_mb=$(( NODES * NODE_MB ))
+need_mb=$(( NODES * NODE_MB ))
-
-budget="" work_mb=0 big_mb=0 big=""
-for item in ${MEMORY_BUDGET:-}; do
-    name=${item%=*} mb=${item##*=}
-    if [ "$name" = "$item" ] || ! [[ "$mb" =~ ^[0-9]+$ ]]; then
-        echo "  ! budget    '$item' in MEMORY_BUDGET is not name=MB"
-        continue
-    fi
-    [ "$mb" -gt 0 ] || continue
-    budget+=$(printf '  %-18s %6d MB' "$name" "$mb")$'\n'
-    work_mb=$(( work_mb + mb ))
-    if [ "$mb" -gt "$big_mb" ]; then big_mb=$mb big=$name; fi
-done
-need_mb=$(( work_mb + node_mb ))
 
 rows=$(container_mb)
-# If our cluster is already up, its memory is already out of MemAvailable.
+# If our cluster is already up, its memory is already out of MemAvailable: need nothing more.
 ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
-if [ "$state" = up ]; then
+still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
-    still_mb=$(( need_mb > ours_mb ? need_mb - ours_mb : 0 ))
-else
-    still_mb=$need_mb
-fi
 headroom=$(( avail_mb - still_mb ))
 
 # The biggest things holding memory, other than this cluster: kind clusters summed, the rest by name.
@@ -131,164 +117,82 @@ others=$(awk -F'\t' -v c="$CLUSTER" '
             $2 == "-"                         { k["container \x27" $3 "\x27"]    += $1 }
             END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
 
-echo
+if [ "$still_mb" -eq 0 ] && [ "$headroom" -ge 512 ]; then
-echo "memory budget  (${CLUSTER})"
+    printf "  memory      up, holding %d MB — %d MB headroom for what you deploy\n" "$ours_mb" "$headroom"
-printf '%s' "$budget"
+elif [ "$still_mb" -eq 0 ]; then
-if [ -z "$budget" ]; then
+    printf "  ! memory    up, but only %d MB headroom for anything you deploy\n" "$headroom"
-    echo "  workload                ?      (the overlay lists none: MEMORY_BUDGET in its rig.env)"
+elif [ "$headroom" -ge 512 ]; then
-fi
+    printf "  memory      fits — ~%d MB for %s node(s), %d MB headroom\n" "$need_mb" "$NODES" "$headroom"
-printf '  %-18s %6d MB   (kind control plane, kube-system; %s node(s))\n' "node overhead" "$node_mb" "$NODES"
-echo "                     ---------"
-printf '  %-18s %6d MB\n' "required" "$need_mb"
-if [ "$state" = up ]; then
-    printf '  %-18s %6d MB   (%s is up)\n' "held by it now" "$ours_mb" "$CLUSTER"
-fi
-printf '  %-18s %6d MB\n' "this machine has" "$total_mb"
-printf '  %-18s %6d MB   (right now)\n' "available" "$avail_mb"
-echo
-
-if [ "$headroom" -ge 512 ]; then
-    if [ "$state" = up ]; then
-        echo "fits: ${CLUSTER} holds ${ours_mb} MB of its ${need_mb} MB; ${avail_mb} MB still available — ${headroom} MB headroom."
-    else
-        echo "fits: ${need_mb} MB needed, ${avail_mb} MB available — ${headroom} MB headroom."
-    fi
 elif [ "$headroom" -ge 0 ]; then
-    echo "! tight: ${still_mb} MB still needed, ${avail_mb} MB available — only ${headroom} MB headroom."
+    printf "  ! memory    fits, but only %d MB headroom (~%d MB for %s node(s))\n" "$headroom" "$need_mb" "$NODES"
 else
-    echo "! does not fit: ${still_mb} MB still needed, ${avail_mb} MB available."
+    printf "  ! memory    does not fit: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
     # Two failures with opposite fixes, and telling them apart is the point.
-    if [ "$need_mb" -le "$total_mb" ]; then
+    if [ "$still_mb" -le "$total_mb" ]; then
-        echo "  Something else holds the memory (below): stopping it helps, a bigger machine would not."
+        echo "              something else holds it (below) — stopping that helps, a bigger VM would not."
         if grep -q 'kind cluster' <<< "$others"; then
-            echo "  'make cluster free' stops the other kind clusters. It stops, never deletes."
+            echo "              'make cluster free' stops the other kind clusters. It stops, never deletes."
         fi
     else
-        echo "  The machine itself is too small: ${total_mb} MB in all. Give it more memory"
+        echo "              the machine itself is too small: ${total_mb} MB total."
-        echo "  ('make check mem' has the WSL steps), or switch parts of the workload off."
     fi
 fi
 if [ -n "$others" ] && { [ -n "$VERBOSE" ] || [ "$headroom" -lt 512 ]; }; then
-    echo "  held elsewhere:"
+    echo "              held elsewhere:"
-    head -6 <<< "$others" | awk -F'\t' '{ printf "    %6d MB  %s\n", $1, $2 }'
+    head -6 <<< "$others" | awk -F'\t' '{ printf "              %6d MB  %s\n", $1, $2 }'
     n_others=$(wc -l <<< "$others")
     if [ "$n_others" -gt 6 ]; then
-        echo "    ... and $((n_others - 6)) more"
+        echo "              ... and $((n_others - 6)) more"
     fi
 fi
-# One part that outweighs the rest is the first thing to switch off on a smaller machine.
-if [ "$work_mb" -gt 0 ] && [ $(( big_mb * 2 )) -gt "$work_mb" ] && [ "$big_mb" -ge 1024 ]; then
-    echo
-    echo "${big} is ${big_mb} of the ${work_mb} MB the workload needs — more than the rest"
-    echo "combined. If you are not working on it, ${MEMORY_BUDGET_TIP:-switching it off in the overlay} gets that back."
-fi
 if [ "$swap_used_mb" -gt 0 ]; then
-    fact "  ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing"
+    fact "              ${swap_used_mb} MB already in swap, which 'available' does not count: expect slow before failing"
 fi
 if [ "$overcommit" = "1" ]; then
-    fact "  overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)"
+    fact "              overcommit=1: allocations never fail, so read 'fits' as a ceiling (OOM killer settles up)"
-fi
-
-# ── where things stand, and the next step ──────────────────────────────────
-echo
-case "$state" in
-    none)
-        echo "${CLUSTER} does not exist yet. 'make cluster up' creates it${ADDONS:+ (with ${ADDONS})},"
-        echo "then 'make tilt' starts the dev loop." ;;
-    stopped)
-        echo "${CLUSTER} exists but is stopped (nothing is lost). Start it again with:"
-        echo "    docker start \$(docker ps -aq -f label=io.x-k8s.kind.cluster=${CLUSTER})" ;;
-    up)
-        if [ -n "$tilt_pid" ]; then
-            echo "${CLUSTER} is up, and its Tilt is running (pid ${tilt_pid}): http://localhost:${TILT_PORT}"
-        else
-            echo "${CLUSTER} is up. 'make tilt' starts the dev loop, at http://localhost:${TILT_PORT}."
-        fi
-        echo "Through the gateway: http://localhost:${HTTP_PORT}" ;;
-    *)
-        echo "docker is not reachable, so the cluster's state is unknown (the machine block below says why)." ;;
-esac
-
-# ── the configuration, as it is ────────────────────────────────────────────
-echo
-bash ./deps.sh detect ${VERBOSE:+all}
-
-echo
-echo "config"
-echo "  cluster     ${CLUSTER}  ($([ "$state" = none ] && echo 'not created' || echo "$state"))"
-echo "  context     ${KUBECONTEXT}"
-if [ -n "${OVERLAY:-}" ]; then
-    echo "  overlay     $(_abs_from_ctrl "$OVERLAY_DIR")"
-elif [ -n "$OVERLAY_DIR" ]; then
-    echo "  overlay     none named — rig's own ${OVERLAY_DIR}"
-fi
-if [ -n "$VERBOSE" ] && [ -n "$OVERLAY_DIR" ]; then
-    ov=$(_from_ctrl "$OVERLAY_DIR") pieces=""
-    for piece in rig.env k8s/overlays/dev kind-config.yaml.tpl addons Tiltfile; do
-        [ -e "$ov/$piece" ] && pieces+="$piece "
-    done
-    echo "              provides: ${pieces:-nothing rig reads}"
-fi
-echo "  profile     ${PROFILE_NAME}"
-echo "  nodes       ${NODES}"
-echo "  kind config ${KIND_CONFIG}"
-echo "  manifests   ${MANIFESTS_DIR:-none}"
-echo "  ingress     ${INGRESS_MODE}"
-if [ ! -f ./.env ]; then
-    fact "  .env        none — built-in defaults (cp ctrl/.env.example ctrl/.env to set values)"
-fi
-if [ -n "${STALE_MANIFESTS_DIR:-}" ]; then
-    echo "  ! .env      MANIFESTS_DIR=${STALE_MANIFESTS_DIR} is the old default; rig's examples moved"
-    echo "              to examples/ — delete that line from ctrl/.env (ignored until then)"
-fi
-# registry.sh points containerd at certs.d, which only works if the kind config says so,
-# and a kind config is fixed at creation: a project's own file that drops it fails silently.
-if [ "$REGISTRY_MODE" != none ] && ! grep -q 'config_path *= *"/etc/containerd/certs.d"' "$KIND_CONFIG"; then
-    echo "  ! kind      ${KIND_CONFIG} lacks the containerd config_path patch that registry mode"
-    echo "              '${REGISTRY_MODE}' needs — copy it from ctrl/k8s/kind-config.yaml.tpl"
 fi
 
 # ── ports: checked before creation; docker reports a clash only halfway through. ──
-# Ports held by our own cluster are not clashes: its node containers, and its registry,
+# Ports held by our own cluster are not clashes. Second grep, not `tr -d ':->'` (a tr range).
-# which is a container of its own. Second grep, not `tr -d ':->'` (a tr range).
+ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
-ours=$({ docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" --format '{{.Ports}}'
+        --format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
-         docker ps --filter "name=^${CLUSTER}-registry\$" --format '{{.Ports}}'; } 2>/dev/null \
+        | grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
-        | tr ',' '\n' | grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
+
-
+clash=0 list="" mine=0
-clash=0 list=""
+for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
-for entry in "http:${HTTP_PORT}" "https:${HTTPS_PORT}" \
+             "TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
-             "tilt:${TILT_PORT}" "registry:${REGISTRY_PORT}"; do
     name="${entry%%:*}"; p="${entry#*:}"
     [ -n "$p" ] || continue
-    list+="$name $p  "
+    list+="$p "
     if ! port_busy "$p"; then
         fact "$(printf "    %-9s %-6s free" "$name" "$p")"
     elif echo "$ours" | grep -qx "$p"; then
+        mine=1
         fact "$(printf "    %-9s %-6s in use by this environment's cluster" "$name" "$p")"
-    elif [ "$name" = tilt ] && [ -n "$tilt_pid" ]; then
-        fact "$(printf "    %-9s %-6s in use by this environment's Tilt" "$name" "$p")"
     else
-        owner=$(port_owner "$p")
+        printf "  ! ports     %s %s IN USE by something else\n" "$name" "$p"
-        if [ "$name" = tilt ] && [ "${owner#* }" = tilt ]; then
+        clash=1
-            printf "  ! ports     tilt %s is held by another Tilt (pid %s, in %s)\n" "$p" "${owner%% *}" \
-                "$(readlink "/proc/${owner%% *}/cwd" 2>/dev/null || echo '?')"
-            echo "              stop it (kill ${owner%% *}) before 'make tilt'"
-        else
-            printf "  ! ports     %s %s IN USE by %s\n" "$name" "$p" "${owner:+pid ${owner}}${owner:-something else}"
-            clash=1
-        fi
     fi
 done
-echo "  ports       ${list%  }"
 if [ "$clash" -eq 1 ]; then
     echo "              override it in ctrl/.env (e.g. HTTP_PORT=21080), or rename this directory"
+elif [ "$mine" -eq 1 ]; then
+    echo "  ports       ${list% }  held by this cluster"
+else
+    echo "  ports       ${list% }  free"
 fi
+if [ -n "${OVERLAY:-}" ]; then
+    fact "              derived from the overlay's folder name"
+else
+    fact "              derived from the directory name; pin them: bash ctrl/ports.sh persist"
+fi
+
 # ── what `make cluster up` wires in beside the cluster ─────────────────────
 REG_NAME="${CLUSTER}-registry"
-if reg_state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then
+if state=$(docker inspect -f '{{.State.Status}}' "$REG_NAME" 2>/dev/null); then
-    echo "  registry    ${REGISTRY_MODE}  localhost:${REGISTRY_PORT}  (${reg_state})"
+    echo "  registry    localhost:${REGISTRY_PORT}  ($state)"
 else
-    echo "  registry    ${REGISTRY_MODE}  (no container yet)"
+    fact "  registry    no container yet — 'make cluster up' starts it"
 fi
 echo "  addons      ${ADDONS:-none}"
 if [ -n "$VERBOSE" ]; then

rig/ctrl/docs.sh

@@ -16,30 +16,12 @@ serve() {
         echo "no docs/index.html" >&2
         exit 1
     fi
-    # One worker and no access log: a few static pages need no more, and the
-    # terminal then shows only real errors. The config is a temp file, gone on exit.
-    # Global, not local: the trap runs after serve() has returned.
-    DOCS_CONF=$(mktemp)
-    trap 'rm -f "$DOCS_CONF"' EXIT
-    cat > "$DOCS_CONF" <<'NGINX'
-worker_processes 1;
-error_log stderr error;
-events { worker_connections 64; }
-http {
-    include /etc/nginx/mime.types;
-    access_log off;
-    server { listen 80; root /usr/share/nginx/html; }
-}
-NGINX
-    chmod 644 "$DOCS_CONF"
     echo "docs for '$CLUSTER' on http://localhost:${DOCS_PORT}"
     echo "  (ctrl-c to stop; nothing is installed and nothing persists)"
     docker run --rm \
         --name "${CLUSTER}-docs" \
-        -e NGINX_ENTRYPOINT_QUIET_LOGS=1 \
         -p "${DOCS_PORT}:80" \
         -v "$REPO/docs:/usr/share/nginx/html:ro" \
-        -v "$DOCS_CONF:/etc/nginx/nginx.conf:ro" \
         nginx:alpine
 }
 

rig/ctrl/lib/config.sh

@@ -208,29 +208,6 @@ render_kind_config() {
         "$KIND_CONFIG"
 }
 
-# Render an addon's manifest (a .yaml beside its .sh) to stdout. Every ${NAME}, or
-# ${NAME:-default}, is filled from the loaded config or the calling script's variables;
-# a NAME with neither is an error, so a typo never becomes an empty image. $VAR and
-# $(VAR) are left alone: they belong to the container. Notes: docs/notes/addons.md
-render_manifest() {
-    local file="$1" text ref key def val
-    text=$(<"$file") || return 1
-    while IFS= read -r ref; do
-        key=${ref#\$\{}; key=${key%\}}; def=""
-        if [[ $key == *:-* ]]; then def=${key#*:-}; key=${key%%:-*}; fi
-        if [ -n "${!key:-}" ]; then
-            val=${!key}
-        elif [[ $ref == *:-* ]]; then
-            val=$def
-        else
-            echo "render_manifest: $file needs \${$key}, which nothing sets" >&2
-            return 1
-        fi
-        text=${text//"$ref"/"$val"}
-    done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}' "$file" | sort -u)
-    printf '%s\n' "$text"
-}
-
 _config_restore() {
     local line
     while IFS= read -r line; do

rig/ctrl/selftest.sh

@@ -307,7 +307,7 @@ HOUSE_PAT="$(printf '%s' 'semes' 'ter|local' '\.ar\b')"
 check "✖ S2 no example environment name from the copies era" "0" \
     "$(cd .. && grep -rIlE "$COPIES_PAT" . --exclude-dir=def --exclude-dir=local --exclude=STALE.md 2>/dev/null | wc -l)"
 check "✖ S3 ctrl/addons makes the cluster work, nothing more" "cert-manager metallb metrics-server" \
-    "$(ls addons/*.sh | xargs -n1 basename | sed 's/\.sh$//' | sort | xargs)"   # the scripts; their manifests sit beside them
+    "$(ls addons/ | sed 's/\.sh$//' | sort | xargs)"
 check "✖ S3 versions.env pins no workload image" "0" \
     "$(grep -cE '^(POSTGRES|REDIS|AIRFLOW)_IMAGE=' versions.env)"
 check "✖ S4 no namespace named after the cluster" "0" "$(grep -c "CLUSTER + ':namespace'" Tiltfile)"
@@ -378,16 +378,6 @@ note "the examples are overlays that work as shipped"
 bad=""
 for f in ../examples/*/addons/*.sh; do [ -e "$f" ] && { bash -n "$f" 2>/dev/null || bad+="$f "; }; done
 check "every example addon parses" "" "$bad"
-# An addon's manifest may only name what the config or its own script sets, so a typo
-# fails here rather than as an empty value on a cluster.
-bad=""
-for y in addons/*.yaml ../examples/*/addons/*.yaml; do
-    [ -e "$y" ] || continue
-    for n in $(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' "$y" | tr -d '${}' | sort -u); do
-        grep -qE "(^|[[:space:]])$n=" "${y%.yaml}.sh" versions.env ../examples/*/rig.env 2>/dev/null || bad+="$y:$n "
-    done
-done
-check "every addon manifest names only what is set" "" "$bad"
 if command -v python3 >/dev/null; then
     bad=""
     for f in ../examples/*/dags/*.py; do

rig/docs/graphs/03-architecture.dot

@@ -0,0 +1,56 @@
+// TODO: PLACEHOLDER — replace with the real estate topology.
+//
+// This is where the extracted platform diagrams land. The shape below is
+// illustrative only: it shows how a mocked dependency, a real service and a
+// remote system are meant to sit together, not what the system actually is.
+//
+// The intended end state is that this file stops being hand-written and is
+// generated from the running cluster, so the diagram becomes a report of what
+// exists rather than a drawing of what was once intended.
+digraph estate {
+    rankdir=LR
+    bgcolor="#0a0e17"
+    fontname="Helvetica"
+    node [fontname="Helvetica" fontsize=11 style=filled color="#1e2a4a" fontcolor="#e8eaf0" shape=box]
+    edge [fontname="Helvetica" fontsize=9 fontcolor="#8892a8" color="#4a5568"]
+
+    label="Estate topology — PLACEHOLDER"
+    labelloc=t
+    fontsize=16
+    fontcolor="#ffc107"
+
+    subgraph cluster_new {
+        label="New"
+        style=dashed
+        color="#1e2a4a"
+        fontcolor="#8892a8"
+
+        api     [label="service under work\n(real: built and hot-reloaded)" fillcolor="#1a3a1a" fontcolor="#00c853"]
+    }
+
+    subgraph cluster_core {
+        label="Core (mocked)"
+        style=dashed
+        color="#1e2a4a"
+        fontcolor="#8892a8"
+
+        svc_a   [label="upstream service\n(mock: canned responses)" fillcolor="#121829"]
+        db      [label="datastore\n(mock)" fillcolor="#121829" shape=cylinder]
+    }
+
+    subgraph cluster_legacy {
+        label="Legacy estate (mocked)"
+        style=dashed
+        color="#1e2a4a"
+        fontcolor="#8892a8"
+
+        batch   [label="batch drop\n(mock: writes files on a timer)" fillcolor="#121829"]
+    }
+
+    remote  [label="external system\n(remote: ExternalName,\nreachable only from a VDI)" fillcolor="#3a1a1a" fontcolor="#ffc107" shape=octagon]
+
+    api -> svc_a  [label="HTTP"]
+    api -> db     [label="query"]
+    api -> batch  [label="file handoff" style=dashed]
+    api -> remote [label="only when reachable" style=dashed color="#ffc107"]
+}

rig/docs/graphs/03-architecture.svg

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 14.1.2 (0)
+ -->
+<!-- Title: estate Pages: 1 -->
+<svg width="579pt" height="362pt"
+ viewBox="0.00 0.00 579.00 362.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 357.62)">
+<title>estate</title>
+<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-357.62 575.41,-357.62 575.41,4 -4,4"/>
+<text xml:space="preserve" text-anchor="middle" x="285.7" y="-334.42" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#ffc107">Estate topology — PLACEHOLDER</text>
+<g id="clust1" class="cluster">
+<title>cluster_new</title>
+<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-130.12 8,-210.12 199,-210.12 199,-130.12 8,-130.12"/>
+<text xml:space="preserve" text-anchor="middle" x="103.5" y="-190.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">New</text>
+</g>
+<g id="clust2" class="cluster">
+<title>cluster_core</title>
+<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="357.33,-172.12 357.33,-318.12 534.83,-318.12 534.83,-172.12 357.33,-172.12"/>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-298.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Core (mocked)</text>
+</g>
+<g id="clust3" class="cluster">
+<title>cluster_legacy</title>
+<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="341.83,-84.12 341.83,-164.12 551.33,-164.12 551.33,-84.12 341.83,-84.12"/>
+<text xml:space="preserve" text-anchor="middle" x="446.58" y="-144.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Legacy estate (mocked)</text>
+</g>
+<!-- api -->
+<g id="node1" class="node">
+<title>api</title>
+<polygon fill="#1a3a1a" stroke="#1e2a4a" points="191,-174.12 16,-174.12 16,-138.12 191,-138.12 191,-174.12"/>
+<text xml:space="preserve" text-anchor="middle" x="103.5" y="-159.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">service under work</text>
+<text xml:space="preserve" text-anchor="middle" x="103.5" y="-145.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">(real: built and hot&#45;reloaded)</text>
+</g>
+<!-- svc_a -->
+<g id="node2" class="node">
+<title>svc_a</title>
+<polygon fill="#121829" stroke="#1e2a4a" points="526.83,-282.12 365.33,-282.12 365.33,-246.12 526.83,-246.12 526.83,-282.12"/>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-267.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">upstream service</text>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-253.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: canned responses)</text>
+</g>
+<!-- api&#45;&gt;svc_a -->
+<g id="edge1" class="edge">
+<title>api&#45;&gt;svc_a</title>
+<path fill="none" stroke="#4a5568" d="M149.07,-174.56C167.54,-182.07 189.23,-190.7 209,-198.12 258.25,-216.6 270.12,-222.85 320.75,-237.12 331.43,-240.13 342.71,-243 353.94,-245.67"/>
+<polygon fill="#4a5568" stroke="#4a5568" points="353.12,-249.07 363.66,-247.92 354.71,-242.25 353.12,-249.07"/>
+<text xml:space="preserve" text-anchor="middle" x="255.88" y="-234.24" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">HTTP</text>
+</g>
+<!-- db -->
+<g id="node3" class="node">
+<title>db</title>
+<path fill="#121829" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-226.22 465.18,-228.18 446.08,-228.18 426.97,-228.18 411.45,-226.22 411.45,-223.81 411.45,-223.81 411.45,-184.43 411.45,-184.43 411.45,-182.02 426.97,-180.06 446.08,-180.06 465.18,-180.06 480.7,-182.02 480.7,-184.43 480.7,-184.43 480.7,-223.81 480.7,-223.81"/>
+<path fill="none" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-221.39 465.18,-219.43 446.08,-219.43 426.97,-219.43 411.45,-221.39 411.45,-223.81"/>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-207.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">datastore</text>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-193.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock)</text>
+</g>
+<!-- api&#45;&gt;db -->
+<g id="edge2" class="edge">
+<title>api&#45;&gt;db</title>
+<path fill="none" stroke="#4a5568" d="M191.32,-168.36C257.83,-177.73 346.91,-190.28 399.91,-197.75"/>
+<polygon fill="#4a5568" stroke="#4a5568" points="399.4,-201.22 409.79,-199.15 400.38,-194.29 399.4,-201.22"/>
+<text xml:space="preserve" text-anchor="middle" x="255.88" y="-185.69" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">query</text>
+</g>
+<!-- batch -->
+<g id="node4" class="node">
+<title>batch</title>
+<polygon fill="#121829" stroke="#1e2a4a" points="537.33,-128.12 354.83,-128.12 354.83,-92.12 537.33,-92.12 537.33,-128.12"/>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-113.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">batch drop</text>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-99.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: writes files on a timer)</text>
+</g>
+<!-- api&#45;&gt;batch -->
+<g id="edge3" class="edge">
+<title>api&#45;&gt;batch</title>
+<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M191.32,-144.39C237.67,-138.13 294.98,-130.39 343.4,-123.85"/>
+<polygon fill="#4a5568" stroke="#4a5568" points="343.61,-127.36 353.05,-122.55 342.67,-120.42 343.61,-127.36"/>
+<text xml:space="preserve" text-anchor="middle" x="255.88" y="-143.94" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">file handoff</text>
+</g>
+<!-- remote -->
+<g id="node5" class="node">
+<title>remote</title>
+<polygon fill="#3a1a1a" stroke="#1e2a4a" points="571.41,-21.74 571.41,-52.5 497.99,-74.24 394.17,-74.24 320.75,-52.5 320.75,-21.74 394.17,0 497.99,0 571.41,-21.74"/>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-46.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">external system</text>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-33.42" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">(remote: ExternalName,</text>
+<text xml:space="preserve" text-anchor="middle" x="446.08" y="-19.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">reachable only from a VDI)</text>
+</g>
+<!-- api&#45;&gt;remote -->
+<g id="edge4" class="edge">
+<title>api&#45;&gt;remote</title>
+<path fill="none" stroke="#ffc107" stroke-dasharray="5,2" d="M149.55,-137.68C167.9,-130.34 189.37,-121.97 209,-114.87 254.45,-98.43 305.32,-81.49 348.17,-67.64"/>
+<polygon fill="#ffc107" stroke="#ffc107" points="349,-71.05 357.44,-64.65 346.85,-64.39 349,-71.05"/>
+<text xml:space="preserve" text-anchor="middle" x="255.88" y="-117.57" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">only when reachable</text>
+</g>
+</g>
+</svg>

rig/docs/index.html

@@ -2,7 +2,6 @@
 <html lang="en">
 <head>
 <meta charset="utf-8">
-<link rel="icon" href="data:,">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>rig — local environment installer</title>
 <style>
@@ -238,9 +237,13 @@
   <div class="nav-backdrop"></div>
   <nav>
     <a href="#start">Start here</a>
+    <a href="#steps">The steps</a>
     <a href="#install">Installation</a>
     <a href="#environments">Environments</a>
     <a href="#overlays">Overlays</a>
+    <a href="#profiles">Profiles</a>
+    <a href="#registry">Registry</a>
+    <a href="#architecture">Architecture</a>
     <a href="#troubleshooting">Troubleshooting</a>
   </nav>
 
@@ -248,68 +251,114 @@
 
   <section class="section" id="start">
     <h2>Start here</h2>
+    <p class="lede">A runnable local model of a large, regulated estate — legacy and new side by side.</p>
     <div class="prose">
-      <p>rig builds a disposable Kubernetes environment on your machine, so you can
+      <p>rig builds a disposable Kubernetes environment on your machine so you can
-      see how a system fits together without access to any of it. It is for
+      explore how a system fits together without needing access to any of it. Its
-      <b>onboarding and exploration</b>, not a production replica.</p>
+      job is <b>onboarding and exploration</b>, not a production replica.</p>
 
-      <p>Most services in it are <b>not real</b>. What has to be faithful is the
+      <p>Most services in it are deliberately <b>not real</b>. What has to be faithful
-      topology: names, ports, dependency order, who can reach whom, and how it fails.
+      is the topology — the names, the ports, the dependency order, who can reach whom,
-      That is what makes a whole estate fit on a laptop. A real 20-service platform
+      and how it fails. The workloads themselves are noise. This is what makes the
-      does not fit once in 14&nbsp;GB; stubs of about 30&nbsp;MB each fit three times.</p>
+      whole estate fit on a laptop: a real 20-service platform will not fit even once
+      on 14&nbsp;GB, but mocks are about 30&nbsp;MB each, so three faithful copies do.</p>
 
       <h3>The only prerequisite</h3>
       <p><b>Docker.</b> No curl, no jq, no python, no apt repositories to configure.</p>
-<pre><code><span class="c"># in the environment directory:</span>
+<pre><code><span class="c"># then, in the environment directory:</span>
-make check       <span class="c"># is this machine ready? reports, never changes anything</span>
+make check     <span class="c"># is this machine ready? reports, never fixes</span>
-make deps        <span class="c"># install the pinned toolchain; safe to re-run</span>
+make deps        <span class="c"># install the pinned toolchain</span>
 make cluster up  <span class="c"># cluster + registry + addons; ports derive by themselves</span>
 </code></pre>
-      <p>Read <code>make check</code> before <code>make deps</code>: it lists what is
+      <p>Read <code>make check</code> before <code>make deps</code>. It never changes
-      missing, including the steps only you can do, such as anything needing
+      anything — it prints what it found and, at the end, the steps it cannot perform
-      <code>sudo</code> or a Windows-side restart.</p>
+      for you.</p>
-
+    </div>
-      <h3>make cluster up</h3>
+  </section>
-      <p>It prints what the kind config locks in before spending the time, because
+
-      that config is fixed when the cluster is created. Re-running is safe and
+  <section class="section" id="steps">
-      continues where an interrupted run stopped, rather than reporting that the
+    <h2>The steps</h2>
-      cluster already exists and leaving every node <code>NotReady</code>.</p>
+    <p class="lede">Start to finish, in order, with what each one actually does.</p>
-<pre><code>make cluster up                        <span class="c"># built-in defaults — no profile needed</span>
+    <div class="prose">
-make cluster up PROFILE=mirror         <span class="c"># images through an internal registry</span>
+
+      <h3>1 &middot; make check</h3>
+      <p>Asks whether this machine is ready. It <b>changes nothing</b> — it
+      reports what it found and, at the end, the things only a human can do
+      (anything needing <code>sudo</code>, or a Windows-side restart). Read it
+      before installing anything; it is faster than discovering the same problems
+      one failure at a time.</p>
+<pre><code>make check</code></pre>
+
+      <h3>2 &middot; make deps</h3>
+      <p>Installs the pinned toolchain — only what is missing — and tells you
+      if its directory is not on PATH yet. Running it twice is safe.</p>
+<pre><code>make deps</code></pre>
+
+      <h3>3 &middot; make cluster up</h3>
+      <p>Builds the cluster, starts its registry and installs the profile's
+      addons — there is nothing else to run first. It prints what the profile
+      locks in <i>before</i> spending the time, because the kind config is
+      fixed at creation and cannot be changed afterwards.</p>
+      <p>Re-running is safe and, more importantly, <b>convergent</b>: if a first
+      attempt was interrupted before the CNI was installed, running it again
+      finishes the job rather than reporting "already exists" and leaving every
+      node permanently NotReady.</p>
+<pre><code>make cluster up                 <span class="c"># built-in defaults — no profile needed</span>
+make cluster up PROFILE=mirror  <span class="c"># after copying env.d/mirror.env.example: cached registry</span>
 make cluster up OVERLAY=examples/data  <span class="c"># an overlay: what runs, kept outside rig</span>
-make cluster reset                     <span class="c"># destroy and rebuild: how an edited kind config takes effect</span>
+make cluster reset              <span class="c"># destroy and rebuild — how an edited kind config takes effect</span>
 </code></pre>
 
+      <h3>4 &middot; make docs</h3>
+      <p>Serves this page from a throwaway container. Works with no cluster and
+      no toolchain, which is deliberate: these pages are the instructions for
+      building everything else, so they cannot depend on it.</p>
+<pre><code>make docs</code></pre>
+
       <h3>Checking on things</h3>
       <dl>
-        <dt>make cluster list</dt><dd>Every cluster on the machine, its memory cost and its port block. A new cluster usually fails to start because of an old one; <code>make cluster free</code> stops them without deleting.</dd>
+        <dt>make cluster list</dt><dd>Every cluster on the machine, its memory cost and its port block. The usual reason a new one will not start is an old one you forgot about; <code>make cluster free</code> frees them without deleting.</dd>
-        <dt>make check</dt><dd>Host, toolchain, and whether this cluster fits, its ports, registry and addons. Only problems print; <code>make check all</code> prints everything.</dd>
+        <dt>make check</dt><dd>Short: host, toolchain, and whether this cluster fits, its ports, registry and addons. Details only appear when something needs attention; <code>make check all</code> prints every one.</dd>
-        <dt>make check mem</dt><dd>Memory in depth: what caps it, how far it climbs, and on WSL the <code>.wslconfig</code> backup and restore.</dd>
+        <dt>make check mem</dt><dd>Memory in depth: what caps it, how far it really climbs, and on WSL the <code>.wslconfig</code> backup and restore.</dd>
-        <dt>make docs</dt><dd>Serves these pages from a throwaway container. It needs no cluster and no toolchain, because these are the instructions for building both.</dd>
       </dl>
+
+      <h3>Running more than one</h3>
+      <p>Name another overlay, or copy the directory and rename it. Cluster name,
+      context, image tags and the port block all follow the folder name — the
+      overlay's, or rig's — so the second environment collides with nothing and
+      neither one's teardown can reach the other.</p>
+<pre><code>OVERLAY=local/platform-v2 make cluster up
+cp -r rig ../platform-v3 &amp;&amp; cd ../platform-v3 &amp;&amp; make cluster up
+</code></pre>
     </div>
   </section>
 
   <section class="section" id="install">
     <h2>Installation</h2>
+    <p class="lede">A container installs onto the host and then gets out of the way.</p>
     <div class="graph-container">
-      <a href="viewer.html?src=graphs/01-install.svg"><img src="graphs/01-install.svg" alt="How the installer works"></a>
+      <a href="viewer.html?src=graphs/01-install.svg"><img src="graphs/01-install.svg" alt="Installation flow"></a>
     </div>
     <div class="prose">
-      <p>The installer is a container, not a shell script, because a stock slim Debian
+      <p>The installer is a container, not a shell script, for a specific reason: a
-      has no curl, wget, jq, python3 or CA bundle: a shell installer could not make a
+      stock slim Debian has no <code>curl</code>, no <code>wget</code>, no
-      verified HTTPS request, let alone check one. The container carries its own
+      <code>jq</code>, no <code>python3</code> and <b>no CA bundle</b>. A shell
-      toolchain, so the host needs only Docker. Nothing runs inside it afterwards —
+      installer could not make a verified HTTPS request, let alone check one. The
-      kind, kubectl, tilt and jq run natively.</p>
+      container carries its own toolchain, so the host needs nothing but Docker.</p>
-
+
-      <p>Every tool is a single binary at a pinned version, checked against a published
+      <p>The cluster never runs inside that container. Everything it installs —
-      SHA256. Node images are pinned <b>by digest</b>, so upgrading kind cannot move
+      kind, kubectl, tilt, jq — runs natively afterwards, so nothing pays a
-      your Kubernetes version.</p>
+      container tax during daily work.</p>
-
+
-      <h3>Two tiers</h3>
+      <h3>Pinned and verified</h3>
-      <p>A managed or corporate-issued machine is not somewhere to install development
+      <p>Every tool is a single binary fetched at a pinned version and checked
-      tools by default, so installing cluster tooling is a decision rather than a side
+      against a published SHA256. Node images are pinned <b>by digest</b>, so
-      effect:</p>
+      upgrading kind cannot silently move your Kubernetes version.</p>
+
+      <h3>Not every machine should get cluster tooling</h3>
+      <p>A managed or corporate-issued machine — the kind that holds the access
+      you cannot get anywhere else — is not somewhere to install development
+      tools by default. So the toolchain comes in two tiers:</p>
       <table>
         <tr><th>tier</th><th>installs</th><th>for</th></tr>
         <tr><td><code>core</code></td><td>kubectl, jq</td><td>talk to a cluster someone else runs</td></tr>
@@ -318,6 +367,11 @@ make cluster reset                     <span class="c"># destroy and rebuild: ho
 <pre><code>make deps core   <span class="c"># kubectl and jq only — nothing that creates a cluster</span>
 make deps        <span class="c"># dev, the default</span>
 </code></pre>
+      <p>Testing <i>in situ</i> on a managed machine is still possible — install
+      the <code>dev</code> tier deliberately when you need it. The point is that
+      it should be a decision rather than a side effect of installing.</p>
+      <p>The documentation itself needs neither tier: <code>make docs</code>
+      wants only Docker.</p>
 
       <h3>Air-gapped</h3>
 <pre><code>make deps image full                       <span class="c"># bakes every binary into the image</span>
@@ -330,129 +384,185 @@ docker load &lt; rig.tgz &amp;&amp; make cluster up PROFILE=offline   <span clas
 
   <section class="section" id="environments">
     <h2>Environments</h2>
+    <p class="lede">One folder is one environment — an overlay's, or rig's own. Copy it, rename it, run it.</p>
     <div class="graph-container">
       <a href="viewer.html?src=graphs/02-environment.svg"><img src="graphs/02-environment.svg" alt="Environment derivation"></a>
     </div>
     <div class="prose">
-      <p>One folder is one environment. Running several versions of a system at once
+      <p>Running several versions of a system at once means several clusters on one
-      means several clusters on one machine, and everything that could collide is
+      machine, not several machines. Everything that could collide is derived from
-      derived from the folder name (the overlay's, when one is named):</p>
+      the folder name (the overlay's when one is named):</p>
       <dl>
         <dt>cluster + context</dt><dd><code>platform-v2/</code> builds <code>platform-v2</code> on <code>kind-platform-v2</code>.</dd>
-        <dt>port block</dt><dd>Ten ports from a hash of the name, above 20000 — clear of 80, 443, 3000, 5432, 8000 and 8080.</dd>
+        <dt>port block</dt><dd>Ten ports from a hash of the name, in the 20000+ range — clear of 80, 443, 3000, 5432, 8000 and 8080.</dd>
         <dt>registry + images</dt><dd>Named after the environment, so two copies never share one.</dd>
       </dl>
-      <p>So two copies never collide, and neither one's <code>make cluster down</code>
+      <p>Two copies therefore never collide, and neither one's
-      can touch the other. <code>make check</code> shows the block;
+      <code>make cluster down</code> can touch the other. <code>make check</code>
-      <code>bash ctrl/ports.sh persist</code> freezes it into <code>ctrl/.env</code>.</p>
+      shows the block; <code>bash ctrl/ports.sh persist</code> freezes it into
+      <code>ctrl/.env</code> if you want it fixed rather than derived.</p>
 
       <h3>Configuration layers</h3>
-      <p>Weakest first, later wins: built-in defaults → pinned versions → a profile, if
+      <p>Weakest first, later wins: built-in defaults → pinned versions → a
-      you name one → the overlay's <code>rig.env</code> → <code>ctrl/.env</code> → the
+      profile, if you name one → the overlay's <code>rig.env</code> →
-      caller's environment. So a value on the command line always beats every file.</p>
+      <code>ctrl/.env</code> → the environment. So
-
+      <code>make cluster up PROFILE=&lt;name&gt;</code> always beats every file.</p>
-      <h3>Profiles</h3>
-      <p>A profile says how this machine reaches the world. rig needs none.</p>
-      <table>
-        <tr><th>example</th><th>registry</th><th>for</th></tr>
-        <tr><td>none</td><td><code>local</code></td><td>the built-in defaults</td></tr>
-        <tr><td><code>mirror.env.example</code></td><td><code>mirror</code></td><td>images through an internal registry</td></tr>
-        <tr><td><code>offline.env.example</code></td><td><code>local</code></td><td>air-gapped</td></tr>
-      </table>
-
-      <h3>Registry modes</h3>
-      <table>
-        <tr><th>mode</th><th>what it does</th></tr>
-        <tr><td><code>none</code></td><td>images are built straight into the node</td></tr>
-        <tr><td><code>local</code></td><td>a registry container wired into the cluster</td></tr>
-        <tr><td><code>mirror</code></td><td>that container as a pull-through cache of the corporate registry</td></tr>
-        <tr><td><code>remote</code></td><td>no local container; pull direct with an <code>imagePullSecret</code></td></tr>
-      </table>
-      <p><code>mirror</code> is what a locked-down network looks like: images come from
-      the corporate registry, you do not hammer it, and you keep working when the
-      connection drops. It is also only reachable from where that registry is routable —
-      from a laptop, use <code>local</code> or <code>offline</code>.</p>
-      <p><b>Corporate CA trust is the usual failure.</b> It has to reach three places:
-      the host Docker daemon, every node's containerd (nodes do not inherit host trust),
-      and any in-cluster client. Set <code>REGISTRY_CA_FILE</code>;
-      <code>make check</code> reports which is still missing. Without it the error is
-      <code>x509: certificate signed by unknown authority</code>.</p>
-
-      <h3>LoadBalancer services</h3>
-      <p>Real manifests use <code>type: LoadBalancer</code>. On a bare local cluster
-      those Services sit at <code>EXTERNAL-IP &lt;pending&gt;</code> forever, with no
-      error: the deployment looks healthy and is unreachable. The <code>metallb</code>
-      addon fixes that, so upstream manifests work unchanged. Its address pool is
-      derived from the cluster's Docker network at install time, because Docker picks
-      that subnet and it differs between machines.</p>
-      <p>Those addresses are reachable from the Linux side, including inside WSL. A
-      browser on Windows has no route to them: use the ingress host ports instead.</p>
-
-      <h3>Networking and memory</h3>
-      <p>The cluster uses kind's built-in networking, which does enforce standard
-      NetworkPolicy — verified against a no-policy control. A pluggable CNI was tried
-      and removed: it added policy features nothing here needs, in exchange for a
-      slower boot and one more thing to get right at creation time.</p>
-      <p>Every cluster is a running container tree whether you use it or not.
-      <code>make cluster list</code> shows what each costs, and
-      <code>make cluster free</code> stops the others without deleting them.</p>
     </div>
   </section>
 
   <section class="section" id="overlays">
     <h2>Overlays</h2>
+    <p class="lede">What runs lives outside rig — rig reads it and never writes into it.</p>
     <div class="prose">
-      <p>An overlay is one folder, kept outside rig's version control, holding a use
+      <p>An overlay is one folder, kept outside rig's version control, holding a
-      case. rig reads it and never writes into it. Every piece is optional:</p>
+      use case. Every piece is optional:</p>
       <table>
         <tr><th>in the overlay</th><th>what rig does with it</th></tr>
         <tr><td><code>rig.env</code></td><td>a config layer: addons, namespaces, images — anything a profile could set</td></tr>
         <tr><td><code>k8s/overlays/dev/</code></td><td>the manifests the dev loop applies</td></tr>
         <tr><td><code>kind-config.yaml.tpl</code></td><td>the cluster's shape, when it needs its own (mounts, ports)</td></tr>
-        <tr><td><code>addons/&lt;name&gt;.sh</code></td><td>addons, found before rig's own; each applies its own <code>&lt;name&gt;.yaml</code></td></tr>
+        <tr><td><code>addons/&lt;name&gt;.sh</code></td><td>addons, found before rig's own</td></tr>
         <tr><td><code>Tiltfile</code></td><td>the workload's half of the dev loop, included by rig's</td></tr>
       </table>
 <pre><code>cp -r examples/starter local/myenv   <span class="c"># local/ is gitignored</span>
 OVERLAY=local/myenv make cluster up
 </code></pre>
       <p>With none named, rig runs its own <code>examples/starter</code>.
-      <code>examples/data</code> carries postgres, redis and airflow as an overlay's own
+      <code>examples/data</code> carries postgres, redis and airflow as an
-      addons. A project can also carry rig at <code>&lt;project&gt;/rig/</code> and be
+      overlay's own addons. A project can also carry rig at
-      the overlay itself, with a three-line forwarding Makefile — see
+      <code>&lt;project&gt;/rig/</code> and be the overlay itself, with a
-      <code>docs/notes/overlay.md</code>.</p>
+      three-line forwarding Makefile — see <code>docs/notes/overlay.md</code>.</p>
-
+    </div>
-      <h3>What runs in it</h3>
+  </section>
-      <p>Each component is one of three things, and switching between them is a
+
-      one-line change:</p>
+  <section class="section" id="profiles">
+    <h2>Profiles</h2>
+    <p class="lede">How this machine reaches the world — optional; rig needs none.</p>
+    <div class="prose">
+      <table>
+        <tr><th>example</th><th>registry</th><th>for</th></tr>
+        <tr><td><i>none</i></td><td>local</td><td>the built-in defaults; no profile needed</td></tr>
+        <tr><td><code>mirror.env.example</code></td><td>mirror</td><td>images through an internal registry</td></tr>
+        <tr><td><code>offline.env.example</code></td><td>local</td><td>air-gapped</td></tr>
+      </table>
+
+      <div class="note"><p><b>The kind config cannot be re-applied.</b> Edit
+      <code>ctrl/k8s/kind-config.yaml.tpl</code> (or the overlay's own); it takes effect when the cluster is created. <code>cluster up</code> prints what it
+      locks in before spending the time, and <code>make cluster reset</code> is
+      the way out.</p></div>
+
+      <h3>LoadBalancer services</h3>
+      <p>Real manifests use <code>type: LoadBalancer</code>, because a real
+      cluster has one. On a bare local cluster those Services sit at
+      <code>EXTERNAL-IP &lt;pending&gt;</code> forever, with no error anywhere —
+      the deployment looks healthy and simply is not reachable.</p>
+      <p>The <code>metallb</code> addon fixes that, so the same manifests work
+      here as upstream and nothing has to be rewritten to NodePort. Its address
+      pool is derived from the cluster's Docker network at install time rather
+      than hardcoded, because Docker picks that subnet and it differs between
+      machines.</p>
+      <div class="note"><p><b>Where those addresses are reachable from.</b> The
+      pool lives on the Docker bridge, so LoadBalancer IPs work from the Linux
+      side — including from inside WSL. A browser on Windows has no route to
+      them. Use the ingress host ports for anything you need to open in a
+      browser.</p></div>
+
+      <h3>Networking</h3>
+      <p>The cluster uses kind's built-in networking, which <b>does</b> enforce
+      standard NetworkPolicy — verified against a no-policy control, not assumed.
+      The widely repeated claim that it accepts policies and silently ignores
+      them is out of date.</p>
+      <p>A pluggable CNI was tried and removed: it only added
+      GlobalNetworkPolicy, policy tiers and egress-CIDR rules, none of which are
+      needed yet, in exchange for a slower boot and one more thing that has to be
+      right at creation time. Worth revisiting only when a policy the built-in
+      cannot express actually comes up.</p>
+
+      <h3>Memory</h3>
+      <p>Every cluster is a running container tree whether you are using it or not.
+      <code>make cluster list</code> shows what exists and what it costs;
+      <code>make cluster free</code> stops the others without deleting them.</p>
+    </div>
+  </section>
+
+  <section class="section" id="registry">
+    <h2>Registry</h2>
+    <p class="lede">Local, cached, or straight to the corporate registry.</p>
+    <div class="prose">
+      <table>
+        <tr><th>mode</th><th>what it does</th></tr>
+        <tr><td><code>none</code></td><td>images are built straight into the node</td></tr>
+        <tr><td><code>local</code></td><td>a registry container wired into the cluster</td></tr>
+        <tr><td><code>mirror</code></td><td>that container as a <b>pull-through cache</b> of the corporate registry</td></tr>
+        <tr><td><code>remote</code></td><td>no local container; pull direct with an imagePullSecret</td></tr>
+      </table>
+      <p><code>mirror</code> is what a locked-down network actually looks like:
+      images originate from the corporate registry, you do not hammer it, and you
+      keep working when the connection drops.</p>
+
+      <div class="note"><p><b>The corporate CA will bite you.</b> A corporate
+      registry is usually behind an internal CA, and trust has to reach
+      <b>three</b> places: the host Docker daemon, every cluster node's containerd
+      (nodes do <i>not</i> inherit host trust), and any in-cluster client. Set
+      <code>REGISTRY_CA_FILE</code> and <code>make check</code> reports which is
+      still missing. The symptom otherwise is an opaque
+      <code>x509: certificate signed by unknown authority</code>.</p></div>
+
+      <p>Reachability also depends on where you are: if the registry is only
+      routable from a managed workspace, <code>mirror</code> and <code>remote</code>
+      will not resolve from a laptop at all. That is what <code>local</code> and
+      <code>offline</code> are for.</p>
+    </div>
+  </section>
+
+  <section class="section" id="architecture">
+    <h2>Architecture</h2>
+    <p class="lede">The estate being modelled.</p>
+    <div class="note"><p><b>TODO — placeholder.</b> The diagram below is
+    illustrative only: it shows how a mocked dependency, a service under active
+    work, and an unreachable remote system sit together. It is not the real
+    topology. Replace <code>docs/graphs/03-architecture.dot</code> with the
+    extracted platform diagrams, then run <code>make docs graphs</code>.</p></div>
+    <div class="graph-container">
+      <a href="viewer.html?src=graphs/03-architecture.svg"><img src="graphs/03-architecture.svg" alt="Estate topology (placeholder)"></a>
+    </div>
+    <div class="prose">
+      <p>Each component is one of three things, and switching between them should be
+      a one-line change rather than a rewrite:</p>
       <dl>
-        <dt>real</dt><dd>Built from source and hot-reloaded. What you are working on — usually exactly one.</dd>
+        <dt>real</dt><dd>Built from source and hot-reloaded. The thing you are actually working on — usually exactly one.</dd>
         <dt>mock</dt><dd>A generic stub with canned responses. Everything you do not care about today.</dd>
-        <dt>remote</dt><dd>No pod at all: a Service of type <code>ExternalName</code> pointing at the real system. In-cluster DNS resolves identically, so callers never change.</dd>
+        <dt>remote</dt><dd>No pod at all: a Service of type ExternalName pointing at the real system. In-cluster DNS resolves identically, so callers never change.</dd>
       </dl>
+      <p>The intended end state is that this diagram is <b>generated from the
+      running cluster</b> rather than drawn by hand — so it becomes a report of
+      what exists instead of a picture of what was once intended.</p>
     </div>
   </section>
 
   <section class="section" id="troubleshooting">
     <h2>Troubleshooting</h2>
+    <p class="lede">The failures that are hard to diagnose from their symptoms.</p>
     <div class="prose">
       <h3>Tilt stops noticing file changes</h3>
-      <p>Almost always inotify limits, and it fails silently: nothing errors, changes
+      <p>Almost always <code>inotify</code> limits, and it fails <i>silently</i> —
-      just stop being picked up. Defaults on WSL are far too low.
+      nothing errors, changes just stop being picked up. Defaults on WSL are far too
-      <code>make check</code> reports it and prints the fix.</p>
+      low. <code>make check</code> reports it and prints the fix.</p>
 
       <h3>Cluster creation dies halfway with a port error</h3>
       <p>Docker reports <code>failed to bind host port … address already in use</code>
-      partway through creating the cluster. <code>make check</code> tests every port in
+      partway through creating the cluster. Run <code>make check</code> first — it
-      this environment's block before anything is built.</p>
+      checks every port in this environment's block before anything is built.</p>
 
       <h3>Every node stays NotReady</h3>
-      <p>Usually a cluster created with the default CNI disabled and the real CNI never
+      <p>Usually a cluster created with the default CNI disabled but the real CNI
-      installed, after an interrupted first run. Run <code>make cluster up</code> again:
+      never installed — typically an interrupted first run. Just run
-      it finishes the missing steps instead of stopping because the cluster exists.</p>
+      <code>make cluster up</code> again: it converges rather than exiting early, and
+      will finish the missing steps.</p>
 
       <h3>x509: certificate signed by unknown authority</h3>
-      <p>Corporate CA trust has not reached one of the three places it needs to be. See
+      <p>Corporate CA trust has not reached one of the three places it needs to be.
-      <a href="#environments">Environments</a>.</p>
+      See <a href="#registry">Registry</a>.</p>
 
       <h3>kubectl says the context does not exist</h3>
       <p>The cluster can exist while its context does not — a reset or a switched

rig/docs/notes/addons.md

@@ -4,12 +4,6 @@
 
 Each addon is its own idempotent script — adding one is adding a file, not editing a dispatcher. `ADDONS` names them, in install order; the overlay's `addons/<name>.sh` is found before rig's `ctrl/addons/<name>.sh`, and every one runs from rig's `ctrl/` with `RIG_CTRL` exported, wherever its file lives (see [overlay.md](overlay.md)).
 
-## The manifest beside the script
-
-An addon's Kubernetes objects are a plain `<name>.yaml` beside its `<name>.sh`, not a heredoc inside it: the YAML reads, diffs and lints as YAML, and the script keeps only what YAML cannot do (secrets made once, a database created, waiting for a rollout). The script renders it with `render_manifest "$ADDON_DIR/<name>.yaml"` (`lib/config.sh`) and pipes it to `kubectl apply`.
-
-`render_manifest` fills `${NAME}` and `${NAME:-default}` from the loaded config or the script's own variables, and fails on a name nothing sets, so a typo is an error rather than an empty image. `$VAR` and `$(VAR)` are left alone: they are the container's, as in postgres' `pg_isready -U $POSTGRES_USER`. selftest checks that every name a manifest uses is set somewhere. `ADDON_DIR` is taken before the script's `cd` into rig's `ctrl/`: `ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"`.
-
 ## What rig ships, and what it does not
 
 rig's own addons make the *cluster* work, and are useless outside one: metallb, cert-manager, metrics-server. Things a workload happens to need — a database, a cache, a scheduler — are the workload's, and which workload needs which is not rig's business, so they live with the overlay. `examples/data/addons/` has postgres, redis and airflow as a worked example; an overlay that wants them copies them in.

rig/docs/notes/check.md

@@ -34,37 +34,14 @@ A port held by THIS environment's own cluster is not a clash; it is the thing wo
 
 The ports are extracted with a second grep rather than `tr -d ':->'`: in tr, ':->' is the character RANGE ':' to '>', which does not contain '-', so the trailing dash survives and nothing ever matches.
 
-## Budget first, configuration after (2026-09-23)
+## Compact by default
-
+
-`make check` answers the question people run it for first, in the shape the adapter rig had
+`make check` prints one line per question — host, toolchain, and for this rig: cluster, memory,
-(the user preferred it): a memory budget with one row per part, the node overhead, the total
+ports, registry, addons — and adds detail only where something needs attention (`!` lines, the
-against what the machine has and has available; then a one-line verdict (`fits`, `! tight`,
+"held elsewhere" list when memory is tight, the clashing port). `make check all` prints every fact,
-`! does not fit`, with the two opposite fixes told apart); then where things stand and the next
+as the full report did before 2026-09-17. `deps.sh detect all` is the same switch for the host part,
-command — not created, stopped (with the `docker start` line), up (with the Tilt and gateway
+so the standalone `rigdeps.sh detect` is short too. Changed because the long report buried the few
-URLs, and whether this environment's Tilt is already running).
+lines that mattered.
-
-Below that, the configuration as it is, raw: the host and toolchain (`deps.sh detect`), then
-one `key value` line each for cluster, context, overlay, profile, nodes, kind config, manifests,
-ingress, ports (named), registry, addons. `!` lines always print; `make check all` adds every
-fact (per-port detail, addon list, swap and overcommit notes). `deps.sh detect all` is the same
-switch for the host part, so the standalone `rigdeps.sh detect` is short too.
-
-## MEMORY_BUDGET
-
-rig knows what a node costs (NODE_MB), not what a workload does, so the parts come from the
-overlay's rig.env: `MEMORY_BUDGET="name=MB ..."`, in the order listed; a 0 is left out, and an
-entry that is not `name=MB` is a `!` line. rig.env is sourced, so an overlay can size a part by
-its own knob (poc_rig's airflow follows its mode). When one part outweighs the rest together
-(and is at least 1 GB), check names it as the first thing to switch off; `MEMORY_BUDGET_TIP`
-says how, in the overlay's words. No budget: one `?` row says where to add it.
-
-## Tilt: ours or another
-
-A Tilt still running after `make cluster down` holds the Tilt port, and the next `make tilt`
-fails with "address already in use" (2026-09-23, the Workspace). check reads the listener with
-`ss -p` (our own processes only): a `tilt` whose working directory is this ctrl/ is this
-environment's Tilt, reported as running with its URL; another `tilt` is a `!` line with its pid,
-its folder and the `kill` that frees the port. Anything else is the ordinary port clash.
 
 ## overlay and kind config
 

rig/docs/notes/docs.md

@@ -8,12 +8,6 @@ out `python -m http.server` too — a minimal Debian has no python3. What it doe
 have, by definition, is Docker: the single prerequisite rig already demands. So a
 throwaway nginx container serves a read-only bind mount.
 
-It runs with a config of its own: one worker (nginx's `auto` starts one per core,
-twelve on an ordinary laptop, for a handful of static pages), no access log and a
-quiet entrypoint, so the terminal shows the URL and nothing else unless something
-fails. The pages declare an empty icon, so the browser does not ask for a
-`favicon.ico` that is not there.
-
 ## Committed SVGs
 
 Rendered SVGs are committed alongside their `.dot` sources for the same reason:

rig/docs/notes/overlay.md

@@ -60,9 +60,6 @@ air-gapped install); an overlay says what runs. `ctrl/.env` is still this
 machine's, and the caller still wins over everything.
 
 `rig.env` may not set `PROFILE` or `OVERLAY`: both are chosen before it loads.
-
-`MEMORY_BUDGET` (and optionally `MEMORY_BUDGET_TIP`) is how an overlay tells `make check` what its
-workload costs: rig only knows the node. See check.md.
 The paths it sets (`MANIFESTS_DIR`, `KIND_CONFIG`) are relative to the overlay.
 `MANIFESTS_DIR=none` means rig applies no manifests and the overlay's Tiltfile
 does, e.g. when kustomize needs flags.
@@ -103,8 +100,7 @@ load_config
 ```
 
 `OVERLAY_DIR` is set, so an addon can find files beside it
-(`$(_from_ctrl "$OVERLAY_DIR")/...`). Its manifest is a `<name>.yaml` beside the script,
+(`$(_from_ctrl "$OVERLAY_DIR")/...`). rig's own `ctrl/addons/` holds only what
-rendered with `render_manifest` (see [addons.md](addons.md)). rig's own `ctrl/addons/` holds only what
 makes a cluster work (metallb, cert-manager, metrics-server);
 `examples/data/addons/` shows workload ones.
 
@@ -155,31 +151,6 @@ $(wordlist 2,$(words $(GOALS)),$(GOALS)):
 The cluster is then named after `<project>`, exactly as a copied rig named
 `<project>` was, so moving a copied rig to this layout keeps its cluster and ports.
 
-rig can also live outside the project, which then holds only the overlay. The
-forwarder names it with one variable, a machine path that is never committed (the
-shell, or an ignored `.env.local`), defaulting to the vendored layout:
-
-```make
-RIG_DIR ?= $(HERE)/rig
-override RIG_DIR := $(abspath $(patsubst ~/%,$(HOME)/%,$(RIG_DIR)))
-```
-
-No search path: an unset `RIG_DIR` means `./rig`, and nothing else is tried. Moving rig
-changes nothing in the cluster's name, context or ports; moving or renaming the overlay
-folder does.
-
-`VERSION`, at rig's top, names the rig. It travels with the folder, vendored, a
-worktree or a plain copy, and needs no git. A project records the one it was last run
-against and warns on another; it does not refuse:
-
-```make
-RIG_REF := 0.1.0
-RIG_AT  := $(strip $(shell cat '$(RIG_DIR)/VERSION' 2>/dev/null))
-ifneq ($(RIG_AT),$(RIG_REF))
-$(warning rig at $(RIG_DIR) is $(or $(RIG_AT),unversioned); this overlay was last run against rig $(RIG_REF))
-endif
-```
-
 `rig.env` holds no secrets, by this contract. A repository whose `.gitignore` has a
 broad `*.env` (a common secrets rule) would still hide it, so an overlay living in
 such a repo re-includes it in its own `.gitignore`: `!rig.env`.

rig/docs/viewer.html

@@ -2,7 +2,6 @@
 <html>
 <head>
 <meta charset="UTF-8">
-<link rel="icon" href="data:,">
 <title>Graph Viewer</title>
 <style>
   * { margin: 0; padding: 0; }

rig/examples/data/addons/airflow.sh

@@ -5,7 +5,6 @@
 # Requires the postgres addon; refuses to install without it.
 # Notes: ../README.md
 set -euo pipefail
-ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -71,7 +70,77 @@ if [ -d "$dags" ]; then
 fi
 
 echo "  applying manifests"
-render_manifest "$ADDON_DIR/airflow.yaml" | $K apply -n "$NS" -f - >/dev/null
+$K apply -n "$NS" -f - >/dev/null <<YAML
+apiVersion: v1
+kind: Service
+metadata:
+  name: airflow
+spec:
+  selector:
+    app: airflow
+  ports:
+    - port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airflow
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: airflow
+  template:
+    metadata:
+      labels:
+        app: airflow
+    spec:
+      containers:
+        - name: airflow
+          image: ${AIRFLOW_IMAGE}
+          args: ["standalone"]
+          env:
+            - name: AIRFLOW__CORE__EXECUTOR
+              value: LocalExecutor
+            - name: AIRFLOW__CORE__LOAD_EXAMPLES
+              value: "false"
+            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+              valueFrom:
+                secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
+            - name: AIRFLOW_CONN_APP_DB
+              valueFrom:
+                secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
+            - name: AIRFLOW__CORE__FERNET_KEY
+              valueFrom:
+                secretKeyRef: {name: airflow, key: FERNET_KEY}
+            - name: _AIRFLOW_WWW_USER_USERNAME
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_USER}
+            - name: _AIRFLOW_WWW_USER_PASSWORD
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: dags
+              mountPath: /opt/airflow/dags
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            # First boot runs the whole migration before it serves anything.
+            initialDelaySeconds: 60
+            periodSeconds: 15
+            failureThreshold: 20
+      volumes:
+        - name: dags
+          configMap:
+            name: airflow-dags
+            optional: true
+YAML
 
 echo "  waiting for airflow (the first boot migrates the database, so this is slow)..."
 $K rollout status deployment/airflow -n "$NS" --timeout=600s

rig/examples/data/addons/airflow.yaml

@@ -1,69 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: airflow
-spec:
-  selector:
-    app: airflow
-  ports:
-    - port: 8080
-      targetPort: 8080
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: airflow
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: airflow
-  template:
-    metadata:
-      labels:
-        app: airflow
-    spec:
-      containers:
-        - name: airflow
-          image: ${AIRFLOW_IMAGE}
-          args: ["standalone"]
-          env:
-            - name: AIRFLOW__CORE__EXECUTOR
-              value: LocalExecutor
-            - name: AIRFLOW__CORE__LOAD_EXAMPLES
-              value: "false"
-            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
-              valueFrom:
-                secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
-            - name: AIRFLOW_CONN_APP_DB
-              valueFrom:
-                secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
-            - name: AIRFLOW__CORE__FERNET_KEY
-              valueFrom:
-                secretKeyRef: {name: airflow, key: FERNET_KEY}
-            - name: _AIRFLOW_WWW_USER_USERNAME
-              valueFrom:
-                secretKeyRef: {name: airflow, key: ADMIN_USER}
-            - name: _AIRFLOW_WWW_USER_PASSWORD
-              valueFrom:
-                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
-          ports:
-            - containerPort: 8080
-          volumeMounts:
-            - name: dags
-              mountPath: /opt/airflow/dags
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: 8080
-            # First boot runs the whole migration before it serves anything.
-            initialDelaySeconds: 60
-            periodSeconds: 15
-            failureThreshold: 20
-      volumes:
-        - name: dags
-          configMap:
-            name: airflow-dags
-            optional: true

rig/examples/data/addons/postgres.sh

@@ -2,7 +2,6 @@
 # PostgreSQL for this overlay: plain manifests, one replica on a PVC, password generated once and kept.
 # Notes: ../README.md
 set -euo pipefail
-ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -27,7 +26,77 @@ else
 fi
 
 echo "  applying manifests"
-render_manifest "$ADDON_DIR/postgres.yaml" | $K apply -n "$NS" -f - >/dev/null
+$K apply -n "$NS" -f - >/dev/null <<YAML
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-data
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests:
+      storage: ${POSTGRES_STORAGE:-2Gi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+spec:
+  replicas: 1
+  # One volume, one writer. Rolling would start a second pod against the same
+  # PVC before the first exits, and Postgres refuses to share a data directory.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: ${POSTGRES_IMAGE}
+          envFrom:
+            - secretRef:
+                name: postgres
+          env:
+            # The image initialises into the volume root otherwise, and a
+            # lost+found from the PVC makes it refuse to initdb.
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          ports:
+            - containerPort: 5432
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
+            initialDelaySeconds: 30
+            periodSeconds: 15
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: postgres-data
+YAML
 
 echo "  waiting for postgres..."
 $K rollout status deployment/postgres -n "$NS" --timeout=240s

rig/examples/data/addons/postgres.yaml

@@ -1,69 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: postgres-data
-spec:
-  accessModes: [ReadWriteOnce]
-  resources:
-    requests:
-      storage: ${POSTGRES_STORAGE:-2Gi}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: postgres
-spec:
-  selector:
-    app: postgres
-  ports:
-    - port: 5432
-      targetPort: 5432
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: postgres
-spec:
-  replicas: 1
-  # One volume, one writer. Rolling would start a second pod against the same
-  # PVC before the first exits, and Postgres refuses to share a data directory.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: postgres
-  template:
-    metadata:
-      labels:
-        app: postgres
-    spec:
-      containers:
-        - name: postgres
-          image: ${POSTGRES_IMAGE}
-          envFrom:
-            - secretRef:
-                name: postgres
-          env:
-            # The image initialises into the volume root otherwise, and a
-            # lost+found from the PVC makes it refuse to initdb.
-            - name: PGDATA
-              value: /var/lib/postgresql/data/pgdata
-          ports:
-            - containerPort: 5432
-          volumeMounts:
-            - name: data
-              mountPath: /var/lib/postgresql/data
-          readinessProbe:
-            exec:
-              command: ["sh", "-c", "pg_isready -U $POSTGRES_USER -d $POSTGRES_DB"]
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          livenessProbe:
-            exec:
-              command: ["sh", "-c", "pg_isready -U $POSTGRES_USER -d $POSTGRES_DB"]
-            initialDelaySeconds: 30
-            periodSeconds: 15
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: postgres-data

rig/examples/data/addons/redis.sh

@@ -2,7 +2,6 @@
 # Redis for this overlay: cache and broker (Celery), no persistence.
 # Notes: ../README.md
 set -euo pipefail
-ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -14,7 +13,43 @@ NS="${DATA_NAMESPACE:-data}"
 $K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
 
 echo "  applying manifests"
-render_manifest "$ADDON_DIR/redis.yaml" | $K apply -n "$NS" -f - >/dev/null
+$K apply -n "$NS" -f - >/dev/null <<YAML
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  selector:
+    app: redis
+  ports:
+    - port: 6379
+      targetPort: 6379
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: ${REDIS_IMAGE}
+          ports:
+            - containerPort: 6379
+          readinessProbe:
+            exec:
+              command: ["redis-cli", "ping"]
+            initialDelaySeconds: 3
+            periodSeconds: 5
+YAML
 
 echo "  waiting for redis..."
 $K rollout status deployment/redis -n "$NS" --timeout=180s

rig/examples/data/addons/redis.yaml

@@ -1,35 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-spec:
-  selector:
-    app: redis
-  ports:
-    - port: 6379
-      targetPort: 6379
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis
-  template:
-    metadata:
-      labels:
-        app: redis
-    spec:
-      containers:
-        - name: redis
-          image: ${REDIS_IMAGE}
-          ports:
-            - containerPort: 6379
-          readinessProbe:
-            exec:
-              command: ["redis-cli", "ping"]
-            initialDelaySeconds: 3
-            periodSeconds: 5

rig/examples/data/rig.env

@@ -7,9 +7,6 @@
 # LocalExecutor and needs no broker: add redis (before airflow) only to switch to Celery.
 ADDONS="metallb postgres airflow"
 
-# What each part costs in MB, for `make check`'s memory budget (rig adds the kind node).
-MEMORY_BUDGET="airflow=1800 postgres=300"
-
 # Namespace for the dependency containers (k8s/base/kustomization.yaml names it too).
 DATA_NAMESPACE=data
 

rig/standalone/default/rigdeps.sh

@@ -61,32 +61,6 @@ render_kind_config ()
     fi;
     sed -e "s|\${CLUSTER}|${CLUSTER}|g" -e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" -e "s|\${HTTP_PORT}|${HTTP_PORT}|g" -e "s|\${HOST_WORKDIR}|${host_workdir}|g" -e "s|\${OVERLAY_DIR}|${overlay_dir}|g" "$KIND_CONFIG"
 }
-render_manifest () 
-{ 
-    local file="$1" text ref key def val;
-    text=$(< "$file") || return 1;
-    while IFS= read -r ref; do
-        key=${ref#\$\{};
-        key=${key%\}};
-        def="";
-        if [[ $key == *:-* ]]; then
-            def=${key#*:-};
-            key=${key%%:-*};
-        fi;
-        if [ -n "${!key:-}" ]; then
-            val=${!key};
-        else
-            if [[ $ref == *:-* ]]; then
-                val=$def;
-            else
-                echo "render_manifest: $file needs \${$key}, which nothing sets" 1>&2;
-                return 1;
-            fi;
-        fi;
-        text=${text//"$ref"/"$val"};
-    done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}' "$file" | sort -u);
-    printf '%s\n' "$text"
-}
 
 # ── configuration, frozen for profile 'default' ──
 load_config() {

rig/standalone/default/rigmini.sh

@@ -61,32 +61,6 @@ render_kind_config ()
     fi;
     sed -e "s|\${CLUSTER}|${CLUSTER}|g" -e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" -e "s|\${HTTP_PORT}|${HTTP_PORT}|g" -e "s|\${HOST_WORKDIR}|${host_workdir}|g" -e "s|\${OVERLAY_DIR}|${overlay_dir}|g" "$KIND_CONFIG"
 }
-render_manifest () 
-{ 
-    local file="$1" text ref key def val;
-    text=$(< "$file") || return 1;
-    while IFS= read -r ref; do
-        key=${ref#\$\{};
-        key=${key%\}};
-        def="";
-        if [[ $key == *:-* ]]; then
-            def=${key#*:-};
-            key=${key%%:-*};
-        fi;
-        if [ -n "${!key:-}" ]; then
-            val=${!key};
-        else
-            if [[ $ref == *:-* ]]; then
-                val=$def;
-            else
-                echo "render_manifest: $file needs \${$key}, which nothing sets" 1>&2;
-                return 1;
-            fi;
-        fi;
-        text=${text//"$ref"/"$val"};
-    done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*(:-[^}]*)?\}' "$file" | sort -u);
-    printf '%s\n' "$text"
-}
 
 # ── configuration, frozen for profile 'default' ──
 load_config() {

soleprint/station/tools/dataconvert/.env-example

@@ -0,0 +1,17 @@
+# Environment Variables for dataconvert tool
+
+# Input settings
+DATACONVERT_INPUT="data/*"
+DATACONVERT_EXCLUDE="*.tmp"
+DATACONVERT_CONFIG=
+
+# Output settings
+DATACONVERT_OUT_DIR=datadir/sql
+DATACONVERT_MAX_ROWS=
+DATACONVERT_SQL_FORMAT=batch
+DATACONVERT_BATCH_ROWS=500
+DATACONVERT_MAX_FILE_SIZE=100M
+DATACONVERT_ALL_TEXT=false
+DATACONVERT_KEEP_FOLDERS=true
+DATACONVERT_NO_SCHEMA=false
+DATACONVERT_NO_DDL=false

soleprint/station/tools/dataconvert/README.md

@@ -100,7 +100,7 @@ Shapes are judged on up to 200,000 values spread through the column.
 ## Folders and exclusions
 
 Output is flat by default: a table's file is named after the file or sheet, and two
-source folders with the same table name (two studies' `drm_lb.csv`) write into the
+source folders with the same table name (two exports' `users.csv`) write into the
 same file. `keep_folders` mirrors the source folders under `out_dir` instead, so each
 keeps its own `.sql`; `SCHEMA.md` stays one document at the top, with the folder in
 each table's heading.

soleprint/station/tools/dataconvert/config.py

@@ -17,6 +17,8 @@ header on row 1.
       "batch_rows": 500,
       "max_file_size": "100M",
       "schema": true,
+      "include_ddl": true,
+      "load_order": ["ref_data", "users", "orders"],
       "header_row": 1,
       "layouts": [
         {
@@ -35,11 +37,11 @@ is one of match.in. The first layout that matches wins; a sheet none matches
 uses the top-level header_row/data_row, which default to 1 and 2.
 
 The run settings (out_dir, max_rows, schema, sql_format, batch_rows,
-max_file_size, all_text, keep_folders, exclude) are the command line's flags
+max_file_size, all_text, keep_folders, exclude, include_ddl, load_order) are the
-with the same names, and a flag given on the command line wins over the file;
+command line's flags with the same names, and a flag given on the command line
---exclude patterns are added to the file's. out_dir is relative to the folder
+wins over the file; --exclude patterns are added to the file's. out_dir is relative
-the config file is in; max_file_size takes a number of bytes or a suffixed size
+to the folder the config file is in; max_file_size takes a number of bytes or a
-such as "100M" (decimal: k, M, G). Keys starting with _ are
+suffixed size such as "100M" (decimal: k, M, G). Keys starting with _ are
 comments; any other unknown key is an error, so a typo is not silently ignored.
 """
 
@@ -96,7 +98,8 @@ class Config:
     def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None,
                  fallback=None, max_rows=None, schema=None, out_dir=None,
                  sql_format=None, batch_rows=None, max_file_size=None,
-                 all_text=None, keep_folders=None, exclude=()):
+                 all_text=None, keep_folders=None, exclude=(), include_ddl=None,
+                 load_order=()):
         self.rules = list(rules)
         self.bare_sheet_prefixes = tuple(bare_sheet_prefixes)
         self.source = source
@@ -115,6 +118,8 @@ class Config:
         self.all_text = all_text
         self.keep_folders = keep_folders
         self.exclude = tuple(exclude)
+        self.include_ddl = include_ddl
+        self.load_order = tuple(load_order)
 
     @property
     def probe_rows(self):
@@ -137,6 +142,8 @@ class Config:
             said.append(f"max_rows {self.max_rows}")
         if self.schema is not None:
             said.append(f"schema {'on' if self.schema else 'off'}")
+        if self.include_ddl is not None:
+            said.append(f"include_ddl {'on' if self.include_ddl else 'off'}")
         if self.sql_format is not None:
             said.append(f"sql_format {self.sql_format}")
         if self.batch_rows is not None:
@@ -149,6 +156,8 @@ class Config:
             said.append("keep_folders")
         if self.exclude:
             said.append(f"exclude {', '.join(self.exclude)}")
+        if self.load_order:
+            said.append(f"load_order {', '.join(self.load_order)}")
         if self.fallback is not None:
             said.append(f"header row {self.fallback.header_row}, data from row {self.fallback.data_row}")
         if self.rules:
@@ -158,7 +167,8 @@ class Config:
 
 FILENAME = "dataconvert.json"
 KNOWN_KEYS = {"layouts", "bare_sheet_prefixes", "max_rows", "schema", "out_dir", "header_row", "data_row",
-              "sql_format", "batch_rows", "max_file_size", "all_text", "keep_folders", "exclude"}
+              "sql_format", "batch_rows", "max_file_size", "all_text", "keep_folders", "exclude",
+              "include_ddl", "load_order"}
 
 
 def parse_size(value):
@@ -214,6 +224,9 @@ def load(path=None, forced=None):
     schema = raw.get("schema")
     if schema is not None and not isinstance(schema, bool):
         raise ConfigError(f"{path}: schema must be true or false")
+    include_ddl = raw.get("include_ddl")
+    if include_ddl is not None and not isinstance(include_ddl, bool):
+        raise ConfigError(f"{path}: include_ddl must be true or false")
     out_dir = raw.get("out_dir")
     if out_dir is not None:
         if not isinstance(out_dir, str) or not out_dir.strip():
@@ -242,6 +255,10 @@ def load(path=None, forced=None):
     if not isinstance(exclude, list) or not all(isinstance(p, str) and p for p in exclude):
         raise ConfigError(f'{path}: exclude must be a list of patterns such as "*.xlsx.ods"')
 
+    load_order = raw.get("load_order", [])
+    if not isinstance(load_order, list) or not all(isinstance(p, str) and p for p in load_order):
+        raise ConfigError(f'{path}: load_order must be a list of table identifiers')
+
     fallback = None
     if "header_row" in raw or "data_row" in raw:
         try:
@@ -252,4 +269,5 @@ def load(path=None, forced=None):
     return Config(rules, [str(p) for p in prefixes], source=path, forced=forced,
                   fallback=fallback, max_rows=max_rows, schema=schema, out_dir=out_dir,
                   sql_format=sql_format, batch_rows=batch_rows, max_file_size=max_file_size,
-                  all_text=flags["all_text"], keep_folders=flags["keep_folders"], exclude=exclude)
+                  all_text=flags["all_text"], keep_folders=flags["keep_folders"], exclude=exclude,
+                  include_ddl=include_ddl, load_order=load_order)

soleprint/station/tools/dataconvert/dataconvert-example.json

@@ -2,7 +2,7 @@
   "_comment": "Template for one set of spreadsheets. Copy it to dataconvert.json in the folder that holds them, where it is picked up automatically, or pass any file with --config FILE. Every key is optional; keys starting with _ are comments, any other unknown key is an error. Rows and columns count from 1, as the spreadsheet shows them.",
 
   "_run": "The same settings as the command-line flags, and a flag given there wins. out_dir is relative to this file's folder. max_rows samples each table (leave it out for all rows). schema false skips SCHEMA.md. sql_format: insert (a statement per row, default), batch (one INSERT per batch_rows rows), or copy (COPY FROM stdin: smallest, psql only, load once). max_file_size splits a table into numbered parts past that size, e.g. \"100M\".",
-  "out_dir": "seed",
+  "out_dir": "datadir/sql",
 
   "_values": "all_text reads every cell as the string in the file (only empty cells become NULL), so codes keep leading zeros and NA stays NA; SCHEMA.md then says what each column looks like. keep_folders mirrors the source folders under out_dir, so same-named tables from different folders do not share a file. exclude skips files by glob: no / matches the file name at any depth, with a / the path under the input folder.",
   "all_text": true,

soleprint/station/tools/dataconvert/dataconvert.py

@@ -25,6 +25,7 @@ it is copied: run this script from anywhere, no install step.
 """
 
 import argparse
+import os
 from pathlib import Path
 
 from output import write_tables
@@ -35,6 +36,24 @@ from readers import expand_inputs, iter_sources
 from schema import SchemaReport
 
 
+def load_dotenv():
+    env_path = Path(".env")
+    if env_path.is_file():
+        for line in env_path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                key, val = line.split("=", 1)
+                if key not in os.environ:
+                    os.environ[key] = val.strip().strip('"\'')
+
+
+def get_bool_env(key, default=False):
+    val = os.getenv(key)
+    if val is None:
+        return default
+    return val.strip().lower() in ('true', '1', 'yes', 'on')
+
+
 def size(value):
     try:
         return cfg.parse_size(value)
@@ -50,35 +69,41 @@ def positive_int(value):
 
 
 def main():
+    load_dotenv()
+
+    default_input = os.getenv("DATACONVERT_INPUT")
+    default_exclude = os.getenv("DATACONVERT_EXCLUDE", "").split(",") if os.getenv("DATACONVERT_EXCLUDE") else []
+    
     parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
-    parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
+    parser.add_argument("--input", nargs="+", default=[default_input] if default_input else None, required=not bool(default_input), 
-    parser.add_argument("--out-dir", default=None,
+                        help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
-                        help="Directory where individual .sql files will be written (default: the config's out_dir; one of the two is required)")
+    parser.add_argument("--out-dir", default=os.getenv("DATACONVERT_OUT_DIR"),
-    parser.add_argument("--max-rows", type=positive_int, default=None,
+                        help="Directory where individual .sql files will be written (default: the config's out_dir or env)")
-                        help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size (default: the config's max_rows, else all)")
+    parser.add_argument("--max-rows", type=positive_int, default=os.getenv("DATACONVERT_MAX_ROWS"),
-    parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md, whatever the config says")
+                        help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
-    parser.add_argument("--sql-format", choices=FORMATS, default=None,
+    parser.add_argument("--no-schema", action="store_true", default=get_bool_env("DATACONVERT_NO_SCHEMA"), 
+                        help="Do not write SCHEMA.md, whatever the config says")
+    parser.add_argument("--no-ddl", action="store_true", default=get_bool_env("DATACONVERT_NO_DDL"), 
+                        help="Do not emit CREATE TABLE IF NOT EXISTS statements before seeding")
+    parser.add_argument("--sql-format", choices=FORMATS, default=os.getenv("DATACONVERT_SQL_FORMAT"),
                         help="insert: a statement per row (default). batch: one INSERT per --batch-rows rows. "
                              "copy: COPY FROM stdin, smallest, psql only, load once")
-    parser.add_argument("--batch-rows", type=positive_int, default=None,
+    parser.add_argument("--batch-rows", type=positive_int, default=os.getenv("DATACONVERT_BATCH_ROWS", DEFAULT_BATCH_ROWS),
                         help=f"Rows per INSERT with --sql-format batch (default {DEFAULT_BATCH_ROWS})")
-    parser.add_argument("--max-file-size", type=size, default=None,
+    parser.add_argument("--max-file-size", type=size, default=os.getenv("DATACONVERT_MAX_FILE_SIZE"),
-                        help='Split a table into numbered parts past this size, e.g. "100M" (default: never split; samples are never split)')
+                        help='Split a table into numbered parts past this size, e.g. "100M"')
-    parser.add_argument("--all-text", action="store_true", default=None,
+    parser.add_argument("--all-text", action="store_true", default=get_bool_env("DATACONVERT_ALL_TEXT", None),
-                        help="Read every column as text, exactly as in the file; only empty cells become NULL. "
+                        help="Read every column as text, exactly as in the file; only empty cells become NULL.")
-                             "Types are then the loader's business; SCHEMA.md says what each column looks like")
+    parser.add_argument("--keep-folders", action="store_true", default=get_bool_env("DATACONVERT_KEEP_FOLDERS", None),
-    parser.add_argument("--keep-folders", action="store_true", default=None,
+                        help="Mirror the source folders under the output directory")
-                        help="Mirror the source folders under the output directory, instead of one flat folder "
+    parser.add_argument("--exclude", action="append", default=default_exclude, metavar="GLOB",
-                             "where same-named tables from different folders share a file")
+                        help='Skip matching files, e.g. "*.xlsx.ods". Repeatable, and added to the config\'s')
-    parser.add_argument("--exclude", action="append", default=[], metavar="GLOB",
-                        help='Skip matching files, e.g. "*.xlsx.ods"; a pattern with no / matches the file name '
-                             "at any depth. Repeatable, and added to the config's")
     parser.add_argument("--header-row", type=positive_int, default=1,
                         help="Spreadsheet row holding the column names, for every file; overrides the config's layouts (default 1)")
     parser.add_argument("--data-row", type=positive_int, default=None,
                         help="First row of data, when rows sit between it and the header (default: the row after the header)")
-    parser.add_argument("--config", default=None,
+    parser.add_argument("--config", default=os.getenv("DATACONVERT_CONFIG"),
-                        help="JSON with layouts, naming and run settings, for every input (default: dataconvert.json in each input's folder, if there is one)")
+                        help="JSON with layouts, naming and run settings, for every input")
     args = parser.parse_args()
 
     forced = None
@@ -129,14 +154,16 @@ def main():
 
         # The command line wins, then the config.
         out_dir = Path(args.out_dir) if args.out_dir else config.out_dir
-        max_rows = args.max_rows if args.max_rows is not None else config.max_rows
+        max_rows = int(args.max_rows) if args.max_rows is not None else config.max_rows
         schema = not args.no_schema and config.schema is not False
+        include_ddl = not args.no_ddl and config.include_ddl is not False
         fmt = args.sql_format or config.sql_format or "insert"
-        batch_rows = args.batch_rows or config.batch_rows or DEFAULT_BATCH_ROWS
+        batch_rows = int(args.batch_rows) if args.batch_rows else (config.batch_rows or DEFAULT_BATCH_ROWS)
         max_bytes = args.max_file_size or config.max_file_size
         all_text = bool(args.all_text or config.all_text)
         keep_folders = bool(args.keep_folders or config.keep_folders)
         exclude = tuple(config.exclude) + tuple(args.exclude)
+        load_order = config.load_order
 
         key = out_dir.resolve()
         if key not in reports:
@@ -146,7 +173,8 @@ def main():
 
         for source_name, folder, dfs in iter_sources(in_path, config, all_text, exclude):
             write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes,
-                         fmt, batch_rows, max_bytes, folder if keep_folders else "")
+                         fmt, batch_rows, max_bytes, folder if keep_folders else "", include_ddl=include_ddl,
+                         load_order=load_order)
 
     for out_dir, report, caps in reports.values():
         if report.entries and out_dir.is_dir():

soleprint/station/tools/dataconvert/output.py

@@ -11,16 +11,16 @@ from progress import say
 from sqlgen import DEFAULT_BATCH_ROWS, Shape, render_sample, sanitize_identifier, utf8_len
 
 
-def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=()) -> str:
+def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=(), prefix="") -> str:
     """
     Sheets of a multi-sheet workbook are prefixed with the workbook, so two
     workbooks cannot collide, unless the config names the sheet as already
-    unique (bare_sheet_prefixes).
+    unique (bare_sheet_prefixes). An optional numeric prefix (e.g. 01_) forces
+    strict load order.
     """
     clean = sanitize_identifier(raw_name)
-    if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)):
+    base_name = f"{sanitize_identifier(source_name)}_{clean}.sql" if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)) else f"{clean}.sql"
-        return f"{sanitize_identifier(source_name)}_{clean}.sql"
+    return f"{prefix}{base_name}" if prefix else base_name
-    return f"{clean}.sql"
 
 
 class TableFile:
@@ -111,19 +111,34 @@ def write_full(shape: Shape, df, table_file: TableFile) -> int:
 
 
 def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None, bare_prefixes=(),
-                 fmt="insert", batch_rows=DEFAULT_BATCH_ROWS, max_bytes=None, folder=""):
+                 fmt="insert", batch_rows=DEFAULT_BATCH_ROWS, max_bytes=None, folder="", include_ddl=True,
+                 load_order=()):
     """
     Write individual .sql files per table/sheet into the output directory, or
     into its `folder` subdirectory when the source folders are kept.
+    Respects DDL-first generation and load_order configuration.
     """
     target = out_dir / folder if folder else out_dir
-    for raw_name, df in dfs.items():
+
+    order_map = {sanitize_identifier(name): idx for idx, name in enumerate(load_order, start=1)}
+
+    def table_sort_key(item):
+        raw_name = item[0]
+        sanitized = sanitize_identifier(raw_name)
+        return (order_map.get(sanitized, 9999), sanitized)
+
+    sorted_dfs = sorted(dfs.items(), key=table_sort_key)
+
+    for raw_name, df in sorted_dfs:
         if df.empty:
             continue
 
         table = sanitize_identifier(raw_name)
-        filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes)
+        order_idx = order_map.get(table)
-        shape = Shape(fmt, table, df.columns, batch_rows)
+        file_prefix = f"{order_idx:02d}_" if order_idx is not None else ""
+
+        filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes, prefix=file_prefix)
+        shape = Shape(fmt, table, df.columns, batch_rows, include_ddl=include_ddl, df=df)
         total = len(df)
         exact = max_rows is None or max_rows >= total
         # A sample is never split: it is small, and it is not for loading.

soleprint/station/tools/dataconvert/readers.py

@@ -17,7 +17,7 @@ from pathlib import Path, PurePosixPath
 import pandas as pd
 
 from progress import say, seconds
-from sqlgen import human_bytes
+from sqlgen import human_bytes, sanitize_identifier
 
 SUPPORTED = {".csv", ".xlsx", ".xls", ".ods"}
 
@@ -68,13 +68,33 @@ def read_sheet(read, config):
         if layout is None:
             layout = config.fallback
     if layout is None:
-        return read(), None
+        data = read()
+        data.columns = usable_columns(data.columns)
+        return data, None
     raw = read(header=None)
     data = raw.iloc[layout.data_row - 1:].copy()
-    data.columns = [str(c).strip() for c in raw.iloc[layout.header_row - 1]]
+    data.columns = usable_columns(str(c).strip() for c in raw.iloc[layout.header_row - 1])
     return data, layout
 
 
+def usable_columns(names) -> list:
+    """
+    Header cells as SQL can take them: an empty one (or one of symbols only, which
+    sanitizes to nothing) becomes column_<n>, n counted from 1 as the spreadsheet does,
+    and a name that repeats once sanitized gets _2, _3. Every other name is kept as written.
+    """
+    out, seen = [], set()
+    for n, name in enumerate(names, start=1):
+        name = str(name)
+        base = name if sanitize_identifier(name) else f"column_{n}"
+        candidate, k = base, 2
+        while sanitize_identifier(candidate) in seen:
+            candidate, k = f"{base}_{k}", k + 1
+        seen.add(sanitize_identifier(candidate))
+        out.append(candidate)
+    return out
+
+
 def load_dataframes_from_file(file_path: Path, config, all_text=False) -> dict:
     """Load a file (.csv, .xlsx, .xls, .ods) into {entity_name: DataFrame}."""
     ext = file_path.suffix.lower()

soleprint/station/tools/dataconvert/sqlgen.py

@@ -38,6 +38,37 @@ def sanitize_identifier(identifier: str) -> str:
     return clean.strip("_")
 
 
+def postgres_type(series: pd.Series) -> str:
+    """Infer a PostgreSQL column data type for DDL generation."""
+    vals = series.dropna()
+    if vals.empty:
+        return "TEXT"
+    if pd.api.types.is_bool_dtype(vals):
+        return "BOOLEAN"
+    if pd.api.types.is_integer_dtype(vals):
+        return "BIGINT"
+    if pd.api.types.is_float_dtype(vals):
+        return "NUMERIC"
+    if pd.api.types.is_datetime64_any_dtype(vals):
+        return "TIMESTAMP"
+    return "TEXT"
+
+
+def ddl_statement(table_name: str, df: pd.DataFrame) -> str:
+    """Emit DDL for table creation before seeding data so that loading does not fail."""
+    cols_def = []
+    for col in df.columns:
+        col_id = sanitize_identifier(col)
+        col_type = postgres_type(df[col])
+        cols_def.append(f'    "{col_id}" {col_type}')
+    cols_sql = ",\n".join(cols_def)
+    # Several sources can feed one table (same-named files in different folders), each
+    # block with its own CREATE; only the first creates. Each block then adds the columns
+    # only it has, so the table ends up with every source's columns, NULL where one lacks them.
+    adds = "".join(f'ALTER TABLE "{table_name}" ADD COLUMN IF NOT EXISTS{line};\n' for line in cols_def)
+    return f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n{cols_sql}\n);\n{adds}\n'
+
+
 def sql_value(v) -> str:
     if pd.isna(v):
         return "NULL"
@@ -68,12 +99,14 @@ def utf8_len(text: str) -> int:
 class Shape:
     """How one table's rows are framed in one format."""
 
-    def __init__(self, fmt, table_name, columns, batch_rows=DEFAULT_BATCH_ROWS):
+    def __init__(self, fmt, table_name, columns, batch_rows=DEFAULT_BATCH_ROWS, include_ddl=True, df=None):
         if fmt not in FORMATS:
             raise ValueError(f"unknown sql format: {fmt}")
         self.fmt = fmt
         self.table_name = table_name
         self.batch_rows = batch_rows
+        self.include_ddl = include_ddl
+        self.df = df
         table_ref = f'"{table_name}"'
         cols = ", ".join(f'"{sanitize_identifier(c)}"' for c in columns)
         self.head = f"-- Generated seed data for table: {table_ref}\n"
@@ -84,6 +117,10 @@ class Shape:
 
     def open(self, note="") -> str:
         text = self.head + note + "BEGIN;\n\n"
+        if self.include_ddl and self.df is not None:
+            # IF NOT EXISTS says "skipping" once per column per block; keep the load log to errors.
+            text += "SET LOCAL client_min_messages = warning;\n"
+            text += ddl_statement(self.table_name, self.df)
         return text + self.copy if self.fmt == "copy" else text
 
     def close(self) -> str:

soleprint/station/tools/dataconvert/uv.lock

@@ -13,6 +13,23 @@ resolution-markers = [
     "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
 ]
 
+[[package]]
+name = "dataconvert"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "odfpy" },
+    { name = "openpyxl" },
+    { name = "pandas" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "odfpy", specifier = ">=1.4.1" },
+    { name = "openpyxl", specifier = ">=3.1.2" },
+    { name = "pandas", specifier = ">=2.2.0" },
+]
+
 [[package]]
 name = "defusedxml"
 version = "0.7.1"
@@ -284,23 +301,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
 ]
 
-[[package]]
-name = "dataconvert"
-version = "0.1.0"
-source = { virtual = "." }
-dependencies = [
-    { name = "odfpy" },
-    { name = "openpyxl" },
-    { name = "pandas" },
-]
-
-[package.metadata]
-requires-dist = [
-    { name = "odfpy", specifier = ">=1.4.1" },
-    { name = "openpyxl", specifier = ">=3.1.2" },
-    { name = "pandas", specifier = ">=2.2.0" },
-]
-
 [[package]]
 name = "six"
 version = "1.17.0"

soleprint/station/tools/datasource/.env-example

@@ -0,0 +1,20 @@
+# Environment Variables for datasource tools
+
+# Database Credentials
+POSTGRES_USER=app
+POSTGRES_PASSWORD=secret
+POSTGRES_DB=app
+
+# Full Database URI (Overrides user/pass/db if provided)
+TARGET_DB_URI=postgresql://app:secret@postgres:5432/app
+
+# Data Dump configuration
+DUMP_OUT_DIR=/backups/cleaned_db_dump
+DUMP_IN_DIR=/backups/cleaned_db_dump
+# Pass schemas exactly as required by pg_dump, e.g., "--schema=core --schema=sales"
+DUMP_SCHEMAS="--schema=core"
+
+# Data Load / Seeding Configuration
+SEED_DIR=/rig/datadir/sql
+TOOLS_DIR=/rig/tools/datasource
+CONF_DIR=/rig/datadir/datasource

soleprint/station/tools/datasource/.gitignore

@@ -0,0 +1,4 @@
+# The real mapping: how one producer's raw tables become the logical domains names
+# their tables and columns. It lives in the data folder (datadir/datasource/), not
+# here; this only stops a stray copy being committed. Start from schema_mapping-example.yaml.
+schema_mapping.yaml

soleprint/station/tools/datasource/README.md

@@ -0,0 +1,30 @@
+# datasource
+
+Scripts to dump, restore, and seed the database, including schema migration mapping.
+
+## Schema Mapping
+
+The data loading process converges raw seed tables into logical domains using a YAML configuration.
+This configuration is injected at runtime from `datadir/datasource/schema_mapping.yaml` (never
+committed: it names the producer's tables) rather than hardcoded in the tool. Start from
+`schema_mapping-example.yaml`.
+
+Example mapping structure:
+
+```yaml
+tables:
+  - source: raw_users
+    target: core.users
+    truncate: true
+    columns:
+      user_id: id
+      username: name
+      created_at: etl_utils.safe_cast_date(created_date)
+      is_active: "COALESCE(etl_utils.safe_cast_boolean(status), TRUE)"
+```
+
+A target that does not exist yet is created from the mapping: its schema, then a table whose
+columns are the mapping's names, typed by its expressions (`safe_cast_date` gives a `date`). A
+target that already exists, from a real DDL with keys and constraints, is used as it is.
+
+All operational variables (database credentials, dump directories, active schemas) are read from the environment or a `.env` file at the root. See `.env-example` for available configuration keys.

soleprint/station/tools/datasource/casting_helpers.sql

@@ -0,0 +1,30 @@
+-- Create dedicated schema for ETL migration helpers
+CREATE SCHEMA IF NOT EXISTS etl_utils;
+
+-- 1. Safe String to Boolean Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_boolean(val TEXT)
+RETURNS BOOLEAN AS $$ BEGIN     IF val IS NULL OR TRIM(val) = '' OR UPPER(TRIM(val)) IN ('NULL', 'N/A', '.') THEN         RETURN NULL;     ELSIF UPPER(TRIM(val)) IN ('Y', 'YES', '1', 'TRUE') THEN         RETURN TRUE;     ELSIF UPPER(TRIM(val)) IN ('N', 'NO', '0', 'FALSE') THEN         RETURN FALSE;     ELSE         RETURN NULL;     END IF; END; $$ LANGUAGE plpgsql IMMUTABLE;
+
+-- 2. Safe String to Numeric/Integer Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_numeric(val TEXT)
+RETURNS NUMERIC AS $$ DECLARE     cleaned TEXT; BEGIN     cleaned := REGEXP_REPLACE(TRIM(val), '[^0-9.-]', '', 'g');     IF cleaned = '' OR cleaned = '.' OR cleaned = '-' THEN         RETURN NULL;     END IF;     RETURN cleaned::NUMERIC; EXCEPTION WHEN OTHERS THEN     RETURN NULL; END; $$ LANGUAGE plpgsql IMMUTABLE;
+
+-- 3. Safe String to Date/Timestamp Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_date(val TEXT)
+RETURNS DATE AS $func$
+BEGIN
+    IF val IS NULL OR TRIM(val) = '' OR val ~* '^(UN|UNK|UNKNOWN)' THEN
+        RETURN NULL;
+    END IF;
+
+    BEGIN
+        RETURN val::DATE;
+    EXCEPTION WHEN OTHERS THEN
+        BEGIN
+            RETURN TO_DATE(val, 'DD-MON-YY');
+        EXCEPTION WHEN OTHERS THEN
+            RETURN NULL;
+        END;
+    END;
+END;
+$func$ LANGUAGE plpgsql IMMUTABLE;

soleprint/station/tools/datasource/dump_trasformed.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+DB_USER="${POSTGRES_USER:-postgres}"
+DB_NAME="${POSTGRES_DB:-production_db}"
+OUT_DIR="${DUMP_OUT_DIR:-/backups/cleaned_db_dump}"
+SCHEMAS="${DUMP_SCHEMAS:-}"
+
+# Dump only the transformed, strictly-typed schemas (skipping raw staging tables)
+# Note: Pass schemas in DUMP_SCHEMAS as e.g. "--schema=schema1 --schema=schema2"
+pg_dump -U "$DB_USER" -d "$DB_NAME" \
+  ${SCHEMAS} \
+  --format=directory \
+  --jobs=4 \
+  --file="$OUT_DIR"

soleprint/station/tools/datasource/load_seed_data.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+# Build the URI dynamically using the injected Postgres secrets
+# Fall back to the default 'app' user if the variables are missing
+USER="${POSTGRES_USER:-app}"
+PASS="${POSTGRES_PASSWORD:-}"
+DB="${POSTGRES_DB:-app}"
+
+DB_URI="${TARGET_DB_URI:-postgresql://${USER}:${PASS}@postgres:5432/${DB}}"
+
+# Point to the containerPaths mapped in your kind-config.yaml.tpl
+SEED_DIR="${SEED_DIR:-/rig/datadir/sql}"
+TOOLS_DIR="${TOOLS_DIR:-/rig/tools/datasource}"
+CONF_DIR="${CONF_DIR:-/rig/datadir/datasource}"
+
+echo "Optimizing PostgreSQL for bulk load..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" <<-EOSQL
+    ALTER SYSTEM SET maintenance_work_mem = '1GB';
+    ALTER SYSTEM SET synchronous_commit = 'off';
+    ALTER SYSTEM SET checkpoint_timeout = '15min';
+    ALTER SYSTEM SET max_wal_size = '2GB';
+    SELECT pg_reload_conf();
+EOSQL
+
+echo "Loading casting helpers..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" -f "$TOOLS_DIR/casting_helpers.sql"
+
+echo "Loading seed data files in single transactions..."
+if [ -d "$SEED_DIR" ]; then
+    for f in "$SEED_DIR"/*.sql; do
+        if [ -f "$f" ]; then
+            echo "Processing $f..."
+            psql -v ON_ERROR_STOP=1 -1 "$DB_URI" -f "$f"
+        fi
+    done
+else
+    echo "Warning: Seed directory $SEED_DIR not found. Skipping flat file loads."
+fi
+
+echo "Converging schemas into logical domains using configured mapping..."
+MAPPING_FILE="${SCHEMA_MAPPING_FILE:-$CONF_DIR/schema_mapping.yaml}"
+python3 "$TOOLS_DIR/migrate_schema.py" "$MAPPING_FILE"
+
+echo "Restoring standard PostgreSQL settings..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" <<-EOSQL
+    ALTER SYSTEM RESET maintenance_work_mem;
+    ALTER SYSTEM RESET synchronous_commit;
+    ALTER SYSTEM RESET checkpoint_timeout;
+    ALTER SYSTEM RESET max_wal_size;
+    SELECT pg_reload_conf();
+EOSQL
+
+echo "Data load completed."

soleprint/station/tools/datasource/migrate_schema.py

@@ -0,0 +1,58 @@
+import os
+import sys
+import yaml
+import psycopg
+from psycopg import sql
+
+def main():
+    mapping_file = os.getenv("SCHEMA_MAPPING_FILE", "/rig/datadir/datasource/schema_mapping.yaml")
+    
+    if len(sys.argv) >= 2:
+        mapping_file = sys.argv[1]
+
+    db_uri = os.getenv("TARGET_DB_URI", "postgresql://postgres:postgres@postgres:5432/postgres")
+
+    print(f"Loading schema mapping from {mapping_file}...")
+    with open(mapping_file, 'r') as f:
+        config = yaml.safe_load(f)
+
+    with psycopg.connect(db_uri) as conn:
+        with conn.cursor() as cur:
+            for table in config.get('tables', []):
+                source = table['source']
+                target = table['target']
+                columns = table['columns']
+
+                print(f"Migrating {source} to {target}...")
+
+                target_cols = ", ".join(columns.keys())
+                source_exprs = ", ".join(columns.values())
+
+                # A missing target is made from the mapping itself: its schema, then a table
+                # shaped by the expressions (safe_cast_date gives a date column, and so on).
+                # A target that exists, from a real DDL with keys and constraints, is left alone.
+                if "." in target:
+                    cur.execute(f"CREATE SCHEMA IF NOT EXISTS {target.split('.', 1)[0]};")
+                shaped = ", ".join(f"{expr} AS {col}" for col, expr in columns.items())
+                cur.execute(f"CREATE TABLE IF NOT EXISTS {target} AS SELECT {shaped} FROM {source} WITH NO DATA;")
+
+                if table.get('truncate', False):
+                    cur.execute(f"TRUNCATE TABLE {target} RESTART IDENTITY CASCADE;")
+
+                query = f"""
+                    INSERT INTO {target} ({target_cols})
+                    SELECT {source_exprs}
+                    FROM {source}
+                """
+                try:
+                    cur.execute(query)
+                    print(f"  Successfully migrated {source}.")
+                except Exception as e:
+                    print(f"  Failed to migrate {source}: {e}")
+                    conn.rollback()
+                    raise
+        conn.commit()
+    print("Schema migration complete.")
+
+if __name__ == '__main__':
+    main()

soleprint/station/tools/datasource/restore_transformed.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+DB_USER="${POSTGRES_USER:-postgres}"
+DB_NAME="${POSTGRES_DB:-production_db}"
+IN_DIR="${DUMP_IN_DIR:-/backups/cleaned_db_dump}"
+
+# Multi-threaded restore into the target PostgreSQL cluster pod
+pg_restore -U "$DB_USER" -d "$DB_NAME" \
+  --jobs=4 \
+  --clean \
+  --if-exists \
+  "$IN_DIR"

soleprint/station/tools/datasource/schema_mapping-example.yaml

@@ -0,0 +1,21 @@
+# Schema Conversion and Type Cast Registry
+# Maps raw seed tables to the unified logical domains.
+
+tables:
+  - source: raw_users
+    target: core.users
+    truncate: true
+    columns:
+      user_id: id
+      username: name
+      created_at: etl_utils.safe_cast_date(created_date)
+      is_active: "COALESCE(etl_utils.safe_cast_boolean(status), TRUE)"
+
+  - source: raw_orders
+    target: core.orders
+    truncate: true
+    columns:
+      order_id: id
+      user_id: user_id
+      total_amount: etl_utils.safe_cast_numeric(amount)
+      order_date: etl_utils.safe_cast_date(date)

soleprint/station/tools/distill/distill-example.json

@@ -1,29 +1,97 @@
 {
-  "_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that. Everything that shapes the run is at the top; 'repos' is just a list of paths. Preview any change with: ./distill.sh list",
+  "_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that — which repos a machine distils is not a fact about the tool. Every key the file understands appears below, once, with the value it would fall back to if you deleted the line. Preview any change with: ./distill.sh list  — or ./distill.sh check, which resolves every path and ref in about a second.",
 
-  "_command": "tree = a directory per repo. digest = one .md per repo, flattened into a single readable file. both = each repo as a directory AND a .md. list = write nothing, just report what would be kept.",
+  "_command": "tree = a directory per repo, a copy you can open, grep and build. digest = one .md per repo, the whole thing flattened into a single readable file (header, file tree, manifest, then every file under its path). both = each repo comes out as a directory AND a .md — they answer different questions and the selection work is shared, so the second costs only a copy. list = write nothing, just report what would be kept, weighed. check = write nothing and read nothing but git's index: does every path exist, does every ref resolve, is there room. Every other command runs check first anyway.",
   "command": "list",
 
+  "_out": "Where the run writes. Relative to wherever you invoke the script from, so an absolute path is the safer thing to write down; ~ is expanded. Ignored by list and check.",
   "out": "distilled",
 
-  "_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base (it gets a _PARTIAL.md saying so).",
+  "_root": "Where a bare slug in 'repos' resolves, so entries can read \"path\": \"myrepo\" instead of the whole absolute path. Leave it out and slugs resolve against the current directory, or DISTILL_ROOT. Absolute paths never touch it.",
+  "root": "~/wdir",
+
+  "_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base, which is far smaller but is a changelist, not the branch (it gets a _PARTIAL.md saying so). diff_base must name one of the refs actually listed, and it is the one that comes out whole. Set per entry to override this default.",
   "branch_mode": "full",
   "diff_base": "main",
 
-  "_filters": "Applied to every repo. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, images, minified, maps). max_bytes skips anything larger and names it in MANIFEST.md.",
+  "_filters": "Applied to every repo, and overridable per entry. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, minified, maps, compiled output, caches) that is dropped by default. max_bytes drops anything larger outright, and names it in MANIFEST.md.",
-  "exclude": [],
+  "exclude": ["*.ipynb", "fixtures/**"],
   "include": [],
   "all": false,
-  "max_bytes": null,
+  "max_bytes": "2M",
 
-  "_split_tokens": "A digest over this many tokens is written as NAME.md (an index: the tree, and the manifest saying which part holds each file) plus NAME.part-01.md, NAME.part-02.md... holding the files, cut between files. Each part stands on its own. Default 100k; 0 never splits. Can also be set per entry.",
+  "_budget": "Three different answers to 'this is too much to read', and they do not compete. clip_bytes inlines only the head and tail of any file over that size, with a marker saying how much was cut. max_tokens holds each digest to roughly that many tokens by clipping largest-first — one shared ceiling, lowered until the total fits — so the biggest file pays and the hundred small ones that describe the project keep every line. Both of those lose text; split_tokens loses none, it just stops writing one enormous file: over ~N tokens the digest becomes NAME.md (an index holding the tree and the manifest, saying which part each file is in) plus NAME.part-01.md, NAME.part-02.md… cut between files in path order, each part standing on its own. Default 100k; 0 never splits. Clipping is a statement about the document, not the files: the tree copy always has them whole. Numbers take k/M/G, decimal.",
+  "clip_bytes": null,
+  "max_tokens": null,
   "split_tokens": "100k",
+
+  "_with_root": "Only matters when an entry names a subpath. Keeps the repo's top-level files (README, pyproject.toml, package.json) alongside the subtree, so the copy still says which project it is a part of.",
+  "with_root": false,
+
+  "_keep_secrets": ".env files, private keys, keystores and the like are dropped by default and named in MANIFEST.md. 'all' does not bring them back — only this does, and only deliberately. Set it on one entry instead to keep the secrets of that repo alone.",
+  "keep_secrets": false,
+
+  "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes what an earlier run into 'out' wrote and this run no longer produces, so dropping a repo from the list drops its output too. Anything distill did not write (it keeps the list in out/.distill-owned) is never touched, so 'out' can be a folder shared with other things.",
   "skip_unchanged": true,
   "prune": true,
 
-  "_repos": "'path' is absolute, or a name resolved under --root. 'branches' is optional — leave it out for the working tree as it stands, uncommitted changes included. A path may appear more than once.",
+  "_bundle": "Also write _BUNDLE.md: every .md in the destination concatenated, in one document, behind an index. A directory of files is right for grep and for reading; an upload box takes one file. Costs a second copy of the text on disk and nothing else. Note that it concatenates whatever split_tokens produced, parts included — bundle and split are opposite answers to the same question, and you can have both on disk.",
+  "bundle": false,
+
+  "_refs_patch": "Put the full diff, not just the diffstat, in NAME@REFS.md. Answers 'what changed' instead of 'how much'.",
+  "refs_patch": false,
+
+  "_raw_fences": "Write runs of backticks and tildes into the digest as they are, instead of escaping them as ⟪BT3⟫ / ⟪TL3⟫. Escaping is the default because a fence inside a file otherwise closes the fence the digest is being read inside; explode.sh puts the characters back. Turn this off only when nothing is going to render the result as markdown.",
+  "raw_fences": false,
+
+  "_top": "'list' only: how many heavy files and directories to show. 0 for none.",
+  "top": 10,
+
+  "_repos": "One entry per thing you want out. 'path' is absolute, or a name resolved under 'root'. The same path may appear as many times as you like, which is why this is a list rather than a keyed object — one entry per repo could not hold two branches of the same repo. Every filter and budget key above may be repeated inside an entry, where it replaces the top-level value for that entry alone; a command-line option still wins over both.",
   "repos": [
-    { "path": "/path/to/some-repo" },
+    {
-    { "path": "/path/to/another-repo", "branches": ["origin/main", "origin/feature/example"] }
+      "_note": "The plainest entry: the working tree as it stands right now, uncommitted changes included, under every setting from the top of the file.",
+      "path": "/path/to/some-repo"
+    },
+
+    {
+      "_note": "Named refs. Each is read with ls-tree/archive, so nothing is checked out and a dirty working tree is never touched. In full mode each of these is a complete copy. A hash, a tag or HEAD~3 works anywhere a branch name does.",
+      "path": "/path/to/another-repo",
+      "branches": ["origin/main", "v1.4.0", "3f2a9c1"]
+    },
+
+    {
+      "_note": "Many branches. In full mode this would be one complete copy per branch, most of them the same bytes — the right answer for an archive and the wrong one for anything with a context window. So: origin/main whole, and the rest as changelists against it.",
+      "path": "/path/to/busy-repo",
+      "branch_mode": "diff",
+      "diff_base": "origin/main",
+      "branches": ["origin/main", "origin/topic-a", "origin/topic-b"]
+    },
+
+    {
+      "_note": "A subtree, and only it — with the repo's own ignore rules, refs and deltas still applying, because the repo root is found for you. subpath may be a list: 'the part I am working on' is usually the module plus the tests or docs beside it, and two entries for that would be two outputs to read separately. 'name' overrides what the output is called, which is what you want when the same repo appears more than once.",
+      "path": "slug-resolved-under-root",
+      "subpath": ["src/api", "tests/api"],
+      "with_root": true,
+      "name": "api-only"
+    },
+
+    {
+      "_note": "Per-entry filters and budget. One generated file does not get to spend the whole digest; this entry also keeps the lockfiles the top of the file drops, and splits later than everything else.",
+      "path": "/path/to/heavy-repo",
+      "include": ["src/**", "*.md"],
+      "exclude": ["src/generated/**"],
+      "all": true,
+      "max_bytes": "5M",
+      "clip_bytes": "64k",
+      "max_tokens": "80k",
+      "split_tokens": "250k"
+    },
+
+    {
+      "_note": "Kept in the file, out of the run. Better than deleting the entry when it is something you distil occasionally — the paths and the branch list stay written down.",
+      "path": "/path/to/retired-repo",
+      "enabled": false
+    }
   ]
 }

soleprint/station/tools/distill/distill.sh

@@ -68,8 +68,8 @@
 # rather than a keyed object; one entry per repo could not hold two branches of
 # the same repo. Per-entry keys: path, branches, subpath (a string, or a list
 # of them), name, enabled, branch_mode, diff_base, include, exclude, max_bytes,
-# clip_bytes, max_tokens, split_tokens, with_root — each falling back to the top
+# clip_bytes, max_tokens, split_tokens, with_root, keep_secrets — each falling
-# of the file.
+# back to the top of the file.
 # A command-line option overrides both: one repo in the list wanting a tighter
 # budget should say so in its entry, but `--max-tokens 60k` on the command line
 # is a thing someone just typed, and it wins over the whole file.
@@ -128,8 +128,12 @@
 #   --strict        refuse a dirty working tree (only affects worktree copies)
 #   --skip-unchanged  leave alone anything whose source and settings have not
 #                   moved since the last run into this destination
-#   --prune         delete anything in DEST this run did not produce, so
+#   --prune         delete what an earlier run into DEST wrote and this run no
-#                   dropping a repo from the list drops its output too
+#                   longer produces, so dropping a repo from the list drops its
+#                   output too. Only what distill created is ever removed
+#                   (DEST/.distill-owned records it): other things in DEST, and
+#                   folders that existed before a run first synced into them,
+#                   are never touched, so DEST can be a shared folder
 #   --bundle        also write DEST/_BUNDLE.md: every digest concatenated into
 #                   one document, for anything that takes a single file
 #   --refs-patch    put the full diff, not just the diffstat, in NAME@REFS.md
@@ -422,8 +426,6 @@ if [ -n "$CONFIG" ]; then
     [ "$(jq -r 'if has("bundle") then .bundle else false end' "$CONFIG")" = true ] && BUNDLE=1
     [ "$(jq -r 'if has("refs_patch") then .refs_patch else false end' "$CONFIG")" = true ] && REFS_PATCH=1
     [ "$(jq -r 'if has("raw_fences") then .raw_fences else false end' "$CONFIG")" = true ] && RAW_FENCES=1
-    [ "$(jq -r 'if has("keep_secrets") then .keep_secrets else false end' "$CONFIG")" = true ] \
-        && KEEP_SECRETS=1
     [ "$(jq -r 'if has("skip_unchanged") then .skip_unchanged else false end' "$CONFIG")" = true ] \
         && SKIP_UNCHANGED=1
 
@@ -467,7 +469,12 @@ if [ -n "$CONFIG" ]; then
             split_tokens: (($e.split_tokens // $cfg.split_tokens // "") | tostring),
             with_root:  (if ($e|has("with_root")) then $e.with_root
                          elif ($cfg|has("with_root")) then $cfg.with_root
-                         else false end)
+                         else false end),
+            # Per entry, like everything else: one repo whose .env is the
+            # point of the copy should not mean every repo beside it leaks.
+            keep_secrets: (if ($e|has("keep_secrets")) then $e.keep_secrets
+                           elif ($cfg|has("keep_secrets")) then $cfg.keep_secrets
+                           else false end)
           }
     ' "$CONFIG" >> "$JOBS" || die "could not read the repo list from $CONFIG"
 
@@ -1350,6 +1357,17 @@ STATE_FILE=""
 STATE_OLD="$TMP/state.old"
 STATE_NEW="$TMP/state.new"
 PRODUCED=()
+# Which top-level entries of DEST distill wrote, as of the previous run. Prune
+# reads this, not the directory: DEST may be a folder shared with things that
+# were never distill's, and those must survive a prune however it is set.
+#
+# Owned means created, not merely written to. An entry that was already there
+# the first time a run wrote into it — a folder being synced into, holding files
+# of its own — is updated but never claimed, so dropping it from the list can
+# never delete what was there before distill.
+OWNED_FILE=""
+OWNED_OLD="$TMP/owned.old"
+EXISTED_BEFORE="$TMP/existed.before"
 
 # Saved after every entry, not once at the end. A run over a dozen repos and
 # their branches takes long enough to get interrupted, and a record written only
@@ -1387,10 +1405,11 @@ fingerprint() {
     else
         src="plain:$(find "$dir" -type f -printf '%P %s %T@\n' 2>/dev/null | LC_ALL=C sort | cksum | cut -d" " -f1)"
     fi
-    printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
+    printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
         "$src" "$sub" "$CMD" "$BASE_REF" "$KEEP_NOISE" "$MAX_BYTES" \
         "${INCLUDES[*]-}" "${EXCLUDES[*]-}" "$MIRROR" \
         "$CLIP_BYTES" "$MAX_TOKENS" "$WITH_ROOT" "$RAW_FENCES" "${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS}" \
+        "$KEEP_SECRETS" \
         | cksum | cut -d' ' -f1
 }
 
@@ -1847,6 +1866,10 @@ if [ "$CMD" != list ] && [ "$CMD" != check ] && [ -z "$DRY" ]; then
     : > "$STATE_NEW"
     produced "$DEST/MANIFEST.md"
     produced "$STATE_FILE"
+    OWNED_FILE="$DEST/.distill-owned"
+    [ -f "$OWNED_FILE" ] && cp "$OWNED_FILE" "$OWNED_OLD"
+    find "$DEST" -mindepth 1 -maxdepth 1 -printf '%f\n' > "$EXISTED_BEFORE"
+    produced "$OWNED_FILE"
 fi
 
 # Repos named on the command line run under the global options, as before.
@@ -1866,6 +1889,7 @@ CLI_MAX_TOKENS="$MAX_TOKENS"
 CLI_SPLIT_TOKENS="$SPLIT_TOKENS"
 CLI_KEEP_NOISE="$KEEP_NOISE"
 CLI_WITH_ROOT="$WITH_ROOT"
+CLI_KEEP_SECRETS="$KEEP_SECRETS"
 CLI_INCLUDES=(${INCLUDES[@]+"${INCLUDES[@]}"})
 CLI_EXCLUDES=(${EXCLUDES[@]+"${EXCLUDES[@]}"})
 
@@ -1887,6 +1911,8 @@ if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then
         then KEEP_NOISE=1; else KEEP_NOISE=""; fi
         if [ -n "$CLI_WITH_ROOT" ] || [ "$(job_value "$job" .with_root)" = true ]
         then WITH_ROOT=1; else WITH_ROOT=""; fi
+        if [ -n "$CLI_KEEP_SECRETS" ] || [ "$(job_value "$job" .keep_secrets)" = true ]
+        then KEEP_SECRETS=1; else KEEP_SECRETS=""; fi
 
         INCLUDES=(); EXCLUDES=()
         while IFS= read -r g; do [ -n "$g" ] && INCLUDES+=("$g"); done \
@@ -1950,16 +1976,19 @@ if [ -n "$BUNDLE" ] && [ "$CMD" != tree ] && [ "$CMD" != list ] && [ -z "$DRY" ]
     fi
 fi
 
-# Anything at the top of the destination this run did not produce came from a
+# Something distill wrote on an earlier run and this run did not produce came
-# previous, longer list. Scoped to depth 1 and to a destination we just wrote
+# from a previous, longer list. Only that: an entry distill never wrote is not
-# to: this deletes, so it should never go hunting.
+# an orphan but someone else's, and DEST may well be a folder shared with them.
-if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
+# Scoped to depth 1 and to a destination we just wrote to: this deletes, so it
+# should never go hunting.
+if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ] && [ -f "$OWNED_OLD" ]; then
     while IFS= read -r -d '' entry; do
         keep=""
         # A digest left alone as unchanged registers only its index, not the
         # parts beside it, which belong to it all the same.
         owner="$entry"
         case "$entry" in *.part-[0-9][0-9].md) owner="${entry%.part-[0-9][0-9].md}.md" ;; esac
+        grep -qxF "$(basename "$owner")" "$OWNED_OLD" || continue
         for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
             { [ "$entry" = "$kept" ] || [ "$owner" = "$kept" ]; } && { keep=1; break; }
         done
@@ -1970,6 +1999,27 @@ if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
     done < <(find "$DEST" -mindepth 1 -maxdepth 1 -print0)
 fi
 
+# Written whether or not prune is on: what this run created, plus whatever an
+# earlier run created that is still on disk, so turning prune on later still
+# knows everything distill put here.
+if [ -n "$OWNED_FILE" ]; then
+    {
+        for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
+            kept="$(basename "$kept")"
+            if ! grep -qxF "$kept" "$EXISTED_BEFORE" \
+               || { [ -f "$OWNED_OLD" ] && grep -qxF "$kept" "$OWNED_OLD"; }; then
+                printf '%s\n' "$kept"
+            fi
+        done
+        if [ -f "$OWNED_OLD" ]; then
+            while IFS= read -r o; do
+                if [ -n "$o" ] && [ -e "$DEST/$o" ]; then printf '%s\n' "$o"; fi
+            done < "$OWNED_OLD"
+        fi
+    } | LC_ALL=C sort -u > "$OWNED_FILE.partial"
+    mv "$OWNED_FILE.partial" "$OWNED_FILE"
+fi
+
 if [ -n "$SKIP_UNCHANGED" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
     mv "$STATE_NEW" "$STATE_FILE"
 fi