Merge branch 'rig-work'

distilled tools
installer
2026-09-22 08:18:31 -03:00 · 2026-09-22 08:18:25 -03:00 · 2026-09-22 08:18:02 -03:00
53 changed files with 943 additions and 74 deletions
--- a/rig/BOOTSTRAP.md
+++ b/rig/BOOTSTRAP.md
@@ -143,6 +143,10 @@ From here on this machine has curl, so **`make deps` is the short form** for
 every later run and every later copy of this directory. The container path is
 the first-time path.

+How the installer itself is tested — machine fixtures, clean containers, a
+Workspace snapshot, a throwaway WSL distro — is in
+[`docs/notes/installer-testing.md`](docs/notes/installer-testing.md).
+

 ## Prove the machine before blaming the project

--- a/rig/Makefile
+++ b/rig/Makefile
@@ -70,8 +70,8 @@ tilt:                          ## dev loop  [up|down]                  (default
 # The counterpart to check: that one asks about the MACHINE and never fails,
 # this one asks about RIG and exits 1. Checks are written as the decisions they
 # defend, so a failure names what is being undone.
-selftest:                      ## does rig still do what it says? exits 1 if not
-	bash ctrl/selftest.sh
+selftest:                      ## does rig still do what it says? exits 1 if not  [install]
+	bash ctrl/selftest.sh $(ARGS)

 # The one-file versions of rig's tools, one folder per profile, for machines the
 # full rig is not going to. Generated from rig as it is, never edited by hand;
--- a/rig/README.md
+++ b/rig/README.md
@@ -110,7 +110,9 @@ cluster name, kube context, ports and paths — the same values every other rig
 script resolves through `ctrl/lib/config.sh` — so a copied and renamed rig, or a
 moved overlay, deploys into its own cluster with no edits.

-`make help` lists every target.
+`make help` lists every target. `make selftest` checks rig itself, the installer's
+detection included; `make selftest install` runs the installer on clean containers
+([`docs/notes/installer-testing.md`](docs/notes/installer-testing.md)).

 On a machine where Docker really is the only thing installed, `make deps` has
 nothing to download with — see [BOOTSTRAP.md](BOOTSTRAP.md), which runs the
--- a/rig/ctrl/deps.sh
+++ b/rig/ctrl/deps.sh
@@ -3,7 +3,7 @@
 # Toolchain installer: detect the host, install pinned tools into $OUT_BIN, report
 # host actions it will not perform (no sudo, no apt). Usually via `make deps`.
 # Usage: deps.sh [detect [all] | list | verify [core|dev] | fetch [core|dev] [--to DIR] | install [core|dev]
-#                 | manifest NAME | manifests [--to DIR]]
+#                 | manifest NAME | manifests [--to DIR] | snapshot [DIR]]
 # Notes: docs/notes/deps.md

 set -euo pipefail
@@ -27,6 +27,14 @@ abspath() {

 OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
 HOST_ROOT="${HOST_ROOT:-/}"
+# A host fixture (docs/notes/installer-testing.md) is a root whose kernel files stand in
+# for this machine's; UNAME_S does the same for the one fact a file cannot carry.
+if [ "$HOST_ROOT" != / ]; then
+    if [ -r "$HOST_ROOT/proc/meminfo" ]; then MEMINFO="${MEMINFO:-$HOST_ROOT/proc/meminfo}"; fi
+    if [ -r "$HOST_ROOT/proc/sys/vm/overcommit_memory" ]; then
+        OVERCOMMIT_FILE="${OVERCOMMIT_FILE:-$HOST_ROOT/proc/sys/vm/overcommit_memory}"
+    fi
+fi
 DEPS_SOURCE="${DEPS_SOURCE:-upstream}"
 DEPS_ARTIFACTORY_URL="${DEPS_ARTIFACTORY_URL:-}"
 BAKED_BIN="${BAKED_BIN:-/opt/rig/bin}"
@@ -138,7 +146,7 @@ docker_pkg() {

 # Windows outside WSL (Git Bash, MSYS, Cygwin) fails confusingly; name it instead.
 require_linux() {
-    case "$(uname -s)" in
+    case "${UNAME_S:-$(uname -s)}" in
        MINGW*|MSYS*|CYGWIN*)
            cat >&2 <<'EOF'
 This has to run inside WSL, not Git Bash / MSYS / Cygwin.
@@ -157,7 +165,7 @@ EOF
    esac
 }

-is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
+is_wsl() { grep -qi microsoft "$(host_file /proc/version)" 2>/dev/null; }

 detect() {
    echo "host"
@@ -765,6 +773,54 @@ fetch_manifests() {  # [--to DIR]
    done
 }

+# ── snapshot: this machine as a host fixture ───────────────────────────────
+# Writes what detect reads, cut down to what it needs — never the environment, the
+# home directory or the host name — plus the lines detect prints for it. A fact of
+# one machine: keep it with an overlay or in rig's local/, never in rig itself, and
+# replay it with ctrl/hosttest.sh. Notes: docs/notes/installer-testing.md
+snapshot() {
+    local dest r f
+    dest="$(abspath "${1:-host-snapshot}")"
+    if [ -n "$(ls -A "$dest" 2>/dev/null)" ]; then
+        echo "snapshot: $dest already holds something — pick an empty directory" >&2
+        exit 1
+    fi
+    r="$dest/root"
+    mkdir -p "$r/etc" "$r/proc/sys/vm"
+    f=$(host_file /etc/os-release)
+    if [ -r "$f" ]; then grep -E '^(PRETTY_NAME|NAME|VERSION_ID|ID|ID_LIKE)=' "$f" > "$r/etc/os-release"; fi
+    # The kernel release says WSL or not; the full build string names build hosts.
+    echo "Linux version $(awk '{print $3; exit}' "$(host_file /proc/version)" 2>/dev/null || uname -r)" > "$r/proc/version"
+    grep -E '^(MemTotal|MemAvailable|SwapTotal|SwapFree):' "${MEMINFO:-/proc/meminfo}" > "$r/proc/meminfo"
+    cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" > "$r/proc/sys/vm/overcommit_memory" 2>/dev/null || true
+    f=$(host_file /etc/wsl.conf)
+    if [ -r "$f" ]; then
+        # Section headers and the two keys detect reads; a [user] default= names a person.
+        grep -E '^[[:space:]]*(\[[a-z0-9]+\]|systemd[[:space:]]*=|generateResolvConf[[:space:]]*=)' "$f" > "$r/etc/wsl.conf" || true
+    fi
+    f=$(ls "$HOST_ROOT"/mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
+    if [ -n "$f" ] && grep -qE '^\s*memory\s*=' "$f"; then
+        mkdir -p "$r/mnt/c/Users/user"
+        { echo "[wsl2]"; grep -E '^\s*memory\s*=' "$f"; } > "$r/mnt/c/Users/user/.wslconfig"
+    fi
+    {
+        echo "# for the record; not replayed"
+        echo "arch=$(arch)"
+        echo "glibc=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$' || echo unknown)"
+        echo "taken=$(date -u +%Y-%m-%d)"
+    } > "$dest/facts.txt"
+    # The lines the fixture itself decides, as detect prints them for it now.
+    {
+        echo "# Written by deps.sh snapshot: what detect said about this machine."
+        env -u MEMINFO -u OVERCOMMIT_FILE HOST_ROOT="$r" bash "./$(basename "${BASH_SOURCE[0]}")" detect all 2>/dev/null \
+            | grep -E '^  (distro|memory|overcommit|systemd|resolv\.conf|wslconfig) |^  ! systemd|^  - resolv\.conf' \
+            | sed 's/^  /+ /'
+    } > "$dest/expect.txt"
+    echo "wrote $dest"
+    (cd "$dest" && find . -type f | sort | sed 's|^\./|  |')
+    echo "keep it with an overlay or in rig's local/, never in rig; replay it with rig's hosttest.sh"
+}
+
 # Baked mode copies binaries already in the image, so it needs no downloader.
 need_downloads() {
    require_amd64
@@ -781,10 +837,12 @@ case "$cmd" in
    manifest)  need_downloads
               fetch_manifest "${1:?usage: $0 manifest <METALLB|CERT_MANAGER|METRICS_SERVER>}" "$MANIFESTS_HOME" ;;
    manifests) need_downloads; fetch_manifests "$@" ;;
+    snapshot)  snapshot "${1:-}" ;;
    *)       echo "usage: $0 [detect [all]|list|verify|fetch|install|manifest NAME|manifests]" >&2
             echo "       install [core|dev]           (default dev)" >&2
             echo "       fetch   [core|dev] [--to DIR]" >&2
             echo "       manifests [--to DIR]         the addons' pinned manifests, verified" >&2
+             echo "       snapshot [DIR]               this machine as a host fixture (no secrets)" >&2
             echo "       OUT_BIN=<dir> overrides the install directory" >&2
             exit 1 ;;
 esac
--- a/rig/ctrl/hosttest.sh
+++ b/rig/ctrl/hosttest.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Replay host fixtures: run `deps.sh detect all` against a stand-in machine and check
+# what it must (and must not) say. No docker, no network, no root.
+# Usage: hosttest.sh [FIXTURE_DIR...]    default: tests/hosts/*    exits 1 on a mismatch
+# A fixture is root/ (the files detect reads), expect.txt (+ must appear, - must not,
+# exit N), and an optional env (KEY=value lines). `deps.sh snapshot` writes one.
+# Notes: docs/notes/installer-testing.md
+set -uo pipefail
+cd "$(dirname "$0")"
+
+dirs=("$@")
+if [ ${#dirs[@]} -eq 0 ]; then dirs=(../tests/hosts/*/); fi
+
+rc=0 passed=0 failed=0
+for d in "${dirs[@]}"; do
+    d="${d%/}"
+    if [ ! -f "$d/expect.txt" ]; then
+        echo "  FAIL  $d: no expect.txt — not a host fixture" >&2
+        rc=1; failed=$((failed + 1)); continue
+    fi
+    # Only the fixture's own settings: the caller's HOST_ROOT, MEMINFO or UNAME_S
+    # must not leak into a replay.
+    run=(env -u HOST_ROOT -u MEMINFO -u OVERCOMMIT_FILE -u UNAME_S)
+    if [ -d "$d/root" ]; then run+=(HOST_ROOT="$(cd "$d/root" && pwd)"); fi
+    if [ -f "$d/env" ]; then
+        while IFS= read -r kv; do run+=("$kv"); done < <(grep -vE '^[[:space:]]*(#|$)' "$d/env")
+    fi
+    out=$("${run[@]}" bash ./deps.sh detect all 2>&1)
+    code=$?
+
+    bad=""
+    want_exit=0
+    while IFS= read -r line; do
+        case "$line" in
+            '+ '*)    grep -qF -- "${line#+ }" <<< "$out" || bad+=$'\n'"        missing: ${line#+ }" ;;
+            '- '*)    grep -qF -- "${line#- }" <<< "$out" && bad+=$'\n'"        present: ${line#- }" ;;
+            'exit '*) want_exit="${line#exit }" ;;
+        esac
+    done < <(grep -vE '^[[:space:]]*(#|$)' "$d/expect.txt")
+    if [ "$code" != "$want_exit" ]; then bad+=$'\n'"        exit:    $code, wanted $want_exit"; fi
+
+    if [ -z "$bad" ]; then
+        printf '  ok    %s\n' "$(basename "$d")"
+        passed=$((passed + 1))
+    else
+        printf '  FAIL  %s%s\n' "$(basename "$d")" "$bad"
+        rc=1; failed=$((failed + 1))
+    fi
+done
+
+printf '%d host fixture(s) as expected, %d not\n' "$passed" "$failed"
+exit "$rc"
--- a/rig/ctrl/installtest.sh
+++ b/rig/ctrl/installtest.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# The installer on clean machines: the generated kit (standalone/default/rigdeps.sh) in
+# stock distro containers, as a non-root user, the way it reaches a real machine.
+# Needs docker and the network; takes minutes. Exits 1 on a failure.
+# Usage: installtest.sh [IMAGE...]    default: ubuntu:22.04 debian:trixie-slim, then offline
+# Notes: docs/notes/installer-testing.md
+set -uo pipefail
+cd "$(dirname "$0")"
+
+KIT="$(cd .. && pwd)/standalone/default/rigdeps.sh"
+images=("$@")
+if [ ${#images[@]} -eq 0 ]; then images=(ubuntu:22.04 debian:trixie-slim); fi
+
+rc=0
+passed=0
+check() {   # name, expected, actual
+    if [ "$2" = "$3" ]; then
+        printf '  ok    %s\n' "$1"
+        passed=$((passed + 1))
+    else
+        printf '  FAIL  %s\n        expected: %s\n        got:      %s\n' "$1" "$2" "$3"
+        rc=1
+    fi
+}
+
+if ! docker info >/dev/null 2>&1; then
+    echo "installtest needs a running docker it can reach" >&2
+    exit 1
+fi
+# A stale kit would test yesterday's installer.
+if ! bash ./standalone.sh check >/dev/null 2>&1; then
+    echo "the kit is stale — run: make standalone" >&2
+    exit 1
+fi
+
+for img in "${images[@]}"; do
+    printf '\n%s\n' "$img"
+
+    # A stock image has no curl or wget: the installer must say so and stop, not
+    # half-install. This is the bootstrap paradox BOOTSTRAP.md describes.
+    out=$(docker run --rm -v "$KIT:/kit/rigdeps.sh:ro" "$img" bash /kit/rigdeps.sh install dev 2>&1)
+    code=$?
+    check "bare: install refuses" "1" "$code"
+    check "bare: and names what is missing" "yes" \
+        "$(grep -q 'neither curl nor wget' <<< "$out" && echo yes || echo no)"
+
+    # The one root step a machine owner takes, then everything else as a plain user —
+    # the Workspace's case: no sudo from the installer, tools in ~/.local/bin.
+    out=$(docker run --rm -v "$KIT:/kit/rigdeps.sh:ro" "$img" bash -c '
+        set -e
+        apt-get update -qq >/dev/null
+        DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl ca-certificates >/dev/null
+        useradd -m t
+        su t -s /bin/bash -c "
+            set -e
+            bash /kit/rigdeps.sh install dev
+            echo ==verify; PATH=\$HOME/.local/bin:\$PATH bash /kit/rigdeps.sh verify dev
+            echo ==manifests; bash /kit/rigdeps.sh manifests --to \$HOME/m
+            echo ==bin; ls \$HOME/.local/bin
+            echo ==m; ls \$HOME/m
+        "' 2>&1)
+    code=$?
+    check "user: install, verify and manifests succeed" "0" "$code"
+    check "user: the dev tier lands in ~/.local/bin" "ctlptl docker-compose jq kind kubectl tilt" \
+        "$(sed -n '/^==bin$/,/^==m$/p' <<< "$out" | grep -vE '^==' | sort | xargs)"
+    check "user: tells them to put it on PATH" "yes" \
+        "$(grep -q 'Put the toolchain on your PATH' <<< "$out" && echo yes || echo no)"
+    check "user: the three manifests, verified" "3" \
+        "$(sed -n '/^==m$/,$p' <<< "$out" | grep -c '\.yaml$')"
+    if [ "$code" -ne 0 ]; then printf '%s\n' "$out" | tail -15 | sed 's/^/        | /'; fi
+done
+
+# The air-gapped path: everything baked into the image, then run with no network at all.
+printf '\noffline (deps-full, --network none)\n'
+tag="rig-installtest:full"
+if docker build -q -f Dockerfile.deps --target deps-full -t "$tag" .. >/dev/null 2>&1; then
+    out=$(docker run --rm --network none --entrypoint bash "$tag" -c '
+        set -e
+        /work/rigdeps.sh install dev
+        /work/rigdeps.sh manifests --to /tmp/m
+        echo ==bin; ls /out/bin
+        echo ==m; ls /tmp/m' 2>&1)
+    code=$?
+    check "offline: install and manifests succeed" "0" "$code"
+    check "offline: the dev tier, from the image" "ctlptl docker-compose jq kind kubectl tilt" \
+        "$(sed -n '/^==bin$/,/^==m$/p' <<< "$out" | grep -vE '^==' | sort | xargs)"
+    check "offline: the three manifests, from the image" "3" \
+        "$(sed -n '/^==m$/,$p' <<< "$out" | grep -c '\.yaml$')"
+    if [ "$code" -ne 0 ]; then printf '%s\n' "$out" | tail -15 | sed 's/^/        | /'; fi
+    docker rmi -f "$tag" >/dev/null 2>&1 || true
+else
+    check "offline: the deps-full image builds" "yes" "no"
+fi
+
+printf '\n'
+if [ "$rc" -eq 0 ]; then
+    printf '%d install checks passed\n' "$passed"
+else
+    printf 'FAILED — the installer did not do on a clean machine what it says\n' >&2
+fi
+exit "$rc"
--- a/rig/ctrl/mem.sh
+++ b/rig/ctrl/mem.sh
@@ -20,7 +20,7 @@ BUDGET_EXPLICIT=no    # whether --budget was given, which retires the guess belo

 # Refuse Git Bash / MSYS / Cygwin and kernels without /proc, with a clear message.
 require_linux() {
-    case "$(uname -s)" in
+    case "${UNAME_S:-$(uname -s)}" in
        MINGW*|MSYS*|CYGWIN*)
            cat >&2 <<'EOF'
 This has to run inside WSL, not Git Bash / MSYS / Cygwin.
--- a/rig/ctrl/selftest.sh
+++ b/rig/ctrl/selftest.sh
@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
 # What rig has settled, written down as assertions: one decision per check.
 # No cluster, docker or network; exits 1 on failure (unlike `make check`).
-# Usage: make selftest   (or: bash ctrl/selftest.sh)
+# Usage: make selftest [install]   (or: bash ctrl/selftest.sh [install])
+#   install: the installer in clean containers — docker, network, minutes (installtest.sh)
 # Notes: docs/notes/selftest.md
 set -uo pipefail          # NOT -e: one failing check must not abort the rest
 cd "$(dirname "$0")"

+if [ "${1:-}" = install ]; then shift; exec bash ./installtest.sh "$@"; fi
+
 source ./lib/config.sh

 rc=0
@@ -316,6 +319,26 @@ check "✖ S7 no house path or host name in rig" "0" \
    "$(cd .. && grep -rIlE "$HOUSE_PAT" . --exclude-dir=def --exclude-dir=local --exclude=STALE.md 2>/dev/null | wc -l)"


+note "the installer detects what each kind of machine needs"
+# Host fixtures (tests/hosts/): a stand-in root per machine, detect run against it. The
+# expensive machines — the Workspace, a WSL install — are exactly the ones you cannot
+# rebuild to test on, so their shapes are replayed here instead.
+out="$(bash ./hosttest.sh 2>&1)"
+check "every host fixture detects as expected" "$(ls -d ../tests/hosts/*/ | wc -l) host fixture(s) as expected, 0 not" \
+    "$(tail -1 <<< "$out")"
+# A snapshot is carried off a machine that may be someone else's: it must hold only the
+# files detect reads, and nothing that names the machine or the person.
+SN="$TMP/snapshot/s"
+bash ./deps.sh snapshot "$SN" >/dev/null 2>&1
+check "snapshot writes only what detect reads" \
+    "expect.txt facts.txt root/etc/os-release root/proc/meminfo root/proc/sys/vm/overcommit_memory root/proc/version" \
+    "$( (cd "$SN" 2>/dev/null && find . -type f | sed 's|^\./||' | grep -vE '^root/(etc/wsl\.conf|mnt/c/Users/user/\.wslconfig)$' | sort | xargs) )"
+check "snapshot names no host, user or home" "0" \
+    "$(grep -rlE "$(hostname)|${USER:-nobody}|/home/" "$SN" 2>/dev/null | wc -l)"
+check "and replays as the machine it was taken on" "yes" \
+    "$(bash ./hosttest.sh "$SN" >/dev/null 2>&1 && echo yes || echo no)"
+
+
 note "the dev loop parses — needs tilt and kubectl, not a cluster"
 # A throwaway kubeconfig with kind-named entries (Tilt trusts kind contexts) and a
 # kubectl that swallows `apply`: the Tiltfile evaluates for real, nothing is contacted.
--- a/rig/docs/notes/deps.md
+++ b/rig/docs/notes/deps.md
@@ -160,3 +160,7 @@ Read the command, THEN shift — and shift only if there is something there. A b
 ## manifest / manifests

 The manifests rig's own addons apply are pinned in versions.env like the binaries, and fetched by the same code: `resolve_url` for the source (upstream, artifactory, baked), `verify` for the sum. `manifest <NAME>` makes one present in `vendor/manifests/` (rig's folder, gitignored) and prints only its path, so an addon can apply it; a cached copy whose sum still matches is reused, one that does not is fetched again. `manifests [--to DIR]` fetches all three, which is how the deps-full image bakes them and how an offline machine is given them. Why the addons stopped applying URLs is in versions.md.
+
+## snapshot, and the test hooks
+
+`snapshot [DIR]` writes this machine as a host fixture: the files detect reads, cut down to what it needs (no environment, home or host name), plus the lines detect printed for it. It is in the kit, so the Workspace needs nothing else to take one. Detection reads through `HOST_ROOT`, `MEMINFO`, `OVERCOMMIT_FILE` and `UNAME_S`, so a fixture can stand in for a machine; with `HOST_ROOT` set, the first two default into it. How the fixtures, the clean-container run and the WSL recipe fit together: [installer-testing.md](installer-testing.md).
--- a/rig/docs/notes/installer-testing.md
+++ b/rig/docs/notes/installer-testing.md
@@ -0,0 +1,118 @@
+# Testing the installer
+
+## Why it needs its own tiers
+
+The installer is `ctrl/deps.sh`. What reaches a machine is its generated one-file kit,
+`standalone/default/rigdeps.sh`: the AWS Workspace gets that file and nothing else. The
+machines it must work on are exactly the ones you cannot rebuild to test on:
+- the Workspace cannot be recreated;
+- redoing WSL means rebuilding the machine you work on every day;
+- a fresh install happens once per machine, so a bug there is found by the one person least
+  able to diagnose it.
+
+So each tier is cheaper than the one after it, and each catches what the one before cannot.
+
+| tier | runs | needs | proves | cannot prove |
+| --- | --- | --- | --- | --- |
+| 1 host fixtures | every `make selftest` | nothing | detection reads each kind of machine right: distro, WSL, memory, overcommit, systemd, `.wslconfig`, the Git Bash refusal | that anything installs |
+| 2 clean containers | `make selftest install` | docker, network, ~1–2 min | a stock Ubuntu 22.04 / Debian refuses cleanly when bare, then installs, verifies and fetches the manifests **as a non-root user**; the offline image works with no network | WSL, the Workspace's own policies, docker itself |
+| 3a Workspace snapshot | by hand, when the Workspace changes | the kit, once, on the Workspace | tier 1 replays the real Workspace's shape from then on | its install (that is tier 2's Ubuntu 22.04 plus the real run) |
+| 3b WSL throwaway | by hand, before teaching or after a WSL change | a Windows machine | the whole path in a real WSL distro that is not your daily one | Windows-side setup (not built, see below) |
+
+## Tier 1 — host fixtures
+
+`tests/hosts/<name>/` is a stand-in machine:
+
+```
+root/        the files detect reads: etc/os-release, proc/version, proc/meminfo,
+             proc/sys/vm/overcommit_memory, etc/wsl.conf, mnt/c/Users/<u>/.wslconfig
+env          optional KEY=value lines, e.g. UNAME_S=MINGW64_NT-10.0 (a fact no file carries)
+expect.txt   "+ text" must appear, "- text" must not, "exit N" (default 0)
+```
+
+`bash ctrl/hosttest.sh [DIR...]` runs `deps.sh detect all` with `HOST_ROOT` pointing at the
+root (which also sets `MEMINFO` and `OVERCOMMIT_FILE`) and checks the lines. Only host facts are
+asserted: docker, PATH and the filesystem belong to the machine running the test.
+
+Shipped: `ubuntu-22.04` (the Workspace's shape: 7.6 GB, overcommit 1, swap in use),
+`debian-trixie`, `wsl-bare` (systemd off, no `.wslconfig`), `wsl-ready`, `git-bash`. To add a
+case, add a folder. Anything true of one real machine goes with an overlay or in `local/`,
+never here.
+
+## Tier 2 — clean containers
+
+`make selftest install` (`ctrl/installtest.sh [IMAGE...]`). It first refuses to run on a stale
+kit, then, for `ubuntu:22.04` and `debian:trixie-slim`:
+1. **bare:** `install` must refuse and name curl/wget — the bootstrap paradox;
+2. **after the one root step** (`apt-get install curl ca-certificates`), as a plain user:
+   `install dev`, `verify dev`, `manifests --to ~/m`. The dev tier lands in `~/.local/bin`,
+   the PATH advice is printed, and all three manifests are there, verified.
+
+Then it builds `deps-full` and runs `install` + `manifests` with `--network none`: the
+air-gapped path. Measured 2026-09-22: 15 checks, 75 s.
+
+## Tier 3a — the Workspace snapshot
+
+```bash
+bash rigdeps.sh snapshot ~/rig-host-snapshot     # on the Workspace; the kit is enough
+```
+
+It writes the files detect reads, cut down to what detect needs:
+- the `os-release` name fields;
+- the kernel release;
+- four meminfo numbers;
+- the overcommit value;
+- `wsl.conf`'s section headers and its two keys;
+- a `.wslconfig` memory line, under the neutral user `user`;
+- `facts.txt` (arch, glibc, date);
+- `expect.txt`, the lines detect printed for it.
+
+No environment, no home, no host name: selftest asserts both the file list and the absence
+of those.
+
+Carry the folder back, keep it in `local/hosts/workspace/` (gitignored) or with the overlay,
+and from then on:
+
+```bash
+bash ctrl/hosttest.sh local/hosts/workspace
+```
+
+Add your own `+`/`-` lines to its `expect.txt` for what must stay true there. Take a new one
+when the Workspace changes (an image update, a memory change).
+
+## Tier 3b — WSL, in a throwaway distro
+
+Never the distro you work in: a second one, imported from a rootfs, removed afterwards. WSL
+runs every distro in one VM, so this changes nothing in yours — **except that
+`wsl --shutdown` stops both**; don't run it while you work. From PowerShell:
+
+```powershell
+# a pristine Ubuntu 22.04 rootfs: from cloud-images.ubuntu.com/wsl/, or
+# `wsl --export Ubuntu-22.04 ubuntu-22.04.tar` of an untouched install kept for this
+wsl --import rig-test C:\wsl\rig-test .\ubuntu-22.04.tar
+wsl -d rig-test -u root -- bash -c "apt-get update && apt-get install -y curl ca-certificates && useradd -m t"
+copy .\rigdeps.sh \\wsl$\rig-test\home\t\
+wsl -d rig-test -u t -- bash -lc "cd ~ && bash rigdeps.sh detect all && bash rigdeps.sh install dev && bash rigdeps.sh verify dev && bash rigdeps.sh snapshot ~/snap"
+# carry \\wsl$\rig-test\home\t\snap back as a fixture, then:
+wsl --unregister rig-test
+```
+
+What to look for: detect says `WSL`, and names the systemd and `.wslconfig` steps; install and
+verify pass. This is the tier to run before teaching Windows users, or after a WSL update.
+Nothing in rig automates it: a throwaway-distro harness is the multi-distro machinery that
+was removed on purpose.
+
+## Not built: a Windows-host installer
+
+Everything above starts inside Linux. The Windows side — enabling WSL, getting a distro, and
+the tools a Windows user needs before rig (python and the like) — is still manual, in the
+README's "Starting from plain Windows". A Windows-host installer is recorded for the future,
+for teaching; it would need tier 3b's approach, a machine that is not your daily one, to test.
+
+## The hooks the tiers rely on
+
+`deps.sh` reads host files through `HOST_ROOT` (`host_file()`), memory through `MEMINFO`, the
+overcommit mode through `OVERCOMMIT_FILE`, the kernel name through `UNAME_S`, and WSL through
+`host_file /proc/version`. With `HOST_ROOT` set, `MEMINFO` and `OVERCOMMIT_FILE` default into
+it. The deps image relies on `HOST_ROOT=/host` for the same reason: a container asking about
+its host.
--- a/rig/docs/notes/selftest.md
+++ b/rig/docs/notes/selftest.md
@@ -89,3 +89,11 @@ The offline example profile must install rig's addons with no network. Each addo
 ## the examples are overlays that work as shipped

 `examples/` is what real overlays are copied from, so every addon there must parse, and every DAG must be valid Python. What they deploy is exercised by the parse checks above, not here.
+
+## the installer detects what each kind of machine needs
+
+The machines the installer must work on — the Workspace, a WSL install — are the ones you cannot rebuild to test on, so their shapes are replayed from fixtures (`tests/hosts/`, run by `hosttest.sh`). And a snapshot is carried off a machine that may be someone else's, so it must hold only what detect reads and nothing that names the machine or the person; that, and that it replays as the machine it was taken on, is asserted on a snapshot of the machine running the test.
+
+## make selftest install
+
+The installer on clean machines: stock Ubuntu 22.04 and Debian containers, the generated kit, a plain user; then the offline image with no network. Docker, network and a minute or two, so it is its own verb rather than part of the cheap run. See [installer-testing.md](installer-testing.md).
--- a/rig/standalone/default/Makefile
+++ b/rig/standalone/default/Makefile
@@ -16,7 +16,7 @@ endif
 help:                          ## list targets
 	@grep -hE '^[a-z][a-z-]*:.*?##' $(MAKEFILE_LIST) | sed 's/:.*##/\t/' | expand -t16

-deps:                          ## rigdeps.sh  [detect|list|verify|fetch|install|manifest|manifests]  (default detect)
+deps:                          ## rigdeps.sh  [detect|list|verify|fetch|install|manifest|manifests|snapshot]  (default detect)
 	bash $(HERE)rigdeps.sh $(or $(ARGS),detect)

 mem:                           ## rigmini.sh  [status|push|all|backup|restore]  (default status)
--- a/rig/standalone/default/rigdeps.sh
+++ b/rig/standalone/default/rigdeps.sh
@@ -128,7 +128,7 @@ load_config() {
 # Toolchain installer: detect the host, install pinned tools into $OUT_BIN, report
 # host actions it will not perform (no sudo, no apt). Usually via `make deps`.
 # Usage: deps.sh [detect [all] | list | verify [core|dev] | fetch [core|dev] [--to DIR] | install [core|dev]
-#                 | manifest NAME | manifests [--to DIR]]
+#                 | manifest NAME | manifests [--to DIR] | snapshot [DIR]]
 # Notes: docs/notes/deps.md

 set -euo pipefail
@@ -152,6 +152,14 @@ abspath() {

 OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
 HOST_ROOT="${HOST_ROOT:-/}"
+# A host fixture (docs/notes/installer-testing.md) is a root whose kernel files stand in
+# for this machine's; UNAME_S does the same for the one fact a file cannot carry.
+if [ "$HOST_ROOT" != / ]; then
+    if [ -r "$HOST_ROOT/proc/meminfo" ]; then MEMINFO="${MEMINFO:-$HOST_ROOT/proc/meminfo}"; fi
+    if [ -r "$HOST_ROOT/proc/sys/vm/overcommit_memory" ]; then
+        OVERCOMMIT_FILE="${OVERCOMMIT_FILE:-$HOST_ROOT/proc/sys/vm/overcommit_memory}"
+    fi
+fi
 DEPS_SOURCE="${DEPS_SOURCE:-upstream}"
 DEPS_ARTIFACTORY_URL="${DEPS_ARTIFACTORY_URL:-}"
 BAKED_BIN="${BAKED_BIN:-/opt/rig/bin}"
@@ -263,7 +271,7 @@ docker_pkg() {

 # Windows outside WSL (Git Bash, MSYS, Cygwin) fails confusingly; name it instead.
 require_linux() {
-    case "$(uname -s)" in
+    case "${UNAME_S:-$(uname -s)}" in
        MINGW*|MSYS*|CYGWIN*)
            cat >&2 <<'EOF'
 This has to run inside WSL, not Git Bash / MSYS / Cygwin.
@@ -282,7 +290,7 @@ EOF
    esac
 }

-is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
+is_wsl() { grep -qi microsoft "$(host_file /proc/version)" 2>/dev/null; }

 detect() {
    echo "host"
@@ -890,6 +898,54 @@ fetch_manifests() {  # [--to DIR]
    done
 }

+# ── snapshot: this machine as a host fixture ───────────────────────────────
+# Writes what detect reads, cut down to what it needs — never the environment, the
+# home directory or the host name — plus the lines detect prints for it. A fact of
+# one machine: keep it with an overlay or in rig's local/, never in rig itself, and
+# replay it with ctrl/hosttest.sh. Notes: docs/notes/installer-testing.md
+snapshot() {
+    local dest r f
+    dest="$(abspath "${1:-host-snapshot}")"
+    if [ -n "$(ls -A "$dest" 2>/dev/null)" ]; then
+        echo "snapshot: $dest already holds something — pick an empty directory" >&2
+        exit 1
+    fi
+    r="$dest/root"
+    mkdir -p "$r/etc" "$r/proc/sys/vm"
+    f=$(host_file /etc/os-release)
+    if [ -r "$f" ]; then grep -E '^(PRETTY_NAME|NAME|VERSION_ID|ID|ID_LIKE)=' "$f" > "$r/etc/os-release"; fi
+    # The kernel release says WSL or not; the full build string names build hosts.
+    echo "Linux version $(awk '{print $3; exit}' "$(host_file /proc/version)" 2>/dev/null || uname -r)" > "$r/proc/version"
+    grep -E '^(MemTotal|MemAvailable|SwapTotal|SwapFree):' "${MEMINFO:-/proc/meminfo}" > "$r/proc/meminfo"
+    cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" > "$r/proc/sys/vm/overcommit_memory" 2>/dev/null || true
+    f=$(host_file /etc/wsl.conf)
+    if [ -r "$f" ]; then
+        # Section headers and the two keys detect reads; a [user] default= names a person.
+        grep -E '^[[:space:]]*(\[[a-z0-9]+\]|systemd[[:space:]]*=|generateResolvConf[[:space:]]*=)' "$f" > "$r/etc/wsl.conf" || true
+    fi
+    f=$(ls "$HOST_ROOT"/mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
+    if [ -n "$f" ] && grep -qE '^\s*memory\s*=' "$f"; then
+        mkdir -p "$r/mnt/c/Users/user"
+        { echo "[wsl2]"; grep -E '^\s*memory\s*=' "$f"; } > "$r/mnt/c/Users/user/.wslconfig"
+    fi
+    {
+        echo "# for the record; not replayed"
+        echo "arch=$(arch)"
+        echo "glibc=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$' || echo unknown)"
+        echo "taken=$(date -u +%Y-%m-%d)"
+    } > "$dest/facts.txt"
+    # The lines the fixture itself decides, as detect prints them for it now.
+    {
+        echo "# Written by deps.sh snapshot: what detect said about this machine."
+        env -u MEMINFO -u OVERCOMMIT_FILE HOST_ROOT="$r" bash "./$(basename "${BASH_SOURCE[0]}")" detect all 2>/dev/null \
+            | grep -E '^  (distro|memory|overcommit|systemd|resolv\.conf|wslconfig) |^  ! systemd|^  - resolv\.conf' \
+            | sed 's/^  /+ /'
+    } > "$dest/expect.txt"
+    echo "wrote $dest"
+    (cd "$dest" && find . -type f | sort | sed 's|^\./|  |')
+    echo "keep it with an overlay or in rig's local/, never in rig; replay it with rig's hosttest.sh"
+}
+
 # Baked mode copies binaries already in the image, so it needs no downloader.
 need_downloads() {
    require_amd64
@@ -906,10 +962,12 @@ case "$cmd" in
    manifest)  need_downloads
               fetch_manifest "${1:?usage: $0 manifest <METALLB|CERT_MANAGER|METRICS_SERVER>}" "$MANIFESTS_HOME" ;;
    manifests) need_downloads; fetch_manifests "$@" ;;
+    snapshot)  snapshot "${1:-}" ;;
    *)       echo "usage: $0 [detect [all]|list|verify|fetch|install|manifest NAME|manifests]" >&2
             echo "       install [core|dev]           (default dev)" >&2
             echo "       fetch   [core|dev] [--to DIR]" >&2
             echo "       manifests [--to DIR]         the addons' pinned manifests, verified" >&2
+             echo "       snapshot [DIR]               this machine as a host fixture (no secrets)" >&2
             echo "       OUT_BIN=<dir> overrides the install directory" >&2
             exit 1 ;;
 esac
--- a/rig/standalone/default/rigmini.sh
+++ b/rig/standalone/default/rigmini.sh
@@ -145,7 +145,7 @@ BUDGET_EXPLICIT=no    # whether --budget was given, which retires the guess belo

 # Refuse Git Bash / MSYS / Cygwin and kernels without /proc, with a clear message.
 require_linux() {
-    case "$(uname -s)" in
+    case "${UNAME_S:-$(uname -s)}" in
        MINGW*|MSYS*|CYGWIN*)
            cat >&2 <<'EOF'
 This has to run inside WSL, not Git Bash / MSYS / Cygwin.
--- a/rig/tests/hosts/debian-trixie/expect.txt
+++ b/rig/tests/hosts/debian-trixie/expect.txt
@@ -0,0 +1,7 @@
+# A plain native Debian with memory to spare and no swap.
+ distro      Debian GNU/Linux 13 (trixie)
+ native linux
+ memory      15861 MB total, 11264 MB available
+ overcommit  0 heuristic
+- in swap
+- WSL
--- a/rig/tests/hosts/debian-trixie/root/etc/os-release
+++ b/rig/tests/hosts/debian-trixie/root/etc/os-release
@@ -0,0 +1,4 @@
+PRETTY_NAME="Debian GNU/Linux 13 (trixie)"
+NAME="Debian GNU/Linux"
+VERSION_ID="13"
+ID=debian
--- a/rig/tests/hosts/debian-trixie/root/proc/meminfo
+++ b/rig/tests/hosts/debian-trixie/root/proc/meminfo
@@ -0,0 +1,4 @@
+MemTotal:       16241664 kB
+MemAvailable:   11534336 kB
+SwapTotal:             0 kB
+SwapFree:              0 kB
--- a/rig/tests/hosts/debian-trixie/root/proc/sys/vm/overcommit_memory
+++ b/rig/tests/hosts/debian-trixie/root/proc/sys/vm/overcommit_memory
@@ -0,0 +1 @@
+0
--- a/rig/tests/hosts/debian-trixie/root/proc/version
+++ b/rig/tests/hosts/debian-trixie/root/proc/version
@@ -0,0 +1 @@
+Linux version 6.12.48+deb13-amd64
--- a/rig/tests/hosts/git-bash/env
+++ b/rig/tests/hosts/git-bash/env
@@ -0,0 +1 @@
+UNAME_S=MINGW64_NT-10.0-19045
--- a/rig/tests/hosts/git-bash/expect.txt
+++ b/rig/tests/hosts/git-bash/expect.txt
@@ -0,0 +1,5 @@
+# Git Bash on Windows: refused before anything runs, pointing at WSL.
+exit 1
+ This has to run inside WSL, not Git Bash / MSYS / Cygwin.
+ wsl --install
+- distro
--- a/rig/tests/hosts/ubuntu-22.04/expect.txt
+++ b/rig/tests/hosts/ubuntu-22.04/expect.txt
@@ -0,0 +1,7 @@
+# A managed Ubuntu 22.04 machine: native, tight on memory, already swapping.
+ distro      Ubuntu 22.04.5 LTS
+ native linux
+ memory      7680 MB total, 1843 MB available, 4096 MB in swap
+ overcommit  1 always
+- WSL
+- systemd not enabled
--- a/rig/tests/hosts/ubuntu-22.04/root/etc/os-release
+++ b/rig/tests/hosts/ubuntu-22.04/root/etc/os-release
@@ -0,0 +1,5 @@
+PRETTY_NAME="Ubuntu 22.04.5 LTS"
+NAME="Ubuntu"
+VERSION_ID="22.04"
+ID=ubuntu
+ID_LIKE=debian
--- a/rig/tests/hosts/ubuntu-22.04/root/proc/meminfo
+++ b/rig/tests/hosts/ubuntu-22.04/root/proc/meminfo
@@ -0,0 +1,4 @@
+MemTotal:        7864320 kB
+MemAvailable:    1887436 kB
+SwapTotal:       8388608 kB
+SwapFree:        4194304 kB
--- a/rig/tests/hosts/ubuntu-22.04/root/proc/sys/vm/overcommit_memory
+++ b/rig/tests/hosts/ubuntu-22.04/root/proc/sys/vm/overcommit_memory
@@ -0,0 +1 @@
+1
--- a/rig/tests/hosts/ubuntu-22.04/root/proc/version
+++ b/rig/tests/hosts/ubuntu-22.04/root/proc/version
@@ -0,0 +1 @@
+Linux version 6.8.0-1030-aws
--- a/rig/tests/hosts/wsl-bare/expect.txt
+++ b/rig/tests/hosts/wsl-bare/expect.txt
@@ -0,0 +1,7 @@
+# Fresh WSL: systemd off, resolv.conf generated, no .wslconfig — all three said.
+ Ubuntu 22.04.5 LTS  amd64, WSL
+ ! systemd not enabled in /etc/wsl.conf
+ - resolv.conf is WSL-generated
+ Enable systemd — add to /etc/wsl.conf:
+ Cap/raise the WSL VM memory
+- native linux
--- a/rig/tests/hosts/wsl-bare/root/etc/os-release
+++ b/rig/tests/hosts/wsl-bare/root/etc/os-release
@@ -0,0 +1,4 @@
+PRETTY_NAME="Ubuntu 22.04.5 LTS"
+NAME="Ubuntu"
+VERSION_ID="22.04"
+ID=ubuntu
--- a/rig/tests/hosts/wsl-bare/root/proc/meminfo
+++ b/rig/tests/hosts/wsl-bare/root/proc/meminfo
@@ -0,0 +1,4 @@
+MemTotal:        8055020 kB
+MemAvailable:    7340032 kB
+SwapTotal:       2097152 kB
+SwapFree:        2097152 kB
--- a/rig/tests/hosts/wsl-bare/root/proc/version
+++ b/rig/tests/hosts/wsl-bare/root/proc/version
@@ -0,0 +1 @@
+Linux version 5.15.167.4-microsoft-standard-WSL2
--- a/rig/tests/hosts/wsl-ready/expect.txt
+++ b/rig/tests/hosts/wsl-ready/expect.txt
@@ -0,0 +1,7 @@
+# WSL set up as BOOTSTRAP says: systemd on, resolv.conf pinned, VM memory set.
+ amd64, WSL
+ systemd     enabled in wsl.conf
+ resolv.conf pinned (generateResolvConf=false)
+ wslconfig   memory set: memory=12GB
+- systemd not enabled
+- Cap/raise the WSL VM memory
--- a/rig/tests/hosts/wsl-ready/root/etc/os-release
+++ b/rig/tests/hosts/wsl-ready/root/etc/os-release
@@ -0,0 +1,4 @@
+PRETTY_NAME="Ubuntu 22.04.5 LTS"
+NAME="Ubuntu"
+VERSION_ID="22.04"
+ID=ubuntu
--- a/rig/tests/hosts/wsl-ready/root/etc/wsl.conf
+++ b/rig/tests/hosts/wsl-ready/root/etc/wsl.conf
@@ -0,0 +1,5 @@
+[boot]
+systemd=true
+
+[network]
+generateResolvConf=false
--- a/rig/tests/hosts/wsl-ready/root/mnt/c/Users/user/.wslconfig
+++ b/rig/tests/hosts/wsl-ready/root/mnt/c/Users/user/.wslconfig
@@ -0,0 +1,2 @@
+[wsl2]
+memory=12GB
--- a/rig/tests/hosts/wsl-ready/root/proc/meminfo
+++ b/rig/tests/hosts/wsl-ready/root/proc/meminfo
@@ -0,0 +1,4 @@
+MemTotal:        8055020 kB
+MemAvailable:    7340032 kB
+SwapTotal:       2097152 kB
+SwapFree:        2097152 kB
--- a/rig/tests/hosts/wsl-ready/root/proc/version
+++ b/rig/tests/hosts/wsl-ready/root/proc/version
@@ -0,0 +1 @@
+Linux version 5.15.167.4-microsoft-standard-WSL2
--- a/soleprint/station/tools/dataconvert/.env-example
+++ b/soleprint/station/tools/dataconvert/.env-example
@@ -0,0 +1,17 @@
+# Environment Variables for dataconvert tool
+
+# Input settings
+DATACONVERT_INPUT="data/*"
+DATACONVERT_EXCLUDE="*.tmp"
+DATACONVERT_CONFIG=
+
+# Output settings
+DATACONVERT_OUT_DIR=datadir/sql
+DATACONVERT_MAX_ROWS=
+DATACONVERT_SQL_FORMAT=batch
+DATACONVERT_BATCH_ROWS=500
+DATACONVERT_MAX_FILE_SIZE=100M
+DATACONVERT_ALL_TEXT=false
+DATACONVERT_KEEP_FOLDERS=true
+DATACONVERT_NO_SCHEMA=false
+DATACONVERT_NO_DDL=false
--- a/soleprint/station/tools/dataconvert/README.md
+++ b/soleprint/station/tools/dataconvert/README.md
@@ -100,7 +100,7 @@ Shapes are judged on up to 200,000 values spread through the column.
 ## Folders and exclusions

 Output is flat by default: a table's file is named after the file or sheet, and two
-source folders with the same table name (two studies' `drm_lb.csv`) write into the
+source folders with the same table name (two exports' `users.csv`) write into the
 same file. `keep_folders` mirrors the source folders under `out_dir` instead, so each
 keeps its own `.sql`; `SCHEMA.md` stays one document at the top, with the folder in
 each table's heading.
--- a/soleprint/station/tools/dataconvert/config.py
+++ b/soleprint/station/tools/dataconvert/config.py
@@ -17,6 +17,8 @@ header on row 1.
      "batch_rows": 500,
      "max_file_size": "100M",
      "schema": true,
+      "include_ddl": true,
+      "load_order": ["ref_data", "users", "orders"],
      "header_row": 1,
      "layouts": [
        {
@@ -35,11 +37,11 @@ is one of match.in. The first layout that matches wins; a sheet none matches
 uses the top-level header_row/data_row, which default to 1 and 2.

 The run settings (out_dir, max_rows, schema, sql_format, batch_rows,
-max_file_size, all_text, keep_folders, exclude) are the command line's flags
-with the same names, and a flag given on the command line wins over the file;
--exclude patterns are added to the file's. out_dir is relative to the folder
-the config file is in; max_file_size takes a number of bytes or a suffixed size
-such as "100M" (decimal: k, M, G). Keys starting with _ are
+max_file_size, all_text, keep_folders, exclude, include_ddl, load_order) are the
+command line's flags with the same names, and a flag given on the command line
+wins over the file; --exclude patterns are added to the file's. out_dir is relative
+to the folder the config file is in; max_file_size takes a number of bytes or a
+suffixed size such as "100M" (decimal: k, M, G). Keys starting with _ are
 comments; any other unknown key is an error, so a typo is not silently ignored.
 """

@@ -96,7 +98,8 @@ class Config:
    def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None,
                 fallback=None, max_rows=None, schema=None, out_dir=None,
                 sql_format=None, batch_rows=None, max_file_size=None,
-                 all_text=None, keep_folders=None, exclude=()):
+                 all_text=None, keep_folders=None, exclude=(), include_ddl=None,
+                 load_order=()):
        self.rules = list(rules)
        self.bare_sheet_prefixes = tuple(bare_sheet_prefixes)
        self.source = source
@@ -115,6 +118,8 @@ class Config:
        self.all_text = all_text
        self.keep_folders = keep_folders
        self.exclude = tuple(exclude)
+        self.include_ddl = include_ddl
+        self.load_order = tuple(load_order)

    @property
    def probe_rows(self):
@@ -137,6 +142,8 @@ class Config:
            said.append(f"max_rows {self.max_rows}")
        if self.schema is not None:
            said.append(f"schema {'on' if self.schema else 'off'}")
+        if self.include_ddl is not None:
+            said.append(f"include_ddl {'on' if self.include_ddl else 'off'}")
        if self.sql_format is not None:
            said.append(f"sql_format {self.sql_format}")
        if self.batch_rows is not None:
@@ -149,6 +156,8 @@ class Config:
            said.append("keep_folders")
        if self.exclude:
            said.append(f"exclude {', '.join(self.exclude)}")
+        if self.load_order:
+            said.append(f"load_order {', '.join(self.load_order)}")
        if self.fallback is not None:
            said.append(f"header row {self.fallback.header_row}, data from row {self.fallback.data_row}")
        if self.rules:
@@ -158,7 +167,8 @@ class Config:

 FILENAME = "dataconvert.json"
 KNOWN_KEYS = {"layouts", "bare_sheet_prefixes", "max_rows", "schema", "out_dir", "header_row", "data_row",
-              "sql_format", "batch_rows", "max_file_size", "all_text", "keep_folders", "exclude"}
+              "sql_format", "batch_rows", "max_file_size", "all_text", "keep_folders", "exclude",
+              "include_ddl", "load_order"}


 def parse_size(value):
@@ -214,6 +224,9 @@ def load(path=None, forced=None):
    schema = raw.get("schema")
    if schema is not None and not isinstance(schema, bool):
        raise ConfigError(f"{path}: schema must be true or false")
+    include_ddl = raw.get("include_ddl")
+    if include_ddl is not None and not isinstance(include_ddl, bool):
+        raise ConfigError(f"{path}: include_ddl must be true or false")
    out_dir = raw.get("out_dir")
    if out_dir is not None:
        if not isinstance(out_dir, str) or not out_dir.strip():
@@ -242,6 +255,10 @@ def load(path=None, forced=None):
    if not isinstance(exclude, list) or not all(isinstance(p, str) and p for p in exclude):
        raise ConfigError(f'{path}: exclude must be a list of patterns such as "*.xlsx.ods"')

+    load_order = raw.get("load_order", [])
+    if not isinstance(load_order, list) or not all(isinstance(p, str) and p for p in load_order):
+        raise ConfigError(f'{path}: load_order must be a list of table identifiers')
+
    fallback = None
    if "header_row" in raw or "data_row" in raw:
        try:
@@ -252,4 +269,5 @@ def load(path=None, forced=None):
    return Config(rules, [str(p) for p in prefixes], source=path, forced=forced,
                  fallback=fallback, max_rows=max_rows, schema=schema, out_dir=out_dir,
                  sql_format=sql_format, batch_rows=batch_rows, max_file_size=max_file_size,
-                  all_text=flags["all_text"], keep_folders=flags["keep_folders"], exclude=exclude)
+                  all_text=flags["all_text"], keep_folders=flags["keep_folders"], exclude=exclude,
+                  include_ddl=include_ddl, load_order=load_order)
--- a/soleprint/station/tools/dataconvert/dataconvert-example.json
+++ b/soleprint/station/tools/dataconvert/dataconvert-example.json
@@ -2,7 +2,7 @@
  "_comment": "Template for one set of spreadsheets. Copy it to dataconvert.json in the folder that holds them, where it is picked up automatically, or pass any file with --config FILE. Every key is optional; keys starting with _ are comments, any other unknown key is an error. Rows and columns count from 1, as the spreadsheet shows them.",

  "_run": "The same settings as the command-line flags, and a flag given there wins. out_dir is relative to this file's folder. max_rows samples each table (leave it out for all rows). schema false skips SCHEMA.md. sql_format: insert (a statement per row, default), batch (one INSERT per batch_rows rows), or copy (COPY FROM stdin: smallest, psql only, load once). max_file_size splits a table into numbered parts past that size, e.g. \"100M\".",
-  "out_dir": "seed",
+  "out_dir": "datadir/sql",

  "_values": "all_text reads every cell as the string in the file (only empty cells become NULL), so codes keep leading zeros and NA stays NA; SCHEMA.md then says what each column looks like. keep_folders mirrors the source folders under out_dir, so same-named tables from different folders do not share a file. exclude skips files by glob: no / matches the file name at any depth, with a / the path under the input folder.",
  "all_text": true,
--- a/soleprint/station/tools/dataconvert/dataconvert.py
+++ b/soleprint/station/tools/dataconvert/dataconvert.py
@@ -25,6 +25,7 @@ it is copied: run this script from anywhere, no install step.
 """

 import argparse
+import os
 from pathlib import Path

 from output import write_tables
@@ -35,6 +36,24 @@ from readers import expand_inputs, iter_sources
 from schema import SchemaReport


+def load_dotenv():
+    env_path = Path(".env")
+    if env_path.is_file():
+        for line in env_path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                key, val = line.split("=", 1)
+                if key not in os.environ:
+                    os.environ[key] = val.strip().strip('"\'')
+
+
+def get_bool_env(key, default=False):
+    val = os.getenv(key)
+    if val is None:
+        return default
+    return val.strip().lower() in ('true', '1', 'yes', 'on')
+
+
 def size(value):
    try:
        return cfg.parse_size(value)
@@ -50,35 +69,41 @@ def positive_int(value):


 def main():
+    load_dotenv()
+
+    default_input = os.getenv("DATACONVERT_INPUT")
+    default_exclude = os.getenv("DATACONVERT_EXCLUDE", "").split(",") if os.getenv("DATACONVERT_EXCLUDE") else []
+    
    parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
-    parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
-    parser.add_argument("--out-dir", default=None,
-                        help="Directory where individual .sql files will be written (default: the config's out_dir; one of the two is required)")
-    parser.add_argument("--max-rows", type=positive_int, default=None,
-                        help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size (default: the config's max_rows, else all)")
-    parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md, whatever the config says")
-    parser.add_argument("--sql-format", choices=FORMATS, default=None,
+    parser.add_argument("--input", nargs="+", default=[default_input] if default_input else None, required=not bool(default_input), 
+                        help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
+    parser.add_argument("--out-dir", default=os.getenv("DATACONVERT_OUT_DIR"),
+                        help="Directory where individual .sql files will be written (default: the config's out_dir or env)")
+    parser.add_argument("--max-rows", type=positive_int, default=os.getenv("DATACONVERT_MAX_ROWS"),
+                        help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
+    parser.add_argument("--no-schema", action="store_true", default=get_bool_env("DATACONVERT_NO_SCHEMA"), 
+                        help="Do not write SCHEMA.md, whatever the config says")
+    parser.add_argument("--no-ddl", action="store_true", default=get_bool_env("DATACONVERT_NO_DDL"), 
+                        help="Do not emit CREATE TABLE IF NOT EXISTS statements before seeding")
+    parser.add_argument("--sql-format", choices=FORMATS, default=os.getenv("DATACONVERT_SQL_FORMAT"),
                        help="insert: a statement per row (default). batch: one INSERT per --batch-rows rows. "
                             "copy: COPY FROM stdin, smallest, psql only, load once")
-    parser.add_argument("--batch-rows", type=positive_int, default=None,
+    parser.add_argument("--batch-rows", type=positive_int, default=os.getenv("DATACONVERT_BATCH_ROWS", DEFAULT_BATCH_ROWS),
                        help=f"Rows per INSERT with --sql-format batch (default {DEFAULT_BATCH_ROWS})")
-    parser.add_argument("--max-file-size", type=size, default=None,
-                        help='Split a table into numbered parts past this size, e.g. "100M" (default: never split; samples are never split)')
-    parser.add_argument("--all-text", action="store_true", default=None,
-                        help="Read every column as text, exactly as in the file; only empty cells become NULL. "
-                             "Types are then the loader's business; SCHEMA.md says what each column looks like")
-    parser.add_argument("--keep-folders", action="store_true", default=None,
-                        help="Mirror the source folders under the output directory, instead of one flat folder "
-                             "where same-named tables from different folders share a file")
-    parser.add_argument("--exclude", action="append", default=[], metavar="GLOB",
-                        help='Skip matching files, e.g. "*.xlsx.ods"; a pattern with no / matches the file name '
-                             "at any depth. Repeatable, and added to the config's")
+    parser.add_argument("--max-file-size", type=size, default=os.getenv("DATACONVERT_MAX_FILE_SIZE"),
+                        help='Split a table into numbered parts past this size, e.g. "100M"')
+    parser.add_argument("--all-text", action="store_true", default=get_bool_env("DATACONVERT_ALL_TEXT", None),
+                        help="Read every column as text, exactly as in the file; only empty cells become NULL.")
+    parser.add_argument("--keep-folders", action="store_true", default=get_bool_env("DATACONVERT_KEEP_FOLDERS", None),
+                        help="Mirror the source folders under the output directory")
+    parser.add_argument("--exclude", action="append", default=default_exclude, metavar="GLOB",
+                        help='Skip matching files, e.g. "*.xlsx.ods". Repeatable, and added to the config\'s')
    parser.add_argument("--header-row", type=positive_int, default=1,
                        help="Spreadsheet row holding the column names, for every file; overrides the config's layouts (default 1)")
    parser.add_argument("--data-row", type=positive_int, default=None,
                        help="First row of data, when rows sit between it and the header (default: the row after the header)")
-    parser.add_argument("--config", default=None,
-                        help="JSON with layouts, naming and run settings, for every input (default: dataconvert.json in each input's folder, if there is one)")
+    parser.add_argument("--config", default=os.getenv("DATACONVERT_CONFIG"),
+                        help="JSON with layouts, naming and run settings, for every input")
    args = parser.parse_args()

    forced = None
@@ -129,14 +154,16 @@ def main():

        # The command line wins, then the config.
        out_dir = Path(args.out_dir) if args.out_dir else config.out_dir
-        max_rows = args.max_rows if args.max_rows is not None else config.max_rows
+        max_rows = int(args.max_rows) if args.max_rows is not None else config.max_rows
        schema = not args.no_schema and config.schema is not False
+        include_ddl = not args.no_ddl and config.include_ddl is not False
        fmt = args.sql_format or config.sql_format or "insert"
-        batch_rows = args.batch_rows or config.batch_rows or DEFAULT_BATCH_ROWS
+        batch_rows = int(args.batch_rows) if args.batch_rows else (config.batch_rows or DEFAULT_BATCH_ROWS)
        max_bytes = args.max_file_size or config.max_file_size
        all_text = bool(args.all_text or config.all_text)
        keep_folders = bool(args.keep_folders or config.keep_folders)
        exclude = tuple(config.exclude) + tuple(args.exclude)
+        load_order = config.load_order

        key = out_dir.resolve()
        if key not in reports:
@@ -146,7 +173,8 @@ def main():

        for source_name, folder, dfs in iter_sources(in_path, config, all_text, exclude):
            write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes,
-                         fmt, batch_rows, max_bytes, folder if keep_folders else "")
+                         fmt, batch_rows, max_bytes, folder if keep_folders else "", include_ddl=include_ddl,
+                         load_order=load_order)

    for out_dir, report, caps in reports.values():
        if report.entries and out_dir.is_dir():
--- a/soleprint/station/tools/dataconvert/output.py
+++ b/soleprint/station/tools/dataconvert/output.py
@@ -11,16 +11,16 @@ from progress import say
 from sqlgen import DEFAULT_BATCH_ROWS, Shape, render_sample, sanitize_identifier, utf8_len


-def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=()) -> str:
+def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=(), prefix="") -> str:
    """
    Sheets of a multi-sheet workbook are prefixed with the workbook, so two
    workbooks cannot collide, unless the config names the sheet as already
-    unique (bare_sheet_prefixes).
+    unique (bare_sheet_prefixes). An optional numeric prefix (e.g. 01_) forces
+    strict load order.
    """
    clean = sanitize_identifier(raw_name)
-    if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)):
-        return f"{sanitize_identifier(source_name)}_{clean}.sql"
-    return f"{clean}.sql"
+    base_name = f"{sanitize_identifier(source_name)}_{clean}.sql" if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)) else f"{clean}.sql"
+    return f"{prefix}{base_name}" if prefix else base_name


 class TableFile:
@@ -111,19 +111,34 @@ def write_full(shape: Shape, df, table_file: TableFile) -> int:


 def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None, bare_prefixes=(),
-                 fmt="insert", batch_rows=DEFAULT_BATCH_ROWS, max_bytes=None, folder=""):
+                 fmt="insert", batch_rows=DEFAULT_BATCH_ROWS, max_bytes=None, folder="", include_ddl=True,
+                 load_order=()):
    """
    Write individual .sql files per table/sheet into the output directory, or
    into its `folder` subdirectory when the source folders are kept.
+    Respects DDL-first generation and load_order configuration.
    """
    target = out_dir / folder if folder else out_dir
-    for raw_name, df in dfs.items():
+
+    order_map = {sanitize_identifier(name): idx for idx, name in enumerate(load_order, start=1)}
+
+    def table_sort_key(item):
+        raw_name = item[0]
+        sanitized = sanitize_identifier(raw_name)
+        return (order_map.get(sanitized, 9999), sanitized)
+
+    sorted_dfs = sorted(dfs.items(), key=table_sort_key)
+
+    for raw_name, df in sorted_dfs:
        if df.empty:
            continue

        table = sanitize_identifier(raw_name)
-        filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes)
-        shape = Shape(fmt, table, df.columns, batch_rows)
+        order_idx = order_map.get(table)
+        file_prefix = f"{order_idx:02d}_" if order_idx is not None else ""
+
+        filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes, prefix=file_prefix)
+        shape = Shape(fmt, table, df.columns, batch_rows, include_ddl=include_ddl, df=df)
        total = len(df)
        exact = max_rows is None or max_rows >= total
        # A sample is never split: it is small, and it is not for loading.
--- a/soleprint/station/tools/dataconvert/sqlgen.py
+++ b/soleprint/station/tools/dataconvert/sqlgen.py
@@ -38,6 +38,33 @@ def sanitize_identifier(identifier: str) -> str:
    return clean.strip("_")


+def postgres_type(series: pd.Series) -> str:
+    """Infer a PostgreSQL column data type for DDL generation."""
+    vals = series.dropna()
+    if vals.empty:
+        return "TEXT"
+    if pd.api.types.is_bool_dtype(vals):
+        return "BOOLEAN"
+    if pd.api.types.is_integer_dtype(vals):
+        return "BIGINT"
+    if pd.api.types.is_float_dtype(vals):
+        return "NUMERIC"
+    if pd.api.types.is_datetime64_any_dtype(vals):
+        return "TIMESTAMP"
+    return "TEXT"
+
+
+def ddl_statement(table_name: str, df: pd.DataFrame) -> str:
+    """Emit DDL for table creation before seeding data so that loading does not fail."""
+    cols_def = []
+    for col in df.columns:
+        col_id = sanitize_identifier(col)
+        col_type = postgres_type(df[col])
+        cols_def.append(f'    "{col_id}" {col_type}')
+    cols_sql = ",\n".join(cols_def)
+    return f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n{cols_sql}\n);\n\n'
+
+
 def sql_value(v) -> str:
    if pd.isna(v):
        return "NULL"
@@ -68,12 +95,14 @@ def utf8_len(text: str) -> int:
 class Shape:
    """How one table's rows are framed in one format."""

-    def __init__(self, fmt, table_name, columns, batch_rows=DEFAULT_BATCH_ROWS):
+    def __init__(self, fmt, table_name, columns, batch_rows=DEFAULT_BATCH_ROWS, include_ddl=True, df=None):
        if fmt not in FORMATS:
            raise ValueError(f"unknown sql format: {fmt}")
        self.fmt = fmt
        self.table_name = table_name
        self.batch_rows = batch_rows
+        self.include_ddl = include_ddl
+        self.df = df
        table_ref = f'"{table_name}"'
        cols = ", ".join(f'"{sanitize_identifier(c)}"' for c in columns)
        self.head = f"-- Generated seed data for table: {table_ref}\n"
@@ -84,6 +113,8 @@ class Shape:

    def open(self, note="") -> str:
        text = self.head + note + "BEGIN;\n\n"
+        if self.include_ddl and self.df is not None:
+            text += ddl_statement(self.table_name, self.df)
        return text + self.copy if self.fmt == "copy" else text

    def close(self) -> str:
--- a/soleprint/station/tools/dataconvert/uv.lock
+++ b/soleprint/station/tools/dataconvert/uv.lock
@@ -13,6 +13,23 @@ resolution-markers = [
    "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
 ]

+[[package]]
+name = "dataconvert"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "odfpy" },
+    { name = "openpyxl" },
+    { name = "pandas" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "odfpy", specifier = ">=1.4.1" },
+    { name = "openpyxl", specifier = ">=3.1.2" },
+    { name = "pandas", specifier = ">=2.2.0" },
+]
+
 [[package]]
 name = "defusedxml"
 version = "0.7.1"
@@ -284,23 +301,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
 ]

-[[package]]
-name = "dataconvert"
-version = "0.1.0"
-source = { virtual = "." }
-dependencies = [
-    { name = "odfpy" },
-    { name = "openpyxl" },
-    { name = "pandas" },
-]
-
-[package.metadata]
-requires-dist = [
-    { name = "odfpy", specifier = ">=1.4.1" },
-    { name = "openpyxl", specifier = ">=3.1.2" },
-    { name = "pandas", specifier = ">=2.2.0" },
-]
-
 [[package]]
 name = "six"
 version = "1.17.0"
--- a/soleprint/station/tools/datasource/.env-example
+++ b/soleprint/station/tools/datasource/.env-example
@@ -0,0 +1,20 @@
+# Environment Variables for datasource tools
+
+# Database Credentials
+POSTGRES_USER=app
+POSTGRES_PASSWORD=secret
+POSTGRES_DB=app
+
+# Full Database URI (Overrides user/pass/db if provided)
+TARGET_DB_URI=postgresql://app:secret@postgres:5432/app
+
+# Data Dump configuration
+DUMP_OUT_DIR=/backups/cleaned_db_dump
+DUMP_IN_DIR=/backups/cleaned_db_dump
+# Pass schemas exactly as required by pg_dump, e.g., "--schema=core --schema=sales"
+DUMP_SCHEMAS="--schema=core"
+
+# Data Load / Seeding Configuration
+SEED_DIR=/rig/datadir/sql
+TOOLS_DIR=/rig/tools/datasource
+CONF_DIR=/rig/datadir/datasource
--- a/soleprint/station/tools/datasource/.gitignore
+++ b/soleprint/station/tools/datasource/.gitignore
@@ -0,0 +1,4 @@
+# The real mapping: how one producer's raw tables become the logical domains names
+# their tables and columns. It lives in the data folder (datadir/datasource/), not
+# here; this only stops a stray copy being committed. Start from schema_mapping-example.yaml.
+schema_mapping.yaml
--- a/soleprint/station/tools/datasource/README.md
+++ b/soleprint/station/tools/datasource/README.md
@@ -0,0 +1,26 @@
+# datasource
+
+Scripts to dump, restore, and seed the database, including schema migration mapping.
+
+## Schema Mapping
+
+The data loading process converges raw seed tables into logical domains using a YAML configuration.
+This configuration is injected at runtime from `datadir/datasource/schema_mapping.yaml` (never
+committed: it names the producer's tables) rather than hardcoded in the tool. Start from
+`schema_mapping-example.yaml`.
+
+Example mapping structure:
+
+```yaml
+tables:
+  - source: raw_users
+    target: core.users
+    truncate: true
+    columns:
+      user_id: id
+      username: name
+      created_at: etl_utils.safe_cast_date(created_date)
+      is_active: "COALESCE(etl_utils.safe_cast_boolean(status), TRUE)"
+```
+
+All operational variables (database credentials, dump directories, active schemas) are read from the environment or a `.env` file at the root. See `.env-example` for available configuration keys.
--- a/soleprint/station/tools/datasource/casting_helpers.sql
+++ b/soleprint/station/tools/datasource/casting_helpers.sql
@@ -0,0 +1,30 @@
+-- Create dedicated schema for ETL migration helpers
+CREATE SCHEMA IF NOT EXISTS etl_utils;
+
+-- 1. Safe String to Boolean Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_boolean(val TEXT)
+RETURNS BOOLEAN AS $$ BEGIN     IF val IS NULL OR TRIM(val) = '' OR UPPER(TRIM(val)) IN ('NULL', 'N/A', '.') THEN         RETURN NULL;     ELSIF UPPER(TRIM(val)) IN ('Y', 'YES', '1', 'TRUE') THEN         RETURN TRUE;     ELSIF UPPER(TRIM(val)) IN ('N', 'NO', '0', 'FALSE') THEN         RETURN FALSE;     ELSE         RETURN NULL;     END IF; END; $$ LANGUAGE plpgsql IMMUTABLE;
+
+-- 2. Safe String to Numeric/Integer Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_numeric(val TEXT)
+RETURNS NUMERIC AS $$ DECLARE     cleaned TEXT; BEGIN     cleaned := REGEXP_REPLACE(TRIM(val), '[^0-9.-]', '', 'g');     IF cleaned = '' OR cleaned = '.' OR cleaned = '-' THEN         RETURN NULL;     END IF;     RETURN cleaned::NUMERIC; EXCEPTION WHEN OTHERS THEN     RETURN NULL; END; $$ LANGUAGE plpgsql IMMUTABLE;
+
+-- 3. Safe String to Date/Timestamp Converter
+CREATE OR REPLACE FUNCTION etl_utils.safe_cast_date(val TEXT)
+RETURNS DATE AS $func$
+BEGIN
+    IF val IS NULL OR TRIM(val) = '' OR val ~* '^(UN|UNK|UNKNOWN)' THEN
+        RETURN NULL;
+    END IF;
+
+    BEGIN
+        RETURN val::DATE;
+    EXCEPTION WHEN OTHERS THEN
+        BEGIN
+            RETURN TO_DATE(val, 'DD-MON-YY');
+        EXCEPTION WHEN OTHERS THEN
+            RETURN NULL;
+        END;
+    END;
+END;
+$func$ LANGUAGE plpgsql IMMUTABLE;
--- a/soleprint/station/tools/datasource/dump_trasformed.sh
+++ b/soleprint/station/tools/datasource/dump_trasformed.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+DB_USER="${POSTGRES_USER:-postgres}"
+DB_NAME="${POSTGRES_DB:-production_db}"
+OUT_DIR="${DUMP_OUT_DIR:-/backups/cleaned_db_dump}"
+SCHEMAS="${DUMP_SCHEMAS:-}"
+
+# Dump only the transformed, strictly-typed schemas (skipping raw staging tables)
+# Note: Pass schemas in DUMP_SCHEMAS as e.g. "--schema=schema1 --schema=schema2"
+pg_dump -U "$DB_USER" -d "$DB_NAME" \
+  ${SCHEMAS} \
+  --format=directory \
+  --jobs=4 \
+  --file="$OUT_DIR"
--- a/soleprint/station/tools/datasource/load_seed_data.sh
+++ b/soleprint/station/tools/datasource/load_seed_data.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+# Build the URI dynamically using the injected Postgres secrets
+# Fall back to the default 'app' user if the variables are missing
+USER="${POSTGRES_USER:-app}"
+PASS="${POSTGRES_PASSWORD:-}"
+DB="${POSTGRES_DB:-app}"
+
+DB_URI="${TARGET_DB_URI:-postgresql://${USER}:${PASS}@postgres:5432/${DB}}"
+
+# Point to the containerPaths mapped in your kind-config.yaml.tpl
+SEED_DIR="${SEED_DIR:-/rig/datadir/sql}"
+TOOLS_DIR="${TOOLS_DIR:-/rig/tools/datasource}"
+CONF_DIR="${CONF_DIR:-/rig/datadir/datasource}"
+
+echo "Optimizing PostgreSQL for bulk load..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" <<-EOSQL
+    ALTER SYSTEM SET maintenance_work_mem = '1GB';
+    ALTER SYSTEM SET synchronous_commit = 'off';
+    ALTER SYSTEM SET checkpoint_timeout = '15min';
+    ALTER SYSTEM SET max_wal_size = '2GB';
+    SELECT pg_reload_conf();
+EOSQL
+
+echo "Loading casting helpers..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" -f "$TOOLS_DIR/casting_helpers.sql"
+
+echo "Loading seed data files in single transactions..."
+if [ -d "$SEED_DIR" ]; then
+    for f in "$SEED_DIR"/*.sql; do
+        if [ -f "$f" ]; then
+            echo "Processing $f..."
+            psql -v ON_ERROR_STOP=1 -1 "$DB_URI" -f "$f"
+        fi
+    done
+else
+    echo "Warning: Seed directory $SEED_DIR not found. Skipping flat file loads."
+fi
+
+echo "Converging schemas into logical domains using configured mapping..."
+MAPPING_FILE="${SCHEMA_MAPPING_FILE:-$CONF_DIR/schema_mapping.yaml}"
+python3 "$TOOLS_DIR/migrate_schema.py" "$MAPPING_FILE"
+
+echo "Restoring standard PostgreSQL settings..."
+psql -v ON_ERROR_STOP=1 "$DB_URI" <<-EOSQL
+    ALTER SYSTEM RESET maintenance_work_mem;
+    ALTER SYSTEM RESET synchronous_commit;
+    ALTER SYSTEM RESET checkpoint_timeout;
+    ALTER SYSTEM RESET max_wal_size;
+    SELECT pg_reload_conf();
+EOSQL
+
+echo "Data load completed."
--- a/soleprint/station/tools/datasource/migrate_schema.py
+++ b/soleprint/station/tools/datasource/migrate_schema.py
@@ -0,0 +1,54 @@
+import os
+import sys
+import yaml
+import psycopg
+from psycopg import sql
+
+def main():
+    mapping_file = os.getenv("SCHEMA_MAPPING_FILE", "/rig/datadir/datasource/schema_mapping.yaml")
+    
+    if len(sys.argv) >= 2:
+        mapping_file = sys.argv[1]
+
+    db_uri = os.getenv("TARGET_DB_URI", "postgresql://postgres:postgres@postgres:5432/postgres")
+
+    print(f"Loading schema mapping from {mapping_file}...")
+    with open(mapping_file, 'r') as f:
+        config = yaml.safe_load(f)
+
+    with psycopg.connect(db_uri) as conn:
+        with conn.cursor() as cur:
+            for table in config.get('tables', []):
+                source = table['source']
+                target = table['target']
+                columns = table['columns']
+
+                print(f"Migrating {source} to {target}...")
+
+                if table.get('truncate', False):
+                    try:
+                        cur.execute(f"TRUNCATE TABLE {target} RESTART IDENTITY CASCADE;")
+                    except psycopg.errors.UndefinedTable:
+                        print(f"  Target table {target} does not exist yet; skipping truncate.")
+                        conn.rollback()
+
+                target_cols = ", ".join(columns.keys())
+                source_exprs = ", ".join(columns.values())
+
+                query = f"""
+                    INSERT INTO {target} ({target_cols})
+                    SELECT {source_exprs}
+                    FROM {source}
+                """
+                try:
+                    cur.execute(query)
+                    print(f"  Successfully migrated {source}.")
+                except Exception as e:
+                    print(f"  Failed to migrate {source}: {e}")
+                    conn.rollback()
+                    raise
+        conn.commit()
+    print("Schema migration complete.")
+
+if __name__ == '__main__':
+    main()
--- a/soleprint/station/tools/datasource/restore_transformed.sh
+++ b/soleprint/station/tools/datasource/restore_transformed.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+# Load environment variables from .env if present
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs)
+fi
+
+DB_USER="${POSTGRES_USER:-postgres}"
+DB_NAME="${POSTGRES_DB:-production_db}"
+IN_DIR="${DUMP_IN_DIR:-/backups/cleaned_db_dump}"
+
+# Multi-threaded restore into the target PostgreSQL cluster pod
+pg_restore -U "$DB_USER" -d "$DB_NAME" \
+  --jobs=4 \
+  --clean \
+  --if-exists \
+  "$IN_DIR"
--- a/soleprint/station/tools/datasource/schema_mapping-example.yaml
+++ b/soleprint/station/tools/datasource/schema_mapping-example.yaml
@@ -0,0 +1,21 @@
+# Schema Conversion and Type Cast Registry
+# Maps raw seed tables to the unified logical domains.
+
+tables:
+  - source: raw_users
+    target: core.users
+    truncate: true
+    columns:
+      user_id: id
+      username: name
+      created_at: etl_utils.safe_cast_date(created_date)
+      is_active: "COALESCE(etl_utils.safe_cast_boolean(status), TRUE)"
+
+  - source: raw_orders
+    target: core.orders
+    truncate: true
+    columns:
+      order_id: id
+      user_id: user_id
+      total_amount: etl_utils.safe_cast_numeric(amount)
+      order_date: etl_utils.safe_cast_date(date)
Author	SHA1	Message	Date
buenosairesam	0b227cbc0e	Merge branch 'rig-work'	2026-09-22 08:18:31 -03:00
buenosairesam	60527645f2	distilled tools	2026-09-22 08:18:25 -03:00
buenosairesam	dee899d86f	installer	2026-09-22 08:18:02 -03:00
				`@@ -0,0 +1 @@`
				`Linux version 5.15.167.4-microsoft-standard-WSL2`