From 966f8fc8218fefcc3764e21eea445eb9c8c4c64b Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Wed, 26 Aug 2026 07:27:12 -0300 Subject: [PATCH] attemp to develop rig in spr without an actual use case --- .gitignore | 4 +- rig/.gitignore | 7 +- rig/BOOTSTRAP.md | 39 +- rig/Makefile | 21 +- rig/README.md | 89 +- rig/ctrl/.env.example | 6 +- .../{Dockerfile.wizard => Dockerfile.deps} | 24 +- rig/ctrl/addons/airflow.sh | 4 +- rig/ctrl/addons/postgres.sh | 13 +- rig/ctrl/addons/redis.sh | 2 +- rig/ctrl/{station.sh => check.sh} | 8 +- rig/ctrl/{wizard.sh => deps.sh} | 102 +- rig/ctrl/env.d/client.env | 2 +- rig/ctrl/env.d/data.env | 17 +- rig/ctrl/env.d/offline.env | 2 +- rig/ctrl/k8s/README.md | 5 +- rig/ctrl/k8s/kind-config.audit.yaml.tpl | 2 +- rig/ctrl/lib/config.sh | 4 +- rig/ctrl/mem.sh | 233 ++++ rig/ctrl/newbox.sh | 7 +- rig/ctrl/registry.sh | 2 +- rig/ctrl/setup.sh | 6 +- rig/ctrl/versions.env | 31 +- rig/docs/graphs/01-install.dot | 10 +- rig/docs/graphs/01-install.svg | 20 +- rig/docs/graphs/02-environment.dot | 2 +- rig/docs/graphs/02-environment.svg | 118 +- rig/docs/index.html | 20 +- rig/sample-rig/.gitignore | 9 - rig/sample-rig/Makefile | 50 - rig/sample-rig/README.md | 140 -- rig/sample-rig/bundle.json | 51 - rig/sample-rig/cluster.mock.json | 40 - rig/sample-rig/ctrl/bundle.sh | 223 ---- rig/sample-rig/ctrl/manifest.py | 141 -- rig/sample-rig/generated/sample-rig.yaml | 406 ------ rig/sample-rig/rig-ui/.gitignore | 3 - rig/sample-rig/rig-ui/index.html | 12 - rig/sample-rig/rig-ui/k8s.yaml | 108 -- rig/sample-rig/rig-ui/package-lock.json | 1164 ----------------- rig/sample-rig/rig-ui/package.json | 14 - rig/sample-rig/rig-ui/src/main.js | 136 -- rig/sample-rig/rig-ui/src/style.css | 139 -- rig/sample-rig/rig-ui/vite.config.js | 9 - soleprint/run.py | 2 +- soleprint/station/tools/distill/distill.sh | 1028 +++++++++++++++ soleprint/station/tools/distill/explode.md | 5 + soleprint/station/tools/distill/explode.sh | 346 +++++ soleprint/station/tools/histgen/.gitignore | 5 + soleprint/station/tools/histgen/Makefile | 137 ++ soleprint/station/tools/histgen/README.md | 545 ++++++++ soleprint/station/tools/histgen/__init__.py | 27 + soleprint/station/tools/histgen/__main__.py | 318 +++++ soleprint/station/tools/histgen/api.py | 98 ++ soleprint/station/tools/histgen/brief.py | 111 ++ soleprint/station/tools/histgen/census.py | 456 +++++++ soleprint/station/tools/histgen/cli.py | 132 ++ soleprint/station/tools/histgen/config.py | 153 +++ soleprint/station/tools/histgen/export.py | 613 +++++++++ soleprint/station/tools/histgen/history.py | 116 ++ soleprint/station/tools/histgen/order.py | 450 +++++++ soleprint/station/tools/histgen/selftest.py | 413 ++++++ soleprint/station/tools/histgen/sift.py | 142 ++ soleprint/station/tools/histgen/snapshot.py | 121 ++ .../tools/histgen/templates/index.html | 77 ++ 65 files changed, 5887 insertions(+), 2853 deletions(-) rename rig/ctrl/{Dockerfile.wizard => Dockerfile.deps} (61%) rename rig/ctrl/{station.sh => check.sh} (95%) rename rig/ctrl/{wizard.sh => deps.sh} (77%) create mode 100755 rig/ctrl/mem.sh delete mode 100644 rig/sample-rig/.gitignore delete mode 100644 rig/sample-rig/Makefile delete mode 100644 rig/sample-rig/README.md delete mode 100644 rig/sample-rig/bundle.json delete mode 100644 rig/sample-rig/cluster.mock.json delete mode 100755 rig/sample-rig/ctrl/bundle.sh delete mode 100644 rig/sample-rig/ctrl/manifest.py delete mode 100644 rig/sample-rig/generated/sample-rig.yaml delete mode 100644 rig/sample-rig/rig-ui/.gitignore delete mode 100644 rig/sample-rig/rig-ui/index.html delete mode 100644 rig/sample-rig/rig-ui/k8s.yaml delete mode 100644 rig/sample-rig/rig-ui/package-lock.json delete mode 100644 rig/sample-rig/rig-ui/package.json delete mode 100644 rig/sample-rig/rig-ui/src/main.js delete mode 100644 rig/sample-rig/rig-ui/src/style.css delete mode 100644 rig/sample-rig/rig-ui/vite.config.js create mode 100755 soleprint/station/tools/distill/distill.sh create mode 100644 soleprint/station/tools/distill/explode.md create mode 100755 soleprint/station/tools/distill/explode.sh create mode 100644 soleprint/station/tools/histgen/.gitignore create mode 100644 soleprint/station/tools/histgen/Makefile create mode 100644 soleprint/station/tools/histgen/README.md create mode 100644 soleprint/station/tools/histgen/__init__.py create mode 100644 soleprint/station/tools/histgen/__main__.py create mode 100644 soleprint/station/tools/histgen/api.py create mode 100644 soleprint/station/tools/histgen/brief.py create mode 100644 soleprint/station/tools/histgen/census.py create mode 100644 soleprint/station/tools/histgen/cli.py create mode 100644 soleprint/station/tools/histgen/config.py create mode 100644 soleprint/station/tools/histgen/export.py create mode 100644 soleprint/station/tools/histgen/history.py create mode 100644 soleprint/station/tools/histgen/order.py create mode 100644 soleprint/station/tools/histgen/selftest.py create mode 100644 soleprint/station/tools/histgen/sift.py create mode 100644 soleprint/station/tools/histgen/snapshot.py create mode 100644 soleprint/station/tools/histgen/templates/index.html diff --git a/.gitignore b/.gitignore index 892dd7a..c612318 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,5 @@ cfg/dlt/ # not land here. They are versioned in their own repo. # # Anchored at the ROOT on purpose: a copy is a SIBLING of rig/, so a rule inside -# rig/.gitignore cannot see it. The negation must name the full path for the same -# reason — `*-rig/` is unanchored and matches at any depth, including rig/sample-rig. +# rig/.gitignore cannot see it. *-rig/ -!rig/sample-rig/ diff --git a/rig/.gitignore b/rig/.gitignore index 563c914..a1666f6 100644 --- a/rig/.gitignore +++ b/rig/.gitignore @@ -11,10 +11,9 @@ ctrl/.env arch/*.dot ctrl/Tiltfile.gen -# binaries pulled by `make deps-bundle` for the air-gapped wizard image +# binaries pulled by `make deps-bundle` for the air-gapped installer image vendor # Client rigs are NOT ignored here. A copy is a SIBLING of this directory -# (spr/acme-rig), so a rule in this file cannot see it — the rules live in -# spr/.gitignore, anchored at spr's root, where `*-rig/` matches the siblings and -# `!rig/sample-rig/` keeps the committed stand-in. +# (../acme-rig), so a rule in this file cannot see it — the rules live in the +# parent repo's .gitignore, anchored at its root, where `*-rig/` matches them. diff --git a/rig/BOOTSTRAP.md b/rig/BOOTSTRAP.md index 3a96251..49eb221 100644 --- a/rig/BOOTSTRAP.md +++ b/rig/BOOTSTRAP.md @@ -4,9 +4,8 @@ The README says the prerequisite is Docker and nothing else. This is what that actually looks like end to end: a bare Linux box, and a new project running under Tilt at the end of it. -rig lives inside soleprint, at `spr/rig` — it is soleprint's cluster half, and -a client copy is a sibling (`spr/acme-rig`). Paths below are relative to -soleprint's checkout. +A copy of this directory is a sibling of it, named after the environment it +models (`acme-rig`). Paths below are relative to the parent checkout. It spans three repos because the work does. **rig** prepares the machine — the pinned toolchain, the cluster, the port arithmetic. **all** owns the shape a @@ -50,7 +49,7 @@ isn't. ## Read the docs before installing anything ```bash -cd spr/rig +cd rig make docs ``` @@ -67,11 +66,11 @@ persists; ctrl-c ends it. ## Ask what is wrong with this machine ```bash -make station +make check cp ctrl/.env.example ctrl/.env ``` -`station.sh` reports and instructs, and fixes nothing. It runs bare rather than +`check.sh` reports and instructs, and fixes nothing. It runs bare rather than in a container because host detection only ever reads `/proc` and `/etc` — no dependency beyond coreutils. @@ -80,7 +79,7 @@ port rig binds derives from this directory's name, so the answer is specific to this copy, and a clash here surfaces as an opaque `failed to bind host port` in the middle of cluster creation if you skip it. -Copy the `.env` even though station only warns about it. It is gitignored, it is +Copy the `.env` even though the check only warns about it. It is gitignored, it is where a machine-local override goes, and `ports.sh persist` expects it to exist. @@ -88,33 +87,33 @@ where a machine-local override goes, and `ports.sh persist` expects it to exist. This is the step where "nothing installed" stops being rhetorical. -`make deps` runs `ctrl/wizard.sh install` directly on the host, and the wizard +`make deps` runs `ctrl/deps.sh install` directly on the host, and the installer fetches with `curl`. A stock `debian:trixie-slim` has no curl — detection runs fine, then the first download dies with `curl: command not found` and an exit -code of 127. That is the bootstrap paradox `ctrl/Dockerfile.wizard` exists -to kill — the wizard carries its own toolchain so the host needs only Docker — +code of 127. That is the bootstrap paradox `ctrl/Dockerfile.deps` exists +to kill — the installer carries its own toolchain so the host needs only Docker — but building the image and running it are two different things, and only the build has a Makefile target today. **On a genuinely bare machine, run it by hand:** ```bash -make wizard # builds rig-wizard:wizard +make deps-image # builds rig-deps:deps mkdir -p ~/.local/bin docker run --rm \ -v /:/host:ro \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$HOME/.local/bin:/out/bin" \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ - rig-wizard:wizard install dev + rig-deps:deps install dev ``` -The image name follows the directory, like everything else here: in `spr/rig` -it is `rig-wizard`, in a copy called `spr/acme-rig` it is `acme-rig-wizard`. The -tag is `wizard` (or `full`, below), not `latest`. +The image name follows the directory, like everything else here: in `rig` +it is `rig-deps`, in a copy called `acme-rig` it is `acme-rig-deps`. The +tag is `deps` (or `full`, below), not `latest`. None of the four arguments are guessable, so: -- **`/:/host:ro`** — the wizard reads the *host's* `/etc/os-release` and +- **`/:/host:ro`** — the installer reads the *host's* `/etc/os-release` and `/etc/wsl.conf`, not the container's. `HOST_ROOT=/host` is already baked into the image; this is what it points at. Read-only, and it is the only reason detection inside a container tells you anything about the machine. @@ -122,7 +121,7 @@ None of the four arguments are guessable, so: and how it counts kind clusters already running. - **`/out/bin`** — the image's `OUT_BIN`. Whatever you mount here is where the four binaries land. -- **`HOST_UID` / `HOST_GID`** — the wizard runs as root so it can reach that +- **`HOST_UID` / `HOST_GID`** — the installer runs as root so it can reach that socket, which means everything it writes into a mounted volume is root-owned and useless to you. These drive the `chown` back. Omit them and the install looks like it worked. @@ -131,18 +130,18 @@ None of the four arguments are guessable, so: tooling — which is the right answer on a managed or corporate-issued machine and is why the split exists. -Then put them on PATH, which the wizard will remind you about because it cannot +Then put them on PATH, which the installer will remind you about because it cannot edit your shell for you: ```bash export PATH="$HOME/.local/bin:$PATH" # and add the same line to ~/.bashrc ``` -If something else on this machine already provides `kubectl`, the wizard says so +If something else on this machine already provides `kubectl`, the installer says so by name rather than shadowing it quietly. `OUT_BIN=$PWD/def/bin` installs somewhere private instead. -**Two variants worth knowing before you need them.** `make wizard full` bakes +**Two variants worth knowing before you need them.** `make deps-image full` bakes every pinned binary into the image at build time (`DEPS_SOURCE=baked`), so `docker save` gives you the entire installer as one file to carry into an air-gapped network. And `DEPS_SOURCE=artifactory` with `DEPS_ARTIFACTORY_URL` diff --git a/rig/Makefile b/rig/Makefile index f33c834..87cd018 100644 --- a/rig/Makefile +++ b/rig/Makefile @@ -20,7 +20,7 @@ SLUG := $(shell echo '$(notdir $(CURDIR))' | tr '[:upper:]' '[:lower:]' | t CLUSTER := $(or $(shell sed -n 's/^CLUSTER=//p' ctrl/.env 2>/dev/null),$(SLUG)) KCTX := --context kind-$(CLUSTER) TILT_PORT := $(shell sed -n 's/^TILT_PORT=//p' ctrl/.env 2>/dev/null) -WIZARD := $(SLUG)-wizard +DEPSIMG := $(SLUG)-deps # Words after the target become the script's subcommand. Make would otherwise # treat them as goals of their own, so each gets a no-op rule. @@ -35,7 +35,7 @@ $(eval $(ARGS):;@:) .PHONY: $(ARGS) endif -.PHONY: help setup station deps wizard cluster registry addons ports \ +.PHONY: help setup check mem deps deps-image cluster registry addons ports \ newbox dockerhost docs tilt \ kind-up kind-down kind-reset tilt-up tilt-down @@ -47,16 +47,19 @@ help: ## list targets setup: ## prepare this machine [core] [--share-docker] [--cluster] bash ctrl/setup.sh $(ARGS) -station: ## is this workstation ready? reports, never fixes - bash ctrl/station.sh +check: ## is this machine ready? reports, never fixes + bash ctrl/check.sh + +mem: ## memory, and any cap holding it [status|backup|restore] + bash ctrl/mem.sh $(or $(ARGS),status) deps: ## install the toolchain [core|dev] (default dev) - bash ctrl/wizard.sh install $(or $(ARGS),dev) + bash ctrl/deps.sh install $(or $(ARGS),dev) -wizard: ## build the installer image [full] - docker build -f ctrl/Dockerfile.wizard \ - --target $(if $(filter full,$(ARGS)),wizard-full,wizard) \ - -t $(WIZARD):$(if $(filter full,$(ARGS)),full,wizard) . +deps-image: ## build the installer image [full] + docker build -f ctrl/Dockerfile.deps \ + --target $(if $(filter full,$(ARGS)),deps-full,deps) \ + -t $(DEPSIMG):$(if $(filter full,$(ARGS)),full,deps) . # ── cluster ──────────────────────────────────────────────────────────────── diff --git a/rig/README.md b/rig/README.md index 1c0ed0e..3427ea9 100644 --- a/rig/README.md +++ b/rig/README.md @@ -9,6 +9,46 @@ topology, not the workloads. **Docker.** Nothing else — no curl, no jq, no python, no apt repositories. +### Starting from plain Windows + +Everything here is bash and runs *inside* a Linux shell, so on a Windows machine +that means WSL. Nothing in rig installs WSL, and nothing will: `wsl --install` +enables Windows features and requires a reboot, which is not something a script +should do to a machine on your behalf — and there is no tested undo for it. + +From an elevated PowerShell or Command Prompt, once: + +```powershell +wsl --install +``` + +Then reboot and open the Linux shell it installed. + +**If you cloned this on the Windows side, copy it into WSL before carrying on.** +WSL can reach the Windows drives at `/mnt/c`, and working from there mostly +functions — slowly — but file watching does not: that filesystem raises no +inotify events, so anything watching for edits silently stops seeing them. + +```bash +cp -r /mnt/c/Users//rig ~/rig +cd ~/rig +``` + +`make deps` reports it if you are running from `/mnt/...`. Then carry on below. + +If it fails, the usual causes give unhelpful messages: + +| symptom | cause | +| --- | --- | +| "the virtual machine could not be started" | virtualization disabled in BIOS/UEFI | +| the command is not recognised | Windows build too old — needs 2004 or later | +| the install starts, then nothing works | a reboot is still pending | + +Running the scripts from **Git Bash, MSYS or Cygwin does not work** — those look +close enough to a Linux shell to get started and then fail without `/proc` or a +docker socket. `ctrl/deps.sh` detects that and says so rather than letting you +find out the slow way. + ## Read the docs first ```bash @@ -21,16 +61,43 @@ instructions for everything else. No cluster and no toolchain required. ## Then ```bash -make station # report host and config problems; changes nothing +make check # report host and config problems; changes nothing make deps # install the toolchain (add `core` on a managed machine) make cluster up # build the cluster for the active profile ``` +`make cluster up` also starts this environment's local registry and wires it +into the node, so an image built locally is pullable by the cluster without +going near docker.io: + +```bash +make registry status # prints: endpoint localhost: +docker build -t localhost:/app:1 . +docker push localhost:/app:1 +kubectl --context kind-$(basename $PWD) run app --image=localhost:/app:1 +``` + +The port block is derived from the directory name, so two copies of rig never +collide: + +```bash +make ports show # HTTP / HTTPS / TILT / REGISTRY +make cluster list # every cluster on this machine, with memory +make cluster free # stop the others if memory is tight +make cluster down # remove this cluster and its registry +``` + +**`make tilt` has nothing to run yet.** The target and its `tilt-up` / `tilt-down` +aliases exist so rig answers to the same spelling as every other project here, +but rig ships no `Tiltfile` — it builds the estate, it is not itself a service +with a dev loop. Add a `ctrl/Tiltfile` and the target works; until then it fails +on the missing file, not on anything rig did. + `make help` lists every target. On a machine where Docker really is the only thing installed, `make deps` has nothing to download with — see [BOOTSTRAP.md](BOOTSTRAP.md), which runs the -toolchain through the wizard container and carries on to scaffolding and running +toolchain through the installer container and carries on to scaffolding and running a new project. ## One directory is one environment @@ -39,10 +106,10 @@ Copy this directory, rename it, run it. Cluster name, kubectl context, image tags and the host port block all derive from the directory name, so copies never collide and neither one's teardown can touch the other. -rig lives inside soleprint, at `spr/rig` — it is soleprint's cluster half, and a -copy is a **sibling**: `spr/acme-rig`. That is why the ignore rules for client -rigs sit in `spr/.gitignore` rather than here; a rule in this directory cannot -see a directory beside it. +A copy of this directory is a **sibling** of it, named after the environment it +models (`acme-rig`). That is why the ignore rules for copies sit in the *parent* +repo's `.gitignore` rather than here: a rule in this directory cannot see a +directory beside it. ## Profiles @@ -54,7 +121,7 @@ apiserver audits. They live in `ctrl/env.d/`, and the active one is `PROFILE`. | `minimal` | the default. One node, no addons, boots fast. | | `client` | the regulated-estate shape — multi-node, audit on, registry mirror. | | `offline` | air-gapped: everything from a preloaded local registry. | -| `data` | the dependency containers a soleprint room asks for. | +| `data` | the cabinets an environment asks for. | ```bash PROFILE=data make cluster up @@ -98,10 +165,10 @@ cluster does. | `redis` | cache and broker | | `airflow` | scheduled pipelines; needs postgres and redis | -The last three are the cluster half of **soleprint's cabinets**. A room declares -what it needs once, in `cfg//data/cabinets.json`; soleprint's `build.py` -composes those services into `docker-compose.yml` for a laptop, and these -install the same ones here. The names match on purpose — each cabinet carries a +The last three are **cabinets**: a public service dropped in as-is, the upstream +image unmodified, reachable at a known address. A cabinet is declared once and +installs on either target — a `service.yml` composes it for a laptop, and these +install the same one here. The names match on purpose: each cabinet carries a `rig_addon` field pointing at `ctrl/addons/.sh`. Plain manifests rather than helm charts, like every other addon: a chart repo is diff --git a/rig/ctrl/.env.example b/rig/ctrl/.env.example index 5f07c10..8f482d8 100644 --- a/rig/ctrl/.env.example +++ b/rig/ctrl/.env.example @@ -26,10 +26,10 @@ PROFILE=minimal # MANIFESTS_DIR=../platform-manifests/overlays/dev MANIFESTS_DIR=ctrl/k8s/overlays/dev -# Where the wizard fetches the pinned binaries from. +# Where the installer fetches the pinned binaries from. # upstream GitHub releases / dl.k8s.io (needs internet) # artifactory a generic repo — what a locked-down client usually allows -# baked already inside the wizard image; no network at all +# baked already inside the installer image; no network at all DEPS_SOURCE=upstream DEPS_ARTIFACTORY_URL= @@ -43,7 +43,7 @@ REGISTRY_PASSWORD= # Corporate root CA, if Artifactory is fronted by an internal CA (it usually is). # Trust has to reach THREE places and nothing does it for you: the host docker # daemon, every kind node's containerd, and any in-cluster client. registry.sh -# handles the first two; station.sh reports when it's configured but not trusted. +# handles the first two; check.sh reports when it's configured but not trusted. # Symptom when missing: x509: certificate signed by unknown authority REGISTRY_CA_FILE= diff --git a/rig/ctrl/Dockerfile.wizard b/rig/ctrl/Dockerfile.deps similarity index 61% rename from rig/ctrl/Dockerfile.wizard rename to rig/ctrl/Dockerfile.deps index c2209e7..c434759 100644 --- a/rig/ctrl/Dockerfile.wizard +++ b/rig/ctrl/Dockerfile.deps @@ -1,18 +1,18 @@ -# The installation wizard. It does NOT run the cluster — it installs a toolchain +# The toolchain installer image. It does NOT run the cluster — it installs a toolchain # onto the host and gets out of the way. # # This exists to kill a bootstrap paradox: a plain bash installer needs curl, jq # and sha256sum to already be present, and a minimal Debian has none of them. -# The wizard carries its own toolchain, so the only host prerequisite is Docker. +# It carries its own toolchain, so the only host prerequisite is Docker. # # Two variants from one file: -# docker build -f ctrl/Dockerfile.wizard --target wizard -t -wizard . -# docker build -f ctrl/Dockerfile.wizard --target wizard-full -t -wizard:full . +# docker build -f ctrl/Dockerfile.deps --target deps -t -deps . +# docker build -f ctrl/Dockerfile.deps --target deps-full -t -deps:full . # -# wizard-full bakes every pinned binary in at build time. `docker save` it and +# deps-full bakes every pinned binary in at build time. `docker save` it and # you have the whole installer as one file to carry into an air-gapped network. -FROM debian:trixie-slim AS wizard +FROM debian:trixie-slim AS deps # ca-certificates + curl: fetch and verify. graphviz + python3: render diagrams # and validate the arch model, so the host never needs an apt package. @@ -26,21 +26,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /work COPY ctrl/versions.env /work/ctrl/versions.env -COPY ctrl/wizard.sh /work/ctrl/wizard.sh -RUN chmod +x /work/ctrl/wizard.sh +COPY ctrl/deps.sh /work/ctrl/deps.sh +RUN chmod +x /work/ctrl/deps.sh # Defaults; every one is overridable with -e at run time. ENV DEPS_SOURCE=upstream \ OUT_BIN=/out/bin \ HOST_ROOT=/host -ENTRYPOINT ["/work/ctrl/wizard.sh"] +ENTRYPOINT ["/work/ctrl/deps.sh"] CMD ["install"] # --------------------------------------------------------------------------- -# wizard-full — same wizard, binaries baked in, works with no network at all. -FROM wizard AS wizard-full -RUN /work/ctrl/wizard.sh fetch --to /opt/rig/bin +# deps-full — same image, binaries baked in, works with no network at all. +FROM deps AS deps-full +RUN /work/ctrl/deps.sh fetch --to /opt/rig/bin ENV DEPS_SOURCE=baked \ BAKED_BIN=/opt/rig/bin diff --git a/rig/ctrl/addons/airflow.sh b/rig/ctrl/addons/airflow.sh index eaed535..f3bc0ea 100755 --- a/rig/ctrl/addons/airflow.sh +++ b/rig/ctrl/addons/airflow.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Apache Airflow — the cluster half of soleprint's airflow cabinet. +# Apache Airflow — the cluster half of the airflow cabinet. # # Airflow needs a metadata database before it will start at all, so this refuses # rather than rolls a pod that will CrashLoopBackOff while the real problem @@ -7,7 +7,7 @@ # # One pod on `standalone`, matching the compose cabinet: migration, admin user, # scheduler and webserver in a single container. The official chart's five -# deployments model an installation; a room switching this on wants pipelines. +# deployments model an installation; switching this on means wanting pipelines. set -euo pipefail cd "$(dirname "$0")/.." diff --git a/rig/ctrl/addons/postgres.sh b/rig/ctrl/addons/postgres.sh index c5949da..b694f30 100755 --- a/rig/ctrl/addons/postgres.sh +++ b/rig/ctrl/addons/postgres.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# PostgreSQL — the cluster half of soleprint's postgres cabinet. +# PostgreSQL — the cluster half of the postgres cabinet. # -# A room declares the dependency once, in cfg//data/cabinets.json. On a -# laptop `build.py` composes it into docker-compose.yml; here it becomes a pod, -# so the same declaration works either way and nothing has to be remembered +# A cabinet is a public service dropped into the environment as-is — the +# upstream image, unmodified, reachable at a known address. This is the cluster +# half of it; the compose half is a `service.yml` beside a `cabinet.json`. The +# declaration is made once and both paths read it, so nothing is remembered # twice. # # Plain manifests rather than a helm chart, matching the other addons: a chart @@ -32,8 +33,8 @@ if $K get secret -n "$NS" postgres >/dev/null 2>&1; then else password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24) $K create secret generic postgres -n "$NS" \ - --from-literal=POSTGRES_DB="${POSTGRES_DB:-soleprint}" \ - --from-literal=POSTGRES_USER="${POSTGRES_USER:-soleprint}" \ + --from-literal=POSTGRES_DB="${POSTGRES_DB:-postgres}" \ + --from-literal=POSTGRES_USER="${POSTGRES_USER:-postgres}" \ --from-literal=POSTGRES_PASSWORD="$password" >/dev/null echo " generated a password (read it back with the command printed below)" fi diff --git a/rig/ctrl/addons/redis.sh b/rig/ctrl/addons/redis.sh index a21ca63..3f104d0 100755 --- a/rig/ctrl/addons/redis.sh +++ b/rig/ctrl/addons/redis.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Redis — the cluster half of soleprint's redis cabinet. +# Redis — the cluster half of the redis cabinet. # # Cache, and the broker anything queue-shaped runs on. No persistence: a broker # that loses its queue on restart is the honest local model, and a PVC here buys diff --git a/rig/ctrl/station.sh b/rig/ctrl/check.sh similarity index 95% rename from rig/ctrl/station.sh rename to rig/ctrl/check.sh index c377198..c6af8b3 100755 --- a/rig/ctrl/station.sh +++ b/rig/ctrl/check.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash -# Station check: is this workstation ready to run rig? +# Readiness check: is this machine ready to run rig? # # Reports and instructs; never silently fixes anything. Everything it finds is # either already fine, or something a human has to decide on. # -# Runs the wizard's host detection in a container when Docker is the only thing +# Runs ctrl/deps.sh host detection in a container when Docker is the only thing # installed, or directly when the toolchain is already present. Then adds the # checks that need this repo's config: profile sanity, CA trust, port clashes. set -euo pipefail cd "$(dirname "$0")" -WIZARD_IMAGE="${WIZARD_IMAGE:-$(basename "$(cd .. && pwd)")-wizard}" +DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}" # Host detection. Prefer running it bare — it needs no dependencies beyond # coreutils — and fall back to the container only if this shell can't. -bash ./wizard.sh detect +bash ./deps.sh detect # ── repo-level checks ────────────────────────────────────────────────────── diff --git a/rig/ctrl/wizard.sh b/rig/ctrl/deps.sh similarity index 77% rename from rig/ctrl/wizard.sh rename to rig/ctrl/deps.sh index 532e310..b28a422 100755 --- a/rig/ctrl/wizard.sh +++ b/rig/ctrl/deps.sh @@ -1,17 +1,22 @@ #!/usr/bin/env bash -# The installation wizard: detect the host, install a pinned toolchain onto it, -# then report what it could not do. It never runs the cluster and never mutates -# the host outside the directories mounted into it. +# Toolchain installer: detect the host, install a pinned toolchain onto it, then +# report what it could not do. # -# Usage (normally via `make station` / `make deps`, or directly): -# wizard.sh detect # report host facts only, change nothing -# wizard.sh fetch [core|dev] [--to DIR] # download + verify into DIR -# wizard.sh install [core|dev] # detect, fetch, install, report +# It never runs the cluster, never uses sudo or apt, and writes only into +# $OUT_BIN (default ~/.local/bin). Everything that would touch the host proper — +# systemd, inotify limits, .wslconfig, docker group — is REPORTED for a human to +# decide on, never performed. That is what makes it safe to run on a machine that +# already has a working setup. +# +# Usage (normally via `make deps`, or directly): +# deps.sh detect # report host facts only, change nothing +# deps.sh fetch [core|dev] [--to DIR] # download + verify into DIR +# deps.sh install [core|dev] # detect, fetch, install, report # # Tiers: 'core' is kubectl + jq (talk to a cluster); 'dev' adds kind and tilt # Default is dev. # -# Runs both inside the wizard container and bare on a host. Inside the +# Runs both inside the installer container and bare on a host. Inside the # container, host files are read through $HOST_ROOT (mount / as :ro); bare, it # falls back to /. @@ -55,6 +60,29 @@ host_file() { # ── detect ───────────────────────────────────────────────────────────────── +# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and +# then fails in a pile of confusing ways: no /proc, no docker socket, none of +# the tooling. Detectable, so name it instead. +require_linux() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + cat >&2 <<'EOF' +This has to run inside WSL, not Git Bash / MSYS / Cygwin. + +If WSL is not installed yet, from an elevated PowerShell or Command Prompt: + + wsl --install + +That enables Windows features and needs a reboot, so it is not something this +script will do for you. Afterwards, open the Linux shell it installs and run +this from there. + +See "Starting from plain Windows" in README.md. +EOF + exit 1 ;; + esac +} + is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; } detect() { @@ -76,6 +104,7 @@ detect() { fi detect_wsl + detect_filesystem detect_docker detect_inotify } @@ -115,18 +144,46 @@ detect_wsl() { if [ -n "$wcfg" ] && grep -qE '^\s*memory\s*=' "$wcfg"; then echo " wslconfig memory set: $(grep -E '^\s*memory\s*=' "$wcfg" | tr -d ' ')" else - MANUAL+=("Cap/raise the WSL VM memory — in %USERPROFILE%\\.wslconfig on Windows: - [wsl2] - memory=8GB - then from a WINDOWS terminal: wsl --shutdown") + MANUAL+=("Cap/raise the WSL VM memory — see what is set versus what booted: + make mem status + It prints the edit to make and the command to apply it.") + fi +} + +# Not a path check: /mnt is an ordinary mount point and an ext4 disk mounted +# there is perfectly fine. What matters is the filesystem. The Windows drives +# arrive as 9p (WSL2) or drvfs (WSL1); network and fuse mounts behave the same +# way. None of them deliver inotify events, so anything watching files goes +# quiet without saying why. +watch_hostile_fs() { + local dir="$1" fstype + fstype=$(findmnt -no FSTYPE --target "$dir" 2>/dev/null || true) + [ -n "$fstype" ] || fstype=$(stat -f -c %T "$dir" 2>/dev/null || true) + case "$fstype" in + 9p|v9fs|drvfs|cifs|smb3|nfs|nfs4|fuse.sshfs|fuseblk) echo "$fstype" ;; + *) echo "" ;; + esac +} + +detect_filesystem() { + local root fstype + root=$(cd .. && pwd -P) + fstype=$(watch_hostile_fs "$root") + if [ -n "$fstype" ]; then + echo " ! this directory is on $fstype — file watching will not work" + MANUAL+=("Move this onto the local disk. Nothing watching files sees changes + on a $fstype mount, and everything else is slower: + cp -r \"$root\" ~/ && cd ~/$(basename "$root")") + else + echo " filesystem $root ($(findmnt -no FSTYPE --target "$root" 2>/dev/null || echo local))" fi } detect_docker() { # Reachability of the daemon is the real question, and the CLI is only how - # we ask it. Note that when this runs inside the wizard container, Docker + # we ask it. Note that when this runs inside the installer container, Docker # necessarily exists on the host — otherwise nothing would be executing — - # so a missing CLI in here is a wizard packaging bug, not a host problem. + # so a missing CLI in here is an installer packaging bug, not a host problem. if ! command -v docker >/dev/null 2>&1; then if [ -S /var/run/docker.sock ]; then echo " docker socket present (no cli in this context)" @@ -230,7 +287,7 @@ fetch_tgz() { chmod +x "$dest/$name" } -# The wizard runs as root so it can reach the docker socket, which means +# The installer runs as root so it can reach the docker socket, which means # everything it writes into a mounted volume lands root-owned and unusable from # the host. Hand it back to whoever owns the mount point (the host user created # that directory before mounting it). @@ -257,7 +314,13 @@ fix_ownership() { CORE_TOOLS="kubectl jq" # No helm: every addon installs with `kubectl apply -f `, so nothing here # has ever invoked it. Add it back the day something actually needs a chart. -DEV_TOOLS="kind tilt" +# +# ctlptl is 'dev' rather than 'core' for the same reason kind is: core is "talk +# to a cluster someone else runs", and ctlptl builds them. It earns its place +# because it is what wires a cluster to a local registry — without one, an +# unqualified image name resolves to docker.io/library/ and there is +# nothing structural stopping a push there. +DEV_TOOLS="kind tilt ctlptl" fetch() { local dest="$OUT_BIN" tier="${TIER:-dev}" @@ -284,7 +347,8 @@ fetch() { fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest" if [ "$tier" = "dev" ]; then fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest" - fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0 + fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0 + fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0 fi fix_ownership "$dest" @@ -301,7 +365,7 @@ report_manual() { echo "nothing left to do by hand." return fi - echo "host actions the wizard cannot perform (${#MANUAL[@]}):" + echo "host actions this cannot perform (${#MANUAL[@]}):" echo local n=1 for m in "${MANUAL[@]}"; do @@ -373,6 +437,8 @@ install() { # ── main ─────────────────────────────────────────────────────────────────── +require_linux + case "${1:-install}" in detect) detect; report_manual ;; fetch) shift; fetch "$@" ;; diff --git a/rig/ctrl/env.d/client.env b/rig/ctrl/env.d/client.env index 1297ec8..2c2fff1 100644 --- a/rig/ctrl/env.d/client.env +++ b/rig/ctrl/env.d/client.env @@ -19,7 +19,7 @@ DNS_MODE=hosts # Opt in to the real ports below only when this is the ONLY environment and # nothing else owns :80. They fail to bind otherwise, and docker reports it as an # opaque "failed to bind host port 0.0.0.0:80/tcp: address already in use" -# halfway through cluster creation. `make station` checks before you spend the +# halfway through cluster creation. `make check` checks before you spend the # time. Uncommenting also means only one environment can exist at a time. # HTTP_PORT=80 # HTTPS_PORT=443 diff --git a/rig/ctrl/env.d/data.env b/rig/ctrl/env.d/data.env index 5b1f078..4b98143 100644 --- a/rig/ctrl/env.d/data.env +++ b/rig/ctrl/env.d/data.env @@ -1,11 +1,10 @@ -# data — a cluster with the dependency containers a soleprint room asks for. +# data — a cluster with the cabinets an environment asks for. # -# The point of this profile is that a room declares what it needs once, in -# cfg//data/cabinets.json, and gets it on either target: `build.py` -# composes those services into docker-compose.yml for a laptop, and the addons -# below install the same ones here. The names match deliberately — -# soleprint/station/cabinets//cabinet.json carries a `rig_addon` field -# pointing at ctrl/addons/.sh. +# A cabinet is a public service dropped in as-is — the upstream image, +# unmodified, reachable at a known address. It is declared once and installs on +# either target: a `service.yml` composes it for a laptop, and the addons below +# install the same one here. The names match deliberately — each cabinet.json +# carries a `rig_addon` field pointing at ctrl/addons/.sh. # # Everything lands in the `data` namespace (DATA_NAMESPACE to move it), so # `make cluster reset` on the app namespace leaves the databases alone. @@ -30,8 +29,8 @@ DATA_NAMESPACE=data # Postgres identity. The password is not here: postgres.sh generates one on # first install and keeps it across re-runs, so re-running the addon never # rotates the credential out from under whatever is already connected. -POSTGRES_DB=soleprint -POSTGRES_USER=soleprint +POSTGRES_DB=app +POSTGRES_USER=app POSTGRES_STORAGE=2Gi AIRFLOW_ADMIN_USER=admin diff --git a/rig/ctrl/env.d/offline.env b/rig/ctrl/env.d/offline.env index 71055a1..ae59379 100644 --- a/rig/ctrl/env.d/offline.env +++ b/rig/ctrl/env.d/offline.env @@ -1,5 +1,5 @@ # offline — air-gapped. Everything comes from a local registry that was loaded -# ahead of time; nothing reaches the internet. Pair with the wizard-full image +# ahead of time; nothing reaches the internet. Pair with the deps-full image # (DEPS_SOURCE=baked) so the toolchain install is offline too. # # The heavier addons are left out to keep first boot viable. diff --git a/rig/ctrl/k8s/README.md b/rig/ctrl/k8s/README.md index 9ccaa52..8d906e8 100644 --- a/rig/ctrl/k8s/README.md +++ b/rig/ctrl/k8s/README.md @@ -1,8 +1,7 @@ # `ctrl/k8s` — cluster shape, and what runs on it -Same layout as every other project here (`unt`, `nvi`, `eth`, `mpr`, and -soleprint's generated rooms): a kind config, a kustomize `base/`, and an -`overlays/dev/` that patches it. See ALL `projects/templates/conventions.md`. +Same layout as every other project here: a kind config, a kustomize `base/`, +and an `overlays/dev/` that patches it. ``` kind-config*.yaml.tpl the cluster itself — nodes, ports, audit diff --git a/rig/ctrl/k8s/kind-config.audit.yaml.tpl b/rig/ctrl/k8s/kind-config.audit.yaml.tpl index 94887da..68bf65d 100644 --- a/rig/ctrl/k8s/kind-config.audit.yaml.tpl +++ b/rig/ctrl/k8s/kind-config.audit.yaml.tpl @@ -44,7 +44,7 @@ nodes: - role: control-plane image: ${NODE_IMAGE} # hostPath is resolved by the HOST dockerd, so this must be a host path even - # when cluster.sh runs inside the wizard container. HOST_WORKDIR says where + # when cluster.sh runs inside the installer container. HOST_WORKDIR says where # this rig lives on the host; bare on a host it is just the repo root. extraMounts: - hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml diff --git a/rig/ctrl/lib/config.sh b/rig/ctrl/lib/config.sh index 3911763..31ff780 100644 --- a/rig/ctrl/lib/config.sh +++ b/rig/ctrl/lib/config.sh @@ -112,7 +112,7 @@ load_config() { fi # Read the shape back out of the YAML rather than trusting a profile to - # restate it. station.sh sizes the memory warning on NODES, and cluster.sh + # restate it. check.sh sizes the memory warning on NODES, and cluster.sh # prints AUDIT before spending minutes building something that cannot be # changed afterwards — both would mislead if the numbers drifted. NODES=$(grep -c '^ - role:' "$KIND_CONFIG_PATH") @@ -125,7 +125,7 @@ load_config() { # depending on something the caller does not set. # # hostPath entries are resolved by the HOST dockerd, so HOST_WORKDIR must stay a -# host path even when this runs inside the wizard container. +# host path even when this runs inside the installer container. render_kind_config() { local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}" sed -e "s|\${CLUSTER}|${CLUSTER}|g" \ diff --git a/rig/ctrl/mem.sh b/rig/ctrl/mem.sh new file mode 100755 index 0000000..fa1ea54 --- /dev/null +++ b/rig/ctrl/mem.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# What memory this machine has, what is left, and — where there is one — what +# cap is holding it there. +# +# Runs on native Linux and under WSL, because rig is developed on one and used +# on the other. The difference is not cosmetic: on WSL the memory you see is a +# VM allocation that can be raised, and the commonest failure is raising it +# without restarting, so the number on disk and the number in /proc disagree. +# On native Linux there is no such cap and pretending otherwise sends you to a +# file that does not exist. +# +# This reports and instructs. It never writes a .wslconfig — applying one costs +# a full VM restart that takes every shell, mount and container with it, and +# choosing that moment is yours. +# +# `backup` exists so `restore` has something to read: back up, hand-edit +# following the printed instruction, restore if it goes wrong. Both are +# WSL-only, because .wslconfig is the only thing here worth backing up. +# +# Usage: mem.sh status | backup | restore +set -euo pipefail + +# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and +# then fails in a pile of confusing ways: no /proc, no docker socket, none of +# the tooling. Detectable, so name it instead. +require_linux() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + cat >&2 <<'EOF' +This has to run inside WSL, not Git Bash / MSYS / Cygwin. + +If WSL is not installed yet, from an elevated PowerShell or Command Prompt: + + wsl --install + +That enables Windows features and needs a reboot, so it is not something this +script will do for you. Afterwards, open the Linux shell it installs and run +this from there. + +See "Starting from plain Windows" in README.md. +EOF + exit 1 ;; + esac +} + +is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; } + +require_wsl() { + if ! is_wsl; then + echo "$1 acts on .wslconfig, which only exists under WSL." >&2 + echo "This is native Linux — there is no VM allocation to save or roll back." >&2 + echo "Use 'mem.sh status' to see what the machine actually has." >&2 + exit 1 + fi +} + +mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); } + +# /mnt/c/Users can hold several real accounts — a renamed login leaves the old +# directory behind — so picking the first alphabetically is a coin toss. Ask +# Windows, then fall back to whichever profile actually owns a config. +wslconfig_path() { + local profile winpath found + profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true) + case "$profile" in + ""|*%*) ;; + *) winpath=$(wslpath -u "$profile" 2>/dev/null || true) + if [ -n "$winpath" ] && [ -d "$winpath" ]; then + echo "$winpath/.wslconfig"; return + fi ;; + esac + + found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true) + if [ -n "$found" ]; then echo "$found"; return; fi + + echo "cannot tell which Windows profile owns .wslconfig. Candidates:" >&2 + ls -d /mnt/c/Users/*/ 2>/dev/null \ + | grep -viE "/(All Users|Default|Default User|Public)/$" | sed "s/^/ /" >&2 + exit 1 +} + +configured_memory() { + [ -r "$1" ] || { echo ""; return; } + sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$1" | tail -1 | tr -d '[:space:]' +} + +# "9GB" / "8192MB" / "9G" -> MB, so it can be compared with /proc/meminfo. +to_mb() { + local v="${1^^}" n + n=$(echo "$v" | tr -dc '0-9') + [ -n "$n" ] || { echo ""; return; } + case "$v" in + *GB|*G) echo $(( n * 1024 )) ;; + *MB|*M) echo "$n" ;; + *) echo $(( n / 1024 / 1024 )) ;; + esac +} + +hogs() { + echo "holding the most:" + ps -eo rss,comm --sort=-rss 2>/dev/null | awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}' +} + +status() { + local total avail swap_total swap_free + total=$(mb MemTotal); avail=$(mb MemAvailable) + swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree) + + if is_wsl; then + local cfg conf conf_mb + cfg=$(wslconfig_path) + conf=$(configured_memory "$cfg") + echo "platform WSL" + echo "config $cfg" + if [ -n "$conf" ]; then + conf_mb=$(to_mb "$conf") + echo "configured $conf (${conf_mb} MB)" + else + conf_mb="" + echo "configured (no memory= set — WSL defaults to 50% of host RAM, or 8GB, whichever is less)" + fi + echo "booted ${total} MB" + echo "available ${avail} MB" + echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)" + + if [ -n "$conf_mb" ]; then + # The VM reports a little less than allocated; 15% covers the kernel + # without calling every healthy machine a mismatch. + if [ "$total" -lt $(( conf_mb * 85 / 100 )) ]; then + echo + echo "! configured ${conf_mb} MB but booted ${total} MB." + echo " The change has not been applied. From a WINDOWS terminal:" + echo + echo " wsl --shutdown" + echo + echo " then start the distro again." + fi + else + echo + echo "To raise it, add to $cfg on the Windows side:" + echo + echo " [wsl2]" + echo " memory=8GB" + echo + echo "then, from a WINDOWS terminal: wsl --shutdown" + fi + + local n + n=$(ls "$cfg".*.bak 2>/dev/null | wc -l) + [ "$n" -gt 0 ] && echo "backups $n (newest: $(ls -t "$cfg".*.bak 2>/dev/null | head -1))" + else + echo "platform native linux" + echo "total ${total} MB" + echo "available ${avail} MB" + echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)" + echo + echo "No VM allocation to raise here — this is the machine's own memory." + echo "If it is tight the levers are freeing something or adding swap." + fi + + # Under a fifth left is worth naming wherever you are running. + if [ "$avail" -lt $(( total / 5 )) ]; then + echo + hogs + fi + return 0 +} + +backup() { + require_wsl backup + local cfg dest + cfg=$(wslconfig_path) + [ -r "$cfg" ] || { echo "nothing to back up: $cfg does not exist" >&2; exit 1; } + # Timestamped and never overwritten: a backup that can destroy itself on a + # second run is not a backup. + dest="${cfg}.$(date +%Y%m%d-%H%M%S).bak" + cp "$cfg" "$dest" + echo "backed up $dest" + echo + echo "Edit $cfg by hand, then from a WINDOWS terminal: wsl --shutdown" +} + +restore() { + require_wsl restore + local cfg newest count + cfg=$(wslconfig_path) + newest=$(ls -t "$cfg".*.bak 2>/dev/null | head -1 || true) + [ -n "$newest" ] || { echo "no backups found beside $cfg" >&2; exit 1; } + + echo "restoring $newest" + echo " -> $cfg" + echo + + # Newest is the right default — undo the last edit — but if you backed up + # *after* editing, the state you want is older. Show the rest so a no-op + # restore is obviously a no-op rather than a mystery. + count=$(ls "$cfg".*.bak 2>/dev/null | wc -l) + if [ "$count" -gt 1 ]; then + echo "$count backups exist, newest first:" + ls -t "$cfg".*.bak | sed 's/^/ /' + echo " (restoring the newest; copy another by hand to pick an older one)" + echo + fi + + if [ -r "$cfg" ]; then + echo "what changes:" + if diff "$cfg" "$newest" > /tmp/mem.diff 2>&1 && [ ! -s /tmp/mem.diff ]; then + echo " nothing — that backup is identical to the current config" + else + sed 's/^/ /' /tmp/mem.diff + fi + rm -f /tmp/mem.diff + echo + fi + + printf "proceed? [y/N] " + read -r reply + case "$reply" in + y|Y|yes|Yes) ;; + *) echo "left alone"; return 0 ;; + esac + cp "$newest" "$cfg" + echo "restored. From a WINDOWS terminal: wsl --shutdown" +} + +require_linux + +case "${1:-status}" in + status) status ;; + backup) backup ;; + restore) restore ;; + *) echo "usage: $0 [status|backup|restore]" >&2; exit 1 ;; +esac diff --git a/rig/ctrl/newbox.sh b/rig/ctrl/newbox.sh index b634839..64d8ff1 100755 --- a/rig/ctrl/newbox.sh +++ b/rig/ctrl/newbox.sh @@ -58,10 +58,13 @@ require_wsl() { cat >&2 <<'EOF' newbox is WSL-only for now. +If WSL is not installed, run `wsl --install` from an elevated Windows prompt +first — see "Starting from plain Windows" in README.md. + On native Linux you do not need it: rig already isolates environments by directory (own cluster, context, images and port block), so a second copy in a second directory is the clean slate. To validate the installer itself against a -bare system, run the wizard against a stock Debian container instead. +bare system, run ctrl/deps.sh against a stock Debian container instead. EOF exit 1 fi @@ -250,7 +253,7 @@ create() { echo echo "next:" echo " make newbox shell # a shell inside it" - echo " then: cd ~/rig && make station && make deps && make cluster up" + echo " then: cd ~/rig && make check && make deps && make cluster up" echo echo "For a browser on Windows to resolve the hostnames, paste this into" echo "C:\\Windows\\System32\\drivers\\etc\\hosts (it has no wildcard support):" diff --git a/rig/ctrl/registry.sh b/rig/ctrl/registry.sh index d5d85b7..c812f5c 100755 --- a/rig/ctrl/registry.sh +++ b/rig/ctrl/registry.sh @@ -43,7 +43,7 @@ K="kubectl --context ${KUBECONTEXT}" # 2. every kind node's containerd — nodes do NOT inherit host trust # 3. anything doing HTTPS from inside the cluster, in its own trust store # -# We handle (2) here because it's ours to handle. (1) is reported by station.sh +# We handle (2) here because it's ours to handle. (1) is reported by check.sh # since it needs root. (3) belongs to the workload. install_ca_into_nodes() { [ -n "${REGISTRY_CA_FILE:-}" ] || return 0 diff --git a/rig/ctrl/setup.sh b/rig/ctrl/setup.sh index 3436e15..878e5e0 100755 --- a/rig/ctrl/setup.sh +++ b/rig/ctrl/setup.sh @@ -73,11 +73,11 @@ record() { step_host() { local out - if ! out=$(bash ./wizard.sh detect 2>&1); then + if ! out=$(bash ./deps.sh detect 2>&1); then record host fail "detection failed" return fi - # Anything the wizard flagged with '!' needs a human; surface the count here + # Anything flagged with '!' needs a human; surface the count here # and the detail below rather than burying it. local warns; warns=$(echo "$out" | grep -c '^\s*!' || true) HOST_DETAIL="$out" @@ -102,7 +102,7 @@ step_toolchain() { return fi - if bash ./wizard.sh install "$TIER" >/tmp/rig-deps.$$ 2>&1; then + if bash ./deps.sh install "$TIER" >/tmp/rig-deps.$$ 2>&1; then local still="" for b in $want; do [ -x "${OUT_BIN:-$HOME/.local/bin}/$b" ] || still="$still $b" diff --git a/rig/ctrl/versions.env b/rig/ctrl/versions.env index bf047ab..9167662 100644 --- a/rig/ctrl/versions.env +++ b/rig/ctrl/versions.env @@ -1,4 +1,4 @@ -# Pinned toolchain — the single manifest the wizard installs from. +# Pinned toolchain — the single manifest ctrl/deps.sh installs from. # Every entry is a single binary; none of them needs an apt repo. # kubectl fully static # kind libc only @@ -6,8 +6,19 @@ # jq upstream static build (Debian's is linked against libjq/libonig) # # Checksums are the upstream-published SHA256 of the linux/amd64 artifact. -# To bump: change the version, then re-run `bash ctrl/versions-refresh.sh` and -# commit the result — never hand-edit a checksum. +# +# To bump: change the version, then take the checksum from the release's own +# published list — never hand-edit or hand-copy one from a download you did. +# For anything hosted on GitHub releases that is: +# +# curl -sSL https://github.com///releases/download//checksums.txt \ +# | grep linux.x86_64 +# +# (kubectl publishes its own instead: .sha256.) +# +# There was a `ctrl/versions-refresh.sh` named here that has never existed. If +# bumping stops being rare enough to do by hand, write it — but a comment +# pointing at a missing script is worse than no comment. KIND_VERSION=v0.32.0 KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 @@ -21,6 +32,14 @@ TILT_VERSION=0.37.6 TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6 TILT_URL=https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz +# ctlptl — creates a kind cluster WITH a local registry wired in, which is what +# keeps images off docker.io (an unqualified name means docker.io/library/). +# Same publisher and same archive shape as tilt: binary at the archive root, so +# fetch_tgz handles it with strip=0 and no special case. +CTLPTL_VERSION=0.9.4 +CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e +CTLPTL_URL=https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz + JQ_VERSION=1.8.2 JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64 @@ -44,9 +63,9 @@ CERT_MANAGER_VERSION=v1.21.1 METRICS_SERVER_VERSION=v0.9.0 METALLB_VERSION=v0.16.0 -# Dependency containers. These mirror soleprint's cabinets -# (soleprint/station/cabinets/), so a room that declares postgres gets the same -# thing whether it runs on compose or in the cluster. Pinned by tag rather than +# Cabinets — public services dropped in as-is, the upstream image unmodified. +# The same declaration installs on compose or in the cluster, so a dependency is +# named once and works either way. Pinned by tag rather than # digest because they are ordinary upstream images with no supply chain claim # attached — bump freely, and preload them for the offline profile. POSTGRES_IMAGE=postgres:16-alpine diff --git a/rig/docs/graphs/01-install.dot b/rig/docs/graphs/01-install.dot index baa76f5..ae1f6af 100644 --- a/rig/docs/graphs/01-install.dot +++ b/rig/docs/graphs/01-install.dot @@ -20,13 +20,13 @@ digraph rig_install { bin [label="~/.local/bin\nkind · kubectl · tilt\njq" fillcolor="#121829" shape=cylinder] } - subgraph cluster_wizard { + subgraph cluster_installer { label="Installer container (transient)" style=dashed color="#1e2a4a" fontcolor="#8892a8" - wizard [label="wizard\ncurl · jq · python · graphviz" fillcolor="#121829"] + installer [label="deps installer\ncurl · jq · python · graphviz" fillcolor="#121829"] detect [label="detect host\nWSL · memory · inotify · docker" fillcolor="#121829"] fetch [label="fetch + verify\nSHA256, pinned versions" fillcolor="#121829"] } @@ -34,14 +34,14 @@ digraph rig_install { upstream [label="upstream\nreleases / corporate mirror" fillcolor="#1a3a1a" fontcolor="#00c853" shape=octagon] report [label="report what it\nCANNOT do" fillcolor="#3a1a1a" fontcolor="#ffc107"] - docker -> wizard [label="docker run"] - wizard -> detect + docker -> installer [label="docker run"] + installer -> detect detect -> fetch fetch -> upstream [label="pinned + checksummed" color="#00c853"] fetch -> bin [label="install"] detect -> report [style=dashed label="sudo / Windows-side steps" color="#ffc107"] // The container is gone after this; nothing depends on it at run time. - wizard -> gone [style=dotted label="exits"] + installer -> gone [style=dotted label="exits"] gone [label="(container discarded)" fillcolor="#0a0e17" fontcolor="#4a5568" color="#1e2a4a" style="filled,dashed"] } diff --git a/rig/docs/graphs/01-install.svg b/rig/docs/graphs/01-install.svg index 21613c0..4df9555 100644 --- a/rig/docs/graphs/01-install.svg +++ b/rig/docs/graphs/01-install.svg @@ -16,7 +16,7 @@ Your machine -cluster_wizard +cluster_installer Installer container (transient) @@ -27,16 +27,16 @@ Docker (the one prerequisite) - + -wizard +installer -wizard +deps installer curl · jq · python · graphviz - + -docker->wizard +docker->installer docker run @@ -57,9 +57,9 @@ detect host WSL · memory · inotify · docker - + -wizard->detect +installer->detect @@ -69,9 +69,9 @@ (container discarded) - + -wizard->gone +installer->gone exits diff --git a/rig/docs/graphs/02-environment.dot b/rig/docs/graphs/02-environment.dot index 6dda83b..f0be52e 100644 --- a/rig/docs/graphs/02-environment.dot +++ b/rig/docs/graphs/02-environment.dot @@ -20,7 +20,7 @@ digraph rig_environment { cname [label="cluster name\nacmebank" fillcolor="#121829"] ctx [label="kubectl context\nkind-acmebank" fillcolor="#121829"] - img [label="image tag\nacmebank-wizard" fillcolor="#121829"] + img [label="image tag\nacmebank-deps" fillcolor="#121829"] ports [label="port block\n21300–21309" fillcolor="#121829"] reg [label="registry container\nacmebank-registry" fillcolor="#121829"] } diff --git a/rig/docs/graphs/02-environment.svg b/rig/docs/graphs/02-environment.svg index 9528093..42bf77c 100644 --- a/rig/docs/graphs/02-environment.svg +++ b/rig/docs/graphs/02-environment.svg @@ -4,28 +4,28 @@ - + rig_environment - -One environment per directory — copies never collide + +One environment per directory — copies never collide cluster_derived - -Everything below is derived from it + +Everything below is derived from it cluster_config - -Configuration — weakest first, later wins + +Configuration — weakest first, later wins dirname - -directory name -e.g. acmebank/ + +directory name +e.g. acmebank/ @@ -37,8 +37,8 @@ dirname->cname - - + + @@ -50,120 +50,120 @@ dirname->ctx - - + + img - -image tag -acmebank-wizard + +image tag +acmebank-deps dirname->img - - + + ports - -port block -21300–21309 + +port block +21300–21309 dirname->ports - - + + reg - -registry container -acmebank-registry + +registry container +acmebank-registry dirname->reg - - + + cluster - -kind cluster + +kind cluster cname->cluster - - + + ports->cluster - - + + versions - -versions.env -pinned toolchain + +versions.env +pinned toolchain profile - -env.d/<profile>.env -nodes · CNI · audit · addons + +env.d/<profile>.env +nodes · CNI · audit · addons versions->profile - - -overridden by + + +overridden by localenv - -ctrl/.env -secrets, overrides + +ctrl/.env +secrets, overrides profile->localenv - - -overridden by + + +overridden by shell - -the environment -PROFILE=client make … + +the environment +PROFILE=client make … localenv->shell - - -overridden by + + +overridden by shell->cluster - - + + diff --git a/rig/docs/index.html b/rig/docs/index.html index fe6bbc6..71e890f 100644 --- a/rig/docs/index.html +++ b/rig/docs/index.html @@ -265,11 +265,11 @@

The only prerequisite

Docker. No curl, no jq, no python, no apt repositories to configure.

# then, in the environment directory:
-make station     # is this workstation ready? reports, never fixes
+make check     # is this machine ready? reports, never fixes
 make deps        # install the pinned toolchain
 make cluster up  # build the cluster for the active profile
-

Read make station before make deps. It never changes +

Read make check before make deps. It never changes anything — it prints what it found and, at the end, the steps it cannot perform for you.

@@ -280,13 +280,13 @@ make cluster up # build the cluster for the active profileStart to finish, in order, with what each one actually does.

-

1 · make station

-

Asks whether this workstation is ready. It changes nothing — it +

1 · make check

+

Asks whether this machine is ready. It changes nothing — it reports what it found and, at the end, the things only a human can do (anything needing sudo, or a Windows-side restart). Read it before installing anything; it is faster than discovering the same problems one failure at a time.

-
make station
+
make check

2 · make setup

Does the preparation that can be automated: installs the pinned @@ -381,8 +381,8 @@ make setup core # same distinction, via setup wants only Docker.

Air-gapped

-
make wizard full                       # bakes every binary into the image
-docker save …-wizard:full | gzip > rig.tgz
+
make deps-image full                       # bakes every binary into the image
+docker save …-deps:full | gzip > rig.tgz
 # carry that one file in, then:
 docker load < rig.tgz && make cluster up PROFILE=offline
 

@@ -485,7 +485,7 @@ docker load < rig.tgz && make cluster up PROFILE=offline
       registry is usually behind an internal CA, and trust has to reach
       three places: the host Docker daemon, every cluster node's containerd
       (nodes do not inherit host trust), and any in-cluster client. Set
-      REGISTRY_CA_FILE and make station reports which is
+      REGISTRY_CA_FILE and make check reports which is
       still missing. The symptom otherwise is an opaque
       x509: certificate signed by unknown authority.
@@ -528,11 +528,11 @@ docker load < rig.tgz && make cluster up PROFILE=offline

Tilt stops noticing file changes

Almost always inotify limits, and it fails silently — nothing errors, changes just stop being picked up. Defaults on WSL are far too - low. make station reports it and prints the fix.

+ low. make check reports it and prints the fix.

Cluster creation dies halfway with a port error

Docker reports failed to bind host port … address already in use - partway through creating the cluster. Run make station first — it + partway through creating the cluster. Run make check first — it checks every port in this environment's block before anything is built.

Every node stays NotReady

diff --git a/rig/sample-rig/.gitignore b/rig/sample-rig/.gitignore deleted file mode 100644 index e785dc1..0000000 --- a/rig/sample-rig/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# NOTE: generated/ is deliberately NOT ignored. The artifact IS the deliverable — -# the whole point is a folder you copy, apply and boot without generating -# anything first. Regenerate it with `make manifest` after editing bundle.json -# or app/serve.py, and commit the result. -# -# (This differs from rig's ctrl/k8s/generated, which is a local build artifact.) - -__pycache__/ -*.pyc diff --git a/rig/sample-rig/Makefile b/rig/sample-rig/Makefile deleted file mode 100644 index c4a3905..0000000 --- a/rig/sample-rig/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# Thin control Makefile — one target per ctrl/ script, subcommand as an -# argument. Same shape as rig's, for the same reason: the logic lives in the -# script, never here. -# -# make up -> ctrl/bundle.sh up -# make bundle down -# -# This directory is a BUNDLE, not an installer. It needs a cluster, which rig -# owns: -# -# cd .. && make cluster up # kind cluster for this environment -# make up # then deploy this bundle into it -# -# Start with: make up - -ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) -ifneq ($(ARGS),) -$(eval $(ARGS):;@:) -endif - -.DEFAULT_GOAL := help -.PHONY: help bundle manifest up down status url list dev - -help: ## list targets - @grep -hE '^[a-z]+:.*?##' $(MAKEFILE_LIST) | sed 's/:.*##/\t/' | expand -t16 - -bundle: ## the bundle [manifest|up|down|status|url|list|dev] - bash ctrl/bundle.sh $(or $(ARGS),status) - -# Shorthands for the ones used constantly. -manifest: ## regenerate generated/.yaml — no cluster needed - bash ctrl/bundle.sh manifest - -up: ## deploy this rig (installs MetalLB if absent) - bash ctrl/bundle.sh up - -down: ## remove this rig (leaves cluster, MetalLB, siblings) - bash ctrl/bundle.sh down - -status: ## what is deployed for this rig - bash ctrl/bundle.sh status - -url: ## the address MetalLB assigned - bash ctrl/bundle.sh url - -list: ## every rig in this cluster, with addresses - bash ctrl/bundle.sh list - -dev: ## run the UI locally with vite — no cluster needed - bash ctrl/bundle.sh dev diff --git a/rig/sample-rig/README.md b/rig/sample-rig/README.md deleted file mode 100644 index 198f762..0000000 --- a/rig/sample-rig/README.md +++ /dev/null @@ -1,140 +0,0 @@ -# sample-rig - -A minimal, non-sensitive bundle that proves an installation works and shows what -shipped. Copy it, rename it, and you have another rig. - -```bash -make manifest # generate the artifact — no cluster, no kubectl needed -make up # deploy it into the local cluster -make list # every rig in this cluster, with addresses -make dev # run the UI locally with vite, no cluster at all -``` - -`make up` prints an address. Open it and the page says **IT WORKS**, then lists -the tools and rigs in the bundle. - -## What it is for - -Three jobs, in the order you hit them: - -1. **Prove the install.** kind is there, a cluster exists, MetalLB hands out an - address, a `type: LoadBalancer` Service actually resolves, and a pod serves. - If all of that works, the environment is sound. -2. **Say what shipped.** The page renders [`bundle.json`](bundle.json) — - standalone tools and rigs, flat, with none of soleprint's internal hierarchy. - Editing that file is the only step needed to change the listing. -3. **Stand in for the real thing.** Nothing here is sensitive. The real - architecture connects separately, against a setup already known to work. - -## The UI is a complement, not the product - -`rig-ui/` is just a vite app. It complements a rig; a rig is complete and useful -without it, and nothing depends on it being there. It is deliberately **not** -generated by kind or tilt — you copy the folder into a rig after that rig is -pulled, and apply one manifest: - -```bash -kubectl apply -n -f rig-ui/k8s.yaml -``` - -That file is the whole integration: one Pod running `npm run dev` on -`node:22-alpine`, one Service. A bare Pod rather than a Deployment because this -is a dev-loop convenience, not a workload to keep alive. - -The app and `bundle.json` arrive as a ConfigMap, so nothing is baked into an -image and editing the bundle is the entire update cycle. The container runs -`npm install` at start, which needs egress to a registry — on a locked-down -cluster point npm at the internal one, or bake an image instead. Nothing else -changes if you do. - -## One artifact, two destinations - -`ctrl/manifest.py` emits `generated/.yaml` — namespace, the app and -bundle embedded in a ConfigMap, Pod, Service. It is self-contained and applies -unmodified anywhere: - -```bash -kubectl apply -f generated/sample-rig.yaml # local kind, or an external cluster -``` - -`make up` applies **that same file**. There is no separate local path, so what -works here cannot quietly differ from what is applied elsewhere. - -This is what `type: LoadBalancer` buys. MetalLB answers it on kind; the AWS load -balancer controller answers it on EKS. NodePort would not survive the trip — it -is a single cluster-wide port range, so two rigs would have to negotiate numbers. - -**VPC-agnostic on purpose.** The target is EKS, but the Service carries no -annotations — no `aws-load-balancer-subnets`, no security groups, no `-scheme`, -no `-type: nlb`. Each of those encodes a specific network layout, and one of them -appearing here would pin the artifact to the account and VPC it was written -against, which is precisely what stops it also working on kind. Subnet discovery -is the cluster's business: EKS resolves it from the tags its own subnets carry. - -That leaves one thing genuinely environment-specific — internal versus -internet-facing. A bare `LoadBalancer` provisions internet-facing, which a -regulated account will usually refuse, and should. That belongs in a -per-environment overlay applied on top, never inlined into this artifact. - -**MetalLB only — no ingress-nginx.** Its controller supports a narrow window of -Kubernetes versions, so depending on it constrains which k8s a rig can be built -with. That undercuts running trailing-edge control planes to model a legacy -estate, which is the reason `versions.env` pins v1_33..v1_36. MetalLB carries no -such constraint, so reachability costs nothing in version coverage. - -## Several rigs, one cluster - -Identity follows the **folder name**, the same rule rig uses for cluster -identity. The namespace is the folder; resource names are generic, and names only -have to be unique within a namespace. - -```bash -cp -r sample-rig corporate-rig -cd corporate-rig && make up # its own namespace, its own address -``` - -No edits, no collisions, both in the same local cluster. `make list` shows them -together. `make down` removes only this one — siblings, MetalLB and the cluster -are left alone. - -Client rigs are gitignored (`*-rig/`, with `sample-rig/` the deliberate -exception): a rig's k8s files spell out a real architecture, and that is exactly -what must not land in this repo. - -## Staging workstations - -`ctrl/manifest.py` is stdlib-only on purpose: it runs on a bare machine before -anything is installed. The toolchain itself is rig's job — `make deps` installs -the pinned kind and tilt binaries, which is what makes a staging AWS workspace -reachable from the same commands as a laptop. - -## Layout - -``` -sample-rig/ -├── Makefile # thin — one target per ctrl/ script -├── bundle.json # what shipped; the UI renders THIS -├── rig-ui/ # the vite app — optional, copied into a rig to enable it -│ ├── k8s.yaml # how to plug it in: one Pod, one Service -│ ├── index.html -│ ├── package.json -│ ├── vite.config.js -│ └── src/{main.js,style.css} -├── ctrl/ -│ ├── manifest.py # emits the artifact -│ └── bundle.sh # generate / deploy / inspect -└── generated/ # the artifact — committed, this is the deliverable -``` - -Editing `bundle.json` or anything in `rig-ui/` means re-running `make manifest`. -The ConfigMap carries a checksum of everything embedded, so a stale deployment is -visible rather than silent. - -## Not built, but not foreclosed - -Everything derives from `bundle.json` plus a target namespace. A Pulumi or -Terraform emitter would sit beside `ctrl/manifest.py` consuming the same inputs; -nothing above it assumes the artifact is YAML. - -Licence terms for the compiled UI component belong in the soleprint-generated -bundle, not here — this sample carries no proprietary component. diff --git a/rig/sample-rig/bundle.json b/rig/sample-rig/bundle.json deleted file mode 100644 index 1fa77db..0000000 --- a/rig/sample-rig/bundle.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "_comment": "What this bundle contains. Single source of truth — the landing page renders THIS file, so adding an entry here is the only edit needed. Deliberately FLAT: standalone tools and rigs, with none of soleprint's internal hierarchy (no artery/atlas/station layering). Nothing here is sensitive; the real architecture connects separately.", - "bundle": { - "name": "sample-rig", - "description": "Non-sensitive sample bundle. Proves the kind install works and shows what ships.", - "sensitive": false - }, - "tools": [ - { - "name": "modelgen", - "summary": "Generate models from config", - "standalone": true - }, - { - "name": "datagen", - "summary": "Generate test data from rig-owned generators", - "standalone": true - }, - { - "name": "graphgen", - "summary": "Generate navigable model graphs", - "standalone": true - }, - { - "name": "tester", - "summary": "HTTP contract test runner — one suite, any environment", - "standalone": true - }, - { - "name": "databrowse", - "summary": "SQL data browser", - "standalone": true - }, - { - "name": "sbwrapper", - "summary": "Sandbox wrapper", - "standalone": true - } - ], - "rigs": [ - { - "name": "sample-rig", - "summary": "This bundle — a minimal, copyable environment", - "active": true - } - ], - "next": [ - "Point MANIFESTS_DIR at the real manifests to connect the actual architecture.", - "Real k8s files are versioned separately and are not part of this bundle." - ] -} diff --git a/rig/sample-rig/cluster.mock.json b/rig/sample-rig/cluster.mock.json deleted file mode 100644 index 2a80708..0000000 --- a/rig/sample-rig/cluster.mock.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "_comment": "MOCKED cluster state. Nothing here is read from a live cluster — it exists so the UI can be shown when there is no cluster at all (a locked-down machine, a laptop with no memory to spare, a demo where kind will not start). The page labels it as mocked; a demo that looks live but is not is worse than one that says so. When a real cluster is present the same shapes come from kubectl.", - - "mocked": true, - - "cluster": { - "name": "sample-rig", - "context": "kind-sample-rig", - "provider": "kind", - "profile": "minimal", - "k8s": "v1.36.1", - "nodes": 1 - }, - - "workloads": [ - { - "name": "rig-ui", - "summary": "Pod · node:22-alpine · vite on :5173", - "state": "Running" - }, - { - "name": "metallb-system/controller", - "summary": "Deployment · assigns LoadBalancer addresses", - "state": "Running" - }, - { - "name": "metallb-system/speaker", - "summary": "DaemonSet · answers ARP in layer 2 mode", - "state": "Running" - } - ], - - "services": [ - { - "name": "rig-ui", - "summary": "LoadBalancer · 80 -> 5173 · no annotations, so it resolves on kind and on EKS alike", - "state": "172.18.255.200" - } - ] -} diff --git a/rig/sample-rig/ctrl/bundle.sh b/rig/sample-rig/ctrl/bundle.sh deleted file mode 100755 index f6e0e06..0000000 --- a/rig/sample-rig/ctrl/bundle.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env bash -# The rig bundle: generate it, deploy it, tear it down, find it. -# -# Usage: bundle.sh manifest | up | down | status | url | list | dev -# -# What `up` proves, in order: kind installed and a cluster exists, MetalLB can -# hand out an address, a Service of type LoadBalancer actually resolves, and a -# pod serves the bundle listing. If all of that works the installation is sound, -# and the only thing missing is the real architecture. -# -# ONE ARTIFACT -# `up` applies generated/.yaml — the same self-contained file you would -# hand to an external cluster. There is no separate local path, so what works -# here cannot quietly differ from the master deployment applied elsewhere. -# -# ONE CLUSTER, SEVERAL RIGS -# Identity follows the FOLDER NAME, exactly as rig's cluster identity does. This -# directory deploys into a namespace named after itself, so copying it to -# corporate-rig/ yields a second rig in the SAME local cluster with no edits and -# no collisions — different namespace, its own MetalLB address. `list` shows all -# of them. The cluster itself is rig's business; this only ever owns a namespace. -# -# MetalLB is installed by calling rig's own addon script rather than -# reimplementing it — deriving the pool from the kind Docker network is the -# fiddly part and there should be exactly one copy of it. -set -euo pipefail -cd "$(dirname "$0")/.." - -BUNDLE_ROOT="$(pwd)" -RIG_CTRL="$(cd .. && pwd)/ctrl" - -# The containing folder's name, reduced to a DNS label (same rule as rig's -# default_cluster_name and ctrl/manifest.py, so all three agree on the slug). -slug() { - local n - n=$(basename "$BUNDLE_ROOT") - n=$(echo "$n" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-') - n=$(echo "$n" | sed 's/^-*//; s/-*$//') - echo "${n:-rig-bundle}" -} -NS="$(slug)" -ARTIFACT="generated/${NS}.yaml" - -# Resolved lazily, not at load time: `manifest` and `dev` deliberately work -# with no cluster and no kubectl at all, and a top-level check would break that. -# -# Follows whatever context rig's cluster.sh selected, so this bundle works in a -# copied-and-renamed environment without being told which cluster it is in. -init_kube() { - KUBECONTEXT="${KUBECONTEXT:-$(kubectl config current-context 2>/dev/null || true)}" - if [ -z "$KUBECONTEXT" ]; then - echo "no kubectl context — bring a cluster up first: (cd .. && make cluster up)" >&2 - exit 1 - fi - KCTX="kubectl --context ${KUBECONTEXT}" - K="kubectl --context ${KUBECONTEXT} --namespace ${NS}" -} - -require_cluster() { - if ! $KCTX cluster-info >/dev/null 2>&1; then - echo "context '$KUBECONTEXT' does not reach a cluster" >&2 - echo "bring one up: (cd .. && make cluster up)" >&2 - exit 1 - fi -} - -ensure_metallb() { - if $KCTX get deployment -n metallb-system controller >/dev/null 2>&1; then - echo "metallb: present" - return 0 - fi - - # Only kind needs it. On a real cluster the cloud load balancer answers a - # `type: LoadBalancer` Service, and installing MetalLB there would be wrong. - case "$KUBECONTEXT" in - kind-*) ;; - *) - echo "metallb: skipped — '$KUBECONTEXT' is not a kind context" - echo " (a cloud load balancer answers LoadBalancer services there)" - return 0 - ;; - esac - - if [ ! -f "$RIG_CTRL/addons/metallb.sh" ]; then - echo "metallb is not installed and rig's addon script was not found at" >&2 - echo " $RIG_CTRL/addons/metallb.sh" >&2 - echo "a Service of type LoadBalancer will sit at without it." >&2 - exit 1 - fi - - # rig's addons derive their target cluster from RIG'S OWN folder name via - # load_config, so left alone this bundle would install into `kind-rig` — - # a cluster that need not exist — while deploying everything else into the - # context actually selected. CLUSTER is in load_config's overridable set, - # so passing it here points the addon at the same cluster we are using. - local target="${KUBECONTEXT#kind-}" - echo "metallb: installing via rig's addon into '$target'" - CLUSTER="$target" bash "$RIG_CTRL/addons/metallb.sh" -} - -# Regenerate the artifact. No cluster and no kubectl required — this is the step -# a staging workstation runs before anything is installed. -manifest() { - mkdir -p generated - python3 ctrl/manifest.py "$NS" > "$ARTIFACT" - echo "wrote $ARTIFACT ($(wc -l < "$ARTIFACT") lines)" - echo " applies as-is anywhere: kubectl apply -f ${BUNDLE_ROOT}/${ARTIFACT}" -} - -up() { - manifest - init_kube - require_cluster - ensure_metallb - - echo - echo "applying '${NS}' to context '${KUBECONTEXT}'" - $KCTX apply -f "$ARTIFACT" - - # `rollout status` does not work on a bare Pod — it only understands - # Deployments, StatefulSets and DaemonSets. Wait on the condition instead. - # This is the slow step: the container npm-installs before vite serves. - echo "waiting for the pod to be ready (npm install runs first)..." - $K wait --for=condition=Ready pod/rig-ui --timeout=300s - echo - url -} - -down() { - init_kube - # Delete the namespace and everything in it goes with it. Scoped to THIS - # rig — a sibling rig in the same cluster is untouched. - $KCTX delete namespace "$NS" --ignore-not-found - echo "'${NS}' removed (cluster, metallb and any sibling rig are left alone)" -} - -status() { - init_kube - require_cluster - if ! $KCTX get namespace "$NS" >/dev/null 2>&1; then - echo "'${NS}' is not deployed — run: make up" - return 0 - fi - $K get pod,svc,configmap -o wide -} - -# Every rig in this cluster, not just this one — the point of the namespace -# split is that several coexist, so there has to be a way to see them together. -list() { - init_kube - require_cluster - local names - names=$($KCTX get namespace -l rig.bundle/name \ - -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true) - if [ -z "$names" ]; then - echo "no rigs deployed in context '${KUBECONTEXT}'" - return 0 - fi - printf "%-20s %-16s %s\n" RIG ADDRESS "" - local n ip - for n in $names; do - ip=$($KCTX -n "$n" get svc rig-ui \ - -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) - printf "%-20s %-16s %s\n" "$n" "${ip:-}" \ - "$([ "$n" = "$NS" ] && echo '<- this one')" - done -} - -# The address MetalLB (or a cloud load balancer) assigned. here is the -# classic silent failure: everything reports healthy and nothing is reachable. -url() { - init_kube - local ip - ip=$($K get svc rig-ui \ - -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) - if [ -z "$ip" ]; then - ip=$($K get svc rig-ui \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) - fi - if [ -z "$ip" ]; then - echo "no external address yet — nothing has assigned one." - echo "on kind: kubectl --context $KUBECONTEXT -n metallb-system get pods" - return 1 - fi - echo "IT WORKS -> http://${ip}/" - echo " bundle http://${ip}/bundle.json" -} - -# Run the UI locally with no cluster at all — the fast way to iterate on -# bundle.json. Same vite command the pod runs, so what you see here is what -# gets served there. -dev() { - if ! command -v npm >/dev/null 2>&1; then - echo "npm not found — the UI needs node locally for this." >&2 - echo "(in-cluster it runs on the node:22-alpine image instead)" >&2 - exit 1 - fi - # bundle.json lives one level up so it stays the rig's data rather than the - # app's; vite serves public/ at the root, which is where the app fetches it. - mkdir -p rig-ui/public - cp bundle.json rig-ui/public/bundle.json - - # The mocked cluster is a DEMO asset and is deliberately not embedded in the - # deployed artifact — on a real rig the UI would then show canned values - # beside a live cluster, which is precisely the lie its banner warns about. - # It is served here, and in the static build for the public UI-only page. - cp cluster.mock.json rig-ui/public/cluster.mock.json - - cd rig-ui - [ -d node_modules ] || npm install --no-audit --no-fund - VITE_RIG_NAME="$NS" npm run dev -} - -case "${1:-status}" in - manifest) manifest ;; - up) up ;; - down) down ;; - status) status ;; - url) url ;; - list) list ;; - dev) dev ;; - *) echo "usage: $0 [manifest|up|down|status|url|list|dev]" >&2; exit 1 ;; -esac diff --git a/rig/sample-rig/ctrl/manifest.py b/rig/sample-rig/ctrl/manifest.py deleted file mode 100644 index cd26199..0000000 --- a/rig/sample-rig/ctrl/manifest.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -"""Emit the complete, self-contained deployment for this rig. - - python3 ctrl/manifest.py [namespace] > generated/.yaml - -The output is the ARTIFACT. It carries everything — namespace, the vite app and -bundle.json embedded in a ConfigMap, the Pod and the Service — so it applies -unmodified to any cluster: - - kubectl apply -f generated/sample-rig.yaml - -On kind, MetalLB answers the `type: LoadBalancer` Service. On a real external -cluster the cloud load balancer does. Same file, no edits, no branch — which is -the point: what runs locally is byte-identical to the deployment applied -elsewhere, so local success actually means something. - -`ctrl/bundle.sh up` applies this same generated output rather than a separate -code path, so the local convenience wrapper can never drift from the artifact. - -Stdlib only, deliberately: this must run on a bare staging workstation before -anything is installed, so it cannot depend on PyYAML or a template engine. - -Open seam — not built: everything here derives from bundle.json plus a target -namespace. A Pulumi or Terraform emitter would sit beside this file consuming the -same inputs; nothing above it assumes the artifact is YAML. -""" - -import json -import re -import sys -import zlib -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -UI = ROOT / "rig-ui" - -# Files embedded into the ConfigMap, mounted read-only at /src in the pod and -# copied into vite's layout at start (see rig-ui/k8s.yaml). Flat on purpose: -# ConfigMap keys cannot contain '/'. -EMBEDDED = { - "bundle.json": ROOT / "bundle.json", - "package.json": UI / "package.json", - "vite.config.js": UI / "vite.config.js", - "index.html": UI / "index.html", - "main.js": UI / "src" / "main.js", - "style.css": UI / "src" / "style.css", -} - - -def slug(name: str) -> str: - """Reduce a folder name to a DNS label, matching ctrl/bundle.sh's rule.""" - out = re.sub(r"[^a-z0-9-]", "-", name.lower()).strip("-") - return out or "rig-bundle" - - -def block(text: str, indent: int) -> str: - """Indent a file's contents for a YAML literal block scalar. - - Blank lines are emitted truly empty rather than as whitespace: trailing - spaces on an otherwise blank line are legal YAML but show up as diff noise - in a committed artifact. - """ - pad = " " * indent - return "\n".join(pad + line if line.strip() else "" for line in text.splitlines()) - - -def checksum(parts: list[str]) -> str: - """Stable content hash of everything embedded, stamped as a label. - - A mounted ConfigMap updates in place without restarting anything, so without - a visible change nothing signals that the pod is serving stale content. - """ - return str(zlib.crc32("".join(parts).encode()) & 0xFFFFFFFF) - - -def build(namespace: str) -> str: - contents = {} - for key, path in EMBEDDED.items(): - if not path.exists(): - sys.exit(f"missing input: {path}") - contents[key] = path.read_text() - - # Fail loudly here rather than shipping an artifact that renders an error. - try: - json.loads(contents["bundle.json"]) - except json.JSONDecodeError as exc: - sys.exit(f"bundle.json is not valid JSON: {exc}") - - app = (UI / "k8s.yaml").read_text() - app = app.replace("__RIG_NAME__", namespace) - - data = "\n".join( - f" {key}: |\n{block(text, 4)}" for key, text in sorted(contents.items()) - ) - - return f"""# GENERATED by ctrl/manifest.py — do not edit. -# Regenerate with: make manifest -# -# Self-contained: applies as-is to any cluster, local kind or external. -# kubectl apply -f this-file.yaml -# -# Namespace carries the identity, so several rigs coexist in one cluster. -apiVersion: v1 -kind: Namespace -metadata: - name: {namespace} - labels: - rig.bundle/name: {namespace} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rig-ui - namespace: {namespace} - labels: - rig.bundle/checksum: "{checksum(list(contents.values()))}" -data: -{data} ---- -{_namespaced(app, namespace)} -""" - - -def _namespaced(doc: str, namespace: str) -> str: - """Add `namespace:` to each resource so the artifact applies without -n. - - rig-ui/k8s.yaml omits it on purpose — applied by hand it should land in - whatever namespace you choose. Pinning it belongs to the generated artifact, - which has to be self-contained. - """ - return re.sub( - r"^(metadata:\n(?:[ \t]+.*\n)*?)([ \t]+)(name: rig-ui)$", - lambda m: f"{m.group(1)}{m.group(2)}{m.group(3)}\n{m.group(2)}namespace: {namespace}", - doc.strip(), - flags=re.MULTILINE, - ) - - -if __name__ == "__main__": - target = sys.argv[1] if len(sys.argv) > 1 else slug(ROOT.name) - sys.stdout.write(build(target)) diff --git a/rig/sample-rig/generated/sample-rig.yaml b/rig/sample-rig/generated/sample-rig.yaml deleted file mode 100644 index 73fe7e3..0000000 --- a/rig/sample-rig/generated/sample-rig.yaml +++ /dev/null @@ -1,406 +0,0 @@ -# GENERATED by ctrl/manifest.py — do not edit. -# Regenerate with: make manifest -# -# Self-contained: applies as-is to any cluster, local kind or external. -# kubectl apply -f this-file.yaml -# -# Namespace carries the identity, so several rigs coexist in one cluster. -apiVersion: v1 -kind: Namespace -metadata: - name: sample-rig - labels: - rig.bundle/name: sample-rig ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rig-ui - namespace: sample-rig - labels: - rig.bundle/checksum: "2074194964" -data: - bundle.json: | - { - "_comment": "What this bundle contains. Single source of truth — the landing page renders THIS file, so adding an entry here is the only edit needed. Deliberately FLAT: standalone tools and rigs, with none of soleprint's internal hierarchy (no artery/atlas/station layering). Nothing here is sensitive; the real architecture connects separately.", - "bundle": { - "name": "sample-rig", - "description": "Non-sensitive sample bundle. Proves the kind install works and shows what ships.", - "sensitive": false - }, - "tools": [ - { - "name": "modelgen", - "summary": "Generate models from config", - "standalone": true - }, - { - "name": "datagen", - "summary": "Generate test data from rig-owned generators", - "standalone": true - }, - { - "name": "graphgen", - "summary": "Generate navigable model graphs", - "standalone": true - }, - { - "name": "tester", - "summary": "HTTP contract test runner — one suite, any environment", - "standalone": true - }, - { - "name": "databrowse", - "summary": "SQL data browser", - "standalone": true - }, - { - "name": "sbwrapper", - "summary": "Sandbox wrapper", - "standalone": true - } - ], - "rigs": [ - { - "name": "sample-rig", - "summary": "This bundle — a minimal, copyable environment", - "active": true - } - ], - "next": [ - "Point MANIFESTS_DIR at the real manifests to connect the actual architecture.", - "Real k8s files are versioned separately and are not part of this bundle." - ] - } - index.html: | - - - - - - IT WORKS - - -
- - - - main.js: | - import "./style.css"; - - /* The IT WORKS page: renders bundle.json as the list of what shipped. - * - * Plain vite, no framework — this is a complement to the rig, not part of it, - * and it should stay small enough that nobody has to adopt a stack to read it. - * - * bundle.json is fetched at runtime rather than imported, so the same built app - * serves whatever rig it was copied into. Editing the ConfigMap changes the page - * without rebuilding. - * - * Styling is a handful of rules on purpose. The real visual identity lives in - * the soleprint UI package; nothing here should grow into a theme. - */ - - const esc = (s) => - String(s).replace(/&/g, "&").replace(//g, ">"); - - const tag = (text, on = false) => - `${esc(text)}`; - - function items(list, activeKey) { - if (!list?.length) return `
  • nothing listed
    • `; - return list - .map((it) => { - const tags = [ - it.standalone ? tag("standalone") : "", - it.state ? tag(it.state) : "", - activeKey && it[activeKey] ? tag("active", true) : "", - ].join(""); - return `
  • ${esc(it.name ?? "?")} - ${esc(it.summary ?? "")}${tags}
    • `; - }) - .join(""); - } - - /* Cluster state, when there is any to show. - * - * Fetched separately and allowed to fail: the bundle listing is the point, and a - * rig with no cluster reachable is a normal state, not an error. Renders nothing - * at all when absent. - * - * When the payload says `mocked`, say so loudly. This exists to demo the UI on a - * machine where kind will not run — and a demo that looks live but is not is - * worse than one that admits it. */ - function clusterSection(c) { - if (!c) return ""; - const m = c.cluster ?? {}; - const banner = c.mocked - ? `

    mocked — no cluster was queried; these are canned values

    ` - : ""; - const meta = [m.context, m.k8s, m.profile ? `profile ${m.profile}` : "", - m.nodes ? `${m.nodes} node${m.nodes > 1 ? "s" : ""}` : ""] - .filter(Boolean).join(" · "); - - return ` -

    Cluster${c.mocked ? " (mocked)" : ""}

    - ${banner} - ${meta ? `

    ${esc(meta)}

    ` : ""} -
      ${items(c.workloads)}
    -

    Services (${c.services?.length ?? 0})

    -
      ${items(c.services)}
    `; - } - - function render(b, name, cluster) { - const meta = b.bundle ?? {}; - const next = (b.next ?? []).map((n) => `
  • ${esc(n)}
    • `).join(""); - return ` -

    IT WORKS — ${esc(name || meta.name || "rig")}

    -

    ${esc(meta.description ?? "")}

    - -

    Tools (${b.tools?.length ?? 0})

    -
      ${items(b.tools)}
    - -

    Rigs (${b.rigs?.length ?? 0})

    -
      ${items(b.rigs, "active")}
    - - ${clusterSection(cluster)} - - ${next ? `` : ""}`; - } - - const app = document.getElementById("app"); - - const json = (path, required) => - fetch(path).then((r) => { - if (r.ok) return r.json(); - if (required) throw new Error(`${path} -> HTTP ${r.status}`); - return null; // optional: absent is a normal state, not an error - }).catch((err) => { - if (required) throw err; - return null; - }); - - Promise.all([json("/bundle.json", true), json("/cluster.mock.json", false)]) - // RIG_NAME is injected by vite from the pod env, so two rigs sharing a - // cluster are distinguishable even if a copied bundle.json kept its old name. - .then(([b, cluster]) => { - app.innerHTML = render(b, import.meta.env.VITE_RIG_NAME, cluster); - }) - .catch((err) => { - app.innerHTML = `

    bundle unavailable

    -

    ${esc(err.message)}

    `; - }); - package.json: | - { - "name": "rig-ui", - "private": true, - "version": "0.1.0", - "type": "module", - "scripts": { - "dev": "vite --host 0.0.0.0 --port 5173", - "build": "vite build", - "preview": "vite preview --host 0.0.0.0 --port 5173" - }, - "devDependencies": { - "vite": "^6" - } - } - style.css: | - /* Minimal, self-contained. The real visual identity ships with the soleprint UI - package, which is a separate artifact — nothing here should grow into a theme. */ - - body { - margin: 0; - padding: 2.5rem 1.5rem; - background: #0d0d0f; - color: #e8e8f0; - font: 14px/1.6 ui-monospace, "JetBrains Mono", Menlo, monospace; - } - main { max-width: 52rem; margin: 0 auto; } - - h1 { margin: 0; font-size: 1.6rem; letter-spacing: 0.02em; } - h1 .ok { color: #3ecf8e; } - h1.err { color: #f06565; } - .sub { color: #8888a0; margin: 0.35rem 0 2.25rem; } - - h2 { - font-size: 0.75rem; - text-transform: uppercase; - letter-spacing: 0.12em; - color: #8888a0; - margin: 2rem 0 0.75rem; - font-weight: 600; - } - - ul { list-style: none; margin: 0; padding: 0; } - li { - display: flex; - gap: 0.75rem; - align-items: baseline; - padding: 0.5rem 0.75rem; - border: 1px solid #2e2e38; - border-radius: 6px; - margin-bottom: 0.4rem; - background: #16161a; - } - .name { font-weight: 600; min-width: 9rem; } - .summary { color: #8888a0; flex: 1; } - - .tag { - font-size: 0.7rem; - padding: 0.1rem 0.45rem; - border-radius: 3px; - background: #26262f; - color: #8888a0; - white-space: nowrap; - } - .tag.on { background: #3ecf8e; color: #0d0d0f; } - - .next { - color: #555568; - font-size: 0.8rem; - margin-top: 2.5rem; - border-top: 1px solid #2e2e38; - padding-top: 1rem; - } - .next li { - display: list-item; - border: 0; - background: none; - padding: 0.15rem 0; - margin: 0 0 0 1.1rem; - list-style: disc; - } - - /* Mocked-data banner. Deliberately loud: this only appears when the cluster - payload is canned, and a demo that looks live but is not is worse than one - that says so. */ - .mock { - margin: 0 0 0.75rem; - padding: 0.4rem 0.75rem; - border: 1px dashed #f5a623; - border-radius: 6px; - color: #f5a623; - font-size: 0.8rem; - } - vite.config.js: | - import { defineConfig } from "vite"; - - /* Serves on 0.0.0.0 so the pod is reachable through the Service, and allows any - * Host header because the address is assigned at runtime (MetalLB locally, a - * cloud load balancer on EKS) and is never known at build time. */ - export default defineConfig({ - server: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true }, - preview: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true }, - }); ---- -# How to plug the UI into whatever k8s you generated. THIS IS THE WHOLE THING: -# one Pod running the vite app, one Service to reach it. -# -# Optional by design. The UI complements a rig; it is not part of the end -# product, and a rig is complete and useful without it. Apply this only when you -# want the listing: -# -# kubectl apply -n -f rig-ui/k8s.yaml -# -# A bare Pod, not a Deployment — this is a dev-loop convenience, not a workload -# to keep alive. If it dies you re-apply it; nothing depends on it staying up. -# -# The app and bundle.json arrive as a ConfigMap named `rig-ui`, which -# ctrl/manifest.py generates from the folder. Nothing is baked into an image, so -# editing bundle.json and re-applying is the whole update cycle. -apiVersion: v1 -kind: Pod -metadata: - name: rig-ui - namespace: sample-rig - labels: - app: rig-ui -spec: - containers: - - name: vite - image: node:22-alpine - workingDir: /app - # npm install at start: no image to build and no registry to publish to, - # which is the point of a minimal plug-in. It needs egress to a registry — - # on a locked-down cluster point npm at the internal one, or bake an image - # instead. Nothing else here changes if you do. - command: ["sh", "-c"] - # A ConfigMap mounts flat (keys cannot contain '/'), so the files are - # placed into vite's expected layout here. bundle.json goes to public/ - # because that is what vite serves at /bundle.json, which is where the - # app fetches it. - args: - - | - mkdir -p /app/src /app/public && - cp /src/package.json /src/vite.config.js /src/index.html /app/ && - cp /src/main.js /src/style.css /app/src/ && - cp /src/bundle.json /app/public/ && - npm install --no-audit --no-fund && - npm run dev - env: - # Rendered in the heading so two rigs sharing a cluster stay - # distinguishable. Set from the namespace by ctrl/manifest.py. - - name: VITE_RIG_NAME - value: sample-rig - ports: - - name: http - containerPort: 5173 - volumeMounts: - # /src is read-only from the ConfigMap; the app is copied to a writable - # /app because npm install has to create node_modules. - - name: rig-ui - mountPath: /src - - name: app - mountPath: /app - readinessProbe: - httpGet: { path: /, port: 5173 } - # npm install decides how long this takes, and it is the slow part. - initialDelaySeconds: 15 - periodSeconds: 5 - failureThreshold: 30 - resources: - requests: { memory: 128Mi, cpu: 50m } - limits: { memory: 512Mi } - volumes: - - name: rig-ui - configMap: - name: rig-ui - - name: app - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: rig-ui - namespace: sample-rig - labels: - app: rig-ui - # No annotations, deliberately — see k8s/app.yaml. The target is EKS but this - # stays VPC-agnostic: no subnets, no security groups, no -scheme, no -type. - # A bare LoadBalancer is what lets one manifest work on kind and on EKS. -spec: - type: LoadBalancer - selector: - app: rig-ui - ports: - - name: http - port: 80 - targetPort: 5173 - protocol: TCP - # Pinned, because a LoadBalancer Service also allocates a NodePort and - # this is the only address that works everywhere. - # - # On WSL the MetalLB address is on a docker bridge INSIDE the Linux VM, - # and Windows has no route to it — the page looks broken while the - # cluster is perfectly healthy. 30080 is what rig's `hostport` ingress - # mode publishes to the host, so this is reachable at - # localhost:$HTTP_PORT from a Windows browser with nothing configured. - # - # Costs nothing elsewhere: MetalLB still assigns an external IP on Linux, - # and on EKS the load balancer targets this NodePort anyway. One Service, - # no per-environment branch. - # - # A pinned NodePort is cluster-unique, so two rigs must live in separate - # clusters — which is how they are run anyway. - nodePort: 30080 diff --git a/rig/sample-rig/rig-ui/.gitignore b/rig/sample-rig/rig-ui/.gitignore deleted file mode 100644 index bea2f92..0000000 --- a/rig/sample-rig/rig-ui/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -node_modules/ -public/ -dist/ diff --git a/rig/sample-rig/rig-ui/index.html b/rig/sample-rig/rig-ui/index.html deleted file mode 100644 index 3230fbf..0000000 --- a/rig/sample-rig/rig-ui/index.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - IT WORKS - - -
    - - - diff --git a/rig/sample-rig/rig-ui/k8s.yaml b/rig/sample-rig/rig-ui/k8s.yaml deleted file mode 100644 index 721765c..0000000 --- a/rig/sample-rig/rig-ui/k8s.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# How to plug the UI into whatever k8s you generated. THIS IS THE WHOLE THING: -# one Pod running the vite app, one Service to reach it. -# -# Optional by design. The UI complements a rig; it is not part of the end -# product, and a rig is complete and useful without it. Apply this only when you -# want the listing: -# -# kubectl apply -n -f rig-ui/k8s.yaml -# -# A bare Pod, not a Deployment — this is a dev-loop convenience, not a workload -# to keep alive. If it dies you re-apply it; nothing depends on it staying up. -# -# The app and bundle.json arrive as a ConfigMap named `rig-ui`, which -# ctrl/manifest.py generates from the folder. Nothing is baked into an image, so -# editing bundle.json and re-applying is the whole update cycle. -apiVersion: v1 -kind: Pod -metadata: - name: rig-ui - labels: - app: rig-ui -spec: - containers: - - name: vite - image: node:22-alpine - workingDir: /app - # npm install at start: no image to build and no registry to publish to, - # which is the point of a minimal plug-in. It needs egress to a registry — - # on a locked-down cluster point npm at the internal one, or bake an image - # instead. Nothing else here changes if you do. - command: ["sh", "-c"] - # A ConfigMap mounts flat (keys cannot contain '/'), so the files are - # placed into vite's expected layout here. bundle.json goes to public/ - # because that is what vite serves at /bundle.json, which is where the - # app fetches it. - args: - - | - mkdir -p /app/src /app/public && - cp /src/package.json /src/vite.config.js /src/index.html /app/ && - cp /src/main.js /src/style.css /app/src/ && - cp /src/bundle.json /app/public/ && - npm install --no-audit --no-fund && - npm run dev - env: - # Rendered in the heading so two rigs sharing a cluster stay - # distinguishable. Set from the namespace by ctrl/manifest.py. - - name: VITE_RIG_NAME - value: __RIG_NAME__ - ports: - - name: http - containerPort: 5173 - volumeMounts: - # /src is read-only from the ConfigMap; the app is copied to a writable - # /app because npm install has to create node_modules. - - name: rig-ui - mountPath: /src - - name: app - mountPath: /app - readinessProbe: - httpGet: { path: /, port: 5173 } - # npm install decides how long this takes, and it is the slow part. - initialDelaySeconds: 15 - periodSeconds: 5 - failureThreshold: 30 - resources: - requests: { memory: 128Mi, cpu: 50m } - limits: { memory: 512Mi } - volumes: - - name: rig-ui - configMap: - name: rig-ui - - name: app - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: rig-ui - labels: - app: rig-ui - # No annotations, deliberately — see k8s/app.yaml. The target is EKS but this - # stays VPC-agnostic: no subnets, no security groups, no -scheme, no -type. - # A bare LoadBalancer is what lets one manifest work on kind and on EKS. -spec: - type: LoadBalancer - selector: - app: rig-ui - ports: - - name: http - port: 80 - targetPort: 5173 - protocol: TCP - # Pinned, because a LoadBalancer Service also allocates a NodePort and - # this is the only address that works everywhere. - # - # On WSL the MetalLB address is on a docker bridge INSIDE the Linux VM, - # and Windows has no route to it — the page looks broken while the - # cluster is perfectly healthy. 30080 is what rig's `hostport` ingress - # mode publishes to the host, so this is reachable at - # localhost:$HTTP_PORT from a Windows browser with nothing configured. - # - # Costs nothing elsewhere: MetalLB still assigns an external IP on Linux, - # and on EKS the load balancer targets this NodePort anyway. One Service, - # no per-environment branch. - # - # A pinned NodePort is cluster-unique, so two rigs must live in separate - # clusters — which is how they are run anyway. - nodePort: 30080 diff --git a/rig/sample-rig/rig-ui/package-lock.json b/rig/sample-rig/rig-ui/package-lock.json deleted file mode 100644 index ae894f1..0000000 --- a/rig/sample-rig/rig-ui/package-lock.json +++ /dev/null @@ -1,1164 +0,0 @@ -{ - "name": "rig-ui", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "rig-ui", - "version": "0.1.0", - "devDependencies": { - "vite": "^6" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@napi-rs/lzma-linux-x64-gnu": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", - "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.20 || ^24.12 || >=25" - } - }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.4.tgz", - "integrity": "sha512-RrPokAb7dmbxFoeO3TloqHyOjgye8RkBhSqmp4aJMIex4c9r46ZstPnleDQOq1t46VOVjwIuwNogIqbodV1Vvg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.4.tgz", - "integrity": "sha512-JKuJc+pnpks2pjy7L/N3v/cAkZxYlnmuZoD840ldbMI5KDbC4iO9NKwPKYdjYFCMAIIlBzYSFHxIJVYzRo2/8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.4.tgz", - "integrity": "sha512-krw5uS2STmvJ02x0uTXHbqQNuz+9eZ1iw+qXk9dmW2gvV4jV7O2hEoOnuhFrpOPiel1mBFtqbxYZZtC46hXLOw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.4.tgz", - "integrity": "sha512-wsTxtgApb4PrOsNJIm0FZ1h3WvCC+k9uxLJ4ad75hgoS4NiRes2SoJFlDAyMwiUY8IssDqGcHbXuN0sx1tfF1A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.4.tgz", - "integrity": "sha512-GUOnQlyZe3yAXhWOtOMsn5Qkrv5E5mZXa0thbARWi5Ei2szlVXJFQhddZ4HbAzh8q92w5twp+CQvs/eFanz9YQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.4.tgz", - "integrity": "sha512-/Y7f3QuxjzPKsjA/rfEDa3+0vXqyjmJ50Ln8dPpCmWkKTrUoWHG1cWhTqaAMLob2m2nESWuC7yGrREz019Ztqg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.4.tgz", - "integrity": "sha512-81wiiX3v7aqy+T+bT61TJ78yJjRquqFFTTbAPt08imfQQzkPIW8t6aJbkTagtCCrXMNc9D66+geqlK7ydLPNqA==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.4.tgz", - "integrity": "sha512-9kmDIvNZqdoHOBZgNtpTBeLWYO/LVipM3H/j62P8848/l/VPEQL6N3uxU9pvP1oZAsXyC2MEnFP3ovRjo7WYNQ==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.4.tgz", - "integrity": "sha512-CcnXHWnXg69g+DX5VWL3FHts3qMRN2uVEHX+BZvGLdd07/gXkn3ePjYtO1LDJvxkGKVHMclKBRa1QUTH+6toYQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.4.tgz", - "integrity": "sha512-iFOibiHnTRuhrWLlRsOQFdZJJIa7S8OwkneJr4ocALP16u5yk6lWLINFwhHaEqBFMsKDUZofLkGos7+CPzGB3g==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.4.tgz", - "integrity": "sha512-XnWYMI7euHlb5a871xPja+Gm7DRCFU+FGRrtS2sMq9N8FvqtpagUy6gD4YOemC5MRk9xbh8+jYMEJbigFQwsgA==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.4.tgz", - "integrity": "sha512-qGDAlO0U8xedCcsdRm9oaoQY8DAx/QT7uIxJWhCdx0ceIWX783UC9QSYkdpzAe29wNiVfp24+bZdQmn49o45SQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.4.tgz", - "integrity": "sha512-ru4H6ezD7ysA5EiEK6qkkaEb4modH8CTej6kUy/gQi20u3kB3G7Zn8snXXkeJSCOFKG/rbPPtM/+9Wgas1961w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.4.tgz", - "integrity": "sha512-2W4MO5WQVJnbJaZdvDb9rhBDuFU1nKIepPFpJUBsTh2k1YY2g+ODViaWuyOAjQ5cOP7NvrvLzt3wvHOoiAvc7w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.4.tgz", - "integrity": "sha512-+fxjfuoAmVMCYV5QyjoIpu0cp5DOiOTeqYFk1AVaxGr+/ravWLX89XfQmptsoWcaVy/TGf2hexzbUOrCQIL1CQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.4.tgz", - "integrity": "sha512-jTn8JfHGL4djjFxPuM06LmNUJDsst2jeVlsd9OmIH6zc5sC9K6rIuO4YajXatLUpBmBKl6b35ro1QZocLi+tcA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.4.tgz", - "integrity": "sha512-oCJCJL4pXsoDcP2QZ+JVlPTIRc6266zsIaeJJsWImmF7HO0W8nb6HuSgZlMWxJwaPf8ehbSw8yo0EUw925hKsA==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.4.tgz", - "integrity": "sha512-W69hukhZ3KKNRCaMIEzKvcFye42hh0FE1+YoYaf5+Ikacuftoco6yO/xouz0hc5d5W/s3yBro5jRiuEE/Q5vUw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.4.tgz", - "integrity": "sha512-qiXbGG2jkjXhzXpsFZSR2Xpb8DN/UaxYsbb/STbuR/6fpaDgRmmaq1B/LmtF2wQFOFOSsK2jdE0RZ3a0zHn4QA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.4.tgz", - "integrity": "sha512-nWeM//hxv8mIo6jD7Hu4o48DVmV9pbV6gsKaWU+4NFyqHoPKwrkRiZGLKUhOBk8qNmDmpwFtPKg80Bo/Tn4xiQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] - }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.4.tgz", - "integrity": "sha512-s62SQ/vgsRSvMwDkOEfTqfgASF0f26ZNaQuTA6Aok5lrikf89yI2W0gFHvZb2Jpgc6N8JnOKZgCK2iciO3CsxQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ] - }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.4.tgz", - "integrity": "sha512-J6wGf8TVGbXJq+HH+ttTvrcfNKPbuZecV6KT1B8I18BC5IURUh5kl4Yl5OEP5eFIUoI5BWxCsyYMhFsDx8kekw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.4.tgz", - "integrity": "sha512-zmfrQd/0wu6oJs8Vq8KwY/YtsKSsLtKe/HwAP4Wqy8LhWjeT55fHRAkOhYQ12wI3ayS4Tt12d5CDRD7N96SAYQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.4.tgz", - "integrity": "sha512-qPzHqdj9rfUD+w79dtE07zi/kFwKyCJqplp5K5ygeLTp7jLpAoc16OAH39HSmRC9UpozaecsleI8uAdEj6v2yw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.4.tgz", - "integrity": "sha512-zD6NdeWEByGE9QF9vCrlJ5YQB4oq9q91kPZS37Jwj5hOkvR1lTBSpsKhKDw4IJtbQ35LsTS1HD9DZYGKIshU1Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@types/estree": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", - "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/postcss": { - "version": "8.5.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", - "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.17", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/rollup": { - "version": "4.62.4", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.4.tgz", - "integrity": "sha512-RXOqwaPsBGjMNMa4sQjDjHieHEZDFoj/Rdr46l2MU5DfEs16wHJPC2RPTPHWhNl+M3aI472LLqFkFKut4SblOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "1.0.9" - }, - "bin": { - "rollup": "dist/bin/rollup" - }, - "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" - }, - "optionalDependencies": { - "@napi-rs/lzma-linux-x64-gnu": "1.5.1", - "@rollup/rollup-android-arm-eabi": "4.62.4", - "@rollup/rollup-android-arm64": "4.62.4", - "@rollup/rollup-darwin-arm64": "4.62.4", - "@rollup/rollup-darwin-x64": "4.62.4", - "@rollup/rollup-freebsd-arm64": "4.62.4", - "@rollup/rollup-freebsd-x64": "4.62.4", - "@rollup/rollup-linux-arm-gnueabihf": "4.62.4", - "@rollup/rollup-linux-arm-musleabihf": "4.62.4", - "@rollup/rollup-linux-arm64-gnu": "4.62.4", - "@rollup/rollup-linux-arm64-musl": "4.62.4", - "@rollup/rollup-linux-loong64-gnu": "4.62.4", - "@rollup/rollup-linux-loong64-musl": "4.62.4", - "@rollup/rollup-linux-ppc64-gnu": "4.62.4", - "@rollup/rollup-linux-ppc64-musl": "4.62.4", - "@rollup/rollup-linux-riscv64-gnu": "4.62.4", - "@rollup/rollup-linux-riscv64-musl": "4.62.4", - "@rollup/rollup-linux-s390x-gnu": "4.62.4", - "@rollup/rollup-linux-x64-gnu": "4.62.4", - "@rollup/rollup-linux-x64-musl": "4.62.4", - "@rollup/rollup-openbsd-x64": "4.62.4", - "@rollup/rollup-openharmony-arm64": "4.62.4", - "@rollup/rollup-win32-arm64-msvc": "4.62.4", - "@rollup/rollup-win32-ia32-msvc": "4.62.4", - "@rollup/rollup-win32-x64-gnu": "4.62.4", - "@rollup/rollup-win32-x64-msvc": "4.62.4", - "fsevents": "~2.3.2" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", - "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.4" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/vite": { - "version": "6.4.3", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", - "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "^0.25.0", - "fdir": "^6.4.4", - "picomatch": "^4.0.2", - "postcss": "^8.5.3", - "rollup": "^4.34.9", - "tinyglobby": "^0.2.13" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", - "jiti": ">=1.21.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - } - } -} diff --git a/rig/sample-rig/rig-ui/package.json b/rig/sample-rig/rig-ui/package.json deleted file mode 100644 index 1c54fe6..0000000 --- a/rig/sample-rig/rig-ui/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "rig-ui", - "private": true, - "version": "0.1.0", - "type": "module", - "scripts": { - "dev": "vite --host 0.0.0.0 --port 5173", - "build": "vite build", - "preview": "vite preview --host 0.0.0.0 --port 5173" - }, - "devDependencies": { - "vite": "^6" - } -} diff --git a/rig/sample-rig/rig-ui/src/main.js b/rig/sample-rig/rig-ui/src/main.js deleted file mode 100644 index bf23b29..0000000 --- a/rig/sample-rig/rig-ui/src/main.js +++ /dev/null @@ -1,136 +0,0 @@ -import "./style.css"; - -/* The IT WORKS page: renders bundle.json as the list of what shipped. - * - * Plain vite, no framework — this is a complement to the rig, not part of it, - * and it should stay small enough that nobody has to adopt a stack to read it. - * - * Laid out like soleprint's templated vein pages, because it does the same job: - * name each component, list what it exposes, show what comes back. Tool chrome - * and output are styled apart on purpose (see style.css) — that separation is - * what tells you whether you are reading the tool or its result. - * - * bundle.json is fetched at runtime rather than imported, so the same built app - * serves whatever rig it was copied into. Editing the ConfigMap changes the page - * without rebuilding. - */ - -const esc = (s) => - String(s).replace(/&/g, "&").replace(//g, ">"); - -const tag = (text, on = false) => - `${esc(text)}`; - -/* Tool chrome: one bordered card per component. */ -function components(list, activeKey) { - if (!list?.length) - return `

    nothing listed

    `; - return list - .map((it) => { - const tags = [ - it.standalone ? tag("standalone") : "", - it.state ? tag(it.state) : "", - activeKey && it[activeKey] ? tag("active", true) : "", - ].join(""); - return `
    -

    ${esc(it.name ?? "?")} ${tags}

    -

    ${esc(it.summary ?? "")}

    -
    `; - }) - .join(""); -} - -/* Endpoint rows: path on the left, what it returns on the right. */ -function endpoints(list) { - return list - .map( - (e) => `
  • ${esc(e.path)} - ${esc(e.desc)}
    • ` - ) - .join(""); -} - -/* Output: what the endpoint above actually returns, so the page demonstrates - itself rather than describing what a demonstration would look like. */ -function example(bundle) { - const sample = { - bundle: bundle.bundle?.name, - tools: (bundle.tools ?? []).map((t) => t.name), - rigs: (bundle.rigs ?? []).map((r) => r.name), - }; - return `
    ${esc(JSON.stringify(sample, null, 2))}
    `; -} - -/* Cluster state, when there is any to show. - * - * Fetched separately and allowed to fail: the bundle listing is the point, and a - * rig with no cluster reachable is a normal state, not an error. Renders nothing - * at all when absent. When the payload says `mocked`, say so loudly. */ -function clusterSection(c) { - if (!c) return ""; - const m = c.cluster ?? {}; - const meta = [m.context, m.k8s, m.profile && `profile ${m.profile}`, - m.nodes && `${m.nodes} node${m.nodes > 1 ? "s" : ""}`] - .filter(Boolean).join(" · "); - - return ` -

    Cluster${c.mocked ? " (mocked)" : ""}

    - ${c.mocked ? `

    mocked — no cluster was queried; these are canned values

    ` : ""} - ${meta ? `

    ${esc(meta)}

    ` : ""} -
    ${components(c.workloads)}
    - -

    Services

    -
    ${components(c.services)}
    `; -} - -function render(b, name, cluster) { - const meta = b.bundle ?? {}; - const next = (b.next ?? []).map((n) => `
  • ${esc(n)}
    • `).join(""); - return ` -

    IT WORKS — ${esc(name || meta.name || "rig")}

    -

    ${esc(meta.description ?? "")}

    - -

    Tools (${b.tools?.length ?? 0})

    -
    ${components(b.tools)}
    - -

    Rigs (${b.rigs?.length ?? 0})

    -
    ${components(b.rigs, "active")}
    - -

    Endpoints

    -
      ${endpoints([ - { path: "/", desc: "this page" }, - { path: "/bundle.json", desc: "the manifest it renders" }, - ])}
    - -

    Example — GET /bundle.json

    - ${example(b)} - - ${clusterSection(cluster)} - - ${next ? `` : ""}`; -} - -const app = document.getElementById("app"); - -const json = (path, required) => - fetch(path) - .then((r) => { - if (r.ok) return r.json(); - if (required) throw new Error(`${path} -> HTTP ${r.status}`); - return null; // optional: absent is a normal state, not an error - }) - .catch((err) => { - if (required) throw err; - return null; - }); - -Promise.all([json("/bundle.json", true), json("/cluster.mock.json", false)]) - // RIG_NAME is injected by vite from the pod env, so two rigs sharing a - // cluster are distinguishable even if a copied bundle.json kept its old name. - .then(([b, cluster]) => { - app.innerHTML = render(b, import.meta.env.VITE_RIG_NAME, cluster); - }) - .catch((err) => { - app.innerHTML = `

    bundle unavailable

    -

    ${esc(err.message)}

    `; - }); diff --git a/rig/sample-rig/rig-ui/src/style.css b/rig/sample-rig/rig-ui/src/style.css deleted file mode 100644 index 5591a90..0000000 --- a/rig/sample-rig/rig-ui/src/style.css +++ /dev/null @@ -1,139 +0,0 @@ -/* Minimal and self-contained — no framework dependency. - * - * The visual language follows soleprint's templated vein pages, because this - * page does the same job: say what a component is, list what it exposes, and - * show what comes back. Two treatments, deliberately distinct: - * - * TOOL CHROME bordered cards on the darker background, accent-coloured - * titles, endpoint rows separated by rules. - * OUTPUT a lighter raised block, monospace, pre-wrap and selectable — - * it is data, not furniture, and should read as a payload. - * - * Keeping them apart matters more than either looks: it is what tells you at a - * glance whether you are reading the tool or the thing it produced. */ - -:root { - --bg: #0d0d0f; - --surface: #16161a; - --surface-raised: #1e1e24; - --border: #2e2e38; - --border-strong: #3d3d4a; - --text: #e8e8f0; - --muted: #8888a0; - --accent: #3ecf8e; - --accent-dim: #f5a623; - --mono: "JetBrains Mono", "Cascadia Mono", Consolas, ui-monospace, monospace; -} - -body { - margin: 0; - padding: 2.5rem 1.5rem; - background: var(--bg); - color: var(--text); - font: 14px/1.6 var(--mono); -} -main { max-width: 56rem; margin: 0 auto; } - -h1 { margin: 0; font-size: 1.6rem; letter-spacing: 0.02em; } -h1 .ok { color: var(--accent); } -h1.err { color: #f06565; } -.tagline { color: var(--muted); margin: 0.35rem 0 2.25rem; } - -h2 { - font-size: 0.75rem; - text-transform: uppercase; - letter-spacing: 0.12em; - color: var(--muted); - margin: 2.25rem 0 0.75rem; - font-weight: 600; -} - -/* ── tool chrome ────────────────────────────────────────────────────────── */ - -.components { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); - gap: 0.75rem; -} - -.component { - background: var(--bg); - border: 1px solid var(--border-strong); - border-radius: 8px; - padding: 0.75rem; -} -.component h4 { - margin: 0 0 0.25rem; - font-size: 0.95rem; - color: var(--accent); -} -.component p { - margin: 0; - font-size: 0.85rem; - color: var(--muted); -} - -.endpoints { list-style: none; margin: 0; padding: 0; } -.endpoints li { - display: flex; - justify-content: space-between; - flex-wrap: wrap; - gap: 0.5rem; - padding: 0.6rem 0; - border-bottom: 1px solid var(--border-strong); -} -.endpoints li:last-child { border-bottom: none; } -.endpoints code { - background: var(--surface); - color: var(--accent); - padding: 0.25rem 0.5rem; - border-radius: 4px; -} -.endpoints .desc { color: var(--muted); font-size: 0.9rem; } - -.tag { - font-size: 0.7rem; - padding: 0.1rem 0.45rem; - border-radius: 3px; - background: var(--border); - color: var(--muted); - white-space: nowrap; -} -.tag.on { background: var(--accent); color: var(--bg); } - -/* ── output ─────────────────────────────────────────────────────────────── */ - -.output { - background: var(--surface); - border: 1px solid var(--border); - border-radius: 8px; - padding: 1rem; - font-size: 0.85rem; - white-space: pre-wrap; - word-break: break-word; - user-select: text; - color: var(--text); - margin: 0; -} -.output .k { color: var(--accent); } - -/* Mocked-data banner. Loud on purpose: it only appears when the payload is - canned, and a demo that looks live but is not is worse than one that says so. */ -.mock { - margin: 0 0 0.75rem; - padding: 0.4rem 0.75rem; - border: 1px dashed var(--accent-dim); - border-radius: 6px; - color: var(--accent-dim); - font-size: 0.8rem; -} - -.next { - color: #555568; - font-size: 0.8rem; - margin-top: 2.5rem; - border-top: 1px solid var(--border); - padding-top: 1rem; -} -.next ul { margin: 0; padding: 0 0 0 1.1rem; } -.next li { list-style: disc; padding: 0.15rem 0; } diff --git a/rig/sample-rig/rig-ui/vite.config.js b/rig/sample-rig/rig-ui/vite.config.js deleted file mode 100644 index 1b1c6e4..0000000 --- a/rig/sample-rig/rig-ui/vite.config.js +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from "vite"; - -/* Serves on 0.0.0.0 so the pod is reachable through the Service, and allows any - * Host header because the address is assigned at runtime (MetalLB locally, a - * cloud load balancer on EKS) and is never known at build time. */ -export default defineConfig({ - server: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true }, - preview: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true }, -}); diff --git a/soleprint/run.py b/soleprint/run.py index 2b309ee..d4d3e3a 100644 --- a/soleprint/run.py +++ b/soleprint/run.py @@ -577,7 +577,7 @@ def station_index(request: Request): # not the server. Route registration raises more than ImportError — FastAPI # turns a missing optional dependency into a RuntimeError at decoration time — # and catching only ImportError meant one such tool took the whole app down. -for _tool in ("tester", "graphgen", "datagen", "shuntgen"): +for _tool in ("tester", "graphgen", "datagen", "shuntgen", "histgen"): try: _module = importlib.import_module(f"station.tools.{_tool}.api") app.include_router(_module.router, prefix="/station") diff --git a/soleprint/station/tools/distill/distill.sh b/soleprint/station/tools/distill/distill.sh new file mode 100755 index 0000000..32817f4 --- /dev/null +++ b/soleprint/station/tools/distill/distill.sh @@ -0,0 +1,1028 @@ +#!/usr/bin/env bash +# Distil repos — or branches of repos — into a local directory. +# +# Takes a list of repos and a destination, and keeps only what the project +# actually is. Everything stays on this machine: no remote, no upload, nothing +# to restart. +# +# What gets thrown away is most of the volume. Asking git what it tracks, rather +# than copying the checkout, takes mpr from 17G to 1.1M; repos that are not git +# fall back to rsync's per-directory .gitignore filter, which is what keeps +# mts's 8.6G of sample and output data out. On top of that, lockfiles, images +# and minified output go, since they are bulk that says nothing about the code. +# +# Usage: +# distill.sh tree [opts] -o DEST REPO... # a directory per repo +# distill.sh digest [opts] -o DEST REPO... # one concatenated .md per repo +# distill.sh both [opts] -o DEST REPO... # both, from a single pass +# distill.sh list [opts] REPO... # what would be kept, + sizes +# distill.sh [tree|digest|both|list] -c FILE # read the whole job from JSON +# +# tree and digest answer different questions. tree gives you files — open them, +# grep them, build them. digest gives you one document to read or hand over: +# a header, a file tree, then every file under a '## path' heading. Neither +# replaces the other, so 'both' produces the pair; the expensive part is the +# selection, which is shared, so the second one costs a copy. +# +# Naming a dozen repos on the command line gets unwieldy fast, and the same set +# tends to be distilled again and again — so the list and the settings can live +# in a JSON file instead. With no REPO and no -c, the distill.json beside this +# script is used if it exists. Everything that shapes the run sits at the top of +# it; 'repos' is just a list of places: +# +# { +# "command": "both", +# "out": "~/distilled", +# "branch_mode": "full", // or "diff", against diff_base +# "diff_base": "main", +# "exclude": [], "include": [], "all": false, "max_bytes": null, +# "skip_unchanged": false, "prune": false, +# "repos": [ +# { "path": "/abs/path/to/repo" }, +# { "path": "/abs/path/to/repo", "branches": ["featA", "featB"] }, +# { "path": "other", "enabled": false } +# ] +# } +# +# A path may appear as many times as you like — that is the point of a list +# rather than a keyed object; one entry per repo could not hold two branches of +# the same repo. Per-entry keys: path, branches, subpath, name, enabled. +# A command-line option overrides the file. +# +# REPO [@[,...]][:] +# +# foo the working tree, as it is now (uncommitted included) +# foo@main one ref, read straight out of the object store +# foo@main,topic several refs, each distilled separately +# foo@all every local branch +# foo:src/api only that subtree +# foo@topic:src/api both +# ../elsewhere@v1.2 a path instead of a slug; any ref git resolves +# +# Refs are read with ls-tree/archive, so nothing is checked out and a +# dirty working tree is never touched. +# +# Options: +# -c FILE read the repo list and settings from JSON (needs jq) +# -o DEST output directory (tree, digest, both) +# --root DIR where bare slugs resolve (default: the dir holding this project) +# --base REF delta mode: distill REF whole, and every other ref as only +# the files that differ from it +# --include GLOB keep only matching paths (repeatable) +# --exclude GLOB drop matching paths (repeatable) +# --all keep the derived output too (lockfiles, minified, caches) +# --max-bytes N skip files larger than N, and say so in the manifest +# --strict refuse a dirty working tree (only affects worktree copies) +# --skip-unchanged leave alone anything whose source and settings have not +# moved since the last run into this destination +# --prune delete anything in DEST this run did not produce, so +# dropping a repo from the list drops its output too +# -n dry run — say what would happen, write nothing +# -d mirror mode — delete extraneous files in DEST (tree, both) +# +# Examples: +# distill.sh list /path/to/repo +# distill.sh both -o ~/out /path/to/repo /path/to/other +# distill.sh digest -o ~/out --base main /path/to/repo@all +# distill.sh tree -o /mnt/stick /path/to/repo@featA,featB +# distill.sh -c distill.json # command and destination from the file +# distill.sh list -c distill.json # preview that same set without writing +# +# Where the copy goes afterwards — a stick, a share, an upload — is not this +# script's business. It writes a local directory and stops. +set -euo pipefail + +SELF="$(basename "$0")" + +usage() { + # The header comment above IS the usage; keeping one copy means they cannot + # drift apart. + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0" +} + +die() { echo "$SELF: $*" >&2; exit 1; } + +# ── what counts as noise ─────────────────────────────────────────────────── +# Tracked, legitimately, and still worth nothing in a copy: things a build +# regenerates. Lockfiles dominate — uv.lock alone is 568K in mpr, and ppl drops +# 1.9M to 384K once its theme lockfile and source maps are gone. +# +# The line here is derived-vs-content, NOT text-vs-binary. That distinction was +# wrong before and cost real files: images, fonts, spreadsheets and PDFs were +# listed here and deleted, so a logo, a font the site loads, or a downloadable +# kit simply vanished from the copy. None of those can be regenerated from what +# is left, which is the only thing that makes a file safe to drop. +# +# So binaries are no longer an extension question at all. Whatever survives this +# list gets copied; the digest, being text, lists the binary ones instead of +# inlining them. If size rather than kind is the worry, --max-bytes is the knob, +# because size is the thing actually being worried about. +# +# One list, one place to edit. --all turns it off wholesale. +NOISE_RE='(^|/)(package-lock\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|yarn\.lock|bun\.lock|bun\.lockb|uv\.lock|poetry\.lock|Pipfile\.lock|Cargo\.lock|composer\.lock|Gemfile\.lock|go\.sum|\.DS_Store|Thumbs\.db)$' +NOISE_RE="$NOISE_RE"'|\.(map|min\.js|min\.css)$' +# Compiled and cached build output — regenerable by definition. +NOISE_RE="$NOISE_RE"'|\.(pyc|pyo|pyd|class|o|obj|a|lib|so|dylib|dll|wasm|pack|idx)$' +NOISE_RE="$NOISE_RE"'|(^|/)(__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|node_modules)/' + +lang_for() { + case "$1" in + *.sh|*.bash|*.zsh) echo bash ;; + *.py) echo python ;; + *.js|*.mjs|*.cjs) echo javascript ;; + *.ts) echo typescript ;; + *.tsx) echo tsx ;; + *.jsx) echo jsx ;; + *.vue) echo vue ;; + *.go) echo go ;; + *.rs) echo rust ;; + *.rb) echo ruby ;; + *.php) echo php ;; + *.java) echo java ;; + *.c|*.h) echo c ;; + *.cc|*.cpp|*.hpp|*.cxx) echo cpp ;; + *.cs) echo csharp ;; + *.swift) echo swift ;; + *.kt|*.kts) echo kotlin ;; + *.sql) echo sql ;; + *.html|*.htm) echo html ;; + *.css) echo css ;; + *.scss|*.sass) echo scss ;; + *.json) echo json ;; + *.yml|*.yaml) echo yaml ;; + *.toml) echo toml ;; + *.ini|*.cfg|*.conf) echo ini ;; + *.xml|*.svg) echo xml ;; + *.md|*.markdown) echo markdown ;; + *.tf|*.tfvars) echo terraform ;; + *.hcl) echo hcl ;; + *.lua) echo lua ;; + *.pl|*.pm) echo perl ;; + *.r|*.R) echo r ;; + *.tex) echo latex ;; + *.env|.env.*) echo dotenv ;; + *Dockerfile*|*.dockerfile) echo dockerfile ;; + *Makefile*|*.mk) echo makefile ;; + *Tiltfile*|*.star|*.bzl) echo python ;; + *.gitignore|*.gitattributes) echo gitignore ;; + *) echo "" ;; + esac +} + +# ── argument parsing ─────────────────────────────────────────────────────── + +# The command is optional: with -c it can come out of the file instead, so a +# leading option is not an error here. +CMD="" +case "${1:-}" in + tree|digest|both|list) CMD="$1"; shift ;; + -h|--help|help) usage; exit 0 ;; + "") usage >&2; exit 1 ;; + -*) ;; + *) die "unknown command: $1 (expected tree, digest, both or list)" ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG="" +DEST="" +DEST_SET="" +ROOT_SET="" +ROOT="${DISTILL_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}" +BASE_REF="" +KEEP_NOISE="" +MAX_BYTES="" +STRICT="" +DRY="" +MIRROR="" +PRUNE="" +SKIP_UNCHANGED="" +INCLUDES=() +EXCLUDES=() +SPECS=() + +while [ $# -gt 0 ]; do + case "$1" in + -c|--config) shift; CONFIG="${1:-}" ;; + -o) shift; DEST="${1:-}"; DEST_SET=1 ;; + --root) shift; ROOT="${1:-}"; ROOT_SET=1 ;; + --base) shift; BASE_REF="${1:-}" ;; + --include) shift; INCLUDES+=("${1:-}") ;; + --exclude) shift; EXCLUDES+=("${1:-}") ;; + --max-bytes) shift; MAX_BYTES="${1:-}" ;; + --all) KEEP_NOISE=1 ;; + --strict) STRICT=1 ;; + --prune) PRUNE=1 ;; + --skip-unchanged) SKIP_UNCHANGED=1 ;; + -n) DRY=1 ;; + -d) MIRROR=1 ;; + -h|--help) usage; exit 0 ;; + --) shift; while [ $# -gt 0 ]; do SPECS+=("$1"); shift; done; break ;; + -*) die "unknown option: $1" ;; + *) SPECS+=("$1") ;; + esac + shift +done + +# ── the config file ──────────────────────────────────────────────────────── +# Bundling the same dozen repos is the normal case, so the default is to read +# the distill.json sitting beside this script when nothing is named on the +# command line. Naming repos still works and still wins — the file is a saved +# default, not a mode. +DEFAULT_CONFIG="$SCRIPT_DIR/distill.json" +if [ -z "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ] && [ -f "$DEFAULT_CONFIG" ]; then + CONFIG="$DEFAULT_CONFIG" + echo "using $CONFIG" +fi + +expand_tilde() { + case "$1" in + "~") printf '%s' "$HOME" ;; + "~/"*) printf '%s/%s' "$HOME" "${1#\~/}" ;; + *) printf '%s' "$1" ;; + esac +} + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +JOBS="$TMP/jobs.ndjson" +: > "$JOBS" + +if [ -n "$CONFIG" ]; then + [ -f "$CONFIG" ] || die "no such config file: $CONFIG" + command -v jq >/dev/null || die "reading $CONFIG needs jq" + jq -e . "$CONFIG" >/dev/null 2>&1 || die "$CONFIG is not valid JSON" + + [ -n "$CMD" ] || CMD="$(jq -r '.command // ""' "$CONFIG")" + [ -n "$DEST_SET" ] || DEST="$(expand_tilde "$(jq -r '.out // ""' "$CONFIG")")" + if [ -z "$ROOT_SET" ]; then + cfg_root="$(jq -r '.root // ""' "$CONFIG")" + [ -n "$cfg_root" ] && ROOT="$(expand_tilde "$cfg_root")" + fi + + cfg_mode="$(jq -r '.branch_mode // "full"' "$CONFIG")" + case "$cfg_mode" in + full|diff) ;; + *) die "branch_mode in $CONFIG must be \"full\" or \"diff\", got: $cfg_mode" ;; + esac + + [ "$(jq -r 'if has("prune") then .prune else false end' "$CONFIG")" = true ] && PRUNE=1 + [ "$(jq -r 'if has("skip_unchanged") then .skip_unchanged else false end' "$CONFIG")" = true ] \ + && SKIP_UNCHANGED=1 + + # Every setting that shapes the run lives at the top of the file; an entry + # is just a repo, and optionally which of its branches. Each becomes the + # same spec string the command line would have used, so there is one parser + # for both routes rather than two that drift. + jq -c ' + def arr($v): if $v == null then [] elif ($v|type) == "array" then $v else [$v] end; + . as $cfg + | (if ($cfg.branch_mode // "full") == "diff" + then ($cfg.diff_base // "main") else "" end) as $base + | (.repos // [])[] + | . as $e + | select(if ($e|has("enabled")) then $e.enabled else true end) + | (arr($e.branches // $e.refs // $e.ref)) as $refs + | ($e.subpath // "") as $sub + | (($e.path // $e.repo) | tostring) as $where + | { + spec: ( $where + + (if ($refs|length) > 0 then "@" + ($refs|join(",")) else "" end) + + (if $sub != "" then ":" + $sub else "" end) ), + name: ($e.name // ""), + base: $base, + include: (arr($cfg.include)), + exclude: (arr($cfg.exclude)), + all: (if ($cfg|has("all")) then $cfg.all else false end), + max_bytes: (($cfg.max_bytes // "") | tostring) + } + ' "$CONFIG" >> "$JOBS" || die "could not read the repo list from $CONFIG" + + [ -s "$JOBS" ] || die "$CONFIG selected no repos (is every entry \"enabled\": false?)" + if grep -q '"spec":"null' "$JOBS"; then + die "an entry in $CONFIG has no \"path\"" + fi +fi + +if [ -z "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then + die "no repos given, and no $DEFAULT_CONFIG to fall back on. try: $SELF --help" +fi +[ -n "$CMD" ] || die "no command given, and none in the config. expected tree, digest, both or list" +[ -d "$ROOT" ] || die "root is not a directory: $ROOT" + +case "$CMD" in + tree|digest|both|list) ;; + *) die "unknown command: $CMD (expected tree, digest, both or list)" ;; +esac + +if [ "$CMD" != list ]; then + [ -n "$DEST" ] || die "an output directory is required for $CMD (-o DEST, or \"out\" in the config)" +fi +case "$CMD" in tree|both) ;; *) [ -z "$MIRROR" ] || die "-d only applies to 'tree' or 'both'" ;; esac + +# ── spec parsing ─────────────────────────────────────────────────────────── +# [@[,...]][:] +# +# Order matters here: split the subpath off first. Refs can contain neither ':' +# nor '@' in the shapes we accept, but a subpath certainly can contain neither +# '@' (rare but legal in filenames) nor anything else we would mistake for one. +# Splitting ':' first and '@' second keeps `foo@topic:src/api` unambiguous. +SPEC_DIR=""; SPEC_NAME=""; SPEC_SUB=""; SPEC_REFS=() +parse_spec() { + local spec="$1" repo="" refs="" + SPEC_SUB=""; SPEC_REFS=() + + case "$spec" in + *:*) SPEC_SUB="${spec#*:}"; spec="${spec%%:*}" ;; + esac + case "$spec" in + *@*) refs="${spec#*@}"; repo="${spec%%@*}" ;; + *) repo="$spec" ;; + esac + + case "$repo" in + /*|./*|../*) SPEC_DIR="$repo" ;; + *) SPEC_DIR="$ROOT/$repo" ;; + esac + [ -d "$SPEC_DIR" ] || die "no such repo: $SPEC_DIR (from '$1')" + SPEC_DIR="$(cd "$SPEC_DIR" && pwd)" + SPEC_NAME="$(basename "$SPEC_DIR")" + + if [ -n "$refs" ]; then + is_git "$SPEC_DIR" || die "$SPEC_NAME is not a git repo, so '@$refs' means nothing" + if [ "$refs" = all ]; then + local b + while IFS= read -r b; do SPEC_REFS+=("$b"); done \ + < <(git -C "$SPEC_DIR" for-each-ref --format='%(refname:short)' refs/heads) + [ ${#SPEC_REFS[@]} -gt 0 ] || die "$SPEC_NAME has no local branches" + else + local IFS=, + read -ra SPEC_REFS <<< "$refs" + fi + local r + for r in "${SPEC_REFS[@]}"; do + git -C "$SPEC_DIR" rev-parse --verify --quiet "$r^{commit}" >/dev/null \ + || die "$SPEC_NAME has no ref '$r'" + done + fi +} + +is_git() { git -C "$1" rev-parse --git-dir >/dev/null 2>&1; } + +# ── listing: source -> NUL-separated relative paths ──────────────────────── + +# What git tracks, and nothing else. This is the whole reason a 17G checkout +# distills to 2.8M: it follows every nested .gitignore and cannot drift the way +# a hand-written exclude list does. +list_worktree_git() { + local dir="$1" sub="$2" + if [ -n "$sub" ]; then git -C "$dir" ls-files -z -- "$sub" + else git -C "$dir" ls-files -z + fi +} + +# A ref, straight out of the object store — no checkout, no worktree, and the +# dirty tree in front of us stays untouched. +list_ref() { + local dir="$1" ref="$2" sub="$3" + if [ -n "$sub" ]; then git -C "$dir" ls-tree -r -z --name-only "$ref" -- "$sub" + else git -C "$dir" ls-tree -r -z --name-only "$ref" + fi +} + +# Not a git repo (lng, meetus, mts here). rsync's per-directory merge filter +# reads a .gitignore in every directory it visits, which is what keeps mts's +# 4.4G samples/ and 4G of *-data/ out. Enumerated with a dry run first so the +# filtering below happens before anything is copied, not after. +list_worktree_plain() { + local dir="$1" sub="$2" src="$dir" + [ -n "$sub" ] && src="$dir/$sub" + rsync -a -n --out-format='%n' \ + --exclude='.git/' --exclude='.DS_Store' \ + --filter=':- .gitignore' \ + "$src/" "$TMP/.rsync-probe/" 2>/dev/null \ + | sed '/\/$/d; /^\.$/d' \ + | { [ -n "$sub" ] && sed "s|^|$sub/|" || cat; } \ + | tr '\n' '\0' +} + +# ── filtering ────────────────────────────────────────────────────────────── +# Path-level only. Whether a file is binary, or too big, is decided later +# against the staged copy, where the bytes actually exist and one code path +# serves both worktrees and refs. +# +# Filtering happens through a temp file rather than a variable: bash strips NUL +# bytes out of a command substitution, silently, so `out="$(cat)"` collapses the +# whole NUL-separated list into one run-on path. (A path containing a literal +# newline would defeat this, but git refuses to produce one without being asked +# very rudely.) +filter_paths() { + local work="$TMP/filter" g re + tr '\0' '\n' > "$work" + + if [ -z "$KEEP_NOISE" ]; then + grep -vE "$NOISE_RE" "$work" > "$work.next" || true + mv "$work.next" "$work" + fi + for g in ${INCLUDES[@]+"${INCLUDES[@]}"}; do + re="$(glob_to_re "$g")" + grep -E "$re" "$work" > "$work.next" || true + mv "$work.next" "$work" + done + for g in ${EXCLUDES[@]+"${EXCLUDES[@]}"}; do + re="$(glob_to_re "$g")" + grep -vE "$re" "$work" > "$work.next" || true + mv "$work.next" "$work" + done + + tr '\n' '\0' < "$work" +} + +# A glob the way a person means it on a path: a pattern with no '/' in it also +# matches basenames at any depth, so --include '*.py' finds core/gpu/worker.py +# and not just top-level files. A pattern that does contain '/' is anchored at +# the repo root, so --exclude 'ui/*' means that directory and not any ui/ +# nested somewhere. +glob_to_re() { + local glob="$1" re + # Escape everything that is not plainly safe, then bring back * and ?. Doing + # it by allowlist avoids two traps that a metacharacter blocklist walks + # straight into: '[.' inside a bracket expression opens a POSIX collating + # symbol rather than matching a dot, and GNU sed expands \xHH in the + # replacement (which is how an earlier version injected a literal NUL). + re=$(printf '%s' "$glob" | sed -e 's/[^[:alnum:]_/-]/\\&/g' -e 's/\\\*/.*/g' -e 's/\\?/./g') + case "$glob" in + */*) printf '^%s$' "$re" ;; + *) printf '^(%s|.*/%s)$' "$re" "$re" ;; + esac +} + +# ── staging ──────────────────────────────────────────────────────────────── +# Everything downstream reads a real directory of real files, so tree, digest +# and list share one selection path instead of three. + +STAGED=0; DROPPED_NOISE=0; DROPPED_BIG=0; DROPPED_BINARY=0; DROPPED_GONE=0 +OMITTED=(); BINARY_FILES=(); BINARY_BYTES=0 + +stage() { + local dir="$1" ref="$2" sub="$3" into="$4" + local listfile="$TMP/list" all_n filtered_n + + mkdir -p "$into" + DROPPED_GONE=0; OMITTED=() + + if [ -n "$ref" ]; then + list_ref "$dir" "$ref" "$sub" > "$TMP/all" + elif is_git "$dir"; then + # A file deleted but not yet committed is still tracked, so it is still + # in this list — and rsync then fails the whole run on the first one. + # Bundling a dirty tree is the normal case here (that is often the state + # you want to ask about), so drop the ghosts and report them rather than + # demanding a clean tree. + list_worktree_git "$dir" "$sub" > "$TMP/all.raw" + : > "$TMP/all" + while IFS= read -r -d '' p; do + if [ -e "$dir/$p" ]; then + printf '%s\0' "$p" >> "$TMP/all" + else + DROPPED_GONE=$((DROPPED_GONE + 1)) + OMITTED+=("$p (tracked but deleted in the working tree)") + fi + done < "$TMP/all.raw" + else + list_worktree_plain "$dir" "$sub" > "$TMP/all" + fi + + # Delta mode: keep only what actually differs from the base. Generic — the + # base is whatever ref you name, and a ref equal to it distills whole. + if [ -n "$BASE_REF" ] && [ -n "$ref" ] && [ "$ref" != "$BASE_REF" ]; then + git -C "$dir" diff --name-only -z "$BASE_REF" "$ref" > "$TMP/changed" 2>/dev/null || : > "$TMP/changed" + comm -12 \ + <(tr '\0' '\n' < "$TMP/all" | sort) \ + <(tr '\0' '\n' < "$TMP/changed" | sort) \ + | tr '\n' '\0' > "$TMP/all.delta" + mv "$TMP/all.delta" "$TMP/all" + fi + + all_n=$(tr '\0' '\n' < "$TMP/all" | grep -c . || true) + filter_paths < "$TMP/all" > "$listfile" + filtered_n=$(tr '\0' '\n' < "$listfile" | grep -c . || true) + DROPPED_NOISE=$(( all_n - filtered_n )) + + if [ "$filtered_n" -eq 0 ]; then STAGED=0; return 0; fi + + if [ -n "$ref" ]; then + # One archive, extracting only the members we kept. tar needs the list + # in a file because the archive itself is on stdin. + git -C "$dir" archive --format=tar "$ref" \ + | tar -x -C "$into" --null -T "$listfile" 2>/dev/null || true + else + rsync -a --files-from="$listfile" --from0 "$dir/" "$into/" + fi + + prune_staged "$into" + STAGED=$(find "$into" -type f | wc -l) +} + +# Content-level work, once the bytes are on disk. The extension list above +# cannot know that some .txt is a 40M blob, or that a .json is really binary. +# +# Only --max-bytes deletes here. Binary files are RECORDED, not removed, and the +# difference matters: a tree is a copy, and a spreadsheet, a font or an icon is +# part of the project. An earlier version ran `grep -I` over the staged tree and +# deleted whatever came back binary, which quietly ate every .xlsx, .ods and +# .docx — files nobody had put on any exclude list. A copy that drops the +# spreadsheets is not a copy. +# +# The digest is the one output that genuinely cannot take them: it is text, and +# there is nothing sensible to inline. So it skips them and says which, rather +# than the tree losing them too. +prune_staged() { + local into="$1" f rel size + DROPPED_BIG=0; DROPPED_BINARY=0; BINARY_FILES=(); BINARY_BYTES=0 + while IFS= read -r -d '' f; do + rel="${f#$into/}" + if [ -n "$MAX_BYTES" ]; then + size=$(stat -c%s "$f") + if [ "$size" -gt "$MAX_BYTES" ]; then + rm -f "$f"; DROPPED_BIG=$((DROPPED_BIG+1)) + OMITTED+=("$rel (over --max-bytes: $size bytes)") + continue + fi + fi + if [ -s "$f" ] && ! grep -Iq . "$f" 2>/dev/null; then + BINARY_FILES+=("$rel") + DROPPED_BINARY=$((DROPPED_BINARY+1)) + BINARY_BYTES=$((BINARY_BYTES + $(stat -c%s "$f"))) + fi + done < <(find "$into" -type f -print0) + find "$into" -type d -empty -delete 2>/dev/null || true +} + +is_binary_file() { + local needle="$1" b + for b in ${BINARY_FILES[@]+"${BINARY_FILES[@]}"}; do + [ "$b" = "$needle" ] && return 0 + done + return 1 +} + +# ── digest ───────────────────────────────────────────────────────────────── + +# A markdown fence has to be longer than the longest run of backticks inside +# the file, or a file that itself contains fenced code — every README here — +# gets silently cut off at its first inner fence. +fence_for() { + local longest + longest=$(grep -o '`\+' "$1" 2>/dev/null | awk '{ if (length($0) > m) m = length($0) } END { print m+0 }') + [ "$longest" -lt 3 ] && longest=2 + printf '`%.0s' $(seq $((longest + 1))) +} + +# Paths grouped under their directory. The point of putting this before the +# contents is that a reader — human or model — can decide what to look at +# without scrolling through the whole thing first. +render_tree() { + (cd "$1" && find . -type f | sed 's|^\./||' | LC_ALL=C sort) | awk -F/ ' + { + dir = (NF > 1) ? substr($0, 1, length($0) - length($NF) - 1) "/" : "./" + if (dir != last) { print dir; last = dir } + print " " $NF + }' +} + +write_digest() { + local staged="$1" out="$2" title="$3" subtitle="$4" + local f rel fence lang bytes + + bytes=$(du -sb "$staged" | cut -f1) + { + echo "# $title" + echo + echo "$subtitle · $(find "$staged" -type f | wc -l) files · $(numfmt --to=iec "$bytes")$( + [ ${#BINARY_FILES[@]} -gt 0 ] && printf ' · %d binary, listed but not inlined' "${#BINARY_FILES[@]}" || true)" + echo + echo "## Tree" + echo + render_tree "$staged" + echo + } > "$out" + + # Named, not silently absent. Something reading only this file would + # otherwise have no idea the spreadsheets exist at all. + if [ ${#BINARY_FILES[@]} -gt 0 ]; then + { + echo "## Binary files (present in the copy, not inlined here)" + echo + for rel in ${BINARY_FILES[@]+"${BINARY_FILES[@]}"}; do + echo "- \`$rel\` ($(numfmt --to=iec "$(stat -c%s "$staged/$rel")"))" + done + echo + } >> "$out" + fi + + while IFS= read -r -d '' f; do + rel="${f#$staged/}" + is_binary_file "$rel" && continue + fence="$(fence_for "$f")" + lang="$(lang_for "$rel")" + { + echo "## $rel" + echo + echo "${fence}${lang}" + cat "$f" + # A file with no trailing newline would otherwise weld its last line + # to the closing fence. + [ -n "$(tail -c1 "$f")" ] && echo + echo "$fence" + echo + } >> "$out" + done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | sed "s|^|$staged/|" | tr '\n' '\0') +} + +# When several refs of one repo are distilled, ship the comparison too. Whatever +# reads this cannot run git, so "what is different on this branch" has to be in +# the text or it is not knowable. +write_refs_summary() { + local dir="$1" name="$2" out="$3" base="$4"; shift 4 + local refs=("$@") r + { + echo "# $name — refs" + echo + echo "Base for comparison: \`$base\`" + echo + for r in "${refs[@]}"; do + echo "## $r" + echo + echo "commit \`$(git -C "$dir" rev-parse --short "$r")\` — $(git -C "$dir" log -1 --format=%s "$r")" + echo + if [ "$r" != "$base" ]; then + echo "$(git -C "$dir" rev-list --count "$base".."$r" 2>/dev/null || echo 0) commits ahead of \`$base\`, $(git -C "$dir" rev-list --count "$r".."$base" 2>/dev/null || echo 0) behind." + echo + echo '```' + git -C "$dir" diff --stat "$base".."$r" 2>/dev/null || true + echo '```' + echo + fi + done + } > "$out" +} + +# ── doing only what changed ──────────────────────────────────────────────── +# Re-running is already safe — the output is a pure function of the sources, so +# two runs produce identical bytes. What re-running is NOT is cheap, and it does +# not notice removals: take a repo out of the config and its directory sits in +# the destination forever, looking current. +# +# Both are opt-in, because both surprise you otherwise: skipping hides work you +# may have wanted redone, and pruning deletes. Off by default the command stays +# the dumb, obvious thing. +# +# The state lives in the destination rather than next to the script, so a +# destination carries its own history and two different jobs cannot confuse each +# other. Shape: label fingerprint files bytes b64(row). +STATE_FILE="" +STATE_OLD="$TMP/state.old" +STATE_NEW="$TMP/state.new" +PRODUCED=() + +state_lookup() { # label -> prints the stored record, or nothing + [ -f "$STATE_OLD" ] || return 0 + grep -F -m1 "$(printf '%s\t' "$1")" "$STATE_OLD" 2>/dev/null || true +} + +# What the output depends on. If any of it moves, the copy is stale. +# +# Deliberately cheap: a ref is its commit, and a worktree is its commit plus the +# porcelain status, which covers uncommitted edits without walking the tree. The +# filters are folded in too, so changing an exclude re-runs rather than quietly +# serving the old answer. Only a non-git tree has to be walked, and there git +# has told us nothing. +fingerprint() { + local dir="$1" ref="$2" sub="$3" src="" + if [ -n "$ref" ]; then + src="ref:$(git -C "$dir" rev-parse "$ref")" + elif is_git "$dir"; then + src="wt:$(git -C "$dir" rev-parse HEAD):$(git -C "$dir" status --porcelain | cksum | cut -d" " -f1)" + else + src="plain:$(find "$dir" -type f -printf '%P %s %T@\n' 2>/dev/null | LC_ALL=C sort | cksum | cut -d" " -f1)" + fi + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + "$src" "$sub" "$CMD" "$BASE_REF" "$KEEP_NOISE" "$MAX_BYTES" \ + "${INCLUDES[*]-}" "${EXCLUDES[*]-}" "$MIRROR" \ + | cksum | cut -d' ' -f1 +} + +# Everything this run is responsible for. Anything else in the destination is +# left over from a previous list, and --prune is what removes it. +produced() { PRODUCED+=("$1"); } + +# ── the run ──────────────────────────────────────────────────────────────── + +MANIFEST_ROWS=() +MANIFEST_NOTES=() +TOTAL_BYTES=0 +TOTAL_TEXT=0 +TOTAL_FILES=0 + +process() { + local dir="$1" name="$2" ref="$3" sub="$4" label="$5" + local staged="$TMP/stage/$label" bytes=0 tokens=0 kind desc + local fp="" prior="" want_dir="" want_md="" + + case "$CMD" in + tree) want_dir="$DEST/$label" ;; + digest) want_md="$DEST/$label.md" ;; + both) want_dir="$DEST/$label"; want_md="$DEST/$label.md" ;; + esac + [ -n "$want_dir" ] && produced "$want_dir" + [ -n "$want_md" ] && produced "$want_md" + + if [ -n "$SKIP_UNCHANGED" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then + fp="$(fingerprint "$dir" "$ref" "$sub")" + prior="$(state_lookup "$label")" + # The recorded outputs have to still BE there. Without this check a + # deleted destination would be reported as up to date and rebuilt as + # nothing, which is the one failure worse than redoing the work. + if [ -n "$prior" ] && [ "$(printf '%s' "$prior" | cut -f2)" = "$fp" ] \ + && { [ -z "$want_dir" ] || [ -d "$want_dir" ]; } \ + && { [ -z "$want_md" ] || [ -f "$want_md" ]; }; then + local p_files p_bytes p_row + p_files="$(printf '%s' "$prior" | cut -f3)" + p_bytes="$(printf '%s' "$prior" | cut -f4)" + p_row="$(printf '%s' "$prior" | cut -f5 | base64 -d)" + local p_text; p_text="$(printf '%s' "$prior" | cut -f6)" + [ -n "$p_text" ] || p_text="$p_bytes" + printf ' %-28s unchanged\n' "$label" + # Carried forward verbatim: a manifest that listed only the repos + # that happened to change would misrepresent what is in the folder. + MANIFEST_ROWS+=("$p_row") + TOTAL_FILES=$((TOTAL_FILES + p_files)) + TOTAL_BYTES=$((TOTAL_BYTES + p_bytes)) + TOTAL_TEXT=$((TOTAL_TEXT + p_text)) + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$p_files" "$p_bytes" \ + "$(printf '%s' "$p_row" | base64 -w0)" "$p_text" >> "$STATE_NEW" + return 0 + fi + fi + + rm -rf "$staged" + stage "$dir" "$ref" "$sub" "$staged" + + if [ "$STAGED" -eq 0 ]; then + echo " $label — nothing selected" + MANIFEST_ROWS+=("| \`$label\` | $dir | — | 0 | — | — |") + return 0 + fi + + bytes=$(du -sb "$staged" | cut -f1) + # Binary bytes are copied but never inlined, so counting them as tokens + # would overstate every digest by the weight of its images. + tokens=$(( (bytes - BINARY_BYTES) / 4 )) + TOTAL_BYTES=$((TOTAL_BYTES + bytes)) + TOTAL_TEXT=$((TOTAL_TEXT + bytes - BINARY_BYTES)) + TOTAL_FILES=$((TOTAL_FILES + STAGED)) + + local is_delta="" + [ -n "$BASE_REF" ] && [ -n "$ref" ] && [ "$ref" != "$BASE_REF" ] && is_delta=1 + + if [ -n "$ref" ]; then + kind="$ref @ $(git -C "$dir" rev-parse --short "$ref")" + [ -n "$is_delta" ] && kind="$kind · DELTA vs $BASE_REF" + elif is_git "$dir"; then + kind="worktree ($(git -C "$dir" rev-parse --abbrev-ref HEAD) @ $(git -C "$dir" rev-parse --short HEAD))" + else + kind="worktree (not git)" + fi + + printf ' %-28s %4d files %8s ~%sk tok\n' \ + "$label" "$STAGED" "$(numfmt --to=iec "$bytes")" "$((tokens / 1000))" + + local dropped=$((DROPPED_NOISE + DROPPED_BIG + DROPPED_GONE)) + MANIFEST_ROWS+=("| \`$label\` | $dir | $kind | $STAGED | $dropped | $(numfmt --to=iec "$bytes") | ~$((tokens / 1000))k |") + + if [ ${#BINARY_FILES[@]} -gt 0 ]; then + MANIFEST_NOTES+=("### $label — binary (copied, not inlined in the digest)") + local b + for b in "${BINARY_FILES[@]}"; do + MANIFEST_NOTES+=("- $b ($(numfmt --to=iec "$(stat -c%s "$staged/$b")"))") + done + MANIFEST_NOTES+=("") + fi + + if [ ${#OMITTED[@]} -gt 0 ]; then + MANIFEST_NOTES+=("### $label — omitted") + local o + for o in "${OMITTED[@]}"; do MANIFEST_NOTES+=("- $o"); done + MANIFEST_NOTES+=("") + fi + + [ -z "$DRY" ] || return 0 + + # Not elif: 'both' does each in turn. They answer different questions — one + # gives you files, the other gives you something to read — and the staging + # work they share is already done by the time we get here, so producing both + # costs a copy rather than a second pass over the repo. + case "$CMD" in + tree|both) + local target="$DEST/$label" + mkdir -p "$target" + rsync -a ${MIRROR:+--delete} "$staged/" "$target/" + # A delta is a directory of source files that looks exactly like a + # working copy and is not one — the files identical to the base are + # simply absent, with nothing to say so. Anyone who opens it later, + # or hands it to something that reads it, has no way to tell. Say it + # in the tree itself, not only in a manifest that travels separately. + if [ -n "$is_delta" ]; then + { + echo "# Partial copy — not a working tree" + echo + echo "This is \`$ref\` of \`${name%% (*}\` reduced to **only the files that differ**" + echo "from \`$BASE_REF\` ($STAGED files)." + echo + echo "Every other file is unchanged from \`$BASE_REF\` and was left out, so this" + echo "will not build and is not the branch. Read it against the \`$BASE_REF\` copy." + echo + echo "To get the branch whole instead, distil it without a base." + } > "$target/_PARTIAL.md" + fi + ;;& + digest|both) + desc="$kind" + [ -n "$sub" ] && desc="$desc · scope $sub" + [ -n "$is_delta" ] && desc="$desc · ONLY files differing from $BASE_REF" + write_digest "$staged" "$DEST/$label.md" "$name" "$desc" + ;; + esac + + if [ -n "$SKIP_UNCHANGED" ]; then + [ -n "$fp" ] || fp="$(fingerprint "$dir" "$ref" "$sub")" + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$STAGED" "$bytes" \ + "$(printf '%s' "${MANIFEST_ROWS[-1]}" | base64 -w0)" \ + "$((bytes - BINARY_BYTES))" >> "$STATE_NEW" + fi +} + +# One spec, with whatever options are currently in effect. Both entry points — +# the command line and the config file — land here, so a distilled repo means the same +# thing however it was asked for. +# Each output needs its own name in DEST, and the repo slug alone is not enough: +# the same repo can legitimately appear twice — once whole, once narrowed to a +# subtree — and both would land on the same file. Fold the ref and the subpath +# into the label, then refuse to reuse one, because the failure otherwise is a +# copy quietly overwritten by the next. +# +# Sets LABEL rather than printing it: called in a $( ) it would run in a +# subshell, and every USED_LABELS update would be thrown away — which is +# precisely the bookkeeping it exists to do. +USED_LABELS="" +LABEL="" +label_for() { + local slug="$1" ref="$2" sub="$3" override="$4" label n + + if [ -n "$override" ]; then + label="$override" + else + label="$slug" + [ -n "$ref" ] && label="$label@${ref//\//-}" + [ -n "$sub" ] && label="$label:${sub//\//-}" + fi + + case " $USED_LABELS " in + *" $label "*) + n=2 + while case " $USED_LABELS " in *" $label-$n "*) true ;; *) false ;; esac; do + n=$((n + 1)) + done + echo " note: '$label' is taken, using '$label-$n' (set \"name\" to choose)" + label="$label-$n" + ;; + esac + USED_LABELS="$USED_LABELS $label" + LABEL="$label" +} + +run_spec() { + local spec="$1" override="${2:-}" ref dirty label + + if [ -n "$MAX_BYTES" ] && ! [[ "$MAX_BYTES" =~ ^[0-9]+$ ]]; then + die "max_bytes wants a plain byte count, got: $MAX_BYTES" + fi + + parse_spec "$spec" + echo "$SPEC_NAME ($SPEC_DIR)" + + if [ -n "$STRICT" ] && [ ${#SPEC_REFS[@]} -eq 0 ] && is_git "$SPEC_DIR"; then + dirty="$(git -C "$SPEC_DIR" status --porcelain)" + [ -z "$dirty" ] || die "$SPEC_NAME has uncommitted changes and --strict is set" + fi + + if [ ${#SPEC_REFS[@]} -eq 0 ]; then + label_for "$SPEC_NAME" "" "$SPEC_SUB" "$override" + process "$SPEC_DIR" "$SPEC_NAME" "" "$SPEC_SUB" "$LABEL" + else + for ref in "${SPEC_REFS[@]}"; do + # An explicit name with several refs still has to tell them apart, + # so the ref is appended to it rather than replacing it. + label_for "${override:-$SPEC_NAME}" "$ref" "$SPEC_SUB" "" + process "$SPEC_DIR" "$SPEC_NAME ($ref)" "$ref" "$SPEC_SUB" "$LABEL" + done + if [ ${#SPEC_REFS[@]} -gt 1 ] && [ "$CMD" != tree ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then + write_refs_summary "$SPEC_DIR" "$SPEC_NAME" \ + "$DEST/${override:-$SPEC_NAME}@REFS.md" \ + "${BASE_REF:-${SPEC_REFS[0]}}" "${SPEC_REFS[@]}" + produced "$DEST/${override:-$SPEC_NAME}@REFS.md" + fi + fi +} + +if [ -n "$DRY" ]; then + echo "dry run — nothing will be written" + [ "$CMD" != list ] && echo "would write to: $DEST" +fi + +[ "$CMD" = list ] || [ -n "$DRY" ] || mkdir -p "$DEST" + +if [ "$CMD" != list ] && [ -z "$DRY" ]; then + STATE_FILE="$DEST/.distill-state" + [ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$STATE_OLD" + : > "$STATE_NEW" + produced "$DEST/MANIFEST.md" + produced "$STATE_FILE" +fi + +# Repos named on the command line run under the global options, as before. +for spec in ${SPECS[@]+"${SPECS[@]}"}; do + run_spec "$spec" +done + +# Config entries each carry their own options, so the globals are reloaded per +# entry. A command-line option, having been parsed already, is left to win. +if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then + while IFS= read -r job; do + [ -n "$job" ] || continue + spec="$(printf '%s' "$job" | jq -r '.spec')" + name="$(printf '%s' "$job" | jq -r '.name')" + + BASE_REF="$(printf '%s' "$job" | jq -r '.base')" + MAX_BYTES="$(printf '%s' "$job" | jq -r '.max_bytes')" + [ "$MAX_BYTES" = null ] && MAX_BYTES="" + [ "$(printf '%s' "$job" | jq -r '.all')" = true ] && KEEP_NOISE=1 || KEEP_NOISE="" + + INCLUDES=(); EXCLUDES=() + while IFS= read -r g; do [ -n "$g" ] && INCLUDES+=("$g"); done \ + < <(printf '%s' "$job" | jq -r '.include[]?') + while IFS= read -r g; do [ -n "$g" ] && EXCLUDES+=("$g"); done \ + < <(printf '%s' "$job" | jq -r '.exclude[]?') + + run_spec "$spec" "$name" + done < "$JOBS" +fi + +echo +printf 'total: %d files, %s, ~%sk tokens\n' \ + "$TOTAL_FILES" "$(numfmt --to=iec "$TOTAL_BYTES")" "$((TOTAL_TEXT / 4000))" + +# Anything at the top of the destination this run did not produce came from a +# previous, longer list. Scoped to depth 1 and to a destination we just wrote +# to: this deletes, so it should never go hunting. +if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then + while IFS= read -r -d '' entry; do + keep="" + for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do + [ "$entry" = "$kept" ] && { keep=1; break; } + done + if [ -z "$keep" ]; then + echo " pruned $(basename "$entry")" + rm -rf "$entry" + fi + done < <(find "$DEST" -mindepth 1 -maxdepth 1 -print0) +fi + +if [ -n "$SKIP_UNCHANGED" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then + mv "$STATE_NEW" "$STATE_FILE" +fi + +if [ "$CMD" != list ] && [ -z "$DRY" ]; then + { + echo "# Distill manifest" + echo + echo "Generated by \`$SELF $CMD\` from \`$ROOT\`." + [ -n "$BASE_REF" ] && echo "Delta mode: non-base refs carry only files differing from \`$BASE_REF\`." + [ -n "$KEEP_NOISE" ] && echo "Noise filter OFF (\`--all\`): lockfiles and binaries included." + echo + echo "| output | source | ref | files | dropped | size | tokens |" + echo "|---|---|---|---|---|---|---|" + printf '%s\n' "${MANIFEST_ROWS[@]}" + echo + printf 'Total: %d files, %s, ~%sk tokens.\n' \ + "$TOTAL_FILES" "$(numfmt --to=iec "$TOTAL_BYTES")" "$((TOTAL_TEXT / 4000))" + if [ ${#MANIFEST_NOTES[@]} -gt 0 ]; then + echo + echo "## Omitted files" + echo + printf '%s\n' "${MANIFEST_NOTES[@]}" + fi + } > "$DEST/MANIFEST.md" + echo "wrote $DEST/MANIFEST.md" +fi diff --git a/soleprint/station/tools/distill/explode.md b/soleprint/station/tools/distill/explode.md new file mode 100644 index 0000000..e647684 --- /dev/null +++ b/soleprint/station/tools/distill/explode.md @@ -0,0 +1,5 @@ +```bash +./ctrl/explode.sh --list bundle.txt # what is in there, write nothing +./ctrl/explode.sh -o ./restored bundle.txt # write the tree +./ctrl/explode.sh -o ./restored --force x.md # overwrite what is already there +``` diff --git a/soleprint/station/tools/distill/explode.sh b/soleprint/station/tools/distill/explode.sh new file mode 100755 index 0000000..d18534b --- /dev/null +++ b/soleprint/station/tools/distill/explode.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +# Explode one file back into the tree of files it describes. +# +# The inverse of ctrl/distill.sh's digest: something hands you a single text +# file with many files inside it, each introduced by its path, and you want the +# directory back. +# +# Three layouts are understood, picked automatically. Prefer the first if you +# control what writes the file: +# +# === FILE: pkg/models/domain.py explicit open and close. Nothing has to be +# counted or inferred, and a block that is +# === END never closed is an error rather than a +# file quietly missing its tail. +# +# === ./pkg/models/domain.py a marker line, then the file, until the +# next marker or the end +# +# ## pkg/models/domain.py distill.sh's own digest: a heading, then +# ```python a fenced block. The fence may be longer +# than three backticks, and the closing one +# ``` has to match it exactly. +# +# Usage: +# explode.sh [opts] FILE +# +# Options: +# -o DEST where to write the tree (default: the current directory) +# --list print what the file contains and write nothing +# -n same as --list +# --force overwrite files that already exist +# --format F fenced | marker | digest | auto (default: auto) +# --selftest check this copy of the script against known input and exit +# +# Examples: +# explode.sh --list bundle.txt +# explode.sh -o ./restored bundle.txt +# explode.sh -o ./restored --force repo.md +# +# Why the explicit form is worth asking for: a writer that emits plain three- +# backtick fences truncates any file that itself contains a fence — every README +# with a shell example — and does it silently, because the nested fence looks +# exactly like the closing one. distill.sh avoids that by making its fences +# longer than anything inside the file, but nothing else will bother. +# +# Two limits worth knowing. A file whose last line has no trailing newline comes +# back with one: the digest has to put a newline before the closing fence, so the +# distinction is not in the input to recover. And in marker layout a line +# starting with "=== " inside a file's own content is indistinguishable from a +# real marker — there are no fences to say otherwise. The digest layout has no +# such ambiguity, which is the reason to prefer it when something else is +# generating the file. +# +# Paths come out of a text file, so they are treated as untrusted: anything +# absolute, or reaching upward with .., is refused and nothing is written. A +# file that describes /etc/cron.d/x is not a file you want to expand blindly. +set -euo pipefail + +SELF="$(basename "$0")" +usage() { awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"; } +die() { echo "$SELF: $*" >&2; exit 1; } + +DEST="." +LIST="" +FORCE="" +FORMAT="auto" +SRC="" +SELFTEST="" + +while [ $# -gt 0 ]; do + case "$1" in + -o) shift; DEST="${1:-}" ;; + --list|-n) LIST=1 ;; + --force) FORCE=1 ;; + --format) shift; FORMAT="${1:-}" ;; + --selftest) SELFTEST=1 ;; + -h|--help) usage; exit 0 ;; + -*) die "unknown option: $1" ;; + *) [ -z "$SRC" ] && SRC="$1" || die "one input file at a time (got '$SRC' and '$1')" ;; + esac + shift +done + +# ── self-test ────────────────────────────────────────────────────────────── +# So a copy of this script on another machine can be checked without any real +# input, and without asking whether it is the version that knows a given format. +# Every case here is one that has actually gone wrong. +selftest() { + local t rc=0 got want + t="$(mktemp -d)"; trap 'rm -rf "$t"' RETURN + + check() { # name, expected, actual + if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1" + else printf ' FAIL %s\n expected: %s\n got: %s\n' "$1" "$2" "$3"; rc=1 + fi + } + + # A file containing a fence and a stray === line: the two things that break + # naive parsers. + cat > "$t/a.txt" <<'FIXTURE' +Here is the code. + +=== FILE: pkg/core/client.py +class Client: + pass +=== END + +=== FILE: README.md +# proj + +```bash +pip install proj +``` + +A line that says === not a marker +=== END +FIXTURE + "$0" -o "$t/a" "$t/a.txt" >/dev/null 2>&1 || true + check "fenced: file count" "2" "$(find "$t/a" -type f 2>/dev/null | wc -l)" + check "fenced: nested fences" "2" "$(grep -c '```' "$t/a/README.md" 2>/dev/null || echo 0)" + check "fenced: === in content" "1" "$(grep -c 'not a marker' "$t/a/README.md" 2>/dev/null || echo 0)" + check "fenced: subdirectory" "class Client:" "$(head -1 "$t/a/pkg/core/client.py" 2>/dev/null)" + check "fenced: prose skipped" "0" "$(find "$t/a" -name 'Here*' 2>/dev/null | wc -l)" + + # An unclosed block must be refused, not written short. + printf '=== FILE: a.py\nx = 1\n=== END\n\n=== FILE: b.py\ny = 2\n' > "$t/b.txt" + "$0" -o "$t/b" "$t/b.txt" >/dev/null 2>&1 || true + check "unterminated: refused" "1" "$([ -e "$t/b" ] && echo 0 || echo 1)" + + # Paths out of a text file are untrusted. + printf '=== FILE: ../escape.py\nx\n=== END\n' > "$t/c.txt" + "$0" -o "$t/c" "$t/c.txt" >/dev/null 2>&1 || true + check "traversal: refused" "1" "$([ -e "$t/c" ] && echo 0 || echo 1)" + + # The stale-copy failure: explicit format read by the marker parser. + "$0" --format marker -o "$t/d" "$t/a.txt" >/dev/null 2>&1 || true + check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)" + + # The other two layouts still work. + printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt" + "$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true + check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)" + + printf '# d\n\n## x/y.py\n\n```python\nz = 1\n```\n' > "$t/f.txt" + "$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true + check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)" + + echo + if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current" + else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2 + fi + return "$rc" +} + +if [ -n "$SELFTEST" ]; then selftest; exit $?; fi + +[ -n "$SRC" ] || { usage >&2; exit 1; } +[ -f "$SRC" ] || die "no such file: $SRC" +case "$FORMAT" in fenced|marker|digest|auto) ;; *) die "--format must be fenced, marker, digest or auto" ;; esac + +# Which layout is it? Count the two shapes and take the commoner one, rather +# than trusting the first line that happens to match: a digest of a repo full of +# markdown will contain plenty of '=== ' inside its own fenced content, and a +# marker file can quote a '## ' heading just as easily. +if [ "$FORMAT" = auto ]; then + n_fenced=$(grep -cE '^=== +FILE: +[^ ]' "$SRC" || true) + n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true) + n_marker=$((n_marker - n_fenced - $(grep -cE '^=== +END[ \t]*$' "$SRC" || true))) + [ "$n_marker" -lt 0 ] && n_marker=0 + n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true) + if [ "$n_fenced" -gt 0 ]; then + FORMAT=fenced + elif [ "$n_marker" -eq 0 ] && [ "$n_digest" -eq 0 ]; then + die "found no '=== FILE:' blocks, no '=== path' markers and no '## path' headings in $SRC" + elif [ "$n_marker" -ge "$n_digest" ]; then FORMAT=marker + else FORMAT=digest + fi + echo "format: $FORMAT" +fi + +# ── the parser ───────────────────────────────────────────────────────────── +# One awk, two modes. In list mode it prints "pathlines"; otherwise it +# writes each file under DEST. Reading the whole thing in awk rather than a +# bash read-loop matters once the input is a few megabytes. +# +# In digest mode a heading only opens a file if a fence follows it. distill.sh +# writes '## Tree' and '## Binary files ...' sections that are prose, and +# treating those as files would scatter junk through the output. +parse() { + awk -v dest="$DEST" -v mode="$1" -v fmt="$FORMAT" ' + function flush() { + if (path != "") { + if (mode == "list") { printf "%s\t%d\n", path, n } + path = "" + } + n = 0 + } + function clean(p) { + sub(/^\.\//, "", p) + sub(/[ \t\r]+$/, "", p) + return p + } + function unsafe(p) { + return (p == "" || p ~ /^\// || p ~ /^[A-Za-z]:/ || p ~ /(^|\/)\.\.(\/|$)/) + } + function open_file(p) { + path = p + n = 0 + if (mode == "write") { + out = dest "/" path + d = out; sub(/\/[^\/]*$/, "", d) + system("mkdir -p \"" d "\"") + printf "" > out + } + } + function emit(line) { + n++ + if (mode == "write") print line >> (dest "/" path) + } + + # Explicit open/close. The whole point is that nothing is inferred: + # content is content until the END line, whatever it looks like. + fmt == "fenced" && path == "" && /^=== +FILE: +/ { + p = substr($0, index($0, "FILE:") + 5) + sub(/^[ \t]+/, "", p) + p = clean(p) + if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next } + open_file(p) + next + } + fmt == "fenced" && path != "" && /^=== +END[ \t]*$/ { flush(); next } + fmt == "fenced" && path == "" { next } # anything between blocks is prose + + fmt == "marker" && /^=== +/ { + flush() + p = clean(substr($0, index($0, " ") + 1)) + # "FILE: ./x.py" and "END" are not paths, they are the explicit + # format being read by the wrong parser. Left alone this writes a + # directory literally called "FILE: ." and a file called "END", + # which is what an out-of-date copy of this script did once. + if (p ~ /^FILE:/ || p == "END") { print "WRONGFMT\t" p; bad = 1; next } + if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next } + open_file(p) + next + } + + # Only when NOT already inside a file. The content of any markdown file + # in the input is full of "## " headings, and the fence is the only + # thing that says which ones are structure and which are text. Once a + # file is open, the matching close fence is the sole way out. + fmt == "digest" && path == "" && /^## +/ { + flush() + pending = clean(substr($0, 4)) + expect = 1 + next + } + fmt == "digest" && expect == 1 { + if ($0 ~ /^[ \t]*$/) next # blank line between the two + if ($0 ~ /^`{3,}/) { # a fence: this is a file + match($0, /^`+/) + fence = substr($0, 1, RLENGTH) + expect = 0 + if (unsafe(pending)) { print "UNSAFE\t" pending; bad = 1; next } + open_file(pending) + next + } + expect = 0 # prose section, not a file + pending = "" + next + } + fmt == "digest" && path != "" && $0 == fence { flush(); next } + + + { if (path != "") emit($0) } + + END { + if (fmt == "fenced" && path != "") { + print "UNTERMINATED\t" path + bad = 1 + } + flush() + exit (bad ? 3 : 0) + } + ' "$SRC" +} + +# Validate before writing anything: a refusal after half the tree is on disk is +# not a refusal. +# awk exits non-zero when it found something wrong; that is the signal, not a +# crash, so let it through and report it properly below. +scan="$(parse list || true)" + +refused="$(printf '%s\n' "$scan" | grep '^UNSAFE' || true)" +if [ -n "$refused" ]; then + echo "$SELF: refusing — these paths escape the destination:" >&2 + printf '%s\n' "$refused" | sed 's/^UNSAFE\t/ /' >&2 + exit 1 +fi + +# A block that never closed means the input is malformed, or a file contained +# the END line. Either way the tail is missing, and a truncated source file that +# looks complete is the failure this format exists to prevent. +wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)" +if [ -n "$wrongfmt" ]; then + echo "$SELF: this file uses '=== FILE: path' / '=== END', but it was read as" >&2 + echo "the plain marker format, which would create a directory called 'FILE: .'" >&2 + echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2 + exit 1 +fi + +unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)" +if [ -n "$unterminated" ]; then + echo "$SELF: refusing — this block was never closed with '=== END':" >&2 + printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2 + echo "the file it describes would be silently truncated" >&2 + exit 1 +fi + +listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT)' || true)" +[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)" +count=$(printf '%s\n' "$listing" | grep -c . ) + +if [ -n "$LIST" ]; then + printf '%s\n' "$listing" | awk -F'\t' '{ printf " %-60s %5d lines\n", $1, $2 }' + echo "$count files" + exit 0 +fi + +# Existing files are someone's work until proven otherwise. +if [ -z "$FORCE" ]; then + clashes="" + while IFS=$'\t' read -r p _; do + [ -e "$DEST/$p" ] && clashes="$clashes $p"$'\n' + done <<< "$listing" + if [ -n "$clashes" ]; then + echo "$SELF: these already exist under $DEST:" >&2 + printf '%s' "$clashes" >&2 + echo "re-run with --force to overwrite" >&2 + exit 1 + fi +fi + +mkdir -p "$DEST" +parse write >/dev/null +printf '%s\n' "$listing" | awk -F'\t' '{ printf " %s\n", $1 }' +echo "wrote $count files to $DEST" diff --git a/soleprint/station/tools/histgen/.gitignore b/soleprint/station/tools/histgen/.gitignore new file mode 100644 index 0000000..9939bf1 --- /dev/null +++ b/soleprint/station/tools/histgen/.gitignore @@ -0,0 +1,5 @@ +# Local settings: whose repo this machine points at is not a fact about the +# tool. Copy the folder, run `make init-config`, and the answer stays here. +histgen.json +def +__pycache__/ diff --git a/soleprint/station/tools/histgen/Makefile b/soleprint/station/tools/histgen/Makefile new file mode 100644 index 0000000..4c72086 --- /dev/null +++ b/soleprint/station/tools/histgen/Makefile @@ -0,0 +1,137 @@ +# histgen — one target per verb. +# +# The folder is meant to be copied out of soleprint and used on its own, so +# everything here is derived from where this file sits rather than written down: +# copy the directory anywhere, `cd` into it, and `make` works. Renaming it works +# too, since the package name comes from the directory. +# +# Two directories, and the whole tool hangs off the difference: +# +# SOURCE the tree to read. Read-only, always. Nothing is written into it. +# OUT the plan, the briefs, and OUT// — a copy of the source with +# the designed history committed into it. +# +# make check prove it works, on its own fixture +# make copy SOURCE=~/work/x OUT=~/out just the files, no repo, no keys +# make run SOURCE=~/work/x OUT=~/out scan + plan + brief +# make list the commits, to confirm +# make commands copy the files, hand back git commands +# make export ...or have it commit them for you +# +# Set them once and the verbs take no arguments: +# +# make init-config SOURCE=... OUT=... +# make config what everything resolves to +# make status what is in OUT, and what is left +# +# The logic lives in the Python, never here. Each target is one invocation. + +HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +PKG := $(notdir $(HERE)) +PARENT := $(patsubst %/,%,$(dir $(HERE))) +PY ?= python3 +PREFIX ?= $(HOME)/.local + +# Run the package from its parent, which is what `python -m` needs and what +# lets this work without installing anything. +HISTGEN := PYTHONPATH=$(PARENT) $(PY) -m $(PKG) + +# Extra flags for the verb being run: make plan REPO=x ARGS=--max-files=12 +ARGS ?= + +# Left empty, these say nothing and the config file decides. Passing REPO= or +# OUT= on the command line overrides it, which is the precedence the tool +# already applies — the Makefile just has to not invent a default of its own. +SOURCE ?= +OUT ?= +CONFIG ?= + +WHERE := $(if $(SOURCE),--source $(SOURCE)) $(if $(OUT),--out $(OUT)) \ + $(if $(CONFIG),--config $(CONFIG)) + +.PHONY: help check run copy scan plan list brief export keep commands dry-run verify status \ + against-history clean install uninstall doctor config init-config + +help: ## List every target + @echo "histgen — seed a clean, logical history into a repo" + @echo + @grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-16s\033[0m %s\n", $$1, $$2}' + @echo + @echo " SOURCE=/path/to/tree what to read (read-only, never written to)" + @echo " OUT=/path/to/out what to write (plan, briefs, and OUT//)" + @echo " ARGS=... extra flags, e.g. ARGS=\"--max-files 25\"" + @echo + @echo " Both can live in histgen.json instead: make init-config SOURCE=.. OUT=.." + +check: ## Prove the whole pipeline works, needing no repo and nothing installed + @$(PY) $(HERE)/selftest.py + +config: ## Show what repo, out and max-files resolve to + @$(HISTGEN) config $(WHERE) + +init-config: ## Write a starter histgen.json beside the tool + @$(HISTGEN) config --init $(WHERE) + +doctor: ## Report whether this machine can run it + @printf 'python : '; $(PY) --version 2>&1 || echo MISSING + @printf 'git : '; git --version 2>&1 || echo MISSING + @printf 'package: %s (from %s)\n' '$(PKG)' '$(PARENT)' + @$(HISTGEN) --help >/dev/null 2>&1 \ + && echo 'import : ok' || echo 'import : FAILED — is the folder intact?' + +run: ## scan + plan + brief, everything before the messages are needed + @$(HISTGEN) run $(WHERE) $(ARGS) + +list: ## Print the commits, to confirm before exporting + @$(HISTGEN) list $(WHERE) $(ARGS) + +status: ## What is in OUT, what state it is in, and what is left to do + @$(HISTGEN) status $(WHERE) $(ARGS) + +copy: ## Copy the files out: no .git, nothing ignored, no keys, no build output + @$(HISTGEN) copy $(WHERE) $(ARGS) + +scan: ## Read the source and cache what was read + @$(HISTGEN) scan $(WHERE) $(ARGS) + +plan: ## Order the files and cut them into commits + @$(HISTGEN) plan $(WHERE) $(ARGS) + +brief: ## Write one brief per commit, for the messages + @$(HISTGEN) brief $(WHERE) $(ARGS) + +against-history: ## Report how an existing history compares. Reads only + @$(HISTGEN) plan $(WHERE) --against-history $(ARGS) + +dry-run: ## Write the export as a reviewable regen.sh instead of running it + @$(HISTGEN) export $(WHERE) --dry-run $(ARGS) + +export: ## Copy the source into OUT and commit the history. Resumes if interrupted + @$(HISTGEN) export $(WHERE) $(ARGS) + +keep: ## Export, carrying an existing history over onto its own branch + @$(HISTGEN) export $(WHERE) --keep-history $(ARGS) + +commands: ## Copy the files, create no repo, print the git commands to run yourself + @$(HISTGEN) export $(WHERE) --commands $(ARGS) + +verify: ## Nothing left untracked, and the exported tree matches the source + @$(HISTGEN) verify $(WHERE) $(ARGS) + +clean: ## Delete the whole OUT directory. The source is not touched + @d=$$($(HISTGEN) config $(WHERE) | awk '/^out /{print $$2}'); \ + test -n "$$d" -a "$$d" != "(unset)" || { echo "Error: no OUT set." >&2; exit 1; }; \ + rm -rf "$$d" && echo "Removed $$d. The source was never written to." + +install: ## Put a `histgen` command on PATH, pointing back at this folder + @mkdir -p $(PREFIX)/bin + @printf '#!/bin/sh\n# Generated by histgen'"'"'s Makefile; points at the folder it was run from.\nPYTHONPATH="%s" exec "%s" -m %s "$$@"\n' \ + '$(PARENT)' '$(shell command -v $(PY))' '$(PKG)' > $(PREFIX)/bin/histgen + @chmod +x $(PREFIX)/bin/histgen + @echo "Installed $(PREFIX)/bin/histgen -> $(HERE)" + @case ":$$PATH:" in *":$(PREFIX)/bin:"*) ;; \ + *) echo "Note: $(PREFIX)/bin is not on PATH." ;; esac + +uninstall: ## Remove that command + @rm -f $(PREFIX)/bin/histgen && echo "Removed $(PREFIX)/bin/histgen" diff --git a/soleprint/station/tools/histgen/README.md b/soleprint/station/tools/histgen/README.md new file mode 100644 index 0000000..9f75a7b --- /dev/null +++ b/soleprint/station/tools/histgen/README.md @@ -0,0 +1,545 @@ +# histgen + +Seeds a clean, logical history into a repo — reading one directory and writing +another, so the tree it reads is never touched. + +```bash +histgen copy # just the files: no .git, nothing ignored, no keys +histgen run # scan the source, plan the commits, write the briefs +histgen list # the commits, to confirm +histgen commands # copy the files, hand back the git add/commit commands +histgen export # ...or have it do the committing itself +``` + +Two directories, and the whole tool hangs off the difference: + +| | | +|---|---| +| **source** | the tree to read. Opened read-only, always. Not a commit, not a `.git`, not a state file is written into it — it can be a checkout you do not own or a read-only mount. | +| **out** | everything produced. The index, the plan, the briefs, and `out//` — a copy of the source with the designed history committed into it. | + +That separation is what makes the history safe to argue with. It is an argument +you will have more than once, and every attempt is a directory you can delete +rather than a repo you have to put back. + +Stdlib only, no network, no API key. Also readable in the browser at +`/station/tools/histgen/`, which shows a plan and never writes one. + +## Copying it out + +Copy the folder anywhere, `cd` into it, and use the Makefile. It derives the +package name and path from where it sits, so the directory can be renamed and +still work, and nothing has to be installed. + +```bash +cp -r histgen ~/tools/ && cd ~/tools/histgen + +make check # prove it works, on its own fixture +make doctor # what this machine has +make init-config SOURCE=~/work/x OUT=~/out # set both once +make run # scan + plan + brief +make list # the commits, to confirm +make export # write them into ~/out/x +make help # every target +``` + +One target per verb, plus `dry-run`, `against-history`, `verify` and `clean`. +Extra flags go in `ARGS`: + +```bash +make plan REPO=/path/to/repo ARGS="--max-files 25" +``` + +`make install` drops a `histgen` command in `~/.local/bin` pointing back at the +folder, if you would rather not `cd` into it. + +## Just the files, without the repo + +```bash +make copy SOURCE=~/code/myproject OUT=~/clean +``` + +Gives you `~/clean/myproject` holding what the project actually is — no `.git`, +nothing gitignored, nothing a build regenerates, and nothing that looks like a +key. No history is planned and nothing is committed; this is the plain utility +underneath the rest. + +``` +14 files tracked, 7 to copy, 7 left behind. + + secret — looks like a key or a credential (2): + .env + certs/server.key + + ignored — tracked, but the ignore rules say they should not be (1): + data/raw/dump.sql + + derived — a build regenerates these (3): + dist/bundle.js.map + dist/bundle.min.js + package-lock.json + + oversize — larger than --max-bytes (1): + data/big.csv + +Copied to /home/you/clean/myproject + no .git — the source has one and it was not copied + what was left behind: /home/you/clean/copied.md +``` + +**Every drop is named**, on screen and in `copied.md`. A file quietly missing +from a copy is the same class of failure as a file quietly missing from a +history, one directory earlier. + +The same filter runs on the history path. `scan` drops secrets and +ignored-but-tracked files before they ever reach a plan, and says so: + +``` +Scanned 5 files: 5 read, 0 reused from cache. + 3 left out of the history: + .env (looks like a key or a credential) + certs/tls.key (looks like a key or a credential) + data/dump.sql (tracked, but the ignore rules say they should not be) +``` + +so `make commands` and `make export` cannot commit a key that `make copy` would +have left behind. `make status` keeps saying it afterwards, and +`--keep-secrets` turns it off. + +**One deliberate difference between the two.** `copy` also drops what a build +regenerates — lockfiles, maps, minified output — because a snapshot is for +reading. `scan` keeps them, because a lockfile is content in a repo somebody is +going to use. `copy --all` keeps them too. + +### What gets left behind, and why + +| | | +|---|---| +| **secret** | `.env`, `*.pem`, `*.key`, `id_rsa`, `.netrc`, `credentials.json`, `service-account*.json`, `.ssh/` | +| **ignored** | tracked *despite* the repo's own ignore rules — someone ran `git add -f` once | +| **derived** | lockfiles, `*.map`, `*.min.js`, `*.pyc`, `*.so`, `__pycache__/`, `node_modules/` | +| **oversize** | whatever `--max-bytes` says | + +The derived list is ported from `ppl/ctrl/distill.sh`, including the lesson in +its comments: **the line is derived-vs-content, not text-vs-binary.** Images, +fonts, spreadsheets and PDFs are content and are kept — none of them can be +regenerated from what is left, which is the only thing that makes a file safe to +drop. That distinction was wrong in distill once and cost real files. + +**Lookalikes are kept on purpose.** `.env.example` is the documented way to say +what the real one needs, and dropping it takes the documentation with the +secret. Same for `server.key.pub` — a public key is not a private one. + +The **ignored** case is the one worth reading. `git add -f` is not always a +mistake, so these are named rather than assumed either way; but a dump or a +credentials file that went in once and was never noticed since looks exactly +like this, and the repo is already contradicting itself about them. + +### Knobs + +`--keep-secrets`, `--exclude` and `--include` work on `copy`, `scan` and `run` +alike, and can live in `histgen.json`. The rest are `copy`'s own. + +```bash +make copy ARGS="--dry-run" # report only, write nothing +make copy ARGS="--max-bytes 100000" # leave anything bigger +make copy ARGS="--exclude '*.csv' --exclude data/" +make copy ARGS="--include package-lock.json" # keep it, whatever the filters say +make copy ARGS="--all --keep-secrets" # turn the two filters off +``` + +`--exclude` follows distill's rule: a pattern with no `/` matches basenames at +any depth. `--include` is checked first and wins outright, so one file can be +rescued without turning a whole filter off. + +To then plan a history from the cleaned tree, point a fresh run at it: + +```bash +make run SOURCE=~/clean/myproject OUT=~/history +``` + +## Recipe: a copy with a clean history + +The common case. You have a repo, you want the same files somewhere new with a +history that reads like the thing was built on purpose, and you do not care what +the old history said. + +Nothing is written to the original. The old history is simply not carried over — +that is the default, and `--keep-history` is the opt-in for when you do want it. + +**`OUT` is the parent directory, not the repo.** The copy keeps the source's own +name underneath it. `OUT` does not have to exist yet; it is created on the first +command. + +```bash +cd ~/tools/histgen # wherever you copied the folder + +make init-config SOURCE=~/code/myproject OUT=~/clean +make config # check both paths before anything runs +``` + +``` +source /home/you/code/myproject read-only +out /home/you/clean +exported to /home/you/clean/myproject <- the copy ends up here +``` + +### 1. Plan it + +```bash +make run # reads the source, groups the files, writes a brief per commit +make list # the proposed commits, in order +``` + +`make list` is the thing to look at. Every commit is marked `*` until it has a +message. If a commit holds two unrelated ideas, move a path between groups in +`~/clean/plan.json` and run `make plan && make list` again — the grouping is a +proposal, and re-planning keeps every message whose group still holds the same +files. + +### 2. Write the messages + +`~/clean/briefs/` has one markdown file per commit, holding each file's opening +comment. Read them and write a `title` and `body` into each group in +`~/clean/plan.json`. + +This is the part worth doing properly: the briefs carry the reasoning already in +the code, which is what makes a message worth reading. A message reconstructed +from the diff just restates the diff. + +```bash +make list # again — the titles you wrote now show instead of the * marks +``` + +To see the shape end to end before writing any of them, use +`ARGS=--allow-untitled` in the next step; the commits get their group name as a +subject, which is fine for a throwaway pass and not fine for anything you keep. + +### 3. Get the commands, and run them yourself + +```bash +make commands +``` + +This copies the planned files into `~/clean/myproject` and **creates no repo** — +no `git init`, no `.git`. What comes back is the list, also saved to +`~/clean/commands.sh`: + +``` +cd /home/you/clean/myproject +git init + +# 01 Repo skeleton: ignore rules and line-endings policy +git add -- .gitattributes .gitignore +git commit -F /home/you/clean/messages/01-skeleton.txt + +# 02 Pin the toolchain in one manifest +git add -- versions.env +git commit -F /home/you/clean/messages/02-versions.txt + +... + +# Worth running afterwards. The first says no file was silently +# missed; the second says the result is byte-identical to the source. +git status --porcelain +git rev-parse HEAD^{tree} # expect 3132703a817922f9f83bafa0e86e6bdf002ce8cb +``` + +`git init` is the first line of the list rather than something already done: a +repo that appeared without you asking is exactly what someone reaching for this +mode does not want. Read the list, edit it, reorder it, run it a line at a time. + +Messages go in files rather than `-m` because bodies are multi-line, and the +body is where the *why* lives. Edit the message files directly if you want to +reword something — nothing has been committed yet. + +The last two commands are worth running when you are done. `git status +--porcelain` printing nothing means no file was silently missed, which is the +failure this whole exercise exists to prevent. The tree hash matching means the +result is byte-identical to the source. + +Only the files in the plan are copied: not the source's `.git`, not anything +gitignored. If `~/clean/myproject` already contains a repo, this refuses rather +than handing you commands that would commit into it. + +### Or let it do the committing + +```bash +make dry-run # optional: writes ~/clean/regen.sh, a script that does everything +make export # copy and commit, checking both guards itself +``` + +``` +18 commits on main in /home/you/clean/myproject. Checking: + nothing left untracked: ok + tree matches source (3132703a8179): ok +``` + +Three modes, and the difference is who does what: + +| | copies the files | makes the repo | commits | +|---|---|---|---| +| `make commands` | yes | no — you run `git init` | you | +| `make dry-run` | no — writes a script that would | in the script | in the script | +| `make export` | yes | yes | yes, and checks both guards | + +```bash +cd ~/clean/myproject +git log --oneline +``` + +### If something goes wrong + +```bash +make status # says which of the four states out is in, and what to do next +``` + +- Interrupted partway? Run `make export` again — it continues from the commit + after the last one recorded, rather than starting over or refusing. (This + applies to `make export`; with `make commands` the repo is yours, so a + half-finished run is yours to continue from the list.) +- Changed the plan after exporting? `make status` says so; `make export + ARGS=--force` discards the copy and redoes it. +- Want to start completely fresh? `make clean` deletes the whole `OUT` + directory. The source is not touched, so there is nothing to put back. + +### Handing this to someone else + +Everything above needs the folder, `python3` and `git` — nothing installed, no +network, no API key. Copy the directory, then: + +```bash +cd histgen && make check # proves the whole pipeline on a fixture it builds +make help # every target +``` + +## What is already in out + +`export` writes, so it starts by working out what it is writing into. Four +states, and they are genuinely different — `histgen status` prints which one: + +| | | +|---|---| +| **absent** | nothing there yet. Copy the tree, init, commit. | +| **unfinished** | commits this tool made, and a record of where it stopped. Something interrupted the run. **Continue from the group after the last one recorded.** | +| **foreign** | commits this tool did *not* make. That history is someone's, so nothing is rewritten, moved or deleted: the designed account goes on its own orphan branch and the existing branch is left exactly where it was. | +| **stale** | the plan changed, or the copy moved underneath us. Refuse, and say which of the two it was. `--force` discards and starts over. | + +Telling **unfinished** from **foreign** is the whole reason `progress.json` +exists. Without it both read as "there are commits here", and the tool either +destroys work it should have kept or refuses to finish work it started — which +is exactly what it used to do. + +``` +$ histgen status +source /home/mariano/wdir/rdir/adapter +out /home/mariano/histories/adapter +census 91 files +plan 24 commits, 6 without a message +export unfinished — 18 group(s) committed by this tool, 6 to go + committed: 1-18 + remaining: 19-24 + +Run `export` again; it continues from where it stopped. +``` + +The record is written after **each** commit, not at the end — the point is to +survive the run not reaching the end. It stores the plan's fingerprint (the +groups and their paths, never the messages, so rewording commit 20 does not +invalidate the nineteen already made) and each commit's sha, which must still +be where the branch tip is or the copy has moved and it says so. + +## Keeping a history that already exists + +```bash +histgen export --keep-history +``` + +Carries the source's `.git` into the copy and commits the designed account to an +orphan branch, leaving the original branch pointing exactly where it did. Two +tiers, which is what `all/ctrl/handover.sh` has been saying all along: + +``` +main o-o-o-o "updates 33.1 139" (untouched) +designed-history o-o-o-o-o-o-o-o the designed account (no shared parent) +``` + +Both are present in `out//`; which one to publish is a decision for later +and by hand. Nothing is rewritten and the source is not touched either way. + +## Settings + +```bash +make init-config SOURCE=~/work/thing OUT=~/histories/thing +make config # what everything resolves to, and where it came from +make run # no arguments +``` + +Writes `histgen.json` beside the tool — the arrangement `ppl/ctrl/distill.sh` +already uses, where the JSON next to the script is picked up when nothing else +says otherwise. + +```json +{ + "source": "~/work/thing", + "out": "~/histories/thing", + "max_files": null, + "keep_history": false, + "branch": null +} +``` + +Precedence, most specific first: + +``` +the command line -> --config FILE -> histgen.json beside the tool -> defaults +``` + +so a config sets a starting point and never wins an argument with a flag typed +deliberately. A mistyped key is refused rather than ignored, because a setting +plainly written in the file and silently not applied is a bad thing to debug. +`out` inside `source` is refused too — the source is read-only by design, and +`out` would end up in its own census. + +`histgen.json` is gitignored by the folder's own `.gitignore`: which tree this +machine points at is not a fact about the tool. + +## The verbs + +Each writes one file under the repo's `.histgen/`, so the step before it is +never repeated. + +| | | | +|---|---|---| +| `scan` | `out/index.json` | what is in the source, cached by content hash | +| `plan` | `out/plan.json` | the order, cut into commits | +| `brief` | `out/briefs/*.md` | one pack per commit, for the messages | +| `copy` | `out//` | the files alone, no repo — needs no plan | +| `list` | — | the commits, printed to confirm | +| `export` | `out//` | the copy and its history, with both guards | +| `status` | — | which of the four states `out` is in | +| `verify` | — | the guards, on their own | + +`plan.json` is the seam. Everything above it is analysis that can be recomputed +from the tree; everything below is git commands. That split is the whole design: +the expensive half is a model reading code, and it should run once. + +## Where the messages come from + +`brief` writes a markdown pack per commit holding each file's **opening +comment** — not the file. An agent reads the packs and writes `title` and +`body` back into `plan.json`. + +That is deliberate. A commit message reconstructed from a diff restates the +diff, and the thing worth recording was never in the diff: it was in the comment +explaining why the ignore rules exist before the code they exclude, or why a +port offset has to mean the same thing in two different projects. Handing over +the reasoning that is already written down produces a message worth reading; +handing over the diff produces `Update files`. + +Keeping the model outside the tool is also what keeps the tool offline, keeps +every message editable before a single commit exists, and keeps the cost of a +500-file repo to the comments rather than the code. + +## The order + +Role first, references second. + +``` +skeleton (.gitignore) -> README -> version pins -> config layer -> profiles + -> templates -> the things that source them -> front door (Makefile) LATE + -> the bootstrap account LAST +``` + +The front door is late because it only dispatches; the bootstrap account is last +because it narrates everything above it. References refine within that, so a +config lands before the script that sources it. + +**References never override roles.** A reference in code is a dependency; a +reference in a comment is a footnote. `.gitignore` names `ctrl/wizard.sh` to say +the opposite of "I need this", and a README names every file in the repo. +Counting those as edges commits the ignore rules after the code they exclude — +consistent, and unreadable. So refs are taken from non-comment lines only, and +narrative files (`.gitignore`, README, docs, BOOTSTRAP) contribute no outgoing +edges at all. + +## The grouping is a proposal + +One coherent idea per commit, not one directory per commit. What holds a group +together is that its files name each other; a hub and the directory named after +it (`addons.sh` and `addons/`) always travel together, because committing a +loader without the things it loads produces a commit that cannot run. + +Where it cannot know — five scripts in one directory that never mention each +other are five ideas or one, and nothing in the text says which — it guesses and +says so. **Moving a path from one group to another in `plan.json` is the +expected way to use this**, and `plan` re-run afterwards keeps every message +whose group still holds the same files. + +`--max-files` sets the cap. The default is about a twentieth of the tree with a +floor of eight, which lands near how these repos were actually built — rig plans +18 against a real 18, spr 77 against a real 78. Raising it gives fewer, larger +commits; lowering it gives more. + +## The two guards + +`export` refuses a plan whose paths are missing, duplicated, or do not cover the +source — before it writes anything. After the last commit it checks both: + +1. **nothing left untracked**, with nothing exempt. The state lives in `out` + and the copy lives inside it, so there is genuinely nothing of ours in the + tree being checked. A file silently missed is the failure this whole tool + exists to prevent. It is quiet at the time and surfaces much later, when + something does not build on a fresh clone and the history offers no clue + which commit should have carried it. +2. **the exported tree still matches the source**, by tree hash. Every path + committed is not the same claim as the same tree: a stale index, a path in + two groups, or a file edited mid-plan all pass the first check and fail this + one. + +`--dry-run` writes `out/regen.sh` and `out/messages/` instead — ordinary git +commands that copy the files and make the commits, both guards included, +reviewable before anything runs. + +## Reporting on a history that already exists + +```bash +python -m station.tools.histgen plan /path/to/repo --against-history +``` + +Maps each of the source's existing commits onto the group holding most of the files it touched, +then reports what agrees and what does not: + +``` + = 01 skeleton a127b1d matched one commit + ~ 03 ctrl split across 2 one idea, committed piecemeal + + 04 ctrl-lib no commit never landed as its own change + ! 4 commit(s) land earlier in the proposed order than work already done + ? 31 commit(s) carry no usable account of the change ("updates 33.1 139") +``` + +It reads and prints. It never rewrites: published history is someone else's +clone. + +## Verified against + +`rig`'s 18-commit history, which was built by hand and is what this reproduces. +Run over the same 64 files, histgen plans 18 commits; the profiles, the cluster +templates, the addons hub, the k8s manifests, the Makefile, `sample-rig` and +`BOOTSTRAP.md` all land as their own commits in the same places. Replaying it +produces a tree hash identical to the one rig ships. + +Where it differs is where the difference is semantic: rig splits its wizard, +host checks, cluster lifecycle and registry into four commits, and nothing in +those four files' text says they are four things. + +## The CLI shape + +`cli.py` is a shared scaffold — subcommand registration, one spelling for +`--source/-o/-n/--force/--dry-run`, `Error: … -> stderr -> exit 1` as the only +exit path, deferred heavy imports so `--help` stays instant, and +`refuse_to_clobber`. It exists because every tool here grew its own slightly +different copy of the same three things. + +histgen is its first user. Nothing else was rewritten to use it: a scaffold +earns adoption by being there when the next tool is written. diff --git a/soleprint/station/tools/histgen/__init__.py b/soleprint/station/tools/histgen/__init__.py new file mode 100644 index 0000000..ef9de7e --- /dev/null +++ b/soleprint/station/tools/histgen/__init__.py @@ -0,0 +1,27 @@ +""" +Histgen — seed a clean, logical history into a repo. + +The general case is a tree with no git at all: read the code, then commit it in +parts, ordered so each commit stands on what came before. Done by hand it means +uploading everything and asking a model; done twice it stops being worth the +time. + +The expensive half (reading the code) and the mechanical half (applying a plan) +are split on purpose, and `plan.json` is the seam. Everything before it is +analysis and can be cached; everything after it is git commands that fail fast. + + python -m station.tools.histgen scan /path/to/repo + python -m station.tools.histgen plan /path/to/repo + python -m station.tools.histgen brief /path/to/repo + # an agent reads briefs/ and writes title+body back into plan.json + python -m station.tools.histgen apply /path/to/repo --dry-run + python -m station.tools.histgen verify /path/to/repo + +Stdlib only, no network. The directory can be copied out of soleprint and run +on its own — a repo that needs a history seeded is, by definition, not one that +already has this framework on its path. +""" + +__version__ = "0.1.0" + +__all__ = ["census", "order", "brief", "snapshot", "export"] diff --git a/soleprint/station/tools/histgen/__main__.py b/soleprint/station/tools/histgen/__main__.py new file mode 100644 index 0000000..a879928 --- /dev/null +++ b/soleprint/station/tools/histgen/__main__.py @@ -0,0 +1,318 @@ +""" +Histgen CLI — seed a clean, logical history into a repo. + + python -m station.tools.histgen run --source ~/work/thing --out ~/out + python -m station.tools.histgen list # the commits, to confirm + python -m station.tools.histgen export # write them into out/thing + +Two directories and one rule: the source is read-only, everything is written +under out. Set both once with `config --init` and the verbs take no arguments. + +Run from the soleprint/ directory so `station.tools...` resolves, or copy the +folder out and use its Makefile. +""" + +import sys +from pathlib import Path + +from .cli import Tool, fail + +VERBS = ( + ("copy", "Copy the files out, with no .git and nothing private."), + ("scan", "Read the source and cache what was read."), + ("plan", "Order the files and cut them into commits."), + ("list", "Print the commits, to confirm before exporting."), + ("brief", "Write one brief per commit, for the messages."), + ("export", "Copy the source into out and commit the history."), + ("verify", "Nothing left untracked, and the tree still matches."), + ("status", "What is in the out directory, and what is left to do."), + ("run", "scan + plan + brief."), + ("config", "Show the resolved settings, or write a starter file."), +) + + +def _settings(args, need_out=True): + """ + Where to read and where to write, with the command line on top. + + Resolved once per invocation and passed down, rather than each module + working it out again — two places deciding where things live is how `scan` + and `plan` end up disagreeing about it. + """ + from . import config + s = config.resolve(args, getattr(args, "config", None)) + + if not s["source"]: + fail("No source given.", + f"Pass --source, or set it in {config.default_path()} " + "(see `histgen config --init`).") + if not s["source"].is_dir(): + fail(f"Not a directory: {s['source']}") + + if need_out and not s["out"]: + fail("No out directory given.", + "Pass --out, or set it in the config. It is where the plan and " + "the exported repo go; the source is never written to.") + if s["out"]: + # The source must stay clean, and an out inside it would be scanned as + # part of the tree it describes on the very next run. + try: + if s["out"] == s["source"] or s["out"].is_relative_to(s["source"]): + fail("out is inside source.", + "The source is read-only by design, and out would end up " + "in its own census. Put out somewhere else.") + except (OSError, ValueError): + pass + return s + + +def _index(out): + """The census, which must already exist — scanning is its own verb.""" + from . import census + index = census.load_index(out) + if not index.get("files"): + fail(f"No census at {census.state_path(out)}.", "Run `scan` first.") + return index + + +def cmd_copy(args): + """The plain utility: files out, repo and private things left behind.""" + from . import snapshot + s = _settings(args) + snapshot.take(s["source"], s["out"], + keep_noise=args.all, keep_secrets=s["keep_secrets"], + max_bytes=args.max_bytes, exclude=s["exclude"], + include=s["include"], force=args.force, dry_run=args.dry_run) + + +def cmd_scan(args): + from . import census + s = _settings(args) + census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"], + exclude=s["exclude"], include=s["include"]) + + +def cmd_plan(args): + from . import order + s = _settings(args) + plan = order.build_plan(_index(s["out"]), s["out"], max_files=s["max_files"]) + if args.against_history: + from . import history + print() + history.compare(s["source"], plan) + + +def cmd_list(args): + """ + Print the commits as a numbered list, which is the thing to confirm. + + Deliberately the plainest output here: a number, a title, and the files + under it. Deciding whether a commit is one idea is done by reading it, and + anything else on the line is in the way. + """ + from . import export as exporter + s = _settings(args) + plan = exporter.load_plan(s["out"]) + files = _index(s["out"])["files"] if args.roles else {} + for g in plan["groups"]: + title = (g.get("title") or "").strip() + mark = " " if title else "*" + print(f"{mark}{g['n']:3}. {title or g['slug'] + ' (no message yet)'}") + for p in g["paths"]: + role = f" [{files[p]['role']}]" if args.roles and p in files else "" + print(f" {p}{role}") + untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()] + print(f"\n{len(plan['groups'])} commits, " + f"{sum(len(g['paths']) for g in plan['groups'])} files.") + if untitled: + print(f"* {len(untitled)} still without a message — read " + f"{s['out']}/briefs/ and write them into {s['out']}/plan.json.") + + +def cmd_brief(args): + from . import brief, order + s = _settings(args) + if not order.plan_path(s["out"]).exists(): + fail(f"No plan at {order.plan_path(s['out'])}.", "Run `plan` first.") + brief.write_briefs(s["out"]) + + +def cmd_export(args): + from . import export as exporter + s = _settings(args) + exporter.export(s["source"], s["out"], dry_run=args.dry_run, + commands=args.commands, allow_untitled=args.allow_untitled, + keep_history=s["keep_history"], branch=s["branch"], + force=args.force) + + +def cmd_verify(args): + from . import export as exporter + s = _settings(args) + plan = exporter.load_plan(s["out"]) + planned = [p for g in plan["groups"] for p in g["paths"]] + copy = exporter.repo_dir(s["source"], s["out"]) + if not copy.is_dir(): + fail(f"Nothing exported at {copy}.", "Run `export` first.") + print("Checking:", flush=True) + if not exporter.verify(copy, exporter.source_tree_hash(s["source"], planned)): + sys.exit(1) + + +def cmd_status(args): + """What is in out, what state it is in, and what is left.""" + from . import census, export as exporter, order + s = _settings(args) + + index = census.load_index(s["out"]) + plan = None + if order.plan_path(s["out"]).exists(): + plan = exporter.load_plan(s["out"]) + + print(f"source {s['source']}") + print(f"out {s['out']}") + if index.get("files"): + left = index.get("left_out", []) + print(f"census {len(index['files'])} files" + + (f", {len(left)} left out" if left else "")) + for item in left: + print(f" left out: {item['path']} ({item['why']})") + else: + print("census none — run `scan`") + if plan: + untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()] + print(f"plan {len(plan['groups'])} commits" + + (f", {len(untitled)} without a message" if untitled else + ", all messages written")) + else: + print("plan none — run `plan`") + + report = exporter.inspect(s["source"], s["out"], plan) + print(f"export {report['state']} — {report['detail']}") + if report["done"]: + print(f" committed: {_ranges(report['done'])}") + if report["remaining"]: + print(f" remaining: {_ranges(report['remaining'])}") + + advice = { + "absent": "Run `export`.", + "unfinished": "Run `export` again; it continues from where it stopped.", + "foreign": "Run `export`; the designed history goes on its own branch " + "and nothing existing is touched.", + "stale": "Run `export --force` to discard and redo, or point --out elsewhere.", + "complete": "Nothing to do.", + } + print(f"\n{advice.get(report['state'], '')}") + + +def _ranges(numbers): + """[1,2,3,7,8] -> '1-3, 7-8'. A list of eighteen numbers is unreadable.""" + if not numbers: + return "none" + out, start, previous = [], numbers[0], numbers[0] + for n in numbers[1:] + [None]: + if n == previous + 1: + previous = n + continue + out.append(str(start) if start == previous else f"{start}-{previous}") + start = previous = n + return ", ".join(out) + + +def cmd_run(args): + from . import brief, census, order + s = _settings(args) + index = census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"], + exclude=s["exclude"], include=s["include"]) + order.build_plan(index, s["out"], max_files=s["max_files"]) + brief.write_briefs(s["out"]) + + +def cmd_config(args): + from . import config + if args.init: + path = config.write_template( + Path(args.config).expanduser() if args.config else config.default_path(), + source=args.source, out=args.out) + print(f"Wrote {path}. Edit \"source\" and \"out\".") + return + s = config.resolve(args, args.config) + print(f"config {s['config_path'] or '(none; using defaults)'}") + print(f"source {s['source'] or '(unset)'} read-only") + print(f"out {s['out'] or '(unset)'}") + if s["source"] and s["out"]: + from .export import repo_dir + print(f"exported to {repo_dir(s['source'], s['out'])}") + print(f"max-files {s['max_files'] or '(scales with the source)'}") + print(f"keep-history {s['keep_history']}") + print(f"keep-secrets {s['keep_secrets']}" + + ("" if s["keep_secrets"] else " keys and credentials are left out")) + if s["exclude"]: + print(f"exclude {', '.join(s['exclude'])}") + if s["include"]: + print(f"include {', '.join(s['include'])}") + + +def main(argv=None): + tool = Tool("histgen", __doc__, package=__package__) + handlers = {name: globals()[f"cmd_{name}"] for name, _ in VERBS} + + for verb, help_text in VERBS: + tool.command(verb, handlers[verb], help_text) + tool.argument(verb, "--source", "-s", default=None, metavar="DIR", + help="The tree to read. Never written to.") + tool.argument(verb, "--out", "-o", default=None, metavar="DIR", + help="Where the plan and the exported repo go.") + tool.argument(verb, "--config", "-c", default=None, metavar="FILE", + help="Settings file, instead of histgen.json beside the tool.") + + for verb in ("plan", "run"): + tool.argument(verb, "--max-files", type=int, default=None, metavar="N", + help="Files per commit before a group is cut. Default " + "scales with the source (about N/20, floor 8).") + tool.argument("plan", "--against-history", action="store_true", + help="Also report how the source's existing history compares.") + tool.argument("list", "--roles", action="store_true", + help="Show each file's detected role.") + + # Whatever reads the source can filter it, so the same three answers hold + # for a snapshot and for a history. Keeping them on `copy` alone was how a + # tracked key stayed out of one and went straight into the other. + for verb in ("copy", "scan", "run"): + tool.argument(verb, "--keep-secrets", action="store_true", + help="Keep files that look like keys or credentials.") + tool.argument(verb, "--exclude", action="append", default=[], metavar="GLOB", + help="Leave these out. Repeatable; a pattern with no / " + "matches basenames at any depth.") + tool.argument(verb, "--include", action="append", default=[], metavar="GLOB", + help="Keep these whatever the filters say. Repeatable.") + + tool.common("copy", "dry_run") + tool.argument("copy", "--all", action="store_true", + help="Keep what a build regenerates too: lockfiles, maps, " + "minified and compiled output.") + tool.argument("copy", "--max-bytes", type=int, default=None, metavar="N", + help="Leave behind anything larger.") + tool.argument("copy", "--force", action="store_true", + help="Write into a destination that is not empty.") + tool.common("export", "dry_run") + tool.argument("export", "--commands", action="store_true", + help="Copy the files, create no repo, and print the git add " + "and git commit commands to run yourself.") + tool.argument("export", "--allow-untitled", action="store_true", + help="Commit groups whose message was never written.") + tool.argument("export", "--keep-history", action="store_true", + help="Carry the source's existing history into the copy, and " + "put the designed one on its own branch.") + tool.argument("export", "--branch", default=None, metavar="NAME", + help="Branch for the designed history when one is kept.") + tool.argument("export", "--force", action="store_true", + help="Discard an out directory that no longer matches the plan.") + tool.argument("config", "--init", action="store_true", + help="Write a starter config file rather than reading one.") + + tool.run(argv) + + +if __name__ == "__main__": + main() diff --git a/soleprint/station/tools/histgen/api.py b/soleprint/station/tools/histgen/api.py new file mode 100644 index 0000000..078e99e --- /dev/null +++ b/soleprint/station/tools/histgen/api.py @@ -0,0 +1,98 @@ +""" +Histgen's HTTP surface. + +Read-only on purpose. The CLI seeds histories; this shows what a plan looks +like before anyone runs it, because the interesting failure — a group that +holds two unrelated ideas — is one you see by reading, not by testing. + +Nothing here writes commits. A browser tab is the wrong place to decide that a +repository's history is about to be rebuilt. +""" + +import logging +from pathlib import Path + +from fastapi import APIRouter, HTTPException +from fastapi.responses import HTMLResponse + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/tools/histgen", tags=["histgen"]) + +SPR_ROOT = Path(__file__).parents[3] +HERE = Path(__file__).parent + + +# ───────────────────────────────────────────────────────────────────────────── +# Pages +# ───────────────────────────────────────────────────────────────────────────── + +@router.get("/", response_class=HTMLResponse) +def index(): + template = HERE / "templates" / "index.html" + if template.exists(): + return template.read_text() + return "

    histgen

    " + + +@router.get("/health") +def health(): + return {"status": "ok", "tool": "histgen"} + + +# ───────────────────────────────────────────────────────────────────────────── +# API +# ───────────────────────────────────────────────────────────────────────────── + +def _resolve(out: str) -> Path: + """ + An out directory, kept inside the tree this instance was built from. + + The parameter is a filesystem path from a query string, so it is the one + input here worth distrusting: without the containment check, `../..` reads + any directory the server can. + """ + target = Path(out).resolve() if Path(out).is_absolute() else (SPR_ROOT.parent / out).resolve() + root = SPR_ROOT.parent.resolve() + if root not in target.parents and target != root: + raise HTTPException(400, f"Outside the tree: {out}") + if not target.is_dir(): + raise HTTPException(404, f"Not a directory: {out}") + return target + + +@router.get("/api/plan") +def get_plan(out: str): + """The plan as it stands, with each group's files and message.""" + import json + + from .order import plan_path + + path = plan_path(_resolve(out)) + if not path.exists(): + raise HTTPException(404, "No plan yet. Run `histgen run`.") + plan = json.loads(path.read_text()) + return { + "groups": plan["groups"], + "commits": len(plan["groups"]), + "files": sum(len(g["paths"]) for g in plan["groups"]), + "untitled": [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()], + } + + +@router.get("/api/census") +def get_census(out: str): + """What the scan found, without the per-file detail.""" + from collections import Counter + + from .census import load_index + + index = load_index(_resolve(out)) + if not index.get("files"): + raise HTTPException(404, "No census yet. Run `scan` first.") + files = index["files"] + return { + "files": len(files), + "roles": dict(Counter(f["role"] for f in files.values())), + "edges": sum(len(f["refs"]) for f in files.values()), + } diff --git a/soleprint/station/tools/histgen/brief.py b/soleprint/station/tools/histgen/brief.py new file mode 100644 index 0000000..79b06a8 --- /dev/null +++ b/soleprint/station/tools/histgen/brief.py @@ -0,0 +1,111 @@ +""" +One pack per commit, for whoever writes the message. + +This is the interface to the expensive reader, and it is a directory of +markdown rather than a network call. The tool does not know how to reach a +model and does not want to: an agent already in this repo reads the briefs and +writes titles and bodies back into plan.json, which keeps the messages editable +before a single commit exists and keeps an API key out of a tool that otherwise +runs offline. + +What travels is the reasoning already in the code — each file's opening +comment — and not the file. That is both what makes the pack cheap and what +makes the message right: a commit message reconstructed from a diff restates +the diff, and the thing worth recording was never in the diff. It was in the +comment explaining why the port offsets have to match another project's, or why +the ignore rules exist before the code they exclude. +""" + +import json + +from .census import INDEX_FILE, state_dir +from .order import plan_path + +BRIEF_DIR = "briefs" + +WHY_CHARS = 700 # an opening comment past this is an essay; the head carries it +MAX_LISTED = 40 + + +HEADER = """# {n:02d} — {slug} + +**{count} file(s), commit {n} of {total}.** + +Write a title and a body for this commit, then put them in +`{plan}` under group {n} as `"title"` and `"body"`. + +- The title says what this commit establishes, in the repo's own words. +- The body carries the **why** — take it from the reasoning already in the + comments below. Do not restate the diff; the diff is already in the commit. +- If a file below does not belong in this commit, move its path to another + group in plan.json. The grouping is a proposal. +""" + + +def _fmt_why(text): + text = (text or "").strip() + if not text: + return "_(no opening comment)_" + if len(text) > WHY_CHARS: + text = text[:WHY_CHARS].rsplit("\n", 1)[0] + "\n…" + return "\n".join("> " + line if line.strip() else ">" for line in text.split("\n")) + + +def write_briefs(out, quiet=False): + state = state_dir(out) + plan = json.loads(plan_path(out).read_text()) + index = json.loads((state / INDEX_FILE).read_text()) + files = index["files"] + groups = plan["groups"] + + # Which commit each path lands in, so a dependency can be named by the + # commit that introduced it rather than by a bare path. "stands on 04" is + # the sentence the ordering exists to make true. + landed = {p: g["n"] for g in groups for p in g["paths"]} + + out_dir = state / BRIEF_DIR + out_dir.mkdir(parents=True, exist_ok=True) + for stale in out_dir.glob("*.md"): + stale.unlink() + + for g in groups: + lines = [HEADER.format(n=g["n"], slug=g["slug"], count=len(g["paths"]), + total=len(groups), plan=plan_path(out))] + + earlier, later = {}, set() + for p in g["paths"]: + for dep in files.get(p, {}).get("refs", []): + n = landed.get(dep) + if n is None or dep in g["paths"]: + continue + (earlier.setdefault(n, set()).add(dep) if n < g["n"] else later.add(dep)) + + if earlier: + lines.append("\n## Stands on\n") + for n in sorted(earlier): + names = ", ".join(f"`{d}`" for d in sorted(earlier[n])[:MAX_LISTED]) + lines.append(f"- commit {n:02d}: {names}") + if later: + # Worth stating plainly rather than hiding: it is the one thing a + # reader of the finished history would notice and the tool cannot + # fix, because the fix is a judgement about which comes first. + names = ", ".join(f"`{d}`" for d in sorted(later)[:MAX_LISTED]) + lines.append("\n## Forward references (this commit names things not yet committed)\n") + lines.append(f"- {names}") + + lines.append("\n## Files\n") + for p in g["paths"]: + e = files.get(p, {}) + meta = f"{e.get('role', '?')}, {e.get('lines', 0)} lines" + if e.get("binary"): + meta += ", binary" + lines.append(f"\n### `{p}`\n\n_{meta}_\n") + lines.append(_fmt_why(e.get("why"))) + + path = out_dir / f"{g['n']:02d}-{g['slug']}.md" + path.write_text("\n".join(lines) + "\n") + + if not quiet: + print(f"Wrote {len(groups)} briefs -> {out_dir}") + print(f"Read them, then write title and body into {plan_path(out)}.") + return out_dir diff --git a/soleprint/station/tools/histgen/census.py b/soleprint/station/tools/histgen/census.py new file mode 100644 index 0000000..7f94b7d --- /dev/null +++ b/soleprint/station/tools/histgen/census.py @@ -0,0 +1,456 @@ +""" +What is in the tree, and what each file says about itself. + +This is the expensive half. Walking is cheap; reading is not, so what gets read +is cached by content hash and a second run costs only the files that changed. +The cache lives beside the output, in the repo's own .histgen/, because a +destination that carries its own state cannot be confused with another one's. + +Two things are extracted from every file, and both are used twice: + + the opening comment why the file exists, in the author's words. `order` + does not read it; `brief` hands it to whoever writes + the commit message, because that reasoning is the + message. A commit that restates its own diff is noise. + + path references which other files this one names. `order` turns them + into edges, so a config lands before the script that + sources it. `brief` reports them as the seam between + one commit and the last. +""" + +import hashlib +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path + +STATE_DIR = ".histgen" +INDEX_FILE = "index.json" + +# Read caps. A file's opening comment is at the top by definition, and nothing +# below the cap has ever been the reason a file exists. The byte cap is what +# keeps a vendored 2 MB bundle from being tokenised for no reason. +HEAD_LINES = 60 +MAX_BYTES = 400_000 + + +# ── the file set ─────────────────────────────────────────────────────────── +# +# Never parse .gitignore. It has negations, directory semantics, precedence +# across nested files and a global excludes file, and a hand-rolled parser that +# gets 95% of that right is worse than none: it is wrong silently, on exactly +# the files someone took care to exclude. Ask git, which is always installed +# here because the output of this tool is a git repository. + +def _git(args, **kw): + return subprocess.run(["git", *args], capture_output=True, text=True, **kw) + + +def is_git(path: Path) -> bool: + r = _git(["-C", str(path), "rev-parse", "--git-dir"]) + return r.returncode == 0 + + +def file_set(source: Path): + """ + The paths a history would contain, relative to the source, sorted. + + A source with git is asked what it tracks — the same question handover.sh + asks, and for the same reason: a hand-maintained list drifts, and the drift + shows up as a file that silently never got committed. + + A source with NO git is the general case, and the interesting one. Rather + than reimplementing the ignore rules, git is pointed at the tree with its + own directory kept in a temporary path: `ls-files -o --exclude-standard` + then means untracked-and-not-ignored, which is exactly the candidate set. + Nothing is written inside the tree, so a dry run leaves no .git behind to + explain later. + """ + if is_git(source): + listed = _git(["-C", str(source), "ls-files", "-z"]).stdout + paths = [p for p in listed.split("\0") if p] + # A tracked file that has been deleted but not committed is still in + # ls-files. It would abort `apply` partway through with a missing path, + # so drop it here and let `verify` be the thing that complains. + return sorted(p for p in paths if (source / p).is_file()) + + with tempfile.TemporaryDirectory(prefix="histgen-git-") as tmp: + env = dict(os.environ, GIT_DIR=str(Path(tmp) / "git"), GIT_WORK_TREE=str(source)) + subprocess.run(["git", "init", "-q"], env=env, capture_output=True) + listed = subprocess.run( + ["git", "ls-files", "-o", "--exclude-standard", "-z"], + env=env, capture_output=True, text=True, + ).stdout + return sorted(p for p in listed.split("\0") + if p and (source / p).is_file()) + + +# ── roles ────────────────────────────────────────────────────────────────── +# +# A file's role is what decides where it lands when nothing references it, and +# most files reference nothing. The ranks are the ordering heuristic itself, +# read bottom-up off rig's log: ignore rules and README first, then the config +# layer, then the things that source it, the front door late because it only +# dispatches, and the bootstrap account last because it narrates the rest. + +ROLE_RANK = { + "skeleton": 0, # .gitignore, .gitattributes — the rules before the files + "readme": 10, # what this is, and the one prerequisite + "pin": 20, # versions, dependency manifests + "config": 30, # the layer everything else reads + "profile": 40, # named variants of that config + "template": 50, # shapes rendered later + "source": 60, # the work + "test": 70, + "asset": 76, + "lock": 78, # generated from a pin; never interesting, never first + "doc": 80, + "frontdoor": 90, # Makefile, Tiltfile — a dispatcher, so it comes after + "bootstrap": 100, # BOOTSTRAP/INSTALL — the account of everything above +} + +_SKELETON = {".gitignore", ".gitattributes", ".editorconfig", ".dockerignore", + "license", "license.md", "license.txt", "copying", "notice"} +_FRONTDOOR = {"makefile", "gnumakefile", "tiltfile", "justfile", "taskfile.yml", + "dockerfile", "docker-compose.yml", "docker-compose.yaml"} +_LOCK = {"package-lock.json", "poetry.lock", "pnpm-lock.yaml", "yarn.lock", + "cargo.lock", "go.sum", "composer.lock", "gemfile.lock", "uv.lock"} +_PIN = {"requirements.txt", "pyproject.toml", "package.json", "go.mod", + "cargo.toml", "gemfile", "versions.env", "setup.py", "setup.cfg"} +_ASSET_EXT = {".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".webp", ".pdf", + ".woff", ".woff2", ".ttf", ".eot", ".mp4", ".zip", ".ods", ".xlsx", + ".bundle", ".so", ".dylib", ".dll", ".wasm"} +_DOC_EXT = {".md", ".rst", ".txt", ".adoc"} + + +def ignored_but_tracked(source: Path, paths): + """ + Files git tracks that the ignore rules say it should not. + + `git add -f` is how they get there, and it is not always a mistake — a + built artifact committed on purpose looks exactly like this. But so does a + dump, a credentials file or a data directory that someone forced in once + and nobody noticed since, and those are the ones worth catching. + + `--no-index` is the whole trick: without it check-ignore stays quiet about + anything already tracked, which is precisely the set being asked about. + """ + if not paths or not is_git(source): + return set() + r = subprocess.run( + ["git", "-C", str(source), "check-ignore", "--no-index", "--stdin", "-z"], + input="\0".join(paths) + "\0", text=True, capture_output=True) + # 0 = some matched, 1 = none matched, anything else is a real failure and + # not a reason to refuse to copy. + if r.returncode not in (0, 1): + return set() + return {p for p in r.stdout.split("\0") if p} + + +def role_of(path: str) -> str: + p = Path(path) + name, low = p.name, p.name.lower() + parts = [s.lower() for s in p.parts] + stem = p.stem.lower() + + if low in _SKELETON: + return "skeleton" + if low in _LOCK: + return "lock" + if low in _FRONTDOOR or low.startswith("dockerfile"): + return "frontdoor" + if stem in ("bootstrap", "install", "installing", "getting-started", "quickstart"): + return "bootstrap" + if stem == "readme": + # Only the repo's own README opens the history. A README inside a + # subdirectory documents that subdirectory and travels with it. + return "readme" if len(p.parts) == 1 else "doc" + if low in _PIN or low.endswith(".lock"): + return "lock" if low.endswith(".lock") else "pin" + if p.suffix.lower() in _ASSET_EXT: + return "asset" + if "test" in parts or "tests" in parts or stem.startswith("test_") or stem.endswith("_test"): + return "test" + if p.suffix in (".tpl", ".tmpl", ".j2", ".mustache") or low.endswith((".yaml.tpl", ".tmpl")): + return "template" + if "templates" in parts: + return "template" + # A profile is a named variant sitting in a directory of siblings: env.d/, + # profiles/, overlays/. The directory is the signal, not the extension. + if any(d in parts for d in ("env.d", "profiles", "environments")): + return "profile" + if stem in ("config", "settings", "conf", "defaults") or low in (".env.example", "env.example"): + return "config" + if low.endswith(".env") or low.endswith(".env.example"): + return "profile" + if p.suffix.lower() in _DOC_EXT or "docs" in parts or "doc" in parts: + return "doc" + return "source" + + +# ── what a file says about itself ────────────────────────────────────────── + +_COMMENT = { + "#": (".sh", ".bash", ".py", ".yaml", ".yml", ".toml", ".env", ".cfg", ".conf", ".tf", ""), + "//": (".js", ".ts", ".jsx", ".tsx", ".go", ".java", ".c", ".h", ".cpp", ".rs", ".scala"), +} + + +def opening_comment(text: str, path: str) -> str: + """ + The comment block at the top of the file, or the module docstring. + + The shebang and any editor modeline are skipped — they are not prose. The + block ends at the first line that is not a comment, which is what makes it + the file's own statement of purpose rather than a running commentary. + """ + lines = text.split("\n")[:HEAD_LINES] + i = 0 + while i < len(lines) and ( + lines[i].startswith("#!") or not lines[i].strip() + or lines[i].lstrip().startswith(("# -*-", "# vim:", "# shellcheck")) + ): + i += 1 + + # A docstring: take it whole, it is the same statement in another syntax. + rest = "\n".join(lines[i:]).lstrip() + for quote in ('"""', "'''"): + if rest.startswith(quote): + end = rest.find(quote, len(quote)) + if end != -1: + return rest[len(quote):end].strip() + + suffix = Path(path).suffix.lower() + markers = [m for m, exts in _COMMENT.items() if suffix in exts] or ["#"] + block = [] + for line in lines[i:]: + stripped = line.strip() + if not any(stripped.startswith(m) for m in markers): + break + for m in markers: + if stripped.startswith(m): + block.append(stripped[len(m):].strip()) + break + return "\n".join(block).strip() + + +_TOKEN = re.compile(r"[A-Za-z0-9_./+-]{4,}") + +# What opens a comment, by language. Used to tell a reference apart from a +# mention, which is the difference between an edge and a footnote. +_COMMENT_PREFIX = ("#", "//", "--", ";", "*", "/*") + + +def _strip_comment(line: str) -> str: + stripped = line.strip() + if stripped.startswith(_COMMENT_PREFIX): + return "" + # A trailing comment on a real line: keep the code, drop the aside. + for marker in (" #", " //"): + cut = line.find(marker) + if cut != -1: + line = line[:cut] + return line + + +def referenced_paths(text: str, path: str, by_path, by_base): + """ + Which other files in this repo this one names, split by how it names them. + + Deliberately textual rather than per-language. A shell `source + "$DIR/lib/config.sh"`, a Makefile's `ctrl/cluster.sh`, a kustomization's + `- namespace.yaml` and a Dockerfile's `COPY run.py .` are all the same fact + — this file needs that one — and four parsers would find it four ways and + disagree at the edges. + + The split matters more than the matching does. A reference in code is a + dependency: config.sh has to exist before the script that sources it. A + reference in a comment is a footnote — .gitignore names `ctrl/wizard.sh` + to say the opposite of "I need this", and README names every file in the + repo. Counting those as edges puts the ignore rules after the code they + exclude, which is exactly backwards. So prose informs the brief and never + the order. + + Bare filenames only count when they are unique in the repo, and only with + their extension. Without both, every `config.py` in a tree of them becomes + an edge to all the others and the ordering collapses into one cycle. + """ + def hits(blob): + found = set() + for token in set(_TOKEN.findall(blob)): + token = token.strip("./") + if not token: + continue + if token in by_path: + found.update(by_path[token]) + continue + # A bare name is only a reference if it carries an extension and + # names exactly one file. "cluster" is a word; "cluster.sh" is not. + if "." in token: + candidates = by_base.get(token) + if candidates and len(candidates) == 1: + found.update(candidates) + found.discard(path) + return found + + lines = text.split("\n") + code = "\n".join(_strip_comment(l) for l in lines) + strong = hits(code) + mentions = hits(text) - strong + return sorted(strong), sorted(mentions) + + +def read_text(full: Path): + """Text, or None if this is not text. Size is checked before reading.""" + try: + if full.stat().st_size > MAX_BYTES: + return None + raw = full.read_bytes() + except OSError: + return None + if b"\0" in raw[:8000]: + return None + return raw.decode("utf-8", errors="replace") + + +# ── the index ────────────────────────────────────────────────────────────── + +def _digest(full: Path) -> str: + h = hashlib.sha256() + with full.open("rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest()[:16] + + +def state_dir(out) -> Path: + """ + Where this run's index, plan, briefs and messages live. + + Always `out`, never the source. The source is opened read-only and nothing + is written into it — which is what removed a whole class of special cases + that used to be here: skipping the state directory during its own census, + exempting it from `git status`, and writing a .git/info/exclude entry to + keep it quiet. None of that has anywhere to happen now. + """ + return Path(out) + + +def state_path(out) -> Path: + return state_dir(out) / INDEX_FILE + + +def current_file_set(source: Path, index) -> set: + """ + What the source holds right now, filtered the way the census was. + + Recomputed rather than read back, because the guard it feeds exists to + catch a file added after the scan. Reading the stored list would answer the + easy question — "did the plan cover what we saw?" — instead of the one + worth asking, which is "does the plan cover what is there?". + """ + from .sift import sift + settings = (index or {}).get("filter", {}) + tracked = file_set(source) + kept, _ = sift(source, tracked, + keep_noise=True, + keep_secrets=settings.get("keep_secrets", False), + exclude=settings.get("exclude", ()), + include=settings.get("include", ()), + ignored=ignored_but_tracked(source, tracked)) + return set(kept) + + +def load_index(out) -> dict: + p = state_path(out) + if not p.exists(): + return {} + try: + return json.loads(p.read_text()) + except (OSError, json.JSONDecodeError): + # A half-written cache is a cache miss, not a crash. Rebuilding costs + # one run; refusing to start costs an explanation. + return {} + + +def scan(source: Path, out, keep_secrets=False, exclude=(), include=(), + quiet=False) -> dict: + """ + Census the tree, reusing everything whose content has not changed. + + The reuse is per file and keyed on content, not on mtime: a checkout, a + branch switch or a `touch` all move mtimes without changing a byte, and + re-reading the whole tree because git rewrote it is the cost this exists to + avoid. + """ + from .sift import REASONS, sift + + # The filter runs here, not at export time, so a key never reaches a plan + # in the first place. Secrets and files the repo's own ignore rules + # contradict are dropped; what a build regenerates is NOT — a lockfile is + # content in a repo somebody is going to use, however little it says. + # `copy` is the one that drops those, because a snapshot is for reading. + tracked = file_set(source) + paths, left_out = sift(source, tracked, keep_noise=True, + keep_secrets=keep_secrets, exclude=exclude, + include=include, + ignored=ignored_but_tracked(source, tracked)) + previous = load_index(out).get("files", {}) + + by_path, by_base = {}, {} + for p in paths: + by_path.setdefault(p, []).append(p) + by_base.setdefault(Path(p).name, []).append(p) + + files, reused = {}, 0 + for rel in paths: + full = source / rel + try: + digest = _digest(full) + except OSError: + continue + old = previous.get(rel) + if old and old.get("hash") == digest and "mentions" in old: + files[rel] = old + reused += 1 + continue + + text = read_text(full) + refs, mentions = ([], []) if text is None else referenced_paths(text, rel, by_path, by_base) + entry = { + "hash": digest, + "size": full.stat().st_size, + "role": role_of(rel), + "binary": text is None, + "why": "" if text is None else opening_comment(text, rel), + "refs": refs, # in code: a dependency, and an edge + "mentions": mentions, # in prose: context for the brief, never an edge + } + entry["lines"] = 0 if text is None else text.count("\n") + 1 + files[rel] = entry + + # The filter settings travel with the index so the guards can re-derive + # the same set later. Without them `check_plan` has to choose between + # trusting a stale list and flagging every filtered file as missing. + index = {"version": 1, "source": str(source), "files": files, + "filter": {"keep_secrets": bool(keep_secrets), + "exclude": list(exclude), "include": list(include)}, + "left_out": [{"path": p, "why": w} for p, w in left_out]} + destination = state_path(out) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(index, indent=2, sort_keys=True)) + + if not quiet: + fresh = len(files) - reused + print(f"Scanned {len(files)} files: {fresh} read, {reused} reused from cache.") + if left_out: + # Never silent. A key that was tracked is a thing to know about, + # and it stays true after the file stops travelling. + print(f" {len(left_out)} left out of the history:") + for rel, why in left_out: + print(f" {rel} ({REASONS[why]})") + print(f" -> {destination}") + return index diff --git a/soleprint/station/tools/histgen/cli.py b/soleprint/station/tools/histgen/cli.py new file mode 100644 index 0000000..1db03d7 --- /dev/null +++ b/soleprint/station/tools/histgen/cli.py @@ -0,0 +1,132 @@ +""" +The shape a station tool's command line has. + +Every tool here grew its own copy of the same three things: argparse subcommands +wired to `cmd_*` functions, a flag vocabulary that is nearly but not quite the +same between tools, and an error convention. shuntgen spells `-s` as --spec in +one subcommand and --source in another; modelgen calls the same idea --source +everywhere; tester has neither. The differences are accidents, not decisions. + +This is that shared shape, factored out. histgen is the first user. Nothing else +is rewritten to use it — a scaffold earns adoption by being there when the next +tool is written, not by a flag-day. + + from .cli import Tool, fail + + tool = Tool("histgen", __doc__) + tool.command("scan", cmd_scan, "Read the repo and cache what was read.") + tool.argument("scan", "repo", help="The tree to read.") + tool.run() + +Conventions it encodes, so they stop being re-decided: + + --source/-s what to read --output/-o where to write + --name/-n what to call it --force/-f write anyway + --dry-run print, do not do + + Progress goes to stdout as plain print(). Errors go to stderr prefixed + 'Error: ' and exit 1 — never a traceback, which tells a user nothing they + can act on. `fail()` is the only exit path. + + Heavy imports live inside the cmd_* function, never at module top, so + `--help` stays instant and an optional dependency only costs the one + subcommand that needs it. +""" + +import argparse +import sys + +# The flags worth having exactly one spelling of. A tool adds its own on top; +# it does not redefine these. +COMMON = { + "source": dict(flags=("--source", "-s"), help="What to read."), + "output": dict(flags=("--output", "-o"), help="Where to write."), + "name": dict(flags=("--name", "-n"), help="What to call it."), + "force": dict(flags=("--force", "-f"), action="store_true", + help="Write even if the destination is occupied."), + "dry_run": dict(flags=("--dry-run",), action="store_true", + help="Print what would happen; change nothing."), +} + + +def fail(message, hint=None): + """The only way out on error: a line a user can act on, never a traceback.""" + print(f"Error: {message}", file=sys.stderr) + if hint: + print(f" {hint}", file=sys.stderr) + sys.exit(1) + + +def refuse_to_clobber(path, force, marker, what): + """ + Regenerating is fine; overwriting something we did not write is not. + + Lifted from shuntgen, which refuses a non-empty output directory unless it + carries the file its own generator leaves behind. The check is cheap and the + failure it prevents — silently eating a directory someone hand-wrote — is + not recoverable from. + """ + if force or not path.exists(): + return + if not any(path.iterdir()): + return + if (path / marker).exists(): + return + fail(f"{path} already exists and was not written by {what}.", + "Pick another path, or pass --force to write into it anyway.") + + +def _prog(package, name): + """ + How this tool was actually invoked, for the usage line. + + The folder is meant to be copied out and run on its own, so a usage line + hardcoding `python -m station.tools.histgen` is wrong the moment it is — + it names a path that does not exist on the machine reading it. + """ + return f"python -m {package or name}" + + +class Tool: + """A tool's whole command line: subcommands, shared flags, one exit path.""" + + def __init__(self, name, description, package=None): + self.name = name + self.parser = argparse.ArgumentParser( + prog=_prog(package, name), + description=(description or "").strip().split("\n\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + self._subparsers = self.parser.add_subparsers(dest="command", required=True) + self._commands = {} + + def command(self, verb, func, help_text): + """Register a subcommand. `func` takes parsed args and returns None.""" + sub = self._subparsers.add_parser(verb, help=help_text, description=help_text) + sub.set_defaults(func=func) + self._commands[verb] = sub + return sub + + def argument(self, verb, *args, **kwargs): + """Add a positional or flag to one subcommand.""" + self._commands[verb].add_argument(*args, **kwargs) + + def common(self, verb, *names, **overrides): + """Add shared flags by name, so their spelling is decided in one place.""" + for key in names: + spec = dict(COMMON[key]) + flags = spec.pop("flags") + spec.update(overrides.get(key, {})) + self._commands[verb].add_argument(*flags, **spec) + + def run(self, argv=None): + args = self.parser.parse_args(argv) + try: + args.func(args) + except KeyboardInterrupt: + # Ctrl-C is a decision, not a crash. Say so and leave quietly. + print("\nInterrupted.", file=sys.stderr) + sys.exit(130) + except BrokenPipeError: + # `... | head` closes the pipe early; that is the caller's business. + sys.exit(0) diff --git a/soleprint/station/tools/histgen/config.py b/soleprint/station/tools/histgen/config.py new file mode 100644 index 0000000..12f1edd --- /dev/null +++ b/soleprint/station/tools/histgen/config.py @@ -0,0 +1,153 @@ +""" +Where to read from and where to write to. + +Two directories, and the whole tool hangs off the difference between them: + + source the tree to read. Opened read-only, always. Nothing is written + into it, ever — not a commit, not a .git, not a state file. It + can be a checkout you do not own or a read-only mount. + + out everything this produces. The index, the plan, the briefs, and + `out//` — a copy of the source with the designed history + committed into it. Delete the directory and you have lost + nothing but time. + +The same repo gets scanned, planned and re-planned a dozen times while its +grouping is argued with, and passing the pair of paths to every one of five +verbs gets old. So they can live in a file instead — the arrangement +`ppl/ctrl/distill.sh` already uses, where the JSON beside the script is picked +up when nothing else says otherwise. + + { + "source": "~/work/some-project", + "out": "~/histories/some-project", + "max_files": null + } + +That separation is what makes the thing safe to experiment with. The history +is an argument you will have more than once, and every attempt is a directory +you can throw away rather than a repo you have to put back. + +Precedence is the usual one, most specific first: + + the command line -> --config FILE -> histgen.json beside the tool + -> the defaults + +so a config file sets a starting point and never wins an argument with a flag +that was typed deliberately. +""" + +import json +from pathlib import Path + +CONFIG_NAME = "histgen.json" +HERE = Path(__file__).resolve().parent + +KEYS = ("source", "out", "max_files", "keep_history", "branch", + "keep_secrets", "exclude", "include") + +# `repo` was what `source` used to be called, back when the tool committed +# into the tree it read. Accepted rather than rejected, because a config file +# written last week should not be an error message. +ALIASES = {"repo": "source"} + +TEMPLATE = { + "source": None, + "out": None, + "max_files": None, + "keep_history": False, + "branch": None, + "keep_secrets": False, + "exclude": [], + "include": [], +} + + +def default_path() -> Path: + """The config beside the tool, used when nothing else is named.""" + return HERE / CONFIG_NAME + + +def find(explicit=None): + """The config file to read, or None. An explicit one that is missing is an error.""" + if explicit: + path = Path(explicit).expanduser() + if not path.is_file(): + from .cli import fail + fail(f"No such config file: {path}") + return path + beside = default_path() + return beside if beside.is_file() else None + + +def load(explicit=None) -> dict: + """Read the config, or return the defaults. Unknown keys are an error.""" + settings = dict(TEMPLATE) + path = find(explicit) + if not path: + return settings + + from .cli import fail + try: + raw = json.loads(path.read_text()) + except json.JSONDecodeError as e: + fail(f"{path} is not valid JSON: {e}") + if not isinstance(raw, dict): + fail(f"{path} should hold an object, not a {type(raw).__name__}.") + + # A typo in a key would otherwise be silent, and the symptom — the tool + # ignoring a setting that is plainly written in the file — is a bad one to + # debug. Naming the valid keys costs one line. + raw = {ALIASES.get(k, k): v for k, v in raw.items()} + unknown = sorted(set(raw) - set(KEYS)) + if unknown: + fail(f"{path}: unknown key(s) {', '.join(unknown)}.", + f"Known keys: {', '.join(KEYS)}.") + + settings.update({k: v for k, v in raw.items() if v is not None}) + settings["_path"] = str(path) + return settings + + +def resolve(args, explicit=None): + """ + Fold the config under the command line and hand back what to actually use. + + Paths are expanded and made absolute here rather than at each use, so + everything downstream compares like with like — a `~` that survived into a + path comparison is a bug that only shows up on someone else's machine. + """ + settings = load(explicit) + + source = getattr(args, "source", None) or settings.get("source") + out = getattr(args, "out", None) or settings.get("out") + + return { + "source": Path(source).expanduser().resolve() if source else None, + "out": Path(out).expanduser().resolve() if out else None, + "max_files": getattr(args, "max_files", None) or settings.get("max_files"), + "keep_history": (getattr(args, "keep_history", False) + or settings.get("keep_history", False)), + "branch": getattr(args, "branch", None) or settings.get("branch"), + "keep_secrets": (getattr(args, "keep_secrets", False) + or settings.get("keep_secrets", False)), + "exclude": list(getattr(args, "exclude", None) or []) + + list(settings.get("exclude") or []), + "include": list(getattr(args, "include", None) or []) + + list(settings.get("include") or []), + "config_path": settings.get("_path"), + } + + +def write_template(path: Path, source=None, out=None) -> Path: + """Write a starter config, never over one that already exists.""" + from .cli import fail + if path.exists(): + fail(f"{path} already exists.", "Edit it, or name another path.") + body = dict(TEMPLATE) + if source: + body["source"] = str(source) + if out: + body["out"] = str(out) + path.write_text(json.dumps(body, indent=2) + "\n") + return path diff --git a/soleprint/station/tools/histgen/export.py b/soleprint/station/tools/histgen/export.py new file mode 100644 index 0000000..69c6856 --- /dev/null +++ b/soleprint/station/tools/histgen/export.py @@ -0,0 +1,613 @@ +""" +Materialise the source into the out directory and commit the designed history. + +The source is never touched. What gets committed is a copy, made here, and the +copy is the only thing that ends up with a history — so an attempt that goes +wrong costs a `rm -rf` rather than an afternoon putting a real checkout back. + +Everything up to this point is analysis and can be recomputed. This part writes, +so it starts by working out what it is writing into. Four states, and they are +genuinely different: + + absent nothing there yet. Copy the tree, init, commit. + + unfinished a copy is there with commits this tool made and a record of + where it stopped. Something interrupted the run — a signal, a + full disk, a hook that refused. Continue from the group after + the last one recorded. + + foreign a copy is there with commits this tool did not make. That is + history someone else is entitled to, so nothing is rewritten, + moved or deleted: the designed account is committed to its own + orphan branch and the existing branch is left exactly as it + was. Two tiers, which is what `all/ctrl/handover.sh` has been + saying all along. + + stale a copy is there that does not match the plan any more. Refuse, + and say which of the two moved. + +Telling the second apart from the third is the whole reason progress.json +exists. Without it both read as "there are commits here", and the tool either +destroys work it should have kept or refuses to finish work it started. +""" + +import hashlib +import json +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +from .census import current_file_set, is_git, load_index, state_dir +from .cli import fail +from .order import plan_path + +MESSAGE_DIR = "messages" +SCRIPT_FILE = "regen.sh" +PROGRESS_FILE = "progress.json" + +# Where the designed account goes when the copy already carries a history that +# is not ours. A name, not a number, because it has to mean something in a +# branch list six months from now. +DEFAULT_BRANCH = "designed-history" + + +# ── where the copy lives ─────────────────────────────────────────────────── + +def repo_dir(source: Path, out: Path) -> Path: + """ + The copy, named after the source. + + Named rather than called `repo/`, because this directory gets `cd`-ed into, + pushed from and looked at in a file manager, and "adapter" answers a + question there that "repo" does not. + """ + return Path(out) / source.name + + +def progress_path(out) -> Path: + return state_dir(out) / PROGRESS_FILE + + +def plan_fingerprint(plan) -> str: + """ + Identifies the plan a history was built from, by its groups and their paths. + + Messages are deliberately not in it. Rewording a commit that has not been + made yet must not invalidate the twelve that have — that is the normal way + this tool gets used, one group's message at a time. + """ + shape = [[g["n"], sorted(g["paths"])] for g in plan["groups"]] + return hashlib.sha256(json.dumps(shape, sort_keys=True).encode()).hexdigest()[:16] + + +def load_progress(out) -> dict: + p = progress_path(out) + if not p.exists(): + return {} + try: + return json.loads(p.read_text()) + except (OSError, json.JSONDecodeError): + # Unreadable progress means we cannot prove which commits are ours, and + # guessing is exactly the thing this file exists to avoid. + return {} + + +def save_progress(out, data) -> None: + progress_path(out).parent.mkdir(parents=True, exist_ok=True) + progress_path(out).write_text(json.dumps(data, indent=2)) + + +# ── git ──────────────────────────────────────────────────────────────────── + +def _git(repo: Path, *args, check=True): + r = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True) + if check and r.returncode != 0: + fail(f"git {' '.join(args[:2])} failed: {r.stderr.strip() or r.stdout.strip()}") + return r + + +def _head(repo: Path): + r = _git(repo, "rev-parse", "HEAD", check=False) + return r.stdout.strip() if r.returncode == 0 else None + + +def _commit_count(repo: Path) -> int: + r = _git(repo, "rev-list", "--count", "HEAD", check=False) + return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0 + + +# ── what state is the out directory in ───────────────────────────────────── + +def inspect(source: Path, out: Path, plan=None): + """ + Read the out directory and say what is there. Writes nothing. + + `status` prints this; `export` branches on it. One function so the two can + never disagree about what they are looking at, which they would within a + week of being written separately. + """ + copy = repo_dir(source, out) + progress = load_progress(out) + report = { + "copy": copy, + "exists": copy.is_dir(), + "git": copy.is_dir() and is_git(copy), + "commits": 0, + "state": "absent", + "done": [], + "remaining": [], + "branch": None, + "detail": "", + } + if not report["exists"]: + report["detail"] = "nothing exported yet" + if plan: + report["remaining"] = [g["n"] for g in plan["groups"]] + return report + + if not report["git"]: + report["state"] = "stale" + report["detail"] = ("a directory is there but it is not a git repo — " + "an export that died before `git init`") + return report + + report["commits"] = _commit_count(copy) + report["branch"] = _git(copy, "rev-parse", "--abbrev-ref", "HEAD", + check=False).stdout.strip() or None + + if report["commits"] == 0: + report["state"] = "absent" + report["detail"] = "a repo with no commits" + if plan: + report["remaining"] = [g["n"] for g in plan["groups"]] + return report + + ours = progress.get("commits", []) + head = _head(copy) + + if not ours: + report["state"] = "foreign" + report["detail"] = (f"{report['commits']} commit(s) this tool did not make") + if plan: + report["remaining"] = [g["n"] for g in plan["groups"]] + return report + + if plan and progress.get("plan") != plan_fingerprint(plan): + report["state"] = "stale" + report["detail"] = ("the plan changed after this history was started — " + "the groups are not the ones these commits were made from") + report["done"] = [c["n"] for c in ours] + return report + + # The record has to still describe reality. If the branch moved underneath + # us — a rebase, a reset, an amend — continuing would build on something + # other than what was recorded, and quietly. + if head != ours[-1]["sha"]: + report["state"] = "stale" + report["detail"] = ("the copy has moved since this tool last wrote to it " + f"(expected {ours[-1]['sha'][:9]}, found " + f"{(head or '-')[:9]})") + report["done"] = [c["n"] for c in ours] + return report + + report["done"] = [c["n"] for c in ours] + if plan: + report["remaining"] = [g["n"] for g in plan["groups"] + if g["n"] not in set(report["done"])] + report["state"] = "complete" if plan and not report["remaining"] else "unfinished" + report["detail"] = (f"{len(report['done'])} group(s) committed by this tool" + + (f", {len(report['remaining'])} to go" if report["remaining"] else "")) + return report + + +# ── checks ───────────────────────────────────────────────────────────────── + +def load_plan(out): + p = plan_path(out) + if not p.exists(): + fail(f"No plan at {p}.", "Run `scan` then `plan` first.") + try: + return json.loads(p.read_text()) + except json.JSONDecodeError as e: + # Hand-editing plan.json is the expected workflow, so a trailing comma + # is a normal event and deserves a line number rather than a traceback. + fail(f"{p} is not valid JSON: {e}") + + +def check_plan(source: Path, out, plan, require_messages=True): + """ + Refuse a plan that could not produce the tree it claims to. + + What the plan is measured against is the source as it stands *now*, with + the same filter the census used. Not the stored list: a file added after + the scan is exactly what this is here to catch, and a stored list cannot + see it. Not the unfiltered source either, or every deliberately dropped key + comes back as a file no group covers. + """ + planned, dupes = [], [] + for g in plan["groups"]: + for p in g["paths"]: + (dupes if p in planned else planned).append(p) + + problems = [] + if dupes: + problems.append(f"{len(dupes)} path(s) appear in more than one group: " + + ", ".join(sorted(set(dupes))[:5])) + + missing = [p for p in planned if not (source / p).is_file()] + if missing: + problems.append(f"{len(missing)} planned path(s) are not in the source: " + + ", ".join(missing[:5])) + + present = current_file_set(source, load_index(out)) + unplanned = sorted(present - set(planned)) + if unplanned: + problems.append(f"{len(unplanned)} file(s) are in the source but in no group: " + + ", ".join(unplanned[:5]) + + "\n Re-run `scan` and `plan` if the source changed since.") + + empty = [g["n"] for g in plan["groups"] if not g["paths"]] + if empty: + problems.append(f"group(s) {empty} have no paths") + + if require_messages: + unwritten = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()] + if unwritten: + problems.append( + f"{len(unwritten)} group(s) have no title: " + + ", ".join(str(n) for n in unwritten[:8]) + + f"\n Read {state_dir(out) / 'briefs'}/ and fill them in, " + "or pass --allow-untitled.") + + if problems: + for p in problems: + print(f"Error: {p}", file=sys.stderr) + sys.exit(1) + return planned + + +def _message(group): + title = (group.get("title") or "").strip() or f"{group['slug']} ({len(group['paths'])} files)" + body = (group.get("body") or "").strip() + return f"{title}\n\n{body}\n" if body else f"{title}\n" + + +def source_tree_hash(source: Path, paths): + """ + The tree hash the source files would produce, without committing anything. + + Runs against a temporary index and, when the source has no git, a temporary + git directory too. The source's own index is never touched: someone running + this mid-edit must not lose their staging area to a verification step. + """ + import tempfile + with tempfile.TemporaryDirectory(prefix="histgen-idx-") as tmp: + env = dict(os.environ, GIT_INDEX_FILE=str(Path(tmp) / "index")) + if not (source / ".git").exists(): + env["GIT_DIR"] = str(Path(tmp) / "git") + env["GIT_WORK_TREE"] = str(source) + subprocess.run(["git", "init", "-q"], env=env, capture_output=True) + proc = subprocess.run( + ["git", "-C", str(source), "update-index", "--add", "--stdin"], + input="\n".join(paths) + "\n", text=True, capture_output=True, env=env) + if proc.returncode != 0: + return None + r = subprocess.run(["git", "-C", str(source), "write-tree"], + capture_output=True, text=True, env=env) + return r.stdout.strip() if r.returncode == 0 else None + + +# ── the guards ───────────────────────────────────────────────────────────── + +def verify(copy: Path, expected_tree=None, quiet=False): + """ + Nothing left untracked, and the tree still matches the source. + + Nothing is exempt from the first check. The state directory lives in `out` + and the copy lives inside it, so there is genuinely nothing of ours in the + tree being checked — which is stricter than the version that had to forgive + its own scaffolding. + """ + ok = True + status = _git(copy, "status", "--porcelain").stdout.strip() + if status: + ok = False + print("Error: the tree is not clean — these never made it into a commit:", + file=sys.stderr) + for line in status.split("\n")[:20]: + print(f" {line}", file=sys.stderr) + if len(status.split("\n")) > 20: + print(f" ... and {len(status.split(chr(10))) - 20} more", file=sys.stderr) + elif not quiet: + print(" nothing left untracked: ok") + + if expected_tree: + head = _git(copy, "rev-parse", "HEAD^{tree}", check=False).stdout.strip() + if head != expected_tree: + ok = False + print(f"Error: the exported tree does not match the source.\n" + f" source {expected_tree}\n HEAD {head}", file=sys.stderr) + elif not quiet: + print(f" tree matches source ({head[:12]}): ok") + return ok + + +# ── materialising the copy ───────────────────────────────────────────────── + +def materialise(source: Path, copy: Path, paths, keep_history=False, quiet=False): + """ + Put the planned files into the copy, and nothing else. + + Copied file by file from the plan rather than with `cp -r`, because the + plan is the definition of what belongs in the history: anything gitignored, + anything untracked, and the source's own .git are all things the source has + and the export must not. + + `keep_history` is the exception, and the only reason the source's .git ever + comes across: it is what lets an existing history be carried into the copy + so the designed account can sit beside it instead of replacing it. + """ + copy.mkdir(parents=True, exist_ok=True) + + if keep_history and (source / ".git").is_dir() and not (copy / ".git").exists(): + shutil.copytree(source / ".git", copy / ".git", symlinks=True) + # A copied .git still points its index at files that are about to be + # rewritten underneath it; reset so status reflects the copy, not the + # source's staging area at the moment it was cloned. + _git(copy, "reset", "-q", check=False) + + for rel in paths: + target = copy / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / rel, target) + + if not quiet: + print(f" copied {len(paths)} files -> {copy}") + + +def _start_branch(copy: Path, report, branch, quiet=False): + """ + Decide which ref the designed history goes on, and get there. + + A foreign history is not ours to move, so the designed account starts from + an orphan — no parent, nothing shared — and the branch that was there keeps + pointing exactly where it did. + """ + if report["state"] != "foreign": + return None + name = branch or DEFAULT_BRANCH + if _git(copy, "rev-parse", "--verify", name, check=False).returncode == 0: + fail(f"Branch '{name}' already exists in {copy}.", + "Pick another with --branch, or delete it if it was a failed attempt.") + kept = report["branch"] or "the existing branch" + if not quiet: + print(f" {report['commits']} existing commit(s) on {kept}: kept, untouched") + print(f" the designed history goes on a new orphan branch '{name}'") + _git(copy, "checkout", "-q", "--orphan", name) + # --orphan keeps the index, which would make the first designed commit + # carry every file the old branch had staged. + _git(copy, "rm", "-rq", "--cached", ".", check=False) + return name + + +def export(source: Path, out, dry_run=False, commands=False, allow_untitled=False, + keep_history=False, branch=None, force=False, quiet=False): + plan = load_plan(out) + planned = check_plan(source, out, plan, require_messages=not allow_untitled) + expected = source_tree_hash(source, planned) + if not expected: + fail("Could not compute the source tree hash.", + "Without it the export cannot be checked, and an unchecked export " + "is the thing this refuses to produce.") + + copy = repo_dir(source, out) + report = inspect(source, out, plan) + + if dry_run: + return _emit_script(source, out, plan, planned, expected, report, + keep_history, branch, quiet) + + if commands: + return _emit_commands(source, out, plan, planned, expected, force, quiet) + + if report["state"] == "stale" and not force: + fail(f"{copy}: {report['detail']}.", + "Pass --force to discard what is there and export again, or point " + "--out somewhere else to keep it.") + if report["state"] == "complete": + print(f"Already exported: {len(report['done'])} groups committed in {copy}.") + print("Nothing to do. Re-plan, or use --force to start over.") + return True + + if report["state"] == "stale" and force: + if not quiet: + print(f" discarding {copy}") + shutil.rmtree(copy) + save_progress(out, {}) + report = inspect(source, out, plan) + + progress = load_progress(out) + resuming = report["state"] == "unfinished" + + if resuming: + done = set(report["done"]) + if not quiet: + print(f"Resuming: {len(done)} of {len(plan['groups'])} groups already " + f"committed in {copy}.") + # The files are already there from the interrupted run, but a source + # edited since would otherwise be silently ignored. + materialise(source, copy, planned, quiet=quiet) + else: + done = set() + materialise(source, copy, planned, keep_history=keep_history, quiet=quiet) + if not is_git(copy): + _git(copy, "init", "-q") + # Re-read the copy. --keep-history has only just put a history into it, + # so the state worked out before the directory existed cannot have seen + # it — and acting on the stale answer commits the designed account on + # top of the history it was supposed to sit beside. + report = inspect(source, out, plan) + active = _start_branch(copy, report, branch, quiet) + progress = {"plan": plan_fingerprint(plan), "source": str(source), + "branch": active, "commits": []} + save_progress(out, progress) + + for g in plan["groups"]: + if g["n"] in done: + continue + _git(copy, "add", "--", *g["paths"]) + msg = copy / ".git" / "HISTGEN_MSG" + msg.write_text(_message(g)) + _git(copy, "commit", "-q", "-F", str(msg)) + msg.unlink(missing_ok=True) + # Recorded after each commit, not at the end. The whole point is to + # survive the run not reaching the end. + progress.setdefault("commits", []).append({"n": g["n"], "sha": _head(copy)}) + progress["plan"] = plan_fingerprint(plan) + save_progress(out, progress) + if not quiet: + print(f" {g['n']:02d} {_message(g).splitlines()[0]}") + + if not quiet: + # The repo's own count, not the plan's. With a kept history the two + # differ, and the number a reader wants is what is actually in there. + where = _git(copy, "rev-parse", "--abbrev-ref", "HEAD", + check=False).stdout.strip() + print(f"\n{_commit_count(copy)} commits on {where} in {copy}. " + "Checking:", flush=True) + if not verify(copy, expected, quiet=quiet): + sys.exit(1) + return True + + +def _emit_script(source, out, plan, planned, expected, report, + keep_history, branch, quiet): + """ + Write the export as a shell script instead of running it. + + Reviewing plain git commands before they run is worth more here than + anywhere else: this is the one operation whose mistakes are baked into + every commit that follows. + """ + state = state_dir(out) + copy = repo_dir(source, out) + msg_dir = state / MESSAGE_DIR + msg_dir.mkdir(parents=True, exist_ok=True) + for stale in msg_dir.glob("*.txt"): + stale.unlink() + + q = shlex.quote + lines = [ + "#!/usr/bin/env bash", + "# Generated by histgen. Review, then run from anywhere.", + "set -euo pipefail", + "", + f"SOURCE={q(str(source))}", + f"COPY={q(str(copy))}", + "", + 'mkdir -p "$COPY"', + ] + if keep_history and (source / ".git").is_dir(): + lines.append('test -d "$COPY/.git" || cp -a "$SOURCE/.git" "$COPY/.git"') + lines += [ + "# Only the planned files: not the source's .git, not anything ignored.", + 'while IFS= read -r f; do mkdir -p "$COPY/$(dirname "$f")"; ' + 'cp -p "$SOURCE/$f" "$COPY/$f"; done <<\'PATHS\'', + *planned, + "PATHS", + "", + 'cd "$COPY"', + "test -d .git || git init -q", + ] + if report["state"] == "foreign": + name = branch or DEFAULT_BRANCH + lines += [f"# {report['commits']} existing commit(s) stay where they are.", + f"git checkout -q --orphan {q(name)}", + "git rm -rq --cached . || true", ""] + + for g in plan["groups"]: + name = f"{g['n']:02d}-{g['slug']}.txt" + (msg_dir / name).write_text(_message(g)) + lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}", + "git add -- " + " ".join(q(p) for p in g["paths"]), + f"git commit -q -F {q(str(msg_dir / name))}", + ""] + + lines += [ + "# The two guards. A history that fails these is not worth keeping.", + 'test -z "$(git status --porcelain)" || ' + '{ echo "FAILED: files left untracked" >&2; exit 1; }', + f'test "$(git rev-parse HEAD^{{tree}})" = "{expected}" || ' + '{ echo "FAILED: exported tree does not match source" >&2; exit 1; }', + 'echo "ok: $(git rev-list --count HEAD) commits, tree verified"', + "", + ] + + script = state / SCRIPT_FILE + script.write_text("\n".join(lines)) + script.chmod(0o755) + if not quiet: + print(f"Wrote {len(plan['groups'])} commits as commands -> {script}") + print(f" messages -> {msg_dir}") + return True + + +def _emit_commands(source: Path, out, plan, planned, expected, force, quiet): + """ + Copy the files, create no repo, and print the commands to make the history. + + The other two modes each decide something for you: `export` runs the whole + thing, `--dry-run` writes a script that would. This one does the half that + is tedious and gets the other half out of the way — the copy is made, and + what comes back is a list you read, edit and run yourself. + + Nothing here creates a .git. `git init` is the first line of the list rather + than something already done, because a repo that appeared without you asking + is exactly what someone reaching for this mode does not want. + """ + copy = repo_dir(source, out) + + # Asked for no repo, so an existing one is a contradiction worth stopping + # for: the commands below would commit into it rather than into a fresh one. + if (copy / ".git").exists() and not force: + fail(f"{copy} already contains a git repo.", + "This mode creates none and the commands assume none. Delete it, " + "point --out elsewhere, or pass --force to copy the files in anyway.") + + materialise(source, copy, planned, quiet=quiet) + + state = state_dir(out) + msg_dir = state / MESSAGE_DIR + msg_dir.mkdir(parents=True, exist_ok=True) + for stale in msg_dir.glob("*.txt"): + stale.unlink() + + q = shlex.quote + lines = [f"cd {q(str(copy))}", "git init", ""] + for g in plan["groups"]: + name = f"{g['n']:02d}-{g['slug']}.txt" + (msg_dir / name).write_text(_message(g)) + lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}", + "git add -- " + " ".join(q(p) for p in g["paths"]), + f"git commit -F {q(str(msg_dir / name))}", + ""] + + lines += [ + "# Worth running afterwards. The first says no file was silently", + "# missed; the second says the result is byte-identical to the source.", + "git status --porcelain", + f"git rev-parse HEAD^{{tree}} # expect {expected}", + "", + ] + + listing = "\n".join(lines) + (state / "commands.sh").write_text(listing) + + if not quiet: + print(f" messages -> {msg_dir}") + print(f" this list -> {state / 'commands.sh'}\n") + print(listing) + return True diff --git a/soleprint/station/tools/histgen/history.py b/soleprint/station/tools/histgen/history.py new file mode 100644 index 0000000..5b63c2c --- /dev/null +++ b/soleprint/station/tools/histgen/history.py @@ -0,0 +1,116 @@ +""" +What the history already there says, next to what the plan proposes. + +Reads and reports. It never rewrites: published history is someone else's +clone, and the useful output here is an argument about ordering, not a +force-push. + +Most repos this gets pointed at are in the checkpoint tier — "updates 33.1 +139", "working state", "debugging" — where the honest finding is that the log +records when work was saved and nothing about how the thing is built. That is +worth printing plainly, because it is the case for keeping a second, designed +history rather than trying to repair this one. +""" + +import subprocess +from collections import defaultdict + +# Subjects that carry no information about what changed. Matching is on the +# whole subject, lowercased, with trailing numbers dropped — "updates 33.1 139" +# and "updates 33.1 84" are the same non-statement. +NOISE = {"update", "updates", "wip", "fix", "fixes", "changes", "some changes", + "working state", "debugging", "init commit", "initial commit", "misc", + "checkpoint", "save", "final", "for final test", "cleanup", "tmp"} + + +def _subject_is_noise(subject): + s = subject.strip().lower().rstrip("0123456789. ") + return s in NOISE or not s + + +def read_history(repo): + """[(sha, subject, [paths])] oldest first, or [] if there is no history.""" + r = subprocess.run( + ["git", "-C", str(repo), "log", "--reverse", "--name-only", + "--format=%x00%h%x1f%s"], + capture_output=True, text=True) + if r.returncode != 0: + return [] + commits = [] + for chunk in r.stdout.split("\0"): + if not chunk.strip(): + continue + head, _, rest = chunk.partition("\n") + sha, _, subject = head.partition("\x1f") + paths = [l for l in rest.split("\n") if l.strip()] + commits.append((sha, subject, paths)) + return commits + + +def compare(source, plan, quiet=False): + """Print how the existing history lines up with the proposed one.""" + commits = read_history(source) + groups = plan["groups"] + if not commits: + print(f"{len(groups)} groups proposed; the source has no history to compare.") + return {"commits": 0, "groups": len(groups)} + + where = {p: g["n"] for g in groups for p in g["paths"]} + + # A commit maps to the group holding most of the files it touched. Files + # that no longer exist are dropped rather than counted against it — a + # commit that deleted something is not disagreeing about order. + mapped, noise, touches = {}, [], defaultdict(list) + for sha, subject, paths in commits: + hits = [where[p] for p in paths if p in where] + if not hits: + noise.append((sha, subject, "touches nothing that still exists")) + continue + best = max(set(hits), key=lambda n: (hits.count(n), -n)) + spread = len(set(hits)) + mapped[sha] = (best, subject, spread, len(hits)) + touches[best].append(sha) + if _subject_is_noise(subject): + noise.append((sha, subject, f"says nothing; touches {spread} group(s)")) + + print(f"{len(groups)} groups proposed, {len(commits)} existing commits.\n") + + # Order disagreement: walking the real history, does the group number ever + # go backwards? That is the concrete "this was built in a different order". + seen_max, inversions = 0, [] + for sha, subject, _ in commits: + if sha not in mapped: + continue + n = mapped[sha][0] + if n < seen_max: + inversions.append((sha, n, seen_max, subject)) + seen_max = max(seen_max, n) + + for g in groups: + shas = touches.get(g["n"], []) + title = g.get("title") or g["slug"] + if not shas: + mark, note = "+", "no existing commit builds this" + elif len(shas) == 1: + mark, note = "=", f"{shas[0]}" + else: + mark, note = "~", f"split across {len(shas)} commits ({', '.join(shas[:4])})" + print(f" {mark} {g['n']:02d} {title[:44]:46} {note}") + + if inversions: + print(f"\n ! {len(inversions)} commit(s) land earlier in the proposed order " + f"than work already done:") + for sha, n, high, subject in inversions[:10]: + print(f" {sha} group {n:02d} after group {high:02d} {subject[:44]}") + + if noise: + print(f"\n ? {len(noise)} commit(s) carry no usable account of the change:") + for sha, subject, why in noise[:10]: + print(f" {sha} {subject[:44]:46} {why}") + if len(noise) > 10: + print(f" ... and {len(noise) - 10} more") + + print("\n = matched one commit ~ split + not in history " + "! out of order ? uninformative") + return {"commits": len(commits), "groups": len(groups), + "inversions": len(inversions), "noise": len(noise)} diff --git a/soleprint/station/tools/histgen/order.py b/soleprint/station/tools/histgen/order.py new file mode 100644 index 0000000..44b38cd --- /dev/null +++ b/soleprint/station/tools/histgen/order.py @@ -0,0 +1,450 @@ +""" +The order the files go in, and where one commit stops and the next begins. + +Two decisions, and they are not the same decision. Order answers "what can be +understood before what"; grouping answers "what is one idea". Getting the first +right and the second wrong gives you 64 correct commits nobody wants to read. + +The order is role first, references second. Roles carry the heuristic — ignore +rules and README, then the config layer, then the things that source it, the +front door late because it only dispatches, the bootstrap account last because +it narrates everything above it. References refine within that, so a config +lands before the script that sources it. + +References never override roles. A reference in code is a dependency, but roles +already encode dependencies that no reference states: nothing in the repo +*refers to* .gitignore, and the README is named by nothing while naming +everything. Letting edges win produces the ignore rules committed after the +code they exclude — technically consistent, and unreadable. + +Nothing here is authoritative. plan.json is a file, and moving a path from one +group to another is the expected way to use it: this gets the shape right so +the argument is about two or three groups, not sixty-four paths. +""" + +import json +from collections import defaultdict +from pathlib import Path + +from .census import ROLE_RANK, state_dir + +PLAN_FILE = "plan.json" + +# Roles that name other files without depending on them. Their outgoing edges +# are dropped: a README mentioning every script in the tree is a table of +# contents, not a build order. +NARRATIVE = {"skeleton", "readme", "doc", "bootstrap", "asset", "lock"} + +# A directory that carries its own README, Makefile or package manifest is a +# project in its own right. It is committed whole and late — it stands on the +# repo around it, so it cannot be read before it. rig's sample-rig is the case: +# sixteen files, one idea, and it calls rig's own addon script rather than +# reimplementing it. +# A README is NOT one of these. Any directory worth having explains itself, and +# ctrl/k8s/README.md documenting four manifests does not make them a project — +# it made them sort after the Makefile, which is where this rule came from. What +# marks a project is something that builds or resolves it. +SUBPROJECT_MARKERS = {"makefile", "package.json", "pyproject.toml", + "go.mod", "cargo.toml", "gemfile", "build.gradle"} +SUBPROJECT_RANK = 95 # after the front door, before the bootstrap account + +# How many files one commit may hold before it stops being one idea. Soft: a +# subproject and a hub with its satellites are exempt, because splitting those +# produces a commit that does not build. +DEFAULT_MAX_FILES = 8 + +# Roles whose files earn their way into a commit by referring to each other, +# rather than by sitting in the same directory. +CODE_ROLES = {"source", "test", "frontdoor"} + + +def _subprojects(paths): + """Directories that are their own project -> the files beneath them.""" + marked = set() + for p in paths: + parent = str(Path(p).parent) + if parent not in (".", "") and Path(p).name.lower() in SUBPROJECT_MARKERS: + marked.add(parent) + # A subproject inside a subproject belongs to the outer one; one commit, + # not two nested ones. + roots = {d for d in marked + if not any(d != o and d.startswith(o + "/") for o in marked)} + owned = {} + for p in paths: + for root in roots: + if p == root or p.startswith(root + "/"): + owned[p] = root + break + return owned + + +# Roles that are one idea when they sit side by side. A diagram and the source +# it renders from belong in the same commit; so do a pin and the config that +# reads it. Grouping only — the ordering still keeps them apart. +GROUP_TIER = {"asset": "doc", "lock": "pin", "pin": "pin", "config": "pin"} + + +def _cluster(path, owner, dirs=()): + """ + The directory a file is grouped under: its subproject, else its own parent. + + The parent, not the top-level directory. Under `ctrl` everything in a tree + this shape lands in one bucket — eleven scripts, four profiles and a k8s + tree — and the split has to be reconstructed afterwards from references + that were never going to describe it. + """ + if owner: + return owner + # `ctrl/addons.sh` belongs with `ctrl/addons/`, not with its own siblings. + # Clustering is what decides which files are even considered together, so a + # hub parted from its satellites here can never be rejoined later. + hub = _hub_of(path) + if hub and hub in dirs: + return hub + return str(Path(path).parent) + + +def adaptive_cap(count, requested=None): + """ + How many files one commit may hold, for a repo this size. + + A fixed cap does not survive the range. rig is 64 files and wants commits + of three or four; spr is 507 and at a flat eight plans a hundred and + forty-three, which is the same failure as one commit from the other end. + + A twentieth of the tree, floored at the default, lands close to what these + repos were actually built as: rig plans 18 against a real 18, spr 77 + against a real 78. It is a starting point, not a claim — --max-files + overrides it and plan.json is editable either way. + """ + if requested: + return requested + return max(DEFAULT_MAX_FILES, -(-count // 20)) + + +def order_files(index, max_files=None): + """Return an ordered list of groups, each a list of repo-relative paths.""" + files = index["files"] + paths = sorted(files) + max_files = adaptive_cap(len(paths), max_files) + owner = _subprojects(paths) + + def rank(p): + return SUBPROJECT_RANK if p in owner else ROLE_RANK.get(files[p]["role"], 60) + + # ── edges ────────────────────────────────────────────────────────────── + # An edge b -> a means "b must come after a". Only real dependencies count, + # and only between files whose roles do not already disagree: a reference + # pointing backwards up the role order is a mention the extractor could not + # tell from a dependency, and honouring it inverts the tier. + after = defaultdict(set) + for p in paths: + if files[p]["role"] in NARRATIVE or p in owner: + continue + for dep in files[p]["refs"]: + if dep in files and dep != p and rank(dep) <= rank(p): + after[p].add(dep) + + # ── ordering ─────────────────────────────────────────────────────────── + # Kahn's algorithm, taking the lowest (rank, path) that is ready. Ties are + # broken by path so two runs on the same tree produce the same history. + blockers = {p: set(after[p]) for p in paths} + dependents = defaultdict(set) + for p, deps in blockers.items(): + for d in deps: + dependents[d].add(p) + + ready = sorted((p for p in paths if not blockers[p]), key=lambda p: (rank(p), p)) + ordered = [] + while ready: + p = ready.pop(0) + ordered.append(p) + for d in sorted(dependents[p]): + blockers[d].discard(p) + if not blockers[d]: + ready.append(d) + ready.sort(key=lambda q: (rank(q), q)) + + # A cycle leaves files unplaced. Two shell scripts that source each other is + # a real thing and not an error here, so append them in role order rather + # than refusing to produce a plan at all. + if len(ordered) < len(paths): + ordered += sorted(set(paths) - set(ordered), key=lambda p: (rank(p), p)) + + return _coalesce(_group(ordered, files, owner, after, max_files), owner, max_files) + + +def _common_dir(paths): + """The deepest directory every path in the group sits under.""" + parts = list(Path(paths[0]).parent.parts) + for p in paths[1:]: + other = Path(p).parent.parts + keep = [] + for a, b in zip(parts, other): + if a != b: + break + keep.append(a) + parts = keep + return tuple(parts) + + +def _coalesce(groups, owner, max_files): + """ + Merge neighbouring groups that are really one idea in one place. + + Cutting on directory is right for a shallow tree and wrong for a deep one: + a directory holding a single file is not an idea, and a repo of five + hundred files has a lot of them. Left alone spr planned three hundred and + thirty-seven commits, which is the same failure as one commit, from the + other end. + + Only neighbours already adjacent in the order merge, only when they share + a top-level directory, only under the cap, and only when BOTH are small. + + Both, not either. Letting a lone file join whatever it happened to sit next + to put `dockerhost.sh` inside the addons commit — a coherent group of seven + with an eighth file that has nothing to do with it. A run of scattered + singletons is fragmentation and should close up; a group that already says + something should not absorb a stray because it had room. + + What counts as small scales with the cap, so raising --max-files actually + buys fewer commits. Gated at a flat two it did not: on a 500-file repo the + cap could be raised from 8 to 40 and the count moved by four, because + nothing was ever allowed to merge. Asking for bigger commits should widen + what is considered fragmentation, not just what is allowed to survive. + """ + loose = max(2, max_files // 4) + out = [] + for group in groups: + if not out: + out.append(group) + continue + previous = out[-1] + if (len(previous) + len(group) <= max_files + and max(len(previous), len(group)) <= loose + and not any(p in owner for p in previous + group) + and _shares_ancestor(_common_dir(previous), _common_dir(group))): + out[-1] = previous + group + else: + out.append(group) + return out + + +def _shares_ancestor(a, b): + """Both at the root, or under a common top-level directory.""" + if not a and not b: + return True + return bool(a) and bool(b) and a[0] == b[0] + + +def _group(ordered, files, owner, after, max_files): + """ + Cut the ordered list into commits. + + One coherent idea per commit, not one directory per commit. What holds a + group together is that its files refer to each other — ports.sh and the + hosts template it renders, a kustomization and the manifests it lists. What + separates two groups in the same directory is that neither names the other. + + A hub and the directory named after it (addons.sh and addons/) travel + together whatever their references say: committing the loader without the + things it loads produces a commit that cannot run. + """ + dirs = {str(Path(p).parent) for p in ordered} + groups, buf = [], [] + seen_cluster = seen_role = None + + def flush(): + nonlocal buf + if buf: + groups.append(buf) + buf = [] + + for path in ordered: + own = owner.get(path) + cluster = _cluster(path, own, dirs) + role = "subproject" if own else GROUP_TIER.get( + files[path]["role"], files[path]["role"]) + if cluster != seen_cluster or role != seen_role: + flush() + seen_cluster, seen_role = cluster, role + buf.append(path) + flush() + + out = [] + for group in groups: + # A subproject is one commit no matter how many files it holds. + if any(p in owner for p in group): + out.append(group) + continue + out.extend(_split(group, files, after, max_files)) + return out + + +def _hub_of(path): + """`ctrl/addons.sh` is the hub of `ctrl/addons/`; returns that directory.""" + p = Path(path) + return str(p.parent / p.stem) if p.suffix else None + + +def _pure_hub(members, hubs): + """True when the group is exactly one hub and things inside its directory.""" + for hub, owner_file in hubs.items(): + if owner_file in members and all( + m == owner_file or m.startswith(hub + "/") for m in members + ): + return True + return False + + +def _split(group, files, after, max_files): + """ + Break one cluster into commits. + + A run of siblings of the same kind in the same directory is left alone — + four profile files under env.d/ are one idea, and nothing in them refers to + anything, so splitting on references turns them into four commits saying + the same thing four times. References are only asked about a group that is + already too big or already spans directories. + + Past that: connected components, because what holds a commit together is + that its files name each other. A hub and the directory named after it + (addons.sh and addons/) survive the cap, since committing the loader + without the things it loads produces a commit that cannot run. Anything + else over the cap gets its hub peeled off into its own commit and the + remainder reconsidered — a cut at a named seam rather than at a count. + """ + if len(group) <= 1: + return [group] + + # ...but only for roles where sitting side by side IS the relationship. + # Four env.d profiles are one idea. Seven scripts that happen to share a + # directory are seven ideas, and `ctrl/` is full of them, so code always + # gets asked about its references. + parents = {str(Path(p).parent) for p in group} + kinds = {files[p]["role"] for p in group} + if len(parents) == 1 and not (kinds & CODE_ROLES): + if len(group) <= max_files: + return [group] + # Over the cap and still one kind in one directory: cut it into runs. + # Nothing here refers to anything, so asking references to find the + # seam yields one commit per file — thirteen commits each saying "a + # project note", which is worse than an admitted arbitrary cut. + return [group[i:i + max_files] for i in range(0, len(group), max_files)] + + index = {p: i for i, p in enumerate(group)} + parent = list(range(len(group))) + + def find(i): + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + def union(a, b): + ra, rb = find(a), find(b) + if ra != rb: + parent[max(ra, rb)] = min(ra, rb) + + hubs = {} + for p in group: + hub = _hub_of(p) + if hub: + hubs[hub] = p + for p in group: + for hub, owner_file in hubs.items(): + if p.startswith(hub + "/"): + union(index[p], index[owner_file]) + for dep in after.get(p, ()): + if dep in index: + union(index[p], index[dep]) + + components = defaultdict(list) + for p in group: + components[find(index[p])].append(p) + + out = [] + for key in sorted(components): + members = components[key] + while len(members) > max_files and not _pure_hub(members, hubs): + degree = {m: sum(1 for o in members if m in after.get(o, ())) for m in members} + hub = max(degree, key=lambda m: (degree[m], m)) + if degree[hub] == 0: + # Nothing holds this together and nothing names anything: an + # arbitrary cut is the honest answer, so cut on the order we + # already have rather than inventing a reason. + out.extend([members[i:i + max_files] + for i in range(0, len(members), max_files)]) + members = [] + break + members.remove(hub) + out.append([hub]) + if members: + out.append(members) + return [g for g in out if g] + + +# ── the plan ─────────────────────────────────────────────────────────────── + +def plan_path(out) -> Path: + return state_dir(out) / PLAN_FILE + + +def build_plan(index, out, max_files=None, quiet=False): + """ + Write plan.json: ordered groups of paths with empty message slots. + + This is the seam. Everything above is analysis and can be recomputed from + the tree; everything below is git commands. An existing plan's messages are + carried over when the group's paths still match exactly, so re-planning + after editing three files does not throw away sixty written messages. + """ + groups = order_files(index, max_files) + + written = {} + existing = plan_path(out) + if existing.exists(): + try: + for g in json.loads(existing.read_text()).get("groups", []): + if g.get("title"): + written[tuple(sorted(g["paths"]))] = (g.get("title"), g.get("body", "")) + except (OSError, json.JSONDecodeError, KeyError, TypeError): + pass + + document = {"version": 1, "groups": []} + for i, paths in enumerate(groups, 1): + title, body = written.get(tuple(sorted(paths)), ("", "")) + document["groups"].append({ + "n": i, + "slug": _slug(paths, index), + "paths": paths, + "title": title, + "body": body, + }) + + destination = plan_path(out) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(document, indent=2)) + if not quiet: + kept = sum(1 for g in document["groups"] if g["title"]) + print(f"Planned {len(groups)} commits over " + f"{sum(len(g) for g in groups)} files" + + (f", {kept} messages carried over" if kept else "") + ".") + print(f" -> {destination}") + return document + + +def _slug(paths, index): + """A stable handle for a group, for filenames and for talking about it.""" + common = Path(paths[0]).parent + for p in paths[1:]: + parts = [] + for a, b in zip(common.parts, Path(p).parent.parts): + if a != b: + break + parts.append(a) + common = Path(*parts) if parts else Path(".") + base = str(common).strip("./").replace("/", "-") + if not base: + base = Path(paths[0]).stem if len(paths) == 1 else index["files"][paths[0]]["role"] + return base.lower().replace("_", "-").replace(".", "")[:40] or "root" diff --git a/soleprint/station/tools/histgen/selftest.py b/soleprint/station/tools/histgen/selftest.py new file mode 100644 index 0000000..b3aa6e6 --- /dev/null +++ b/soleprint/station/tools/histgen/selftest.py @@ -0,0 +1,413 @@ +""" +Prove the whole pipeline on a tree this builds itself. + +`make check` after copying the folder somewhere new, with no repo to point at +and nothing installed. It builds a small tree with the shapes that matter — +ignore rules, a README, a pin, a config that sources it, a directory of +profiles, a hub with satellites, a front door, an ignored directory — runs all +five verbs over it, and asserts what has to be true. + +The guards get the same treatment as the happy path. A check that only ever +proves things work would have missed the one real bug found while writing this: +the tree-hash guard returned None on exactly the trees it was written for, and +reported success anyway. + + python3 selftest.py # or: make check +""" + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +TREE = { + ".gitignore": "# Generated output; regenerate rather than commit.\nbuild/\n*.log\n", + ".gitattributes": "# LF everywhere: a CRLF checkout breaks the shebang.\n* text=auto eol=lf\n", + "README.md": "# demo\n\nWhat this is, and the one prerequisite.\n", + "ctrl/versions.env": "# Pinned toolchain — one manifest, one answer.\nKUBECTL=1.31.0\n", + "ctrl/lib/config.sh": "# Shared config loading. Sourced, never executed.\n" + '. "$(dirname "$0")/../versions.env"\n', + "ctrl/env.d/minimal.env": "# The smallest profile that still starts.\nADDONS=\n", + "ctrl/env.d/full.env": "# Everything on, for a demo machine.\nADDONS=redis\n", + "ctrl/addons.sh": "# Install the addons the profile asked for.\n" + "# Adding one is adding a file, not editing a dispatcher.\n" + 'for a in ctrl/addons/*.sh; do sh "$a"; done\n', + "ctrl/addons/redis.sh": "# Redis — the broker half, nothing else uses it.\necho redis\n", + "ctrl/addons/postgres.sh": "# Postgres — the metadata store.\necho postgres\n", + "Makefile": "# Thin front door: one target per ctrl/ script.\nup:\n\tsh ctrl/addons.sh\n", + "BOOTSTRAP.md": "# From a bare machine to something running.\n", + "build/generated.txt": "this is ignored and must never be committed", + "noisy.log": "also ignored", +} + +IGNORED = {"build/generated.txt", "noisy.log"} + +# Things a copy should leave behind, and the two lookalikes it must not. Added +# to the tree only for the `copy` checks, and force-added so the ignored-but- +# tracked case is real rather than described. +SIFTABLE = { + "package-lock.json": "lockfile contents", + "dist/app.min.js": "minified", + ".env": "API_KEY=real-secret-value", + ".env.example": "API_KEY=", + "certs/server.key": "-----BEGIN PRIVATE KEY-----", + "certs/server.key.pub": "ssh-rsa AAAA", + "assets/logo.png": "PNG", + "build/forced.bin": "ignored yet tracked", +} +DROPPED = {"package-lock.json", "dist/app.min.js", ".env", + "certs/server.key", "build/forced.bin"} +KEPT_LOOKALIKES = {".env.example", "certs/server.key.pub", "assets/logo.png"} + + +def run(pkg, parent, *args): + return subprocess.run([sys.executable, "-m", pkg, *args], + capture_output=True, text=True, + env={"PYTHONPATH": str(parent), "PATH": "/usr/bin:/bin", + "HOME": str(Path.home())}) + + +def build(root: Path): + for rel, body in TREE.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body) + + +def main(): + here = Path(__file__).resolve().parent + pkg, parent = here.name, here.parent + failures = [] + + def check(label, condition, detail=""): + print(f" {'ok ' if condition else 'FAIL'} {label}") + if not condition: + failures.append(f"{label}{': ' + detail if detail else ''}") + + def hg(*args): + return run(pkg, parent, *args) + + with tempfile.TemporaryDirectory(prefix="histgen-selftest-") as tmp: + source, out = Path(tmp) / "demo", Path(tmp) / "out" + build(source) + W = ["--source", str(source), "--out", str(out)] + copy = out / "demo" + + r = hg("run", *W) + check("run: scan + plan + brief", r.returncode == 0, r.stderr.strip()) + if r.returncode != 0: + print(r.stderr) + return 1 + + index = json.loads((out / "index.json").read_text()) + plan = json.loads((out / "plan.json").read_text()) + planned = [p for g in plan["groups"] for p in g["paths"]] + + check("the ignore rules were honoured", not (set(planned) & IGNORED), + f"ignored files got planned: {sorted(set(planned) & IGNORED)}") + check("every other file is in exactly one group", + sorted(planned) == sorted(set(TREE) - IGNORED) and len(planned) == len(set(planned))) + check("the ignore rules are committed first", + plan["groups"][0]["paths"][0].startswith(".git")) + check("the front door is not", "Makefile" not in plan["groups"][0]["paths"]) + check("the hub travels with its satellites", + any({"ctrl/addons.sh", "ctrl/addons/redis.sh", "ctrl/addons/postgres.sh"} + <= set(g["paths"]) for g in plan["groups"])) + check("a comment is not mistaken for a dependency", + index["files"][".gitignore"]["refs"] == []) + check("a real dependency is found", + "ctrl/versions.env" in index["files"]["ctrl/lib/config.sh"]["refs"]) + check("the reasoning was extracted for the message", + "not editing a dispatcher" in index["files"]["ctrl/addons.sh"]["why"]) + check("a brief exists per commit", + len(list((out / "briefs").glob("*.md"))) == len(plan["groups"])) + check("list prints one line per commit", + all(f"{g['n']:3}." in hg("list", *W).stdout for g in plan["groups"])) + + # The guards refuse before they approve. + check("export refuses a plan with no messages", + "no title" in hg("export", *W).stderr) + (source / "appeared-late.sh").write_text("# added after planning\n") + r = hg("export", *W, "--allow-untitled") + check("export refuses a file no group covers", + r.returncode != 0 and "no group" in r.stderr) + (source / "appeared-late.sh").unlink() + + # --- commands: copy the files, make no repo, hand back the list --- + # Its own out directory, so it needs its own plan in it: the plan is a + # fact about an out directory, not about the source. + hands = Path(str(out) + "-byhand") + H = ["--source", str(source), "--out", str(hands)] + hg("run", *H) + r = hg("export", *H, "--allow-untitled", "--commands") + check("commands: succeeds", r.returncode == 0, r.stderr.strip()) + made = hands / "demo" + check("commands: the files were copied", (made / "README.md").is_file()) + check("commands: NO repo was created", not (made / ".git").exists()) + check("commands: gitignored files did not come across", + not any((made / i).exists() for i in IGNORED)) + check("commands: git init is the first thing offered, not done for you", + "git init" in r.stdout) + check("commands: one add and one commit per group", + r.stdout.count("git add -- ") == len(plan["groups"]) + and r.stdout.count("git commit -F ") == len(plan["groups"])) + check("commands: the list is saved too", (hands / "commands.sh").is_file()) + + # The list has to actually work, which is the only claim that matters. + subprocess.run(["bash", str(hands / "commands.sh")], capture_output=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t"}) + check("commands: running the list builds the history", + subprocess.run(["git", "-C", str(made), "rev-list", "--count", "HEAD"], + capture_output=True, text=True).stdout.strip() + == str(len(plan["groups"]))) + check("commands: it leaves nothing untracked", + not subprocess.run(["git", "-C", str(made), "status", "--porcelain"], + capture_output=True, text=True).stdout.strip()) + expected = [l for l in r.stdout.split("\n") if "expect " in l][0].split("expect ")[1].strip() + check("commands: the tree matches the hash it told you to expect", + subprocess.run(["git", "-C", str(made), "rev-parse", "HEAD^{tree}"], + capture_output=True, text=True).stdout.strip() == expected) + check("commands: refuses when a repo is already there", + hg("export", *H, "--allow-untitled", "--commands").returncode != 0) + + # --- absent -> exported --- + check("status says absent before anything is exported", + "absent" in hg("status", *W).stdout) + r = hg("export", *W, "--allow-untitled") + check("export writes the history", r.returncode == 0, r.stderr.strip()) + check("both guards ran", + "nothing left untracked: ok" in r.stdout and "tree matches source" in r.stdout) + check("the copy is named after the source", copy.is_dir()) + check("one commit per group", + subprocess.run(["git", "-C", str(copy), "rev-list", "--count", "HEAD"], + capture_output=True, text=True).stdout.strip() + == str(len(plan["groups"]))) + check("the ignored files never entered the history", + not (set(subprocess.run(["git", "-C", str(copy), "ls-files"], + capture_output=True, text=True).stdout.split()) & IGNORED)) + + # The invariant the whole design rests on. + check("THE SOURCE WAS NEVER WRITTEN TO", + not (source / ".git").exists() and not (source / ".histgen").exists() + and sorted(p.relative_to(source).as_posix() + for p in source.rglob("*") if p.is_file()) == sorted(TREE)) + + check("a finished export says so and stops", + "Nothing to do" in hg("export", *W, "--allow-untitled").stdout) + check("verify passes on its own", hg("verify", *W).returncode == 0) + + # --- unfinished: interrupt, then resume --- + progress = json.loads((out / "progress.json").read_text()) + # Half of them, whatever the fixture happens to plan. Hardcoding four + # made this pass vacuously the moment the fixture planned exactly four: + # nothing was left to resume, so "complete" was the honest answer and + # the resume path was never entered. + keep = progress["commits"][:max(1, len(progress["commits"]) // 2)] + subprocess.run(["git", "-C", str(copy), "reset", "-q", "--hard", keep[-1]["sha"]]) + progress["commits"] = keep + (out / "progress.json").write_text(json.dumps(progress)) + + check("status spots a half-finished history", + "unfinished" in hg("status", *W).stdout) + r = hg("export", *W, "--allow-untitled") + check("export resumes rather than restarting", + r.returncode == 0 and "Resuming" in r.stdout, r.stdout + r.stderr) + check("resuming did not redo the commits already made", + f"{len(keep)} of {len(plan['groups'])}" in r.stdout, r.stdout) + check("the resumed history is complete and verified", + "tree matches source" in r.stdout) + + # --- stale: the plan moved --- + plan["groups"][1]["paths"].append(plan["groups"][2]["paths"].pop()) + (out / "plan.json").write_text(json.dumps(plan)) + check("status spots a plan that no longer matches", + "stale" in hg("status", *W).stdout) + check("export refuses a stale export", + hg("export", *W, "--allow-untitled").returncode != 0) + check("--force starts over", + hg("export", *W, "--allow-untitled", "--force").returncode == 0) + + # --- foreign: a history that has to be kept --- + with tempfile.TemporaryDirectory(prefix="histgen-selftest-keep-") as tmp: + source, out = Path(tmp) / "demo", Path(tmp) / "out" + build(source) + for cmd in (["init", "-q"], ["add", "-A"], ["commit", "-qm", "init commit"]): + subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t"}) + before = subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() + W = ["--source", str(source), "--out", str(out)] + copy = out / "demo" + + hg("run", *W) + r = hg("export", *W, "--allow-untitled", "--keep-history") + check("keep: export succeeds", r.returncode == 0, r.stderr.strip()) + check("keep: the old history is reported as kept", "kept, untouched" in r.stdout) + + branches = subprocess.run(["git", "-C", str(copy), "branch", "--format=%(refname:short)"], + capture_output=True, text=True).stdout.split() + check("keep: the designed history is on its own branch", + "designed-history" in branches and len(branches) >= 2, str(branches)) + main = [b for b in branches if b != "designed-history"][0] + check("keep: the old branch still points where it did", + subprocess.run(["git", "-C", str(copy), "rev-parse", main], + capture_output=True, text=True).stdout.strip() == before) + check("keep: the two histories share no commit", + subprocess.run(["git", "-C", str(copy), "merge-base", main, "designed-history"], + capture_output=True, text=True).returncode != 0) + check("keep: the source's own history is untouched", + subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"], + capture_output=True, text=True).stdout.strip() == before) + + # --- the filter has to hold on the HISTORY path, not just on copy --- + # This is the case that was wrong: `copy` left a tracked key behind while + # `scan` walked straight past it, so the key stayed out of the snapshot and + # went into the commits. + with tempfile.TemporaryDirectory(prefix="histgen-selftest-secret-") as tmp: + source, out = Path(tmp) / "demo", Path(tmp) / "out" + build(source) + for rel, body in SIFTABLE.items(): + (source / rel).parent.mkdir(parents=True, exist_ok=True) + (source / rel).write_text(body) + env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + for cmd in (["init", "-q"], ["add", "-A", "-f", "."], ["commit", "-qm", "init"]): + subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env) + + W = ["--source", str(source), "--out", str(out)] + r = hg("run", *W) + check("filter: scan says what it left out", "left out of the history" in r.stdout) + + index = json.loads((out / "index.json").read_text()) + plan = json.loads((out / "plan.json").read_text()) + planned = {p for g in plan["groups"] for p in g["paths"]} + secrets = {".env", "certs/server.key"} + check("filter: no secret reached the census", not (set(index["files"]) & secrets)) + check("filter: no secret reached the plan", not (planned & secrets)) + check("filter: the ignored-but-tracked file did not either", + "build/forced.bin" not in planned) + check("filter: the lookalikes are still in the plan", + {".env.example", "certs/server.key.pub"} <= planned) + check("filter: a lockfile IS kept — a repo wants its lockfile", + "package-lock.json" in planned) + + r = hg("export", *W, "--allow-untitled", "--commands") + # Exact tokens, not substrings: ".env" is inside ".env.example", so a + # substring test reports a leak every time the lookalike is kept — + # which is exactly the behaviour that is wanted. + added = {tok for line in r.stdout.split("\n") if line.startswith("git add -- ") + for tok in line[len("git add -- "):].split()} + check("filter: no secret reached the commands", + not (added & secrets), str(sorted(added & secrets))) + check("filter: the commands cover exactly the planned files", + added == planned, str(sorted(added ^ planned))) + check("filter: no secret reached the copy", + not any((out / "demo" / x).exists() for x in secrets)) + + out2 = Path(tmp) / "out2" + hg("run", "--source", str(source), "--out", str(out2)) + r = hg("export", "--source", str(source), "--out", str(out2), "--allow-untitled") + check("filter: the guards still pass on the filtered set", + "tree matches source" in r.stdout, r.stdout + r.stderr) + tracked = subprocess.run(["git", "-C", str(out2 / "demo"), "ls-files"], + capture_output=True, text=True).stdout.split() + check("filter: the committed history holds no secret", + not (set(tracked) & secrets), str(tracked)) + + out3 = Path(tmp) / "out3" + hg("run", "--source", str(source), "--out", str(out3), "--keep-secrets") + index3 = json.loads((out3 / "index.json").read_text()) + check("filter: --keep-secrets brings them back", + secrets <= set(index3["files"])) + + # --- copy: the plain utility, no history involved --- + with tempfile.TemporaryDirectory(prefix="histgen-selftest-copy-") as tmp: + source, out = Path(tmp) / "demo", Path(tmp) / "out" + build(source) + for rel, body in SIFTABLE.items(): + (source / rel).parent.mkdir(parents=True, exist_ok=True) + (source / rel).write_text(body) + env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + for cmd in (["init", "-q"], ["add", "-A", "-f", "."], + ["commit", "-qm", "init"]): + subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env) + + C = ["--source", str(source), "--out", str(out)] + r = hg("copy", *C) + check("copy: succeeds", r.returncode == 0, r.stderr.strip()) + made = out / "demo" + check("copy: no .git came across", not (made / ".git").exists()) + check("copy: nothing gitignored came across", + not any((made / i).exists() for i in IGNORED)) + for rel in sorted(DROPPED): + check(f"copy: left behind {rel}", not (made / rel).exists()) + for rel in sorted(KEPT_LOOKALIKES): + check(f"copy: kept {rel}", (made / rel).is_file()) + check("copy: every drop is named in the output", + all(rel in r.stdout for rel in DROPPED), r.stdout) + check("copy: a manifest records what was left behind", + (out / "copied.md").is_file() + and all(rel in (out / "copied.md").read_text() for rel in DROPPED)) + check("copy: refuses a destination that is not empty", + hg("copy", *C).returncode != 0) + + preview = Path(tmp) / "preview" + r = hg("copy", "--source", str(source), "--out", str(preview), "--dry-run") + check("copy: --dry-run writes nothing", not preview.exists() and r.returncode == 0) + + full = Path(tmp) / "full" + r = hg("copy", "--source", str(source), "--out", str(full), + "--all", "--keep-secrets") + check("copy: --all --keep-secrets keeps what it says", + (full / "demo" / "package-lock.json").is_file() + and (full / "demo" / ".env").is_file()) + + picky = Path(tmp) / "picky" + r = hg("copy", "--source", str(source), "--out", str(picky), + "--exclude", "*.png", "--include", "package-lock.json") + check("copy: --exclude drops by glob at any depth", + not (picky / "demo" / "assets" / "logo.png").exists()) + check("copy: --include overrides the filters", + (picky / "demo" / "package-lock.json").is_file()) + + # --- config --- + with tempfile.TemporaryDirectory(prefix="histgen-selftest-cfg-") as tmp: + source, out = Path(tmp) / "demo", Path(tmp) / "out" + build(source) + cfg = Path(tmp) / "settings.json" + cfg.write_text(json.dumps({"source": str(source), "out": str(out)})) + r = hg("config", "--config", str(cfg)) + check("config: a file supplies source and out", + r.returncode == 0 and str(out) in r.stdout, r.stderr.strip()) + check("config: the command line wins", + "/tmp/override" in hg("config", "--config", str(cfg), + "--out", "/tmp/override").stdout) + cfg.write_text(json.dumps({"source": str(source), "outp": "typo"})) + check("config: a mistyped key is refused", + "unknown key" in hg("config", "--config", str(cfg)).stderr) + cfg.write_text(json.dumps({"repo": str(source), "out": str(out)})) + check("config: the old 'repo' key still works", + hg("config", "--config", str(cfg)).returncode == 0) + check("out inside source is refused", + "inside source" in hg("status", "--source", str(source), + "--out", str(source / "sub")).stderr) + + print() + if failures: + print(f"{len(failures)} check(s) failed:") + for f in failures: + print(f" - {f}") + return 1 + print("all checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/station/tools/histgen/sift.py b/soleprint/station/tools/histgen/sift.py new file mode 100644 index 0000000..fa49095 --- /dev/null +++ b/soleprint/station/tools/histgen/sift.py @@ -0,0 +1,142 @@ +""" +What to leave behind when copying a repo out of itself. + +Three kinds of file get dropped, and they are dropped for three different +reasons. Keeping them apart matters, because only one of the three is safe to +decide silently. + + derived something a build regenerates. Lockfiles, source maps, minified + output, compiled objects, cache directories. The list is ported + from `ppl/ctrl/distill.sh`, including the lesson written into its + comments: the line is **derived-vs-content, not text-vs-binary**. + That distinction was wrong there once and cost real files — a + logo, a font the site loads, a downloadable PDF — because none of + those can be regenerated from what is left, which is the only + thing that makes a file safe to drop. So images, fonts and + spreadsheets are content and are kept. + + secret a private key, a credential store, an .env holding real values. + distill does not do this; .gitignore usually has, and where it has + not, the file is tracked and travels. That is not hypothetical — + soleprint's own notes record an API key that was tracked in a + tool's .env, and untracking it did not unpublish it. + + ignored tracked, and yet matched by the repo's own ignore rules. Someone + ran `git add -f` once. Sometimes deliberate — a built artifact + committed on purpose — and sometimes a dump or a credentials file + that went in and was never noticed again. Reported by name either + way, because the repo is already contradicting itself about them. + + oversize whatever --max-bytes says. Size is its own worry and gets its own + knob rather than being smuggled in as a guess about kind. + +Every drop is reported. A file quietly missing from a copy is the same class of +failure as a file quietly missing from a history, and this tool exists because +that class of failure is expensive. +""" + +import fnmatch +import re +from pathlib import Path + +# ── derived ──────────────────────────────────────────────────────────────── +# One list, one place to edit. `--all` turns it off wholesale. +NOISE = re.compile( + r'(^|/)(package-lock\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|yarn\.lock' + r'|bun\.lock|bun\.lockb|uv\.lock|poetry\.lock|Pipfile\.lock|Cargo\.lock' + r'|composer\.lock|Gemfile\.lock|go\.sum|\.DS_Store|Thumbs\.db)$' + r'|\.(map|min\.js|min\.css)$' + r'|\.(pyc|pyo|pyd|class|o|obj|a|lib|so|dylib|dll|wasm|pack|idx)$' + r'|(^|/)(__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|node_modules)/' +) + +# ── secret ───────────────────────────────────────────────────────────────── +# Deliberately narrow. A pattern that catches a real key once a year and a +# needed file once a week gets turned off, and then it catches nothing. +SECRET = re.compile( + r'(^|/)\.env(\.[A-Za-z0-9_-]+)?$' + r'|(^|/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)$' + r'|\.(pem|key|p12|pfx|jks|keystore|ppk|asc|gpg)$' + r'|(^|/)(\.netrc|\.npmrc|\.pypirc|\.htpasswd|\.dockercfg)$' + r'|(^|/)\.ssh/' + r'|(^|/)credentials(\.json|\.yaml|\.yml)?$' + r'|(^|/)service-account[-_A-Za-z0-9]*\.json$' +) + +# The exceptions matter more than the rule. A committed .env.example is the +# documented way to say what the real one needs, and dropping it takes the +# documentation with the secret. +SECRET_OK = re.compile( + r'\.(example|sample|template|dist|tmpl)$' + r'|(^|/)\.env\.(example|sample|template)$' + r'|\.pub$' +) + + +def is_derived(rel: str) -> bool: + return bool(NOISE.search(rel)) + + +def is_secret(rel: str) -> bool: + return bool(SECRET.search(rel)) and not SECRET_OK.search(rel) + + +def matches(rel: str, patterns) -> bool: + """ + Glob match, with distill's rule: a pattern holding no `/` also matches + basenames at any depth, so `--exclude '*.csv'` means what it looks like. + """ + name = Path(rel).name + for pattern in patterns or (): + if fnmatch.fnmatch(rel, pattern): + return True + if "/" not in pattern and fnmatch.fnmatch(name, pattern): + return True + if pattern.endswith("/") and (rel + "/").startswith(pattern.lstrip("/")): + return True + return False + + +def sift(source: Path, paths, keep_noise=False, keep_secrets=False, + max_bytes=None, exclude=(), include=(), ignored=()): + """ + Split the file list into what to copy and what to leave, with reasons. + + `include` is checked first and wins outright: it is the way to say "yes, I + do want that lockfile" without turning the whole filter off. + """ + kept, dropped = [], [] + for rel in paths: + if include and matches(rel, include): + kept.append(rel) + continue + if exclude and matches(rel, exclude): + dropped.append((rel, "excluded")) + continue + if not keep_secrets and is_secret(rel): + dropped.append((rel, "secret")) + continue + if rel in ignored: + dropped.append((rel, "ignored")) + continue + if not keep_noise and is_derived(rel): + dropped.append((rel, "derived")) + continue + if max_bytes: + try: + if (source / rel).stat().st_size > max_bytes: + dropped.append((rel, "oversize")) + continue + except OSError: + pass + kept.append(rel) + return kept, dropped + + +REASONS = { + "ignored": "tracked, but the ignore rules say they should not be", + "derived": "a build regenerates these", + "secret": "looks like a key or a credential", + "oversize": "larger than --max-bytes", + "excluded": "matched --exclude", +} diff --git a/soleprint/station/tools/histgen/snapshot.py b/soleprint/station/tools/histgen/snapshot.py new file mode 100644 index 0000000..ac0975b --- /dev/null +++ b/soleprint/station/tools/histgen/snapshot.py @@ -0,0 +1,121 @@ +""" +Copy a repo's files out of it, without the repo. + +Named snapshot.py and not copy.py, which is what it was for about ten minutes. +A module called `copy` beside the code shadows the standard library's, and the +directory lands on sys.path whenever anything is run from inside it — so +`dataclasses` imported this file instead, and every command died on an import +error before parsing a single argument. The verb is still `copy`; the file +cannot be. + +The plain utility underneath everything else: point it at a tree, get a folder +holding what the project actually is — no `.git`, nothing gitignored, nothing a +build regenerates, and nothing that looks like a key. + +It is the thing to reach for when the history is not the point. Handing a +snapshot to someone, feeding a tree to something that should not see the +history, or getting a clean starting tree before planning one. + +What is dropped is reported and written to a manifest, never assumed. A file +missing from a copy without a line saying so is the same failure this whole +tool exists to prevent, one directory earlier. +""" + +import shutil +from pathlib import Path + +from .census import file_set, ignored_but_tracked, is_git, state_dir +from .cli import fail +from .sift import REASONS, sift + +MANIFEST = "copied.md" + + +def destination(source: Path, out: Path) -> Path: + """`out/`, so the folder keeps the name the thing already had.""" + return Path(out) / source.name + + +def take(source: Path, out, keep_noise=False, keep_secrets=False, max_bytes=None, + exclude=(), include=(), force=False, dry_run=False, quiet=False): + dest = destination(source, out) + + if dest.exists() and any(dest.iterdir()) and not force and not dry_run: + fail(f"{dest} already exists and is not empty.", + "Pass --force to write into it anyway, or point --out elsewhere.") + + paths = file_set(source) + kept, dropped = sift(source, paths, keep_noise=keep_noise, + keep_secrets=keep_secrets, max_bytes=max_bytes, + exclude=exclude, include=include, + ignored=ignored_but_tracked(source, paths)) + + if not quiet: + print(f"{len(paths)} files tracked, {len(kept)} to copy, {len(dropped)} left behind.") + by_reason = {} + for rel, why in dropped: + by_reason.setdefault(why, []).append(rel) + for why in ("secret", "ignored", "derived", "oversize", "excluded"): + hits = by_reason.get(why) + if not hits: + continue + # Secrets are listed in full however many there are. The others are + # bulk and a count is enough; a key that got dropped is a thing you + # want to see the name of, because it means it was tracked. + shown = hits if why in ("secret", "ignored") else hits[:5] + print(f"\n {why} — {REASONS[why]} ({len(hits)}):") + for rel in shown: + print(f" {rel}") + if len(hits) > len(shown): + print(f" ... and {len(hits) - len(shown)} more") + + if dry_run: + if not quiet: + print(f"\nNothing written. Would copy to {dest}.") + return {"kept": kept, "dropped": dropped, "dest": dest} + + dest.mkdir(parents=True, exist_ok=True) + for rel in kept: + target = dest / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / rel, target) + + _manifest(source, out, dest, paths, kept, dropped) + + if not quiet: + print(f"\nCopied to {dest}") + print(f" no .git — {'the source has one and it was not copied' + if is_git(source) else 'the source has none either'}") + print(f" what was left behind: {state_dir(out) / MANIFEST}") + return {"kept": kept, "dropped": dropped, "dest": dest} + + +def _manifest(source, out, dest, paths, kept, dropped): + """A record of the decision, beside the copy rather than inside it.""" + lines = [ + f"# Copied from `{source}`", "", + f"- source: `{source}`", + f"- copy: `{dest}`", + f"- {len(paths)} files tracked, {len(kept)} copied, {len(dropped)} left behind", + "", + "No `.git` was copied. The file list is what git tracks, so nothing " + "untracked or ignored came across — except where a file was tracked " + "*despite* the ignore rules, which is listed below rather than assumed.", + "", + ] + by_reason = {} + for rel, why in dropped: + by_reason.setdefault(why, []).append(rel) + for why in ("secret", "ignored", "derived", "oversize", "excluded"): + hits = by_reason.get(why) + if not hits: + continue + lines += [f"## {why} — {REASONS[why]}", ""] + lines += [f"- `{rel}`" for rel in hits] + lines.append("") + if not dropped: + lines += ["Nothing was left behind.", ""] + + path = state_dir(out) / MANIFEST + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines)) diff --git a/soleprint/station/tools/histgen/templates/index.html b/soleprint/station/tools/histgen/templates/index.html new file mode 100644 index 0000000..c244116 --- /dev/null +++ b/soleprint/station/tools/histgen/templates/index.html @@ -0,0 +1,77 @@ + + + + +histgen — station + + + + +

    histgen

    +

    A proposed history, before it is a history. Read-only — + python -m station.tools.histgen apply <repo> is what commits.

    + +
    + + +
    + +
    +
    + + + +