Compare commits
2 Commits
rig-fold
...
49a9f8ee57
| Author | SHA1 | Date | |
|---|---|---|---|
| 49a9f8ee57 | |||
| 966f8fc821 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -40,7 +40,5 @@ cfg/dlt/
|
|||||||
# not land here. They are versioned in their own repo.
|
# not land here. They are versioned in their own repo.
|
||||||
#
|
#
|
||||||
# Anchored at the ROOT on purpose: a copy is a SIBLING of rig/, so a rule inside
|
# Anchored at the ROOT on purpose: a copy is a SIBLING of rig/, so a rule inside
|
||||||
# rig/.gitignore cannot see it. The negation must name the full path for the same
|
# rig/.gitignore cannot see it.
|
||||||
# reason — `*-rig/` is unanchored and matches at any depth, including rig/sample-rig.
|
|
||||||
*-rig/
|
*-rig/
|
||||||
!rig/sample-rig/
|
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ Every script stays runnable on its own — the standalone rule holds:
|
|||||||
```bash
|
```bash
|
||||||
python build.py --cfg amar # -> gen/amar/
|
python build.py --cfg amar # -> gen/amar/
|
||||||
cd gen/standalone && python run.py # bare-metal
|
cd gen/standalone && python run.py # bare-metal
|
||||||
./ctrl/kind-up.sh # still works directly
|
./ctrl/cluster.sh up # still runs directly; rig builds the cluster
|
||||||
cd gen/<room> && ./ctrl/start.sh # each room owns its lifecycle scripts
|
cd gen/<room> && ./ctrl/start.sh # each room owns its lifecycle scripts
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
2
Makefile
2
Makefile
@@ -13,7 +13,7 @@
|
|||||||
# make component ARGS="publish soleprint-ui /tmp/out --dist"
|
# make component ARGS="publish soleprint-ui /tmp/out --dist"
|
||||||
# make deploy ARGS="--build"
|
# make deploy ARGS="--build"
|
||||||
#
|
#
|
||||||
# Every script stays runnable on its own (./ctrl/kind-up.sh still works, and each
|
# Every script stays runnable on its own (./ctrl/cluster.sh up still works, and each
|
||||||
# built room keeps its own gen/<room>/ctrl/*.sh) — the standalone rule holds, and
|
# built room keeps its own gen/<room>/ctrl/*.sh) — the standalone rule holds, and
|
||||||
# this only saves typing.
|
# this only saves typing.
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -6,16 +6,49 @@
|
|||||||
# ./ctrl/cluster.sh down # delete it (drops every room's namespace)
|
# ./ctrl/cluster.sh down # delete it (drops every room's namespace)
|
||||||
# ./ctrl/cluster.sh status # what's running on it
|
# ./ctrl/cluster.sh status # what's running on it
|
||||||
#
|
#
|
||||||
# One target, one script — the variants live here. The kind-*.sh files stay
|
# spr depends on rig, never the other way round. Building and deleting a cluster
|
||||||
# exactly as they are and remain runnable on their own; this only dispatches.
|
# is rig's job, so up and down hand straight to rig/ctrl/cluster.sh, carrying the
|
||||||
|
# four things that make this cluster spr's rather than rig's defaults:
|
||||||
|
#
|
||||||
|
# CLUSTER=spr rooms deploy into the kind-spr context
|
||||||
|
# KIND_CONFIG spr's own shape, which maps the rooms' gateway NodePorts
|
||||||
|
# REGISTRY_MODE=none rooms load images straight into the node
|
||||||
|
# PROFILE=minimal pinned here, so a change to rig's own ctrl/.env can never
|
||||||
|
# quietly add addons to spr's cluster
|
||||||
|
#
|
||||||
|
# status stays here: it answers a question about rooms, not about the cluster.
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RIG_CTRL="$SCRIPT_DIR/../rig/ctrl"
|
||||||
|
|
||||||
|
rig() {
|
||||||
|
CLUSTER=spr \
|
||||||
|
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml" \
|
||||||
|
REGISTRY_MODE=none \
|
||||||
|
PROFILE=minimal \
|
||||||
|
bash "$RIG_CTRL/cluster.sh" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
case "${1:-status}" in
|
case "${1:-status}" in
|
||||||
up) exec "$SCRIPT_DIR/kind-up.sh" ;;
|
up)
|
||||||
down) exec "$SCRIPT_DIR/kind-down.sh" ;;
|
rig up
|
||||||
status) exec "$SCRIPT_DIR/kind-status.sh" ;;
|
echo
|
||||||
|
echo "Per-room deploy:"
|
||||||
|
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
||||||
|
;;
|
||||||
|
down)
|
||||||
|
rig down
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
if ! kind get clusters 2>/dev/null | grep -qx spr; then
|
||||||
|
echo "No 'spr' kind cluster — run: make cluster up"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
kubectl --context kind-spr get namespaces -l soleprint-room
|
||||||
|
echo
|
||||||
|
kubectl --context kind-spr get pods -A -l soleprint-room
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unknown subcommand: $1" >&2
|
echo "Unknown subcommand: $1" >&2
|
||||||
echo "Usage: cluster.sh [up|down|status]" >&2
|
echo "Usage: cluster.sh [up|down|status]" >&2
|
||||||
|
|||||||
@@ -3,9 +3,15 @@ apiVersion: kind.x-k8s.io/v1alpha4
|
|||||||
# Single shared cluster for all soleprint rooms.
|
# Single shared cluster for all soleprint rooms.
|
||||||
# Each room deploys into its own namespace; gateway Services pick a
|
# Each room deploys into its own namespace; gateway Services pick a
|
||||||
# NodePort from the 30080-30099 range mapped here.
|
# NodePort from the 30080-30099 range mapped here.
|
||||||
name: spr
|
#
|
||||||
|
# Built by rig, not by spr: ctrl/cluster.sh hands this file to
|
||||||
|
# rig/ctrl/cluster.sh, which substitutes CLUSTER and NODE_IMAGE (named without
|
||||||
|
# braces here so this comment survives the substitution). The shape is spr's —
|
||||||
|
# what its cluster needs is spr's business. Building it is rig's.
|
||||||
|
name: ${CLUSTER}
|
||||||
nodes:
|
nodes:
|
||||||
- role: control-plane
|
- role: control-plane
|
||||||
|
image: ${NODE_IMAGE}
|
||||||
extraPortMappings:
|
extraPortMappings:
|
||||||
# Room gateway NodePorts (one per active room).
|
# Room gateway NodePorts (one per active room).
|
||||||
- {containerPort: 30080, hostPort: 30080, protocol: TCP}
|
- {containerPort: 30080, hostPort: 30080, protocol: TCP}
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Delete the shared `spr` kind cluster (drops every room's namespace too).
|
|
||||||
# Use `gen/<room>/ctrl/k8s-down.sh` instead if you only want to remove
|
|
||||||
# a single room's namespace.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "Deleting kind cluster 'spr'..."
|
|
||||||
kind delete cluster --name spr
|
|
||||||
else
|
|
||||||
echo "No kind cluster 'spr' to delete."
|
|
||||||
fi
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Show what's running on the shared `spr` cluster.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if ! kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "No 'spr' kind cluster — run ctrl/kind-up.sh"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
kubectl --context kind-spr get namespaces -l soleprint-room
|
|
||||||
echo
|
|
||||||
kubectl --context kind-spr get pods -A -l soleprint-room
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Create (or no-op) the single shared `spr` kind cluster used by every
|
|
||||||
# soleprint room. Per-room work happens inside namespaces — see
|
|
||||||
# `gen/<room>/ctrl/k8s-up.sh`.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml"
|
|
||||||
|
|
||||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "Kind cluster 'spr' already exists."
|
|
||||||
else
|
|
||||||
echo "Creating kind cluster 'spr'..."
|
|
||||||
kind create cluster --config "$KIND_CONFIG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
kubectl config use-context kind-spr >/dev/null
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "Cluster ready. Per-room deploy:"
|
|
||||||
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
|
||||||
7
rig/.gitignore
vendored
7
rig/.gitignore
vendored
@@ -11,10 +11,9 @@ ctrl/.env
|
|||||||
arch/*.dot
|
arch/*.dot
|
||||||
ctrl/Tiltfile.gen
|
ctrl/Tiltfile.gen
|
||||||
|
|
||||||
# binaries pulled by `make deps-bundle` for the air-gapped wizard image
|
# binaries pulled by `make deps-bundle` for the air-gapped installer image
|
||||||
vendor
|
vendor
|
||||||
|
|
||||||
# Client rigs are NOT ignored here. A copy is a SIBLING of this directory
|
# Client rigs are NOT ignored here. A copy is a SIBLING of this directory
|
||||||
# (spr/acme-rig), so a rule in this file cannot see it — the rules live in
|
# (../acme-rig), so a rule in this file cannot see it — the rules live in the
|
||||||
# spr/.gitignore, anchored at spr's root, where `*-rig/` matches the siblings and
|
# parent repo's .gitignore, anchored at its root, where `*-rig/` matches them.
|
||||||
# `!rig/sample-rig/` keeps the committed stand-in.
|
|
||||||
|
|||||||
@@ -4,9 +4,8 @@ The README says the prerequisite is Docker and nothing else. This is what that
|
|||||||
actually looks like end to end: a bare Linux box, and a new project running under
|
actually looks like end to end: a bare Linux box, and a new project running under
|
||||||
Tilt at the end of it.
|
Tilt at the end of it.
|
||||||
|
|
||||||
rig lives inside soleprint, at `spr/rig` — it is soleprint's cluster half, and
|
A copy of this directory is a sibling of it, named after the environment it
|
||||||
a client copy is a sibling (`spr/acme-rig`). Paths below are relative to
|
models (`acme-rig`). Paths below are relative to the parent checkout.
|
||||||
soleprint's checkout.
|
|
||||||
|
|
||||||
It spans three repos because the work does. **rig** prepares the machine — the
|
It spans three repos because the work does. **rig** prepares the machine — the
|
||||||
pinned toolchain, the cluster, the port arithmetic. **all** owns the shape a
|
pinned toolchain, the cluster, the port arithmetic. **all** owns the shape a
|
||||||
@@ -50,7 +49,7 @@ isn't.
|
|||||||
## Read the docs before installing anything
|
## Read the docs before installing anything
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd spr/rig
|
cd rig
|
||||||
make docs
|
make docs
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -67,11 +66,11 @@ persists; ctrl-c ends it.
|
|||||||
## Ask what is wrong with this machine
|
## Ask what is wrong with this machine
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make station
|
make check
|
||||||
cp ctrl/.env.example ctrl/.env
|
cp ctrl/.env.example ctrl/.env
|
||||||
```
|
```
|
||||||
|
|
||||||
`station.sh` reports and instructs, and fixes nothing. It runs bare rather than
|
`check.sh` reports and instructs, and fixes nothing. It runs bare rather than
|
||||||
in a container because host detection only ever reads `/proc` and `/etc` — no
|
in a container because host detection only ever reads `/proc` and `/etc` — no
|
||||||
dependency beyond coreutils.
|
dependency beyond coreutils.
|
||||||
|
|
||||||
@@ -80,7 +79,7 @@ port rig binds derives from this directory's name, so the answer is specific to
|
|||||||
this copy, and a clash here surfaces as an opaque `failed to bind host port` in
|
this copy, and a clash here surfaces as an opaque `failed to bind host port` in
|
||||||
the middle of cluster creation if you skip it.
|
the middle of cluster creation if you skip it.
|
||||||
|
|
||||||
Copy the `.env` even though station only warns about it. It is gitignored, it is
|
Copy the `.env` even though the check only warns about it. It is gitignored, it is
|
||||||
where a machine-local override goes, and `ports.sh persist` expects it to exist.
|
where a machine-local override goes, and `ports.sh persist` expects it to exist.
|
||||||
|
|
||||||
|
|
||||||
@@ -88,33 +87,33 @@ where a machine-local override goes, and `ports.sh persist` expects it to exist.
|
|||||||
|
|
||||||
This is the step where "nothing installed" stops being rhetorical.
|
This is the step where "nothing installed" stops being rhetorical.
|
||||||
|
|
||||||
`make deps` runs `ctrl/wizard.sh install` directly on the host, and the wizard
|
`make deps` runs `ctrl/deps.sh install` directly on the host, and the installer
|
||||||
fetches with `curl`. A stock `debian:trixie-slim` has no curl — detection runs
|
fetches with `curl`. A stock `debian:trixie-slim` has no curl — detection runs
|
||||||
fine, then the first download dies with `curl: command not found` and an exit
|
fine, then the first download dies with `curl: command not found` and an exit
|
||||||
code of 127. That is the bootstrap paradox `ctrl/Dockerfile.wizard` exists
|
code of 127. That is the bootstrap paradox `ctrl/Dockerfile.deps` exists
|
||||||
to kill — the wizard carries its own toolchain so the host needs only Docker —
|
to kill — the installer carries its own toolchain so the host needs only Docker —
|
||||||
but building the image and running it are two different things, and only the
|
but building the image and running it are two different things, and only the
|
||||||
build has a Makefile target today. **On a genuinely bare machine, run it by
|
build has a Makefile target today. **On a genuinely bare machine, run it by
|
||||||
hand:**
|
hand:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make wizard # builds rig-wizard:wizard
|
make deps-image # builds rig-deps:deps
|
||||||
mkdir -p ~/.local/bin
|
mkdir -p ~/.local/bin
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
-v /:/host:ro \
|
-v /:/host:ro \
|
||||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||||
-v "$HOME/.local/bin:/out/bin" \
|
-v "$HOME/.local/bin:/out/bin" \
|
||||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||||
rig-wizard:wizard install dev
|
rig-deps:deps install dev
|
||||||
```
|
```
|
||||||
|
|
||||||
The image name follows the directory, like everything else here: in `spr/rig`
|
The image name follows the directory, like everything else here: in `rig`
|
||||||
it is `rig-wizard`, in a copy called `spr/acme-rig` it is `acme-rig-wizard`. The
|
it is `rig-deps`, in a copy called `acme-rig` it is `acme-rig-deps`. The
|
||||||
tag is `wizard` (or `full`, below), not `latest`.
|
tag is `deps` (or `full`, below), not `latest`.
|
||||||
|
|
||||||
None of the four arguments are guessable, so:
|
None of the four arguments are guessable, so:
|
||||||
|
|
||||||
- **`/:/host:ro`** — the wizard reads the *host's* `/etc/os-release` and
|
- **`/:/host:ro`** — the installer reads the *host's* `/etc/os-release` and
|
||||||
`/etc/wsl.conf`, not the container's. `HOST_ROOT=/host` is already baked into
|
`/etc/wsl.conf`, not the container's. `HOST_ROOT=/host` is already baked into
|
||||||
the image; this is what it points at. Read-only, and it is the only reason
|
the image; this is what it points at. Read-only, and it is the only reason
|
||||||
detection inside a container tells you anything about the machine.
|
detection inside a container tells you anything about the machine.
|
||||||
@@ -122,7 +121,7 @@ None of the four arguments are guessable, so:
|
|||||||
and how it counts kind clusters already running.
|
and how it counts kind clusters already running.
|
||||||
- **`/out/bin`** — the image's `OUT_BIN`. Whatever you mount here is where the
|
- **`/out/bin`** — the image's `OUT_BIN`. Whatever you mount here is where the
|
||||||
four binaries land.
|
four binaries land.
|
||||||
- **`HOST_UID` / `HOST_GID`** — the wizard runs as root so it can reach that
|
- **`HOST_UID` / `HOST_GID`** — the installer runs as root so it can reach that
|
||||||
socket, which means everything it writes into a mounted volume is root-owned
|
socket, which means everything it writes into a mounted volume is root-owned
|
||||||
and useless to you. These drive the `chown` back. Omit them and the install
|
and useless to you. These drive the `chown` back. Omit them and the install
|
||||||
looks like it worked.
|
looks like it worked.
|
||||||
@@ -131,18 +130,18 @@ None of the four arguments are guessable, so:
|
|||||||
tooling — which is the right answer on a managed or corporate-issued machine and
|
tooling — which is the right answer on a managed or corporate-issued machine and
|
||||||
is why the split exists.
|
is why the split exists.
|
||||||
|
|
||||||
Then put them on PATH, which the wizard will remind you about because it cannot
|
Then put them on PATH, which the installer will remind you about because it cannot
|
||||||
edit your shell for you:
|
edit your shell for you:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export PATH="$HOME/.local/bin:$PATH" # and add the same line to ~/.bashrc
|
export PATH="$HOME/.local/bin:$PATH" # and add the same line to ~/.bashrc
|
||||||
```
|
```
|
||||||
|
|
||||||
If something else on this machine already provides `kubectl`, the wizard says so
|
If something else on this machine already provides `kubectl`, the installer says so
|
||||||
by name rather than shadowing it quietly. `OUT_BIN=$PWD/def/bin` installs
|
by name rather than shadowing it quietly. `OUT_BIN=$PWD/def/bin` installs
|
||||||
somewhere private instead.
|
somewhere private instead.
|
||||||
|
|
||||||
**Two variants worth knowing before you need them.** `make wizard full` bakes
|
**Two variants worth knowing before you need them.** `make deps-image full` bakes
|
||||||
every pinned binary into the image at build time (`DEPS_SOURCE=baked`), so
|
every pinned binary into the image at build time (`DEPS_SOURCE=baked`), so
|
||||||
`docker save` gives you the entire installer as one file to carry into an
|
`docker save` gives you the entire installer as one file to carry into an
|
||||||
air-gapped network. And `DEPS_SOURCE=artifactory` with `DEPS_ARTIFACTORY_URL`
|
air-gapped network. And `DEPS_SOURCE=artifactory` with `DEPS_ARTIFACTORY_URL`
|
||||||
@@ -195,8 +194,8 @@ follows is only the mechanical part.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
SLUG=<slug> # short, lowercase, no separators
|
SLUG=<slug> # short, lowercase, no separators
|
||||||
cp -r ~/wdir/all/projects/templates/broad ~/wdir/"$SLUG"
|
cp -r ~/wdir/semester/all/projects/templates/broad ~/wdir/semester/"$SLUG"
|
||||||
cd ~/wdir/"$SLUG"
|
cd ~/wdir/semester/"$SLUG"
|
||||||
grep -rl '<slug>' ctrl | xargs sed -i "s/<slug>/$SLUG/g"
|
grep -rl '<slug>' ctrl | xargs sed -i "s/<slug>/$SLUG/g"
|
||||||
cp ctrl/k8s/.env.example ctrl/k8s/.env
|
cp ctrl/k8s/.env.example ctrl/k8s/.env
|
||||||
git init && git add -A && git commit -m "scaffold $SLUG from broad"
|
git init && git add -A && git commit -m "scaffold $SLUG from broad"
|
||||||
@@ -214,7 +213,7 @@ scaffold's Makefile from the directory, so there is nothing to edit for either.
|
|||||||
is already in use, so copying it unchanged puts two projects on one port:
|
is already in use, so copying it unchanged puts two projects on one port:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
grep -h '^TILT_PORT=' ~/wdir/*/ctrl/k8s/.env 2>/dev/null | sort
|
grep -h '^TILT_PORT=' ~/wdir/semester/*/ctrl/k8s/.env 2>/dev/null | sort
|
||||||
```
|
```
|
||||||
|
|
||||||
Choose a free one in `10300–10399` — the range ALL reserves in
|
Choose a free one in `10300–10399` — the range ALL reserves in
|
||||||
@@ -244,7 +243,7 @@ The workload is an nginx placeholder so a fresh copy reaches something that
|
|||||||
answers; replace it. Keep `30080` in step between the overlay patch and
|
answers; replace it. Keep `30080` in step between the overlay patch and
|
||||||
`kind-config.yaml`'s `containerPort` — the hostPort is this project's to pick.
|
`kind-config.yaml`'s `containerPort` — the hostPort is this project's to pick.
|
||||||
Reachability is a plain kind port mapping: no ingress controller and no MetalLB.
|
Reachability is a plain kind port mapping: no ingress controller and no MetalLB.
|
||||||
Caddy maps `<slug>.local.ar` onto the host port (`~/wdir/ppl/local/Caddyfile`),
|
Caddy maps `<slug>.local.ar` onto the host port (`~/wdir/semester/ppl/local/Caddyfile`),
|
||||||
with `*.local.ar` resolving to 127.0.0.1 through dnsmasq. That is the whole chain.
|
with `*.local.ar` resolving to 127.0.0.1 through dnsmasq. That is the whole chain.
|
||||||
|
|
||||||
**The one file the scaffold still does not ship is `ctrl/Tiltfile`** — `make
|
**The one file the scaffold still does not ship is `ctrl/Tiltfile`** — `make
|
||||||
@@ -270,9 +269,9 @@ delete-and-recreate for when a cluster wedges.
|
|||||||
## Register it
|
## Register it
|
||||||
|
|
||||||
The project exists; now it is findable. Add an entry to
|
The project exists; now it is findable. Add an entry to
|
||||||
`~/wdir/all/projects/index.json` and write its `projects/<slug>.md` beside the
|
`~/wdir/semester/all/projects/index.json` and write its `projects/<slug>.md` beside the
|
||||||
others. Structured fields in the index, prose in the markdown.
|
others. Structured fields in the index, prose in the markdown.
|
||||||
|
|
||||||
Putting it on the CI server and deploying it is `ppl`'s half, and it starts at
|
Putting it on the CI server and deploying it is `ppl`'s half, and it starts at
|
||||||
`~/wdir/ppl/ctrl/init-repo.sh` — gitea remote, then Woodpecker. That is a
|
`~/wdir/semester/ppl/ctrl/init-repo.sh` — gitea remote, then Woodpecker. That is a
|
||||||
different document.
|
different document.
|
||||||
|
|||||||
24
rig/Makefile
24
rig/Makefile
@@ -20,7 +20,7 @@ SLUG := $(shell echo '$(notdir $(CURDIR))' | tr '[:upper:]' '[:lower:]' | t
|
|||||||
CLUSTER := $(or $(shell sed -n 's/^CLUSTER=//p' ctrl/.env 2>/dev/null),$(SLUG))
|
CLUSTER := $(or $(shell sed -n 's/^CLUSTER=//p' ctrl/.env 2>/dev/null),$(SLUG))
|
||||||
KCTX := --context kind-$(CLUSTER)
|
KCTX := --context kind-$(CLUSTER)
|
||||||
TILT_PORT := $(shell sed -n 's/^TILT_PORT=//p' ctrl/.env 2>/dev/null)
|
TILT_PORT := $(shell sed -n 's/^TILT_PORT=//p' ctrl/.env 2>/dev/null)
|
||||||
WIZARD := $(SLUG)-wizard
|
DEPSIMG := $(SLUG)-deps
|
||||||
|
|
||||||
# Words after the target become the script's subcommand. Make would otherwise
|
# Words after the target become the script's subcommand. Make would otherwise
|
||||||
# treat them as goals of their own, so each gets a no-op rule.
|
# treat them as goals of their own, so each gets a no-op rule.
|
||||||
@@ -35,7 +35,7 @@ $(eval $(ARGS):;@:)
|
|||||||
.PHONY: $(ARGS)
|
.PHONY: $(ARGS)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: help setup station deps wizard cluster registry addons ports \
|
.PHONY: help setup check mem deps deps-image pins cluster registry addons ports \
|
||||||
newbox dockerhost docs tilt \
|
newbox dockerhost docs tilt \
|
||||||
kind-up kind-down kind-reset tilt-up tilt-down
|
kind-up kind-down kind-reset tilt-up tilt-down
|
||||||
|
|
||||||
@@ -47,16 +47,22 @@ help: ## list targets
|
|||||||
setup: ## prepare this machine [core] [--share-docker] [--cluster]
|
setup: ## prepare this machine [core] [--share-docker] [--cluster]
|
||||||
bash ctrl/setup.sh $(ARGS)
|
bash ctrl/setup.sh $(ARGS)
|
||||||
|
|
||||||
station: ## is this workstation ready? reports, never fixes
|
check: ## is this machine ready? reports, never fixes
|
||||||
bash ctrl/station.sh
|
bash ctrl/check.sh
|
||||||
|
|
||||||
|
mem: ## memory, and any cap holding it [status|backup|restore]
|
||||||
|
bash ctrl/mem.sh $(or $(ARGS),status)
|
||||||
|
|
||||||
deps: ## install the toolchain [core|dev] (default dev)
|
deps: ## install the toolchain [core|dev] (default dev)
|
||||||
bash ctrl/wizard.sh install $(or $(ARGS),dev)
|
bash ctrl/deps.sh install $(or $(ARGS),dev)
|
||||||
|
|
||||||
|
pins: ## standalone/rigdeps.sh still installs what rig pins?
|
||||||
|
bash ctrl/pins.sh
|
||||||
|
|
||||||
wizard: ## build the installer image [full]
|
deps-image: ## build the installer image [full]
|
||||||
docker build -f ctrl/Dockerfile.wizard \
|
docker build -f ctrl/Dockerfile.deps \
|
||||||
--target $(if $(filter full,$(ARGS)),wizard-full,wizard) \
|
--target $(if $(filter full,$(ARGS)),deps-full,deps) \
|
||||||
-t $(WIZARD):$(if $(filter full,$(ARGS)),full,wizard) .
|
-t $(DEPSIMG):$(if $(filter full,$(ARGS)),full,deps) .
|
||||||
|
|
||||||
# ── cluster ────────────────────────────────────────────────────────────────
|
# ── cluster ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,46 @@ topology, not the workloads.
|
|||||||
|
|
||||||
**Docker.** Nothing else — no curl, no jq, no python, no apt repositories.
|
**Docker.** Nothing else — no curl, no jq, no python, no apt repositories.
|
||||||
|
|
||||||
|
### Starting from plain Windows
|
||||||
|
|
||||||
|
Everything here is bash and runs *inside* a Linux shell, so on a Windows machine
|
||||||
|
that means WSL. Nothing in rig installs WSL, and nothing will: `wsl --install`
|
||||||
|
enables Windows features and requires a reboot, which is not something a script
|
||||||
|
should do to a machine on your behalf — and there is no tested undo for it.
|
||||||
|
|
||||||
|
From an elevated PowerShell or Command Prompt, once:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
wsl --install
|
||||||
|
```
|
||||||
|
|
||||||
|
Then reboot and open the Linux shell it installed.
|
||||||
|
|
||||||
|
**If you cloned this on the Windows side, copy it into WSL before carrying on.**
|
||||||
|
WSL can reach the Windows drives at `/mnt/c`, and working from there mostly
|
||||||
|
functions — slowly — but file watching does not: that filesystem raises no
|
||||||
|
inotify events, so anything watching for edits silently stops seeing them.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp -r /mnt/c/Users/<you>/rig ~/rig
|
||||||
|
cd ~/rig
|
||||||
|
```
|
||||||
|
|
||||||
|
`make deps` reports it if you are running from `/mnt/...`. Then carry on below.
|
||||||
|
|
||||||
|
If it fails, the usual causes give unhelpful messages:
|
||||||
|
|
||||||
|
| symptom | cause |
|
||||||
|
| --- | --- |
|
||||||
|
| "the virtual machine could not be started" | virtualization disabled in BIOS/UEFI |
|
||||||
|
| the command is not recognised | Windows build too old — needs 2004 or later |
|
||||||
|
| the install starts, then nothing works | a reboot is still pending |
|
||||||
|
|
||||||
|
Running the scripts from **Git Bash, MSYS or Cygwin does not work** — those look
|
||||||
|
close enough to a Linux shell to get started and then fail without `/proc` or a
|
||||||
|
docker socket. `ctrl/deps.sh` detects that and says so rather than letting you
|
||||||
|
find out the slow way.
|
||||||
|
|
||||||
## Read the docs first
|
## Read the docs first
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -21,16 +61,43 @@ instructions for everything else. No cluster and no toolchain required.
|
|||||||
## Then
|
## Then
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make station # report host and config problems; changes nothing
|
make check # report host and config problems; changes nothing
|
||||||
make deps # install the toolchain (add `core` on a managed machine)
|
make deps # install the toolchain (add `core` on a managed machine)
|
||||||
make cluster up # build the cluster for the active profile
|
make cluster up # build the cluster for the active profile
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`make cluster up` also starts this environment's local registry and wires it
|
||||||
|
into the node, so an image built locally is pullable by the cluster without
|
||||||
|
going near docker.io:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make registry status # prints: endpoint localhost:<port>
|
||||||
|
docker build -t localhost:<port>/app:1 .
|
||||||
|
docker push localhost:<port>/app:1
|
||||||
|
kubectl --context kind-$(basename $PWD) run app --image=localhost:<port>/app:1
|
||||||
|
```
|
||||||
|
|
||||||
|
The port block is derived from the directory name, so two copies of rig never
|
||||||
|
collide:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make ports show # HTTP / HTTPS / TILT / REGISTRY
|
||||||
|
make cluster list # every cluster on this machine, with memory
|
||||||
|
make cluster free # stop the others if memory is tight
|
||||||
|
make cluster down # remove this cluster and its registry
|
||||||
|
```
|
||||||
|
|
||||||
|
**`make tilt` has nothing to run yet.** The target and its `tilt-up` / `tilt-down`
|
||||||
|
aliases exist so rig answers to the same spelling as every other project here,
|
||||||
|
but rig ships no `Tiltfile` — it builds the estate, it is not itself a service
|
||||||
|
with a dev loop. Add a `ctrl/Tiltfile` and the target works; until then it fails
|
||||||
|
on the missing file, not on anything rig did.
|
||||||
|
|
||||||
`make help` lists every target.
|
`make help` lists every target.
|
||||||
|
|
||||||
On a machine where Docker really is the only thing installed, `make deps` has
|
On a machine where Docker really is the only thing installed, `make deps` has
|
||||||
nothing to download with — see [BOOTSTRAP.md](BOOTSTRAP.md), which runs the
|
nothing to download with — see [BOOTSTRAP.md](BOOTSTRAP.md), which runs the
|
||||||
toolchain through the wizard container and carries on to scaffolding and running
|
toolchain through the installer container and carries on to scaffolding and running
|
||||||
a new project.
|
a new project.
|
||||||
|
|
||||||
## One directory is one environment
|
## One directory is one environment
|
||||||
@@ -39,10 +106,10 @@ Copy this directory, rename it, run it. Cluster name, kubectl context, image
|
|||||||
tags and the host port block all derive from the directory name, so copies never
|
tags and the host port block all derive from the directory name, so copies never
|
||||||
collide and neither one's teardown can touch the other.
|
collide and neither one's teardown can touch the other.
|
||||||
|
|
||||||
rig lives inside soleprint, at `spr/rig` — it is soleprint's cluster half, and a
|
A copy of this directory is a **sibling** of it, named after the environment it
|
||||||
copy is a **sibling**: `spr/acme-rig`. That is why the ignore rules for client
|
models (`acme-rig`). That is why the ignore rules for copies sit in the *parent*
|
||||||
rigs sit in `spr/.gitignore` rather than here; a rule in this directory cannot
|
repo's `.gitignore` rather than here: a rule in this directory cannot see a
|
||||||
see a directory beside it.
|
directory beside it.
|
||||||
|
|
||||||
## Profiles
|
## Profiles
|
||||||
|
|
||||||
@@ -54,7 +121,7 @@ apiserver audits. They live in `ctrl/env.d/`, and the active one is `PROFILE`.
|
|||||||
| `minimal` | the default. One node, no addons, boots fast. |
|
| `minimal` | the default. One node, no addons, boots fast. |
|
||||||
| `client` | the regulated-estate shape — multi-node, audit on, registry mirror. |
|
| `client` | the regulated-estate shape — multi-node, audit on, registry mirror. |
|
||||||
| `offline` | air-gapped: everything from a preloaded local registry. |
|
| `offline` | air-gapped: everything from a preloaded local registry. |
|
||||||
| `data` | the dependency containers a soleprint room asks for. |
|
| `data` | the cabinets an environment asks for. |
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
PROFILE=data make cluster up
|
PROFILE=data make cluster up
|
||||||
@@ -98,10 +165,10 @@ cluster does.
|
|||||||
| `redis` | cache and broker |
|
| `redis` | cache and broker |
|
||||||
| `airflow` | scheduled pipelines; needs postgres and redis |
|
| `airflow` | scheduled pipelines; needs postgres and redis |
|
||||||
|
|
||||||
The last three are the cluster half of **soleprint's cabinets**. A room declares
|
The last three are **cabinets**: a public service dropped in as-is, the upstream
|
||||||
what it needs once, in `cfg/<room>/data/cabinets.json`; soleprint's `build.py`
|
image unmodified, reachable at a known address. A cabinet is declared once and
|
||||||
composes those services into `docker-compose.yml` for a laptop, and these
|
installs on either target — a `service.yml` composes it for a laptop, and these
|
||||||
install the same ones here. The names match on purpose — each cabinet carries a
|
install the same one here. The names match on purpose: each cabinet carries a
|
||||||
`rig_addon` field pointing at `ctrl/addons/<name>.sh`.
|
`rig_addon` field pointing at `ctrl/addons/<name>.sh`.
|
||||||
|
|
||||||
Plain manifests rather than helm charts, like every other addon: a chart repo is
|
Plain manifests rather than helm charts, like every other addon: a chart repo is
|
||||||
|
|||||||
@@ -26,10 +26,10 @@ PROFILE=minimal
|
|||||||
# MANIFESTS_DIR=../platform-manifests/overlays/dev
|
# MANIFESTS_DIR=../platform-manifests/overlays/dev
|
||||||
MANIFESTS_DIR=ctrl/k8s/overlays/dev
|
MANIFESTS_DIR=ctrl/k8s/overlays/dev
|
||||||
|
|
||||||
# Where the wizard fetches the pinned binaries from.
|
# Where the installer fetches the pinned binaries from.
|
||||||
# upstream GitHub releases / dl.k8s.io (needs internet)
|
# upstream GitHub releases / dl.k8s.io (needs internet)
|
||||||
# artifactory a generic repo — what a locked-down client usually allows
|
# artifactory a generic repo — what a locked-down client usually allows
|
||||||
# baked already inside the wizard image; no network at all
|
# baked already inside the installer image; no network at all
|
||||||
DEPS_SOURCE=upstream
|
DEPS_SOURCE=upstream
|
||||||
DEPS_ARTIFACTORY_URL=
|
DEPS_ARTIFACTORY_URL=
|
||||||
|
|
||||||
@@ -43,7 +43,7 @@ REGISTRY_PASSWORD=
|
|||||||
# Corporate root CA, if Artifactory is fronted by an internal CA (it usually is).
|
# Corporate root CA, if Artifactory is fronted by an internal CA (it usually is).
|
||||||
# Trust has to reach THREE places and nothing does it for you: the host docker
|
# Trust has to reach THREE places and nothing does it for you: the host docker
|
||||||
# daemon, every kind node's containerd, and any in-cluster client. registry.sh
|
# daemon, every kind node's containerd, and any in-cluster client. registry.sh
|
||||||
# handles the first two; station.sh reports when it's configured but not trusted.
|
# handles the first two; check.sh reports when it's configured but not trusted.
|
||||||
# Symptom when missing: x509: certificate signed by unknown authority
|
# Symptom when missing: x509: certificate signed by unknown authority
|
||||||
REGISTRY_CA_FILE=
|
REGISTRY_CA_FILE=
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,18 @@
|
|||||||
# The installation wizard. It does NOT run the cluster — it installs a toolchain
|
# The toolchain installer image. It does NOT run the cluster — it installs a toolchain
|
||||||
# onto the host and gets out of the way.
|
# onto the host and gets out of the way.
|
||||||
#
|
#
|
||||||
# This exists to kill a bootstrap paradox: a plain bash installer needs curl, jq
|
# This exists to kill a bootstrap paradox: a plain bash installer needs curl, jq
|
||||||
# and sha256sum to already be present, and a minimal Debian has none of them.
|
# and sha256sum to already be present, and a minimal Debian has none of them.
|
||||||
# The wizard carries its own toolchain, so the only host prerequisite is Docker.
|
# It carries its own toolchain, so the only host prerequisite is Docker.
|
||||||
#
|
#
|
||||||
# Two variants from one file:
|
# Two variants from one file:
|
||||||
# docker build -f ctrl/Dockerfile.wizard --target wizard -t <slug>-wizard .
|
# docker build -f ctrl/Dockerfile.deps --target deps -t <slug>-deps .
|
||||||
# docker build -f ctrl/Dockerfile.wizard --target wizard-full -t <slug>-wizard:full .
|
# docker build -f ctrl/Dockerfile.deps --target deps-full -t <slug>-deps:full .
|
||||||
#
|
#
|
||||||
# wizard-full bakes every pinned binary in at build time. `docker save` it and
|
# deps-full bakes every pinned binary in at build time. `docker save` it and
|
||||||
# you have the whole installer as one file to carry into an air-gapped network.
|
# you have the whole installer as one file to carry into an air-gapped network.
|
||||||
|
|
||||||
FROM debian:trixie-slim AS wizard
|
FROM debian:trixie-slim AS deps
|
||||||
|
|
||||||
# ca-certificates + curl: fetch and verify. graphviz + python3: render diagrams
|
# ca-certificates + curl: fetch and verify. graphviz + python3: render diagrams
|
||||||
# and validate the arch model, so the host never needs an apt package.
|
# and validate the arch model, so the host never needs an apt package.
|
||||||
@@ -26,21 +26,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
|
|
||||||
WORKDIR /work
|
WORKDIR /work
|
||||||
COPY ctrl/versions.env /work/ctrl/versions.env
|
COPY ctrl/versions.env /work/ctrl/versions.env
|
||||||
COPY ctrl/wizard.sh /work/ctrl/wizard.sh
|
COPY ctrl/deps.sh /work/ctrl/deps.sh
|
||||||
RUN chmod +x /work/ctrl/wizard.sh
|
RUN chmod +x /work/ctrl/deps.sh
|
||||||
|
|
||||||
# Defaults; every one is overridable with -e at run time.
|
# Defaults; every one is overridable with -e at run time.
|
||||||
ENV DEPS_SOURCE=upstream \
|
ENV DEPS_SOURCE=upstream \
|
||||||
OUT_BIN=/out/bin \
|
OUT_BIN=/out/bin \
|
||||||
HOST_ROOT=/host
|
HOST_ROOT=/host
|
||||||
|
|
||||||
ENTRYPOINT ["/work/ctrl/wizard.sh"]
|
ENTRYPOINT ["/work/ctrl/deps.sh"]
|
||||||
CMD ["install"]
|
CMD ["install"]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# wizard-full — same wizard, binaries baked in, works with no network at all.
|
# deps-full — same image, binaries baked in, works with no network at all.
|
||||||
FROM wizard AS wizard-full
|
FROM deps AS deps-full
|
||||||
RUN /work/ctrl/wizard.sh fetch --to /opt/rig/bin
|
RUN /work/ctrl/deps.sh fetch --to /opt/rig/bin
|
||||||
ENV DEPS_SOURCE=baked \
|
ENV DEPS_SOURCE=baked \
|
||||||
BAKED_BIN=/opt/rig/bin
|
BAKED_BIN=/opt/rig/bin
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Apache Airflow — the cluster half of soleprint's airflow cabinet.
|
# Apache Airflow — the cluster half of the airflow cabinet.
|
||||||
#
|
#
|
||||||
# Airflow needs a metadata database before it will start at all, so this refuses
|
# Airflow needs a metadata database before it will start at all, so this refuses
|
||||||
# rather than rolls a pod that will CrashLoopBackOff while the real problem
|
# rather than rolls a pod that will CrashLoopBackOff while the real problem
|
||||||
@@ -7,7 +7,7 @@
|
|||||||
#
|
#
|
||||||
# One pod on `standalone`, matching the compose cabinet: migration, admin user,
|
# One pod on `standalone`, matching the compose cabinet: migration, admin user,
|
||||||
# scheduler and webserver in a single container. The official chart's five
|
# scheduler and webserver in a single container. The official chart's five
|
||||||
# deployments model an installation; a room switching this on wants pipelines.
|
# deployments model an installation; switching this on means wanting pipelines.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# PostgreSQL — the cluster half of soleprint's postgres cabinet.
|
# PostgreSQL — the cluster half of the postgres cabinet.
|
||||||
#
|
#
|
||||||
# A room declares the dependency once, in cfg/<room>/data/cabinets.json. On a
|
# A cabinet is a public service dropped into the environment as-is — the
|
||||||
# laptop `build.py` composes it into docker-compose.yml; here it becomes a pod,
|
# upstream image, unmodified, reachable at a known address. This is the cluster
|
||||||
# so the same declaration works either way and nothing has to be remembered
|
# half of it; the compose half is a `service.yml` beside a `cabinet.json`. The
|
||||||
|
# declaration is made once and both paths read it, so nothing is remembered
|
||||||
# twice.
|
# twice.
|
||||||
#
|
#
|
||||||
# Plain manifests rather than a helm chart, matching the other addons: a chart
|
# Plain manifests rather than a helm chart, matching the other addons: a chart
|
||||||
@@ -32,8 +33,8 @@ if $K get secret -n "$NS" postgres >/dev/null 2>&1; then
|
|||||||
else
|
else
|
||||||
password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||||
$K create secret generic postgres -n "$NS" \
|
$K create secret generic postgres -n "$NS" \
|
||||||
--from-literal=POSTGRES_DB="${POSTGRES_DB:-soleprint}" \
|
--from-literal=POSTGRES_DB="${POSTGRES_DB:-postgres}" \
|
||||||
--from-literal=POSTGRES_USER="${POSTGRES_USER:-soleprint}" \
|
--from-literal=POSTGRES_USER="${POSTGRES_USER:-postgres}" \
|
||||||
--from-literal=POSTGRES_PASSWORD="$password" >/dev/null
|
--from-literal=POSTGRES_PASSWORD="$password" >/dev/null
|
||||||
echo " generated a password (read it back with the command printed below)"
|
echo " generated a password (read it back with the command printed below)"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Redis — the cluster half of soleprint's redis cabinet.
|
# Redis — the cluster half of the redis cabinet.
|
||||||
#
|
#
|
||||||
# Cache, and the broker anything queue-shaped runs on. No persistence: a broker
|
# Cache, and the broker anything queue-shaped runs on. No persistence: a broker
|
||||||
# that loses its queue on restart is the honest local model, and a PVC here buys
|
# that loses its queue on restart is the honest local model, and a PVC here buys
|
||||||
|
|||||||
221
rig/ctrl/check.sh
Executable file
221
rig/ctrl/check.sh
Executable file
@@ -0,0 +1,221 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Readiness check: is this machine ready to run rig?
|
||||||
|
#
|
||||||
|
# Reports and instructs; never silently fixes anything. Everything it finds is
|
||||||
|
# either already fine, or something a human has to decide on.
|
||||||
|
#
|
||||||
|
# Runs ctrl/deps.sh host detection in a container when Docker is the only thing
|
||||||
|
# installed, or directly when the toolchain is already present. Then adds the
|
||||||
|
# checks that need this repo's config: profile sanity, CA trust, port clashes.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}"
|
||||||
|
|
||||||
|
# Host detection. Prefer running it bare — it needs no dependencies beyond
|
||||||
|
# coreutils — and fall back to the container only if this shell can't.
|
||||||
|
bash ./deps.sh detect
|
||||||
|
|
||||||
|
# ── repo-level checks ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
source ./lib/config.sh
|
||||||
|
load_config
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "config"
|
||||||
|
echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})"
|
||||||
|
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
|
||||||
|
echo " registry ${REGISTRY_MODE}"
|
||||||
|
echo " ingress ${INGRESS_MODE}"
|
||||||
|
|
||||||
|
if [ ! -f ./.env ]; then
|
||||||
|
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── memory ─────────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# A profile on a box that is already full is the most common first failure, and
|
||||||
|
# it presents as pods stuck Pending rather than anything that says "memory".
|
||||||
|
# Warns; never blocks. Whether to try anyway is the user's call.
|
||||||
|
|
||||||
|
# A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist
|
||||||
|
# only so the tight and does-not-fit branches can be exercised against another
|
||||||
|
# machine's real numbers; in normal use they are the kernel's own files.
|
||||||
|
mb_of() {
|
||||||
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||||
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# What one node costs, measured rather than guessed. On 2026-09-11 a minimal
|
||||||
|
# control-plane node ran at 620 MiB idle and ~728 MiB with a small mock, plus
|
||||||
|
# 16 MiB for the local registry — ~745 MiB of working set. 800 rounds that up,
|
||||||
|
# and agrees with the 800 MB observed independently on a larger rig. Worker
|
||||||
|
# nodes carry no etcd or apiserver and are lighter, so for a multi-node shape
|
||||||
|
# this errs high. It is the cluster alone: whatever you deploy comes on top.
|
||||||
|
NODE_MB=800
|
||||||
|
|
||||||
|
# Every running container's working set in MB, tagged with the kind cluster it
|
||||||
|
# belongs to ('-' when it is not kind). docker stats reports usage minus page
|
||||||
|
# cache, which is what actually competes — cache is handed back under pressure.
|
||||||
|
# Counting only kind would hide the usual culprit on a managed workspace, where
|
||||||
|
# the memory is held by other containers entirely.
|
||||||
|
container_mb() {
|
||||||
|
docker info >/dev/null 2>&1 || return 0
|
||||||
|
awk -F'\t' '
|
||||||
|
FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next }
|
||||||
|
{
|
||||||
|
grp = ($1 in cl ? cl[$1] : "-")
|
||||||
|
# A kind cluster'"'"'s local registry is a plain container with no kind
|
||||||
|
# label, named <cluster>-registry, so on its own it would read as a
|
||||||
|
# stranger. It belongs to its cluster — but only if that cluster exists:
|
||||||
|
# a registry whose cluster is gone is a genuine stray, and says so.
|
||||||
|
if (grp == "-" && $1 ~ /-registry$/) {
|
||||||
|
base = $1; sub(/-registry$/, "", base)
|
||||||
|
if (base in isc) grp = base
|
||||||
|
}
|
||||||
|
split($2, u, " "); v = u[1]; mb = 0
|
||||||
|
if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 }
|
||||||
|
else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v }
|
||||||
|
else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 }
|
||||||
|
else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 }
|
||||||
|
printf "%d\t%s\t%s\n", mb, grp, $1
|
||||||
|
}
|
||||||
|
' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \
|
||||||
|
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
||||||
|
}
|
||||||
|
|
||||||
|
total_mb=$(mb_of MemTotal)
|
||||||
|
avail_mb=$(mb_of MemAvailable)
|
||||||
|
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
||||||
|
overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||||
|
need_mb=$(( NODES * NODE_MB ))
|
||||||
|
|
||||||
|
rows=$(container_mb)
|
||||||
|
# Once this environment's own cluster is running, its real footprint is already
|
||||||
|
# out of MemAvailable and the per-node estimate stops being relevant. Subtracting
|
||||||
|
# the measurement from the estimate would count the same memory twice, and a
|
||||||
|
# running cluster that happens to sit under 800 MB would still "need" the gap.
|
||||||
|
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
||||||
|
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "memory"
|
||||||
|
printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \
|
||||||
|
"$need_mb" "$NODES" "$NODE_MB"
|
||||||
|
if [ "$ours_mb" -gt 0 ]; then
|
||||||
|
printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER"
|
||||||
|
fi
|
||||||
|
printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb"
|
||||||
|
|
||||||
|
# The biggest things holding memory right now, other than this cluster: kind
|
||||||
|
# clusters summed per cluster, everything else by container name.
|
||||||
|
others=$(awk -F'\t' -v c="$CLUSTER" '
|
||||||
|
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
||||||
|
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
||||||
|
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
||||||
|
if [ -n "$others" ]; then
|
||||||
|
echo " held elsewhere:"
|
||||||
|
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
||||||
|
n_others=$(wc -l <<< "$others")
|
||||||
|
if [ "$n_others" -gt 6 ]; then
|
||||||
|
echo " ... and $((n_others - 6)) more"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
headroom=$(( avail_mb - still_mb ))
|
||||||
|
if [ "$still_mb" -eq 0 ]; then
|
||||||
|
if [ "$headroom" -ge 512 ]; then
|
||||||
|
printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom"
|
||||||
|
else
|
||||||
|
printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||||
|
fi
|
||||||
|
elif [ "$headroom" -ge 512 ]; then
|
||||||
|
printf " fits — %d MB headroom for what you deploy\n" "$headroom"
|
||||||
|
elif [ "$headroom" -ge 0 ]; then
|
||||||
|
printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||||
|
else
|
||||||
|
printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
||||||
|
# Two failures with opposite fixes, and telling them apart is the point.
|
||||||
|
if [ "$still_mb" -le "$total_mb" ]; then
|
||||||
|
echo " The machine is big enough; something else is holding memory (above)."
|
||||||
|
echo " Stopping that is what helps — a bigger VM would not."
|
||||||
|
if grep -q 'kind cluster' <<< "$others"; then
|
||||||
|
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$swap_used_mb" -gt 0 ]; then
|
||||||
|
printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb"
|
||||||
|
echo " cluster here to be slow well before it fails"
|
||||||
|
fi
|
||||||
|
if [ "$overcommit" = "1" ]; then
|
||||||
|
echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling."
|
||||||
|
echo " A cluster that starts cleanly can still lose processes to the OOM killer."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The CA reaches three places and only one of them is ours. Report the other two.
|
||||||
|
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
||||||
|
echo
|
||||||
|
echo "registry CA"
|
||||||
|
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
||||||
|
echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
||||||
|
else
|
||||||
|
echo " file $REGISTRY_CA_FILE"
|
||||||
|
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
||||||
|
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
||||||
|
echo " ! the HOST docker daemon does not trust it yet:"
|
||||||
|
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
||||||
|
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
||||||
|
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Host ports this environment will try to bind. Checked before cluster creation
|
||||||
|
# because docker reports a clash halfway through, as an opaque
|
||||||
|
# "failed to bind host port ...: address already in use".
|
||||||
|
echo
|
||||||
|
echo "ports (block derived from the directory name — see 'make ports')"
|
||||||
|
|
||||||
|
port_busy() {
|
||||||
|
if command -v ss >/dev/null 2>&1; then
|
||||||
|
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
||||||
|
fi
|
||||||
|
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
||||||
|
# than silently reporting everything as free.
|
||||||
|
local hex; hex=$(printf ':%04X' "$1")
|
||||||
|
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# A port held by THIS environment's own cluster is not a clash — it is the thing
|
||||||
|
# working. Reporting it as a problem every time the cluster is up would train
|
||||||
|
# people to ignore this section, which is the opposite of the point.
|
||||||
|
# Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the
|
||||||
|
# character RANGE ':' to '>', which does not contain '-', so the trailing dash
|
||||||
|
# survives and nothing ever matches.
|
||||||
|
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
||||||
|
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
||||||
|
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
||||||
|
|
||||||
|
clash=0
|
||||||
|
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
||||||
|
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
||||||
|
name="${entry%%:*}"; p="${entry#*:}"
|
||||||
|
[ -n "$p" ] || continue
|
||||||
|
if ! port_busy "$p"; then
|
||||||
|
printf " %-9s %-6s free\n" "$name" "$p"
|
||||||
|
elif echo "$ours" | grep -qx "$p"; then
|
||||||
|
printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p"
|
||||||
|
else
|
||||||
|
printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p"
|
||||||
|
clash=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$clash" -eq 1 ]; then
|
||||||
|
echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080"
|
||||||
|
echo " (or rename this directory — the whole block follows the name)"
|
||||||
|
fi
|
||||||
@@ -25,7 +25,7 @@ up() {
|
|||||||
# Say what this profile locks in BEFORE spending minutes building it:
|
# Say what this profile locks in BEFORE spending minutes building it:
|
||||||
# the audit policy is an apiserver flag and cannot be changed later.
|
# the audit policy is an apiserver flag and cannot be changed later.
|
||||||
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
|
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
|
||||||
echo " shape ctrl/k8s/$KIND_CONFIG"
|
echo " shape ${KIND_CONFIG_SHOWN}"
|
||||||
echo " nodes $NODES"
|
echo " nodes $NODES"
|
||||||
echo " image $NODE_IMAGE"
|
echo " image $NODE_IMAGE"
|
||||||
echo " audit $AUDIT"
|
echo " audit $AUDIT"
|
||||||
|
|||||||
@@ -1,17 +1,22 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# The installation wizard: detect the host, install a pinned toolchain onto it,
|
# Toolchain installer: detect the host, install a pinned toolchain onto it, then
|
||||||
# then report what it could not do. It never runs the cluster and never mutates
|
# report what it could not do.
|
||||||
# the host outside the directories mounted into it.
|
|
||||||
#
|
#
|
||||||
# Usage (normally via `make station` / `make deps`, or directly):
|
# It never runs the cluster, never uses sudo or apt, and writes only into
|
||||||
# wizard.sh detect # report host facts only, change nothing
|
# $OUT_BIN (default ~/.local/bin). Everything that would touch the host proper —
|
||||||
# wizard.sh fetch [core|dev] [--to DIR] # download + verify into DIR
|
# systemd, inotify limits, .wslconfig, docker group — is REPORTED for a human to
|
||||||
# wizard.sh install [core|dev] # detect, fetch, install, report
|
# decide on, never performed. That is what makes it safe to run on a machine that
|
||||||
|
# already has a working setup.
|
||||||
|
#
|
||||||
|
# Usage (normally via `make deps`, or directly):
|
||||||
|
# deps.sh detect # report host facts only, change nothing
|
||||||
|
# deps.sh fetch [core|dev] [--to DIR] # download + verify into DIR
|
||||||
|
# deps.sh install [core|dev] # detect, fetch, install, report
|
||||||
#
|
#
|
||||||
# Tiers: 'core' is kubectl + jq (talk to a cluster); 'dev' adds kind and tilt
|
# Tiers: 'core' is kubectl + jq (talk to a cluster); 'dev' adds kind and tilt
|
||||||
# Default is dev.
|
# Default is dev.
|
||||||
#
|
#
|
||||||
# Runs both inside the wizard container and bare on a host. Inside the
|
# Runs both inside the installer container and bare on a host. Inside the
|
||||||
# container, host files are read through $HOST_ROOT (mount / as :ro); bare, it
|
# container, host files are read through $HOST_ROOT (mount / as :ro); bare, it
|
||||||
# falls back to /.
|
# falls back to /.
|
||||||
|
|
||||||
@@ -44,6 +49,14 @@ MANUAL=()
|
|||||||
# Host FILES (/etc/..., /mnt/c/...) must be read through the mount. Kernel-level
|
# Host FILES (/etc/..., /mnt/c/...) must be read through the mount. Kernel-level
|
||||||
# facts (kernel version, meminfo, inotify) are shared with the container, so the
|
# facts (kernel version, meminfo, inotify) are shared with the container, so the
|
||||||
# container's own view is already the host's.
|
# container's own view is already the host's.
|
||||||
|
# A /proc/meminfo field in MB, 0 if the field is absent. MEMINFO exists so the
|
||||||
|
# tight and does-not-fit branches can be exercised against a real machine's
|
||||||
|
# numbers from somewhere else; in normal use it is always /proc/meminfo.
|
||||||
|
mb_of() {
|
||||||
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||||
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||||
|
}
|
||||||
|
|
||||||
host_file() {
|
host_file() {
|
||||||
local p="${1#/}"
|
local p="${1#/}"
|
||||||
if [ "$HOST_ROOT" != "/" ] && [ -e "$HOST_ROOT/$p" ]; then
|
if [ "$HOST_ROOT" != "/" ] && [ -e "$HOST_ROOT/$p" ]; then
|
||||||
@@ -55,6 +68,29 @@ host_file() {
|
|||||||
|
|
||||||
# ── detect ─────────────────────────────────────────────────────────────────
|
# ── detect ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
|
||||||
|
See "Starting from plain Windows" in README.md.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
detect() {
|
detect() {
|
||||||
@@ -64,20 +100,34 @@ detect() {
|
|||||||
local osr; osr=$(host_file /etc/os-release)
|
local osr; osr=$(host_file /etc/os-release)
|
||||||
[ -r "$osr" ] && echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' "$osr")"
|
[ -r "$osr" ] && echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' "$osr")"
|
||||||
|
|
||||||
local total_kb avail_kb
|
# In MB. Whole gigabytes lose nearly half a GB on exactly the machines where
|
||||||
total_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo)
|
# it matters: 1874 MB available used to print as "1 GB". Facts only — whether
|
||||||
avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
|
# that is enough depends on the profile, which check.sh knows and this does not.
|
||||||
printf " memory %d GB total, %d GB available\n" \
|
local total_mb avail_mb swap_total_mb swap_used_mb om
|
||||||
$((total_kb / 1024 / 1024)) $((avail_kb / 1024 / 1024))
|
total_mb=$(mb_of MemTotal)
|
||||||
|
avail_mb=$(mb_of MemAvailable)
|
||||||
if [ $((avail_kb / 1024 / 1024)) -lt 4 ]; then
|
swap_total_mb=$(mb_of SwapTotal)
|
||||||
echo " ! under 4 GB available — a multi-node profile will struggle."
|
swap_used_mb=$(( swap_total_mb - $(mb_of SwapFree) ))
|
||||||
echo " 'make cluster list' shows the others; 'make cluster free' stops them."
|
printf " memory %d MB total, %d MB available\n" "$total_mb" "$avail_mb"
|
||||||
|
if [ "$swap_total_mb" -gt 0 ]; then
|
||||||
|
printf " swap %d MB used of %d MB\n" "$swap_used_mb" "$swap_total_mb"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# How the kernel answers an allocation it cannot really satisfy. With 1 it
|
||||||
|
# always says yes and settles up later with the OOM killer, so a cluster that
|
||||||
|
# starts cleanly can still lose processes afterwards.
|
||||||
|
om=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||||
|
case "$om" in
|
||||||
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess" ;;
|
||||||
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit" ;;
|
||||||
|
2) echo " overcommit 2 strict — an allocation fails honestly instead of killing later" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
detect_wsl
|
detect_wsl
|
||||||
|
detect_filesystem
|
||||||
detect_docker
|
detect_docker
|
||||||
detect_inotify
|
detect_inotify
|
||||||
|
detect_toolchain
|
||||||
}
|
}
|
||||||
|
|
||||||
detect_wsl() {
|
detect_wsl() {
|
||||||
@@ -115,18 +165,46 @@ detect_wsl() {
|
|||||||
if [ -n "$wcfg" ] && grep -qE '^\s*memory\s*=' "$wcfg"; then
|
if [ -n "$wcfg" ] && grep -qE '^\s*memory\s*=' "$wcfg"; then
|
||||||
echo " wslconfig memory set: $(grep -E '^\s*memory\s*=' "$wcfg" | tr -d ' ')"
|
echo " wslconfig memory set: $(grep -E '^\s*memory\s*=' "$wcfg" | tr -d ' ')"
|
||||||
else
|
else
|
||||||
MANUAL+=("Cap/raise the WSL VM memory — in %USERPROFILE%\\.wslconfig on Windows:
|
MANUAL+=("Cap/raise the WSL VM memory — see what is set versus what booted:
|
||||||
[wsl2]
|
make mem status
|
||||||
memory=8GB
|
It prints the edit to make and the command to apply it.")
|
||||||
then from a WINDOWS terminal: wsl --shutdown")
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Not a path check: /mnt is an ordinary mount point and an ext4 disk mounted
|
||||||
|
# there is perfectly fine. What matters is the filesystem. The Windows drives
|
||||||
|
# arrive as 9p (WSL2) or drvfs (WSL1); network and fuse mounts behave the same
|
||||||
|
# way. None of them deliver inotify events, so anything watching files goes
|
||||||
|
# quiet without saying why.
|
||||||
|
watch_hostile_fs() {
|
||||||
|
local dir="$1" fstype
|
||||||
|
fstype=$(findmnt -no FSTYPE --target "$dir" 2>/dev/null || true)
|
||||||
|
[ -n "$fstype" ] || fstype=$(stat -f -c %T "$dir" 2>/dev/null || true)
|
||||||
|
case "$fstype" in
|
||||||
|
9p|v9fs|drvfs|cifs|smb3|nfs|nfs4|fuse.sshfs|fuseblk) echo "$fstype" ;;
|
||||||
|
*) echo "" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_filesystem() {
|
||||||
|
local root fstype
|
||||||
|
root=$(cd .. && pwd -P)
|
||||||
|
fstype=$(watch_hostile_fs "$root")
|
||||||
|
if [ -n "$fstype" ]; then
|
||||||
|
echo " ! this directory is on $fstype — file watching will not work"
|
||||||
|
MANUAL+=("Move this onto the local disk. Nothing watching files sees changes
|
||||||
|
on a $fstype mount, and everything else is slower:
|
||||||
|
cp -r \"$root\" ~/ && cd ~/$(basename "$root")")
|
||||||
|
else
|
||||||
|
echo " filesystem $root ($(findmnt -no FSTYPE --target "$root" 2>/dev/null || echo local))"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
detect_docker() {
|
detect_docker() {
|
||||||
# Reachability of the daemon is the real question, and the CLI is only how
|
# Reachability of the daemon is the real question, and the CLI is only how
|
||||||
# we ask it. Note that when this runs inside the wizard container, Docker
|
# we ask it. Note that when this runs inside the installer container, Docker
|
||||||
# necessarily exists on the host — otherwise nothing would be executing —
|
# necessarily exists on the host — otherwise nothing would be executing —
|
||||||
# so a missing CLI in here is a wizard packaging bug, not a host problem.
|
# so a missing CLI in here is an installer packaging bug, not a host problem.
|
||||||
if ! command -v docker >/dev/null 2>&1; then
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
if [ -S /var/run/docker.sock ]; then
|
if [ -S /var/run/docker.sock ]; then
|
||||||
echo " docker socket present (no cli in this context)"
|
echo " docker socket present (no cli in this context)"
|
||||||
@@ -230,7 +308,7 @@ fetch_tgz() {
|
|||||||
chmod +x "$dest/$name"
|
chmod +x "$dest/$name"
|
||||||
}
|
}
|
||||||
|
|
||||||
# The wizard runs as root so it can reach the docker socket, which means
|
# The installer runs as root so it can reach the docker socket, which means
|
||||||
# everything it writes into a mounted volume lands root-owned and unusable from
|
# everything it writes into a mounted volume lands root-owned and unusable from
|
||||||
# the host. Hand it back to whoever owns the mount point (the host user created
|
# the host. Hand it back to whoever owns the mount point (the host user created
|
||||||
# that directory before mounting it).
|
# that directory before mounting it).
|
||||||
@@ -257,7 +335,94 @@ fix_ownership() {
|
|||||||
CORE_TOOLS="kubectl jq"
|
CORE_TOOLS="kubectl jq"
|
||||||
# No helm: every addon installs with `kubectl apply -f <url>`, so nothing here
|
# No helm: every addon installs with `kubectl apply -f <url>`, so nothing here
|
||||||
# has ever invoked it. Add it back the day something actually needs a chart.
|
# has ever invoked it. Add it back the day something actually needs a chart.
|
||||||
DEV_TOOLS="kind tilt"
|
#
|
||||||
|
# ctlptl is 'dev' rather than 'core' for the same reason kind is: core is "talk
|
||||||
|
# to a cluster someone else runs", and ctlptl builds them. It earns its place
|
||||||
|
# because it is what wires a cluster to a local registry — without one, an
|
||||||
|
# unqualified image name resolves to docker.io/library/<name> and there is
|
||||||
|
# nothing structural stopping a push there.
|
||||||
|
DEV_TOOLS="kind tilt ctlptl"
|
||||||
|
|
||||||
|
# ── what is already on this machine ───────────────────────────────────────
|
||||||
|
#
|
||||||
|
# A tool already on PATH at its pinned version is left where it is. Without
|
||||||
|
# this, install downloads a second copy into OUT_BIN and then reports the first
|
||||||
|
# one as shadowed — noise, and wrong, when both are the same version. That is
|
||||||
|
# the normal state of any machine someone set up by hand: the AWS Workspace
|
||||||
|
# keeps its toolchain in ~/wdir/bin, all five at exactly these pins.
|
||||||
|
|
||||||
|
pin_of() {
|
||||||
|
case "$1" in
|
||||||
|
kubectl) echo "$KUBECTL_VERSION" ;;
|
||||||
|
jq) echo "$JQ_VERSION" ;;
|
||||||
|
kind) echo "$KIND_VERSION" ;;
|
||||||
|
tilt) echo "$TILT_VERSION" ;;
|
||||||
|
ctlptl) echo "$CTLPTL_VERSION" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# The version string a binary reports. Each tool spells the question
|
||||||
|
# differently, and kubectl has to be told --client or it goes looking for a
|
||||||
|
# server to ask.
|
||||||
|
reported_version() {
|
||||||
|
local tool="$1" path="$2"
|
||||||
|
case "$tool" in
|
||||||
|
kubectl) "$path" version --client 2>/dev/null ;;
|
||||||
|
jq) "$path" --version 2>/dev/null ;;
|
||||||
|
*) "$path" version 2>/dev/null ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# Does the binary at PATH report PIN? Matched as a whole version token, so
|
||||||
|
# 0.37.6 never matches 10.37.60, with the leading v optional either side: kind
|
||||||
|
# says v0.32.0, jq says jq-1.8.2, and tilt says v0.37.6 against a pin of 0.37.6.
|
||||||
|
#
|
||||||
|
# Bash's own regex rather than grep, deliberately. grep is not the same program
|
||||||
|
# on every machine — some builds reject patterns that others accept — and a
|
||||||
|
# failed grep inside a count reads exactly like a zero.
|
||||||
|
version_matches() {
|
||||||
|
local tool="$1" path="$2" pin="$3" out v re
|
||||||
|
out=$(reported_version "$tool" "$path") || return 1
|
||||||
|
v="${pin#v}"
|
||||||
|
v="${v//./\\.}"
|
||||||
|
re="(^|[^0-9.])v?${v}([^0-9.]|\$)"
|
||||||
|
[[ $out =~ $re ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# DEPS_ONLY narrows a fetch to the tools it names. Unset means the whole tier,
|
||||||
|
# which is what an explicit `deps.sh fetch` always gets: "download these into
|
||||||
|
# DIR" must not quietly skip something because this machine happens to have it.
|
||||||
|
# Only install() sets it, to what detect_toolchain found missing or mismatched.
|
||||||
|
want() { [ -z "${DEPS_ONLY:-}" ] || [[ " $DEPS_ONLY " == *" $1 "* ]]; }
|
||||||
|
|
||||||
|
# Every tool in the tier with its state, probed once and reported once. What
|
||||||
|
# still needs fetching is left in TOOLCHAIN_NEED for install() to act on.
|
||||||
|
TOOLCHAIN_NEED=""
|
||||||
|
detect_toolchain() {
|
||||||
|
local tier="${TIER:-dev}" b pin path found
|
||||||
|
TOOLCHAIN_NEED=""
|
||||||
|
echo
|
||||||
|
echo "toolchain (pinned, tier '$tier')"
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
pin=$(pin_of "$b")
|
||||||
|
path=$(command -v "$b" 2>/dev/null || true)
|
||||||
|
if [ -z "$path" ]; then
|
||||||
|
printf " - %-8s %-9s not found\n" "$b" "$pin"
|
||||||
|
TOOLCHAIN_NEED+="$b "
|
||||||
|
elif version_matches "$b" "$path" "$pin"; then
|
||||||
|
printf " %-8s %-9s %s\n" "$b" "$pin" "$path"
|
||||||
|
else
|
||||||
|
found=$(reported_version "$b" "$path" 2>/dev/null | head -1 || true)
|
||||||
|
printf " ! %-8s wants %s, %s reports '%s'\n" "$b" "$pin" "$path" "$found"
|
||||||
|
TOOLCHAIN_NEED+="$b "
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -z "$TOOLCHAIN_NEED" ]; then
|
||||||
|
echo " every pinned tool is already on PATH — nothing to fetch"
|
||||||
|
else
|
||||||
|
echo " 'make deps' fetches only: ${TOOLCHAIN_NEED% }"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
fetch() {
|
fetch() {
|
||||||
local dest="$OUT_BIN" tier="${TIER:-dev}"
|
local dest="$OUT_BIN" tier="${TIER:-dev}"
|
||||||
@@ -279,12 +444,17 @@ fetch() {
|
|||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "fetching '$tier' toolchain (source: $DEPS_SOURCE)"
|
if [ -n "${DEPS_ONLY:-}" ]; then
|
||||||
fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"
|
echo "fetching ${DEPS_ONLY% } (source: $DEPS_SOURCE)"
|
||||||
fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"
|
else
|
||||||
|
echo "fetching '$tier' toolchain (source: $DEPS_SOURCE)"
|
||||||
|
fi
|
||||||
|
if want kubectl; then fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"; fi
|
||||||
|
if want jq; then fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"; fi
|
||||||
if [ "$tier" = "dev" ]; then
|
if [ "$tier" = "dev" ]; then
|
||||||
fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"
|
if want kind; then fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"; fi
|
||||||
fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0
|
if want tilt; then fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0; fi
|
||||||
|
if want ctlptl; then fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0; fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
fix_ownership "$dest"
|
fix_ownership "$dest"
|
||||||
@@ -301,7 +471,7 @@ report_manual() {
|
|||||||
echo "nothing left to do by hand."
|
echo "nothing left to do by hand."
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
echo "host actions the wizard cannot perform (${#MANUAL[@]}):"
|
echo "host actions this cannot perform (${#MANUAL[@]}):"
|
||||||
echo
|
echo
|
||||||
local n=1
|
local n=1
|
||||||
for m in "${MANUAL[@]}"; do
|
for m in "${MANUAL[@]}"; do
|
||||||
@@ -326,6 +496,9 @@ warn_shadowing() {
|
|||||||
command -v "$b" 2>/dev/null || true)
|
command -v "$b" 2>/dev/null || true)
|
||||||
[ -n "$existing" ] || continue
|
[ -n "$existing" ] || continue
|
||||||
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||||
|
# The same version in both places is not a conflict: nothing changes for
|
||||||
|
# any other project whichever copy PATH happens to find first.
|
||||||
|
if version_matches "$b" "$existing" "$(pin_of "$b")"; then continue; fi
|
||||||
shadowed+=" $b $existing"$'\n'
|
shadowed+=" $b $existing"$'\n'
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -348,31 +521,42 @@ warn_shadowing() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
install() {
|
install() {
|
||||||
local tier="${1:-dev}"
|
local tier="${1:-dev}" b
|
||||||
|
TIER="$tier"
|
||||||
detect
|
detect
|
||||||
echo
|
|
||||||
fetch "$tier"
|
|
||||||
echo
|
|
||||||
echo "installed to $OUT_BIN ($tier):"
|
|
||||||
for b in $(tier_tools "$tier"); do
|
|
||||||
[ -x "$OUT_BIN/$b" ] && echo " $b"
|
|
||||||
done
|
|
||||||
if [ "$tier" = "core" ]; then
|
|
||||||
echo " (no kind/tilt — 'make deps dev' adds them)"
|
|
||||||
fi
|
|
||||||
warn_shadowing "$tier"
|
|
||||||
|
|
||||||
case ":${PATH}:" in
|
# detect_toolchain has already probed PATH. Fetch only what it found missing
|
||||||
*":$OUT_BIN:"*) ;;
|
# or at the wrong version; a tool already present at its pin stays where it is.
|
||||||
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
if [ -n "$TOOLCHAIN_NEED" ]; then
|
||||||
|
echo
|
||||||
|
DEPS_ONLY="$TOOLCHAIN_NEED" fetch "$tier"
|
||||||
|
echo
|
||||||
|
echo "installed to $OUT_BIN ($tier):"
|
||||||
|
for b in $TOOLCHAIN_NEED; do
|
||||||
|
if [ -x "$OUT_BIN/$b" ]; then echo " $b"; fi
|
||||||
|
done
|
||||||
|
if [ "$tier" = "core" ]; then
|
||||||
|
echo " (no kind/tilt — 'make deps dev' adds them)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Only worth saying when something actually landed in OUT_BIN. When every
|
||||||
|
# tool was satisfied elsewhere, OUT_BIN may reasonably be off PATH, and
|
||||||
|
# telling the user to add it would be advice to fix nothing.
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||||
export PATH=\"${OUT_BIN}:\$PATH\"") ;;
|
export PATH=\"${OUT_BIN}:\$PATH\"") ;;
|
||||||
esac
|
esac
|
||||||
|
fi
|
||||||
|
warn_shadowing "$tier"
|
||||||
|
|
||||||
report_manual
|
report_manual
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── main ───────────────────────────────────────────────────────────────────
|
# ── main ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
|
||||||
case "${1:-install}" in
|
case "${1:-install}" in
|
||||||
detect) detect; report_manual ;;
|
detect) detect; report_manual ;;
|
||||||
fetch) shift; fetch "$@" ;;
|
fetch) shift; fetch "$@" ;;
|
||||||
@@ -19,7 +19,7 @@ DNS_MODE=hosts
|
|||||||
# Opt in to the real ports below only when this is the ONLY environment and
|
# Opt in to the real ports below only when this is the ONLY environment and
|
||||||
# nothing else owns :80. They fail to bind otherwise, and docker reports it as an
|
# nothing else owns :80. They fail to bind otherwise, and docker reports it as an
|
||||||
# opaque "failed to bind host port 0.0.0.0:80/tcp: address already in use"
|
# opaque "failed to bind host port 0.0.0.0:80/tcp: address already in use"
|
||||||
# halfway through cluster creation. `make station` checks before you spend the
|
# halfway through cluster creation. `make check` checks before you spend the
|
||||||
# time. Uncommenting also means only one environment can exist at a time.
|
# time. Uncommenting also means only one environment can exist at a time.
|
||||||
# HTTP_PORT=80
|
# HTTP_PORT=80
|
||||||
# HTTPS_PORT=443
|
# HTTPS_PORT=443
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
# data — a cluster with the dependency containers a soleprint room asks for.
|
# data — a cluster with the cabinets an environment asks for.
|
||||||
#
|
#
|
||||||
# The point of this profile is that a room declares what it needs once, in
|
# A cabinet is a public service dropped in as-is — the upstream image,
|
||||||
# cfg/<room>/data/cabinets.json, and gets it on either target: `build.py`
|
# unmodified, reachable at a known address. It is declared once and installs on
|
||||||
# composes those services into docker-compose.yml for a laptop, and the addons
|
# either target: a `service.yml` composes it for a laptop, and the addons below
|
||||||
# below install the same ones here. The names match deliberately —
|
# install the same one here. The names match deliberately — each cabinet.json
|
||||||
# soleprint/station/cabinets/<name>/cabinet.json carries a `rig_addon` field
|
# carries a `rig_addon` field pointing at ctrl/addons/<name>.sh.
|
||||||
# pointing at ctrl/addons/<name>.sh.
|
|
||||||
#
|
#
|
||||||
# Everything lands in the `data` namespace (DATA_NAMESPACE to move it), so
|
# Everything lands in the `data` namespace (DATA_NAMESPACE to move it), so
|
||||||
# `make cluster reset` on the app namespace leaves the databases alone.
|
# `make cluster reset` on the app namespace leaves the databases alone.
|
||||||
@@ -30,8 +29,8 @@ DATA_NAMESPACE=data
|
|||||||
# Postgres identity. The password is not here: postgres.sh generates one on
|
# Postgres identity. The password is not here: postgres.sh generates one on
|
||||||
# first install and keeps it across re-runs, so re-running the addon never
|
# first install and keeps it across re-runs, so re-running the addon never
|
||||||
# rotates the credential out from under whatever is already connected.
|
# rotates the credential out from under whatever is already connected.
|
||||||
POSTGRES_DB=soleprint
|
POSTGRES_DB=app
|
||||||
POSTGRES_USER=soleprint
|
POSTGRES_USER=app
|
||||||
POSTGRES_STORAGE=2Gi
|
POSTGRES_STORAGE=2Gi
|
||||||
|
|
||||||
AIRFLOW_ADMIN_USER=admin
|
AIRFLOW_ADMIN_USER=admin
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# offline — air-gapped. Everything comes from a local registry that was loaded
|
# offline — air-gapped. Everything comes from a local registry that was loaded
|
||||||
# ahead of time; nothing reaches the internet. Pair with the wizard-full image
|
# ahead of time; nothing reaches the internet. Pair with the deps-full image
|
||||||
# (DEPS_SOURCE=baked) so the toolchain install is offline too.
|
# (DEPS_SOURCE=baked) so the toolchain install is offline too.
|
||||||
#
|
#
|
||||||
# The heavier addons are left out to keep first boot viable.
|
# The heavier addons are left out to keep first boot viable.
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
# `ctrl/k8s` — cluster shape, and what runs on it
|
# `ctrl/k8s` — cluster shape, and what runs on it
|
||||||
|
|
||||||
Same layout as every other project here (`unt`, `nvi`, `eth`, `mpr`, and
|
Same layout as every other project here: a kind config, a kustomize `base/`,
|
||||||
soleprint's generated rooms): a kind config, a kustomize `base/`, and an
|
and an `overlays/dev/` that patches it.
|
||||||
`overlays/dev/` that patches it. See ALL `projects/templates/conventions.md`.
|
|
||||||
|
|
||||||
```
|
```
|
||||||
kind-config*.yaml.tpl the cluster itself — nodes, ports, audit
|
kind-config*.yaml.tpl the cluster itself — nodes, ports, audit
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ nodes:
|
|||||||
- role: control-plane
|
- role: control-plane
|
||||||
image: ${NODE_IMAGE}
|
image: ${NODE_IMAGE}
|
||||||
# hostPath is resolved by the HOST dockerd, so this must be a host path even
|
# hostPath is resolved by the HOST dockerd, so this must be a host path even
|
||||||
# when cluster.sh runs inside the wizard container. HOST_WORKDIR says where
|
# when cluster.sh runs inside the installer container. HOST_WORKDIR says where
|
||||||
# this rig lives on the host; bare on a host it is just the repo root.
|
# this rig lives on the host; bare on a host it is just the repo root.
|
||||||
extraMounts:
|
extraMounts:
|
||||||
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
|
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
|
||||||
|
|||||||
@@ -103,16 +103,26 @@ load_config() {
|
|||||||
|
|
||||||
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
|
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
|
||||||
# shape is adding a file; there is no dispatcher to edit.
|
# shape is adding a file; there is no dispatcher to edit.
|
||||||
|
#
|
||||||
|
# A host that needs its own shape — extra port mappings, more nodes — passes
|
||||||
|
# an absolute path instead, and rig renders it exactly like one of its own:
|
||||||
|
# ${CLUSTER} and ${NODE_IMAGE} are substituted either way. The shape stays in
|
||||||
|
# the host's tree, because what a host's cluster needs is the host's business;
|
||||||
|
# rig only knows how to build whatever it is handed.
|
||||||
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
|
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
|
||||||
KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"
|
case "$KIND_CONFIG" in
|
||||||
|
/*) KIND_CONFIG_PATH="$KIND_CONFIG"; KIND_CONFIG_SHOWN="$KIND_CONFIG" ;;
|
||||||
|
*) KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"; KIND_CONFIG_SHOWN="ctrl/k8s/${KIND_CONFIG}" ;;
|
||||||
|
esac
|
||||||
if [ ! -f "$KIND_CONFIG_PATH" ]; then
|
if [ ! -f "$KIND_CONFIG_PATH" ]; then
|
||||||
echo "no such cluster shape: ctrl/k8s/${KIND_CONFIG}" >&2
|
echo "no such cluster shape: ${KIND_CONFIG_SHOWN}" >&2
|
||||||
echo "available: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
|
echo "rig's own: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
|
||||||
|
echo "or pass an absolute path to a shape of your own" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Read the shape back out of the YAML rather than trusting a profile to
|
# Read the shape back out of the YAML rather than trusting a profile to
|
||||||
# restate it. station.sh sizes the memory warning on NODES, and cluster.sh
|
# restate it. check.sh sizes the memory warning on NODES, and cluster.sh
|
||||||
# prints AUDIT before spending minutes building something that cannot be
|
# prints AUDIT before spending minutes building something that cannot be
|
||||||
# changed afterwards — both would mislead if the numbers drifted.
|
# changed afterwards — both would mislead if the numbers drifted.
|
||||||
NODES=$(grep -c '^ - role:' "$KIND_CONFIG_PATH")
|
NODES=$(grep -c '^ - role:' "$KIND_CONFIG_PATH")
|
||||||
@@ -125,7 +135,7 @@ load_config() {
|
|||||||
# depending on something the caller does not set.
|
# depending on something the caller does not set.
|
||||||
#
|
#
|
||||||
# hostPath entries are resolved by the HOST dockerd, so HOST_WORKDIR must stay a
|
# hostPath entries are resolved by the HOST dockerd, so HOST_WORKDIR must stay a
|
||||||
# host path even when this runs inside the wizard container.
|
# host path even when this runs inside the installer container.
|
||||||
render_kind_config() {
|
render_kind_config() {
|
||||||
local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}"
|
local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}"
|
||||||
sed -e "s|\${CLUSTER}|${CLUSTER}|g" \
|
sed -e "s|\${CLUSTER}|${CLUSTER}|g" \
|
||||||
|
|||||||
233
rig/ctrl/mem.sh
Executable file
233
rig/ctrl/mem.sh
Executable file
@@ -0,0 +1,233 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# What memory this machine has, what is left, and — where there is one — what
|
||||||
|
# cap is holding it there.
|
||||||
|
#
|
||||||
|
# Runs on native Linux and under WSL, because rig is developed on one and used
|
||||||
|
# on the other. The difference is not cosmetic: on WSL the memory you see is a
|
||||||
|
# VM allocation that can be raised, and the commonest failure is raising it
|
||||||
|
# without restarting, so the number on disk and the number in /proc disagree.
|
||||||
|
# On native Linux there is no such cap and pretending otherwise sends you to a
|
||||||
|
# file that does not exist.
|
||||||
|
#
|
||||||
|
# This reports and instructs. It never writes a .wslconfig — applying one costs
|
||||||
|
# a full VM restart that takes every shell, mount and container with it, and
|
||||||
|
# choosing that moment is yours.
|
||||||
|
#
|
||||||
|
# `backup` exists so `restore` has something to read: back up, hand-edit
|
||||||
|
# following the printed instruction, restore if it goes wrong. Both are
|
||||||
|
# WSL-only, because .wslconfig is the only thing here worth backing up.
|
||||||
|
#
|
||||||
|
# Usage: mem.sh status | backup | restore
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
|
||||||
|
See "Starting from plain Windows" in README.md.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
|
require_wsl() {
|
||||||
|
if ! is_wsl; then
|
||||||
|
echo "$1 acts on .wslconfig, which only exists under WSL." >&2
|
||||||
|
echo "This is native Linux — there is no VM allocation to save or roll back." >&2
|
||||||
|
echo "Use 'mem.sh status' to see what the machine actually has." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
||||||
|
|
||||||
|
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
||||||
|
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
||||||
|
# Windows, then fall back to whichever profile actually owns a config.
|
||||||
|
wslconfig_path() {
|
||||||
|
local profile winpath found
|
||||||
|
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
||||||
|
case "$profile" in
|
||||||
|
""|*%*) ;;
|
||||||
|
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
||||||
|
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
||||||
|
echo "$winpath/.wslconfig"; return
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||||
|
if [ -n "$found" ]; then echo "$found"; return; fi
|
||||||
|
|
||||||
|
echo "cannot tell which Windows profile owns .wslconfig. Candidates:" >&2
|
||||||
|
ls -d /mnt/c/Users/*/ 2>/dev/null \
|
||||||
|
| grep -viE "/(All Users|Default|Default User|Public)/$" | sed "s/^/ /" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
configured_memory() {
|
||||||
|
[ -r "$1" ] || { echo ""; return; }
|
||||||
|
sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$1" | tail -1 | tr -d '[:space:]'
|
||||||
|
}
|
||||||
|
|
||||||
|
# "9GB" / "8192MB" / "9G" -> MB, so it can be compared with /proc/meminfo.
|
||||||
|
to_mb() {
|
||||||
|
local v="${1^^}" n
|
||||||
|
n=$(echo "$v" | tr -dc '0-9')
|
||||||
|
[ -n "$n" ] || { echo ""; return; }
|
||||||
|
case "$v" in
|
||||||
|
*GB|*G) echo $(( n * 1024 )) ;;
|
||||||
|
*MB|*M) echo "$n" ;;
|
||||||
|
*) echo $(( n / 1024 / 1024 )) ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
hogs() {
|
||||||
|
echo "holding the most:"
|
||||||
|
ps -eo rss,comm --sort=-rss 2>/dev/null | awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
||||||
|
}
|
||||||
|
|
||||||
|
status() {
|
||||||
|
local total avail swap_total swap_free
|
||||||
|
total=$(mb MemTotal); avail=$(mb MemAvailable)
|
||||||
|
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
||||||
|
|
||||||
|
if is_wsl; then
|
||||||
|
local cfg conf conf_mb
|
||||||
|
cfg=$(wslconfig_path)
|
||||||
|
conf=$(configured_memory "$cfg")
|
||||||
|
echo "platform WSL"
|
||||||
|
echo "config $cfg"
|
||||||
|
if [ -n "$conf" ]; then
|
||||||
|
conf_mb=$(to_mb "$conf")
|
||||||
|
echo "configured $conf (${conf_mb} MB)"
|
||||||
|
else
|
||||||
|
conf_mb=""
|
||||||
|
echo "configured (no memory= set — WSL defaults to 50% of host RAM, or 8GB, whichever is less)"
|
||||||
|
fi
|
||||||
|
echo "booted ${total} MB"
|
||||||
|
echo "available ${avail} MB"
|
||||||
|
echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||||
|
|
||||||
|
if [ -n "$conf_mb" ]; then
|
||||||
|
# The VM reports a little less than allocated; 15% covers the kernel
|
||||||
|
# without calling every healthy machine a mismatch.
|
||||||
|
if [ "$total" -lt $(( conf_mb * 85 / 100 )) ]; then
|
||||||
|
echo
|
||||||
|
echo "! configured ${conf_mb} MB but booted ${total} MB."
|
||||||
|
echo " The change has not been applied. From a WINDOWS terminal:"
|
||||||
|
echo
|
||||||
|
echo " wsl --shutdown"
|
||||||
|
echo
|
||||||
|
echo " then start the distro again."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo
|
||||||
|
echo "To raise it, add to $cfg on the Windows side:"
|
||||||
|
echo
|
||||||
|
echo " [wsl2]"
|
||||||
|
echo " memory=8GB"
|
||||||
|
echo
|
||||||
|
echo "then, from a WINDOWS terminal: wsl --shutdown"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local n
|
||||||
|
n=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
||||||
|
[ "$n" -gt 0 ] && echo "backups $n (newest: $(ls -t "$cfg".*.bak 2>/dev/null | head -1))"
|
||||||
|
else
|
||||||
|
echo "platform native linux"
|
||||||
|
echo "total ${total} MB"
|
||||||
|
echo "available ${avail} MB"
|
||||||
|
echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||||
|
echo
|
||||||
|
echo "No VM allocation to raise here — this is the machine's own memory."
|
||||||
|
echo "If it is tight the levers are freeing something or adding swap."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Under a fifth left is worth naming wherever you are running.
|
||||||
|
if [ "$avail" -lt $(( total / 5 )) ]; then
|
||||||
|
echo
|
||||||
|
hogs
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
backup() {
|
||||||
|
require_wsl backup
|
||||||
|
local cfg dest
|
||||||
|
cfg=$(wslconfig_path)
|
||||||
|
[ -r "$cfg" ] || { echo "nothing to back up: $cfg does not exist" >&2; exit 1; }
|
||||||
|
# Timestamped and never overwritten: a backup that can destroy itself on a
|
||||||
|
# second run is not a backup.
|
||||||
|
dest="${cfg}.$(date +%Y%m%d-%H%M%S).bak"
|
||||||
|
cp "$cfg" "$dest"
|
||||||
|
echo "backed up $dest"
|
||||||
|
echo
|
||||||
|
echo "Edit $cfg by hand, then from a WINDOWS terminal: wsl --shutdown"
|
||||||
|
}
|
||||||
|
|
||||||
|
restore() {
|
||||||
|
require_wsl restore
|
||||||
|
local cfg newest count
|
||||||
|
cfg=$(wslconfig_path)
|
||||||
|
newest=$(ls -t "$cfg".*.bak 2>/dev/null | head -1 || true)
|
||||||
|
[ -n "$newest" ] || { echo "no backups found beside $cfg" >&2; exit 1; }
|
||||||
|
|
||||||
|
echo "restoring $newest"
|
||||||
|
echo " -> $cfg"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Newest is the right default — undo the last edit — but if you backed up
|
||||||
|
# *after* editing, the state you want is older. Show the rest so a no-op
|
||||||
|
# restore is obviously a no-op rather than a mystery.
|
||||||
|
count=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
||||||
|
if [ "$count" -gt 1 ]; then
|
||||||
|
echo "$count backups exist, newest first:"
|
||||||
|
ls -t "$cfg".*.bak | sed 's/^/ /'
|
||||||
|
echo " (restoring the newest; copy another by hand to pick an older one)"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -r "$cfg" ]; then
|
||||||
|
echo "what changes:"
|
||||||
|
if diff "$cfg" "$newest" > /tmp/mem.diff 2>&1 && [ ! -s /tmp/mem.diff ]; then
|
||||||
|
echo " nothing — that backup is identical to the current config"
|
||||||
|
else
|
||||||
|
sed 's/^/ /' /tmp/mem.diff
|
||||||
|
fi
|
||||||
|
rm -f /tmp/mem.diff
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "proceed? [y/N] "
|
||||||
|
read -r reply
|
||||||
|
case "$reply" in
|
||||||
|
y|Y|yes|Yes) ;;
|
||||||
|
*) echo "left alone"; return 0 ;;
|
||||||
|
esac
|
||||||
|
cp "$newest" "$cfg"
|
||||||
|
echo "restored. From a WINDOWS terminal: wsl --shutdown"
|
||||||
|
}
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
|
||||||
|
case "${1:-status}" in
|
||||||
|
status) status ;;
|
||||||
|
backup) backup ;;
|
||||||
|
restore) restore ;;
|
||||||
|
*) echo "usage: $0 [status|backup|restore]" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
@@ -58,10 +58,13 @@ require_wsl() {
|
|||||||
cat >&2 <<'EOF'
|
cat >&2 <<'EOF'
|
||||||
newbox is WSL-only for now.
|
newbox is WSL-only for now.
|
||||||
|
|
||||||
|
If WSL is not installed, run `wsl --install` from an elevated Windows prompt
|
||||||
|
first — see "Starting from plain Windows" in README.md.
|
||||||
|
|
||||||
On native Linux you do not need it: rig already isolates environments by
|
On native Linux you do not need it: rig already isolates environments by
|
||||||
directory (own cluster, context, images and port block), so a second copy in a
|
directory (own cluster, context, images and port block), so a second copy in a
|
||||||
second directory is the clean slate. To validate the installer itself against a
|
second directory is the clean slate. To validate the installer itself against a
|
||||||
bare system, run the wizard against a stock Debian container instead.
|
bare system, run ctrl/deps.sh against a stock Debian container instead.
|
||||||
EOF
|
EOF
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -250,7 +253,7 @@ create() {
|
|||||||
echo
|
echo
|
||||||
echo "next:"
|
echo "next:"
|
||||||
echo " make newbox shell # a shell inside it"
|
echo " make newbox shell # a shell inside it"
|
||||||
echo " then: cd ~/rig && make station && make deps && make cluster up"
|
echo " then: cd ~/rig && make check && make deps && make cluster up"
|
||||||
echo
|
echo
|
||||||
echo "For a browser on Windows to resolve the hostnames, paste this into"
|
echo "For a browser on Windows to resolve the hostnames, paste this into"
|
||||||
echo "C:\\Windows\\System32\\drivers\\etc\\hosts (it has no wildcard support):"
|
echo "C:\\Windows\\System32\\drivers\\etc\\hosts (it has no wildcard support):"
|
||||||
|
|||||||
57
rig/ctrl/pins.sh
Executable file
57
rig/ctrl/pins.sh
Executable file
@@ -0,0 +1,57 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Do the standalone scripts still install what rig pins?
|
||||||
|
#
|
||||||
|
# standalone/rigdeps.sh carries its toolchain pins inline, because it exists for
|
||||||
|
# a machine that will never have ctrl/versions.env. That makes two copies of the
|
||||||
|
# same versions and checksums, and two copies drift the day one is edited and
|
||||||
|
# the other forgotten. This is the check that notices.
|
||||||
|
#
|
||||||
|
# ctrl/versions.env is the source of truth. Only the keys rigdeps.sh itself
|
||||||
|
# defines are compared: versions.env also pins addon images (cert-manager,
|
||||||
|
# metallb, metrics-server) that rigdeps.sh never installs, and demanding those
|
||||||
|
# would make this fail forever for no reason.
|
||||||
|
#
|
||||||
|
# Exits non-zero on any mismatch — unlike the host checks, this one is a test.
|
||||||
|
#
|
||||||
|
# Usage: pins.sh
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
SOURCE=./versions.env
|
||||||
|
COPY=../standalone/rigdeps.sh
|
||||||
|
|
||||||
|
[ -r "$COPY" ] || { echo "no $COPY to compare" >&2; exit 1; }
|
||||||
|
|
||||||
|
# KEY=value for the pin keys a file defines, quotes stripped. awk rather than a
|
||||||
|
# grep regex, which is not the same program everywhere.
|
||||||
|
pins() {
|
||||||
|
awk -F= '/^[A-Z_]+_(VERSION|SHA256)=/ {
|
||||||
|
v = substr($0, index($0, "=") + 1); gsub(/^["\x27]|["\x27]$/, "", v)
|
||||||
|
print $1 "=" v }' "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "pins: standalone/rigdeps.sh against ctrl/versions.env"
|
||||||
|
bad=0
|
||||||
|
while IFS='=' read -r key copy_val; do
|
||||||
|
[ -n "$key" ] || continue
|
||||||
|
src_val=$(pins "$SOURCE" | sed -n "s/^${key}=//p" | head -1)
|
||||||
|
if [ -z "$src_val" ]; then
|
||||||
|
printf " ! %-16s in rigdeps.sh but not in versions.env\n" "$key"
|
||||||
|
bad=1
|
||||||
|
elif [ "$src_val" = "$copy_val" ]; then
|
||||||
|
printf " %-16s %s\n" "$key" "$( [ ${#src_val} -gt 20 ] && echo "${src_val:0:12}…" || echo "$src_val" )"
|
||||||
|
else
|
||||||
|
printf " ! %-16s versions.env %s\n" "$key" "$src_val"
|
||||||
|
printf " %-16s rigdeps.sh %s\n" "" "$copy_val"
|
||||||
|
bad=1
|
||||||
|
fi
|
||||||
|
done < <(pins "$COPY")
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$bad" -eq 0 ]; then
|
||||||
|
echo "in step — rigdeps.sh installs exactly what rig pins."
|
||||||
|
else
|
||||||
|
echo "DRIFT. versions.env is the source of truth: copy the differing lines from it"
|
||||||
|
echo "into standalone/rigdeps.sh, taking checksums from the publisher's release list."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@@ -43,7 +43,7 @@ K="kubectl --context ${KUBECONTEXT}"
|
|||||||
# 2. every kind node's containerd — nodes do NOT inherit host trust
|
# 2. every kind node's containerd — nodes do NOT inherit host trust
|
||||||
# 3. anything doing HTTPS from inside the cluster, in its own trust store
|
# 3. anything doing HTTPS from inside the cluster, in its own trust store
|
||||||
#
|
#
|
||||||
# We handle (2) here because it's ours to handle. (1) is reported by station.sh
|
# We handle (2) here because it's ours to handle. (1) is reported by check.sh
|
||||||
# since it needs root. (3) belongs to the workload.
|
# since it needs root. (3) belongs to the workload.
|
||||||
install_ca_into_nodes() {
|
install_ca_into_nodes() {
|
||||||
[ -n "${REGISTRY_CA_FILE:-}" ] || return 0
|
[ -n "${REGISTRY_CA_FILE:-}" ] || return 0
|
||||||
|
|||||||
@@ -73,11 +73,11 @@ record() {
|
|||||||
|
|
||||||
step_host() {
|
step_host() {
|
||||||
local out
|
local out
|
||||||
if ! out=$(bash ./wizard.sh detect 2>&1); then
|
if ! out=$(bash ./deps.sh detect 2>&1); then
|
||||||
record host fail "detection failed"
|
record host fail "detection failed"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
# Anything the wizard flagged with '!' needs a human; surface the count here
|
# Anything flagged with '!' needs a human; surface the count here
|
||||||
# and the detail below rather than burying it.
|
# and the detail below rather than burying it.
|
||||||
local warns; warns=$(echo "$out" | grep -c '^\s*!' || true)
|
local warns; warns=$(echo "$out" | grep -c '^\s*!' || true)
|
||||||
HOST_DETAIL="$out"
|
HOST_DETAIL="$out"
|
||||||
@@ -102,7 +102,7 @@ step_toolchain() {
|
|||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if bash ./wizard.sh install "$TIER" >/tmp/rig-deps.$$ 2>&1; then
|
if bash ./deps.sh install "$TIER" >/tmp/rig-deps.$$ 2>&1; then
|
||||||
local still=""
|
local still=""
|
||||||
for b in $want; do
|
for b in $want; do
|
||||||
[ -x "${OUT_BIN:-$HOME/.local/bin}/$b" ] || still="$still $b"
|
[ -x "${OUT_BIN:-$HOME/.local/bin}/$b" ] || still="$still $b"
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Station check: is this workstation ready to run rig?
|
|
||||||
#
|
|
||||||
# Reports and instructs; never silently fixes anything. Everything it finds is
|
|
||||||
# either already fine, or something a human has to decide on.
|
|
||||||
#
|
|
||||||
# Runs the wizard's host detection in a container when Docker is the only thing
|
|
||||||
# installed, or directly when the toolchain is already present. Then adds the
|
|
||||||
# checks that need this repo's config: profile sanity, CA trust, port clashes.
|
|
||||||
set -euo pipefail
|
|
||||||
cd "$(dirname "$0")"
|
|
||||||
|
|
||||||
WIZARD_IMAGE="${WIZARD_IMAGE:-$(basename "$(cd .. && pwd)")-wizard}"
|
|
||||||
|
|
||||||
# Host detection. Prefer running it bare — it needs no dependencies beyond
|
|
||||||
# coreutils — and fall back to the container only if this shell can't.
|
|
||||||
bash ./wizard.sh detect
|
|
||||||
|
|
||||||
# ── repo-level checks ──────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
source ./lib/config.sh
|
|
||||||
load_config
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "config"
|
|
||||||
echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})"
|
|
||||||
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
|
|
||||||
echo " registry ${REGISTRY_MODE}"
|
|
||||||
echo " ingress ${INGRESS_MODE}"
|
|
||||||
|
|
||||||
if [ ! -f ./.env ]; then
|
|
||||||
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# A 3-node profile on a box that's already full is the most common first
|
|
||||||
# failure, and it presents as pods stuck Pending rather than anything obvious.
|
|
||||||
avail=$(awk '/^MemAvailable:/{printf "%d", $2/1024/1024}' /proc/meminfo)
|
|
||||||
need=$((NODES * 2))
|
|
||||||
if [ "$avail" -lt "$need" ]; then
|
|
||||||
echo " ! profile '${PROFILE_NAME}' wants ~${need} GB, ${avail} GB available"
|
|
||||||
echo " 'make cluster list' shows what else is running; 'make cluster free' stops it"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# The CA reaches three places and only one of them is ours. Report the other two.
|
|
||||||
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
|
||||||
echo
|
|
||||||
echo "registry CA"
|
|
||||||
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
|
||||||
echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
|
||||||
else
|
|
||||||
echo " file $REGISTRY_CA_FILE"
|
|
||||||
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
|
||||||
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
|
||||||
echo " ! the HOST docker daemon does not trust it yet:"
|
|
||||||
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
|
||||||
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
|
||||||
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Host ports this environment will try to bind. Checked before cluster creation
|
|
||||||
# because docker reports a clash halfway through, as an opaque
|
|
||||||
# "failed to bind host port ...: address already in use".
|
|
||||||
echo
|
|
||||||
echo "ports (block derived from the directory name — see 'make ports')"
|
|
||||||
|
|
||||||
port_busy() {
|
|
||||||
if command -v ss >/dev/null 2>&1; then
|
|
||||||
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
|
||||||
fi
|
|
||||||
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
|
||||||
# than silently reporting everything as free.
|
|
||||||
local hex; hex=$(printf ':%04X' "$1")
|
|
||||||
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
|
||||||
}
|
|
||||||
|
|
||||||
# A port held by THIS environment's own cluster is not a clash — it is the thing
|
|
||||||
# working. Reporting it as a problem every time the cluster is up would train
|
|
||||||
# people to ignore this section, which is the opposite of the point.
|
|
||||||
# Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the
|
|
||||||
# character RANGE ':' to '>', which does not contain '-', so the trailing dash
|
|
||||||
# survives and nothing ever matches.
|
|
||||||
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
|
||||||
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
|
||||||
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
|
||||||
|
|
||||||
clash=0
|
|
||||||
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
|
||||||
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
|
||||||
name="${entry%%:*}"; p="${entry#*:}"
|
|
||||||
[ -n "$p" ] || continue
|
|
||||||
if ! port_busy "$p"; then
|
|
||||||
printf " %-9s %-6s free\n" "$name" "$p"
|
|
||||||
elif echo "$ours" | grep -qx "$p"; then
|
|
||||||
printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p"
|
|
||||||
else
|
|
||||||
printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p"
|
|
||||||
clash=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$clash" -eq 1 ]; then
|
|
||||||
echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080"
|
|
||||||
echo " (or rename this directory — the whole block follows the name)"
|
|
||||||
fi
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
# Pinned toolchain — the single manifest the wizard installs from.
|
# Pinned toolchain — the single manifest ctrl/deps.sh installs from.
|
||||||
# Every entry is a single binary; none of them needs an apt repo.
|
# Every entry is a single binary; none of them needs an apt repo.
|
||||||
# kubectl fully static
|
# kubectl fully static
|
||||||
# kind libc only
|
# kind libc only
|
||||||
@@ -6,8 +6,19 @@
|
|||||||
# jq upstream static build (Debian's is linked against libjq/libonig)
|
# jq upstream static build (Debian's is linked against libjq/libonig)
|
||||||
#
|
#
|
||||||
# Checksums are the upstream-published SHA256 of the linux/amd64 artifact.
|
# Checksums are the upstream-published SHA256 of the linux/amd64 artifact.
|
||||||
# To bump: change the version, then re-run `bash ctrl/versions-refresh.sh` and
|
#
|
||||||
# commit the result — never hand-edit a checksum.
|
# To bump: change the version, then take the checksum from the release's own
|
||||||
|
# published list — never hand-edit or hand-copy one from a download you did.
|
||||||
|
# For anything hosted on GitHub releases that is:
|
||||||
|
#
|
||||||
|
# curl -sSL https://github.com/<org>/<repo>/releases/download/<tag>/checksums.txt \
|
||||||
|
# | grep linux.x86_64
|
||||||
|
#
|
||||||
|
# (kubectl publishes its own instead: <KUBECTL_URL>.sha256.)
|
||||||
|
#
|
||||||
|
# There was a `ctrl/versions-refresh.sh` named here that has never existed. If
|
||||||
|
# bumping stops being rare enough to do by hand, write it — but a comment
|
||||||
|
# pointing at a missing script is worse than no comment.
|
||||||
|
|
||||||
KIND_VERSION=v0.32.0
|
KIND_VERSION=v0.32.0
|
||||||
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||||
@@ -21,6 +32,14 @@ TILT_VERSION=0.37.6
|
|||||||
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
||||||
TILT_URL=https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz
|
TILT_URL=https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz
|
||||||
|
|
||||||
|
# ctlptl — creates a kind cluster WITH a local registry wired in, which is what
|
||||||
|
# keeps images off docker.io (an unqualified name means docker.io/library/<name>).
|
||||||
|
# Same publisher and same archive shape as tilt: binary at the archive root, so
|
||||||
|
# fetch_tgz handles it with strip=0 and no special case.
|
||||||
|
CTLPTL_VERSION=0.9.4
|
||||||
|
CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e
|
||||||
|
CTLPTL_URL=https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz
|
||||||
|
|
||||||
JQ_VERSION=1.8.2
|
JQ_VERSION=1.8.2
|
||||||
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
||||||
JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64
|
JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64
|
||||||
@@ -44,9 +63,9 @@ CERT_MANAGER_VERSION=v1.21.1
|
|||||||
METRICS_SERVER_VERSION=v0.9.0
|
METRICS_SERVER_VERSION=v0.9.0
|
||||||
METALLB_VERSION=v0.16.0
|
METALLB_VERSION=v0.16.0
|
||||||
|
|
||||||
# Dependency containers. These mirror soleprint's cabinets
|
# Cabinets — public services dropped in as-is, the upstream image unmodified.
|
||||||
# (soleprint/station/cabinets/), so a room that declares postgres gets the same
|
# The same declaration installs on compose or in the cluster, so a dependency is
|
||||||
# thing whether it runs on compose or in the cluster. Pinned by tag rather than
|
# named once and works either way. Pinned by tag rather than
|
||||||
# digest because they are ordinary upstream images with no supply chain claim
|
# digest because they are ordinary upstream images with no supply chain claim
|
||||||
# attached — bump freely, and preload them for the offline profile.
|
# attached — bump freely, and preload them for the offline profile.
|
||||||
POSTGRES_IMAGE=postgres:16-alpine
|
POSTGRES_IMAGE=postgres:16-alpine
|
||||||
|
|||||||
@@ -20,13 +20,13 @@ digraph rig_install {
|
|||||||
bin [label="~/.local/bin\nkind · kubectl · tilt\njq" fillcolor="#121829" shape=cylinder]
|
bin [label="~/.local/bin\nkind · kubectl · tilt\njq" fillcolor="#121829" shape=cylinder]
|
||||||
}
|
}
|
||||||
|
|
||||||
subgraph cluster_wizard {
|
subgraph cluster_installer {
|
||||||
label="Installer container (transient)"
|
label="Installer container (transient)"
|
||||||
style=dashed
|
style=dashed
|
||||||
color="#1e2a4a"
|
color="#1e2a4a"
|
||||||
fontcolor="#8892a8"
|
fontcolor="#8892a8"
|
||||||
|
|
||||||
wizard [label="wizard\ncurl · jq · python · graphviz" fillcolor="#121829"]
|
installer [label="deps installer\ncurl · jq · python · graphviz" fillcolor="#121829"]
|
||||||
detect [label="detect host\nWSL · memory · inotify · docker" fillcolor="#121829"]
|
detect [label="detect host\nWSL · memory · inotify · docker" fillcolor="#121829"]
|
||||||
fetch [label="fetch + verify\nSHA256, pinned versions" fillcolor="#121829"]
|
fetch [label="fetch + verify\nSHA256, pinned versions" fillcolor="#121829"]
|
||||||
}
|
}
|
||||||
@@ -34,14 +34,14 @@ digraph rig_install {
|
|||||||
upstream [label="upstream\nreleases / corporate mirror" fillcolor="#1a3a1a" fontcolor="#00c853" shape=octagon]
|
upstream [label="upstream\nreleases / corporate mirror" fillcolor="#1a3a1a" fontcolor="#00c853" shape=octagon]
|
||||||
report [label="report what it\nCANNOT do" fillcolor="#3a1a1a" fontcolor="#ffc107"]
|
report [label="report what it\nCANNOT do" fillcolor="#3a1a1a" fontcolor="#ffc107"]
|
||||||
|
|
||||||
docker -> wizard [label="docker run"]
|
docker -> installer [label="docker run"]
|
||||||
wizard -> detect
|
installer -> detect
|
||||||
detect -> fetch
|
detect -> fetch
|
||||||
fetch -> upstream [label="pinned + checksummed" color="#00c853"]
|
fetch -> upstream [label="pinned + checksummed" color="#00c853"]
|
||||||
fetch -> bin [label="install"]
|
fetch -> bin [label="install"]
|
||||||
detect -> report [style=dashed label="sudo / Windows-side steps" color="#ffc107"]
|
detect -> report [style=dashed label="sudo / Windows-side steps" color="#ffc107"]
|
||||||
|
|
||||||
// The container is gone after this; nothing depends on it at run time.
|
// The container is gone after this; nothing depends on it at run time.
|
||||||
wizard -> gone [style=dotted label="exits"]
|
installer -> gone [style=dotted label="exits"]
|
||||||
gone [label="(container discarded)" fillcolor="#0a0e17" fontcolor="#4a5568" color="#1e2a4a" style="filled,dashed"]
|
gone [label="(container discarded)" fillcolor="#0a0e17" fontcolor="#4a5568" color="#1e2a4a" style="filled,dashed"]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-170.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Your machine</text>
|
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-170.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Your machine</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust2" class="cluster">
|
<g id="clust2" class="cluster">
|
||||||
<title>cluster_wizard</title>
|
<title>cluster_installer</title>
|
||||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-95 8,-175 745.5,-175 745.5,-95 8,-95"/>
|
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-95 8,-175 745.5,-175 745.5,-95 8,-95"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="376.75" y="-155.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Installer container (transient)</text>
|
<text xml:space="preserve" text-anchor="middle" x="376.75" y="-155.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Installer container (transient)</text>
|
||||||
</g>
|
</g>
|
||||||
@@ -27,16 +27,16 @@
|
|||||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-46.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">Docker</text>
|
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-46.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">Docker</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-32.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">(the one prerequisite)</text>
|
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-32.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">(the one prerequisite)</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- wizard -->
|
<!-- installer -->
|
||||||
<g id="node3" class="node">
|
<g id="node3" class="node">
|
||||||
<title>wizard</title>
|
<title>installer</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="181.25,-139 16,-139 16,-103 181.25,-103 181.25,-139"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="181.25,-139 16,-139 16,-103 181.25,-103 181.25,-139"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">wizard</text>
|
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">deps installer</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">curl · jq · python · graphviz</text>
|
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">curl · jq · python · graphviz</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- docker->wizard -->
|
<!-- docker->installer -->
|
||||||
<g id="edge1" class="edge">
|
<g id="edge1" class="edge">
|
||||||
<title>docker->wizard</title>
|
<title>docker->installer</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M906.12,-45.7C757.47,-50.51 475.98,-63.12 238.25,-94 223.38,-95.93 207.7,-98.48 192.45,-101.24"/>
|
<path fill="none" stroke="#4a5568" d="M906.12,-45.7C757.47,-50.51 475.98,-63.12 238.25,-94 223.38,-95.93 207.7,-98.48 192.45,-101.24"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="192.13,-97.74 182.93,-103.01 193.4,-104.63 192.13,-97.74"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="192.13,-97.74 182.93,-103.01 193.4,-104.63 192.13,-97.74"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="506.62" y="-74.39" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">docker run</text>
|
<text xml:space="preserve" text-anchor="middle" x="506.62" y="-74.39" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">docker run</text>
|
||||||
@@ -57,9 +57,9 @@
|
|||||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">detect host</text>
|
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">detect host</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">WSL · memory · inotify · docker</text>
|
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">WSL · memory · inotify · docker</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- wizard->detect -->
|
<!-- installer->detect -->
|
||||||
<g id="edge2" class="edge">
|
<g id="edge2" class="edge">
|
||||||
<title>wizard->detect</title>
|
<title>installer->detect</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M181.44,-121C195.97,-121 211.28,-121 226.37,-121"/>
|
<path fill="none" stroke="#4a5568" d="M181.44,-121C195.97,-121 211.28,-121 226.37,-121"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="226.33,-124.5 236.33,-121 226.33,-117.5 226.33,-124.5"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="226.33,-124.5 236.33,-121 226.33,-117.5 226.33,-124.5"/>
|
||||||
</g>
|
</g>
|
||||||
@@ -69,9 +69,9 @@
|
|||||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="400.5,-47 266.75,-47 266.75,-11 400.5,-11 400.5,-47"/>
|
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="400.5,-47 266.75,-47 266.75,-11 400.5,-11 400.5,-47"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-25.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#4a5568">(container discarded)</text>
|
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-25.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#4a5568">(container discarded)</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- wizard->gone -->
|
<!-- installer->gone -->
|
||||||
<g id="edge7" class="edge">
|
<g id="edge7" class="edge">
|
||||||
<title>wizard->gone</title>
|
<title>installer->gone</title>
|
||||||
<path fill="none" stroke="#4a5568" stroke-dasharray="1,5" d="M119.29,-102.62C138.26,-86 168.54,-62.29 199.25,-49.75 216.76,-42.6 236.49,-37.9 255.29,-34.81"/>
|
<path fill="none" stroke="#4a5568" stroke-dasharray="1,5" d="M119.29,-102.62C138.26,-86 168.54,-62.29 199.25,-49.75 216.76,-42.6 236.49,-37.9 255.29,-34.81"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="255.67,-38.29 265.04,-33.35 254.64,-31.37 255.67,-38.29"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="255.67,-38.29 265.04,-33.35 254.64,-31.37 255.67,-38.29"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="209.75" y="-52.45" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">exits</text>
|
<text xml:space="preserve" text-anchor="middle" x="209.75" y="-52.45" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">exits</text>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 9.0 KiB After Width: | Height: | Size: 9.0 KiB |
@@ -20,7 +20,7 @@ digraph rig_environment {
|
|||||||
|
|
||||||
cname [label="cluster name\nacmebank" fillcolor="#121829"]
|
cname [label="cluster name\nacmebank" fillcolor="#121829"]
|
||||||
ctx [label="kubectl context\nkind-acmebank" fillcolor="#121829"]
|
ctx [label="kubectl context\nkind-acmebank" fillcolor="#121829"]
|
||||||
img [label="image tag\nacmebank-wizard" fillcolor="#121829"]
|
img [label="image tag\nacmebank-deps" fillcolor="#121829"]
|
||||||
ports [label="port block\n21300–21309" fillcolor="#121829"]
|
ports [label="port block\n21300–21309" fillcolor="#121829"]
|
||||||
reg [label="registry container\nacmebank-registry" fillcolor="#121829"]
|
reg [label="registry container\nacmebank-registry" fillcolor="#121829"]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,28 +4,28 @@
|
|||||||
<!-- Generated by graphviz version 14.1.2 (0)
|
<!-- Generated by graphviz version 14.1.2 (0)
|
||||||
-->
|
-->
|
||||||
<!-- Title: rig_environment Pages: 1 -->
|
<!-- Title: rig_environment Pages: 1 -->
|
||||||
<svg width="971pt" height="481pt"
|
<svg width="962pt" height="481pt"
|
||||||
viewBox="0.00 0.00 971.00 481.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
viewBox="0.00 0.00 962.00 481.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 476.83)">
|
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 476.83)">
|
||||||
<title>rig_environment</title>
|
<title>rig_environment</title>
|
||||||
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-476.83 967,-476.83 967,4 -4,4"/>
|
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-476.83 958,-476.83 958,4 -4,4"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="481.5" y="-453.63" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#0066ff">One environment per directory — copies never collide</text>
|
<text xml:space="preserve" text-anchor="middle" x="477" y="-453.63" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#0066ff">One environment per directory — copies never collide</text>
|
||||||
<g id="clust1" class="cluster">
|
<g id="clust1" class="cluster">
|
||||||
<title>cluster_derived</title>
|
<title>cluster_derived</title>
|
||||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-65 8,-144.5 605,-144.5 605,-65 8,-65"/>
|
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-65 8,-144.5 596,-144.5 596,-65 8,-65"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="306.5" y="-125.3" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Everything below is derived from it</text>
|
<text xml:space="preserve" text-anchor="middle" x="302" y="-125.3" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Everything below is derived from it</text>
|
||||||
</g>
|
</g>
|
||||||
<g id="clust2" class="cluster">
|
<g id="clust2" class="cluster">
|
||||||
<title>cluster_config</title>
|
<title>cluster_config</title>
|
||||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="613,-65 613,-437.33 955,-437.33 955,-65 613,-65"/>
|
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="604,-65 604,-437.33 946,-437.33 946,-65 604,-65"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="784" y="-418.13" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Configuration — weakest first, later wins</text>
|
<text xml:space="preserve" text-anchor="middle" x="775" y="-418.13" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Configuration — weakest first, later wins</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- dirname -->
|
<!-- dirname -->
|
||||||
<g id="node1" class="node">
|
<g id="node1" class="node">
|
||||||
<title>dirname</title>
|
<title>dirname</title>
|
||||||
<polygon fill="#1f6feb" stroke="#1e2a4a" points="375.11,-197.44 375.11,-219.63 329.94,-235.33 266.06,-235.33 220.89,-219.63 220.89,-197.44 266.06,-181.75 329.94,-181.75 375.11,-197.44"/>
|
<polygon fill="#1f6feb" stroke="#1e2a4a" points="370.11,-197.44 370.11,-219.63 324.94,-235.33 261.06,-235.33 215.89,-219.63 215.89,-197.44 261.06,-181.75 324.94,-181.75 370.11,-197.44"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="298" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">directory name</text>
|
<text xml:space="preserve" text-anchor="middle" x="293" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">directory name</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="298" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">e.g. acmebank/</text>
|
<text xml:space="preserve" text-anchor="middle" x="293" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">e.g. acmebank/</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- cname -->
|
<!-- cname -->
|
||||||
<g id="node2" class="node">
|
<g id="node2" class="node">
|
||||||
@@ -37,8 +37,8 @@
|
|||||||
<!-- dirname->cname -->
|
<!-- dirname->cname -->
|
||||||
<g id="edge1" class="edge">
|
<g id="edge1" class="edge">
|
||||||
<title>dirname->cname</title>
|
<title>dirname->cname</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M232.76,-193.05C195.81,-183.01 149.78,-167.28 113,-144.5 101.37,-137.29 90.31,-127.09 81.33,-117.6"/>
|
<path fill="none" stroke="#4a5568" d="M228.68,-192.51C192.88,-182.36 148.52,-166.7 113,-144.5 101.4,-137.25 90.34,-127.04 81.36,-117.55"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="84.04,-115.37 74.73,-110.3 78.85,-120.07 84.04,-115.37"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="84.07,-115.32 74.76,-110.26 78.88,-120.02 84.07,-115.32"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- ctx -->
|
<!-- ctx -->
|
||||||
<g id="node3" class="node">
|
<g id="node3" class="node">
|
||||||
@@ -50,120 +50,120 @@
|
|||||||
<!-- dirname->ctx -->
|
<!-- dirname->ctx -->
|
||||||
<g id="edge2" class="edge">
|
<g id="edge2" class="edge">
|
||||||
<title>dirname->ctx</title>
|
<title>dirname->ctx</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M269.64,-181.32C248.71,-161.98 220.42,-135.83 199.86,-116.83"/>
|
<path fill="none" stroke="#4a5568" d="M265.77,-181.32C245.77,-162.07 218.77,-136.06 199.05,-117.08"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="202.4,-114.41 192.68,-110.19 197.65,-119.55 202.4,-114.41"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="201.55,-114.63 191.91,-110.21 196.69,-119.67 201.55,-114.63"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- img -->
|
<!-- img -->
|
||||||
<g id="node4" class="node">
|
<g id="node4" class="node">
|
||||||
<title>img</title>
|
<title>img</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="354,-109 242,-109 242,-73 354,-73 354,-109"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="344.12,-109 241.88,-109 241.88,-73 344.12,-73 344.12,-109"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="298" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">image tag</text>
|
<text xml:space="preserve" text-anchor="middle" x="293" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">image tag</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="298" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-wizard</text>
|
<text xml:space="preserve" text-anchor="middle" x="293" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-deps</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- dirname->img -->
|
<!-- dirname->img -->
|
||||||
<g id="edge3" class="edge">
|
<g id="edge3" class="edge">
|
||||||
<title>dirname->img</title>
|
<title>dirname->img</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M298,-181.32C298,-163.19 298,-139.07 298,-120.47"/>
|
<path fill="none" stroke="#4a5568" d="M293,-181.32C293,-163.19 293,-139.07 293,-120.47"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="301.5,-120.67 298,-110.67 294.5,-120.67 301.5,-120.67"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="296.5,-120.67 293,-110.67 289.5,-120.67 296.5,-120.67"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- ports -->
|
<!-- ports -->
|
||||||
<g id="node5" class="node">
|
<g id="node5" class="node">
|
||||||
<title>ports</title>
|
<title>ports</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="460.38,-109 371.62,-109 371.62,-73 460.38,-73 460.38,-109"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="451.38,-109 362.62,-109 362.62,-73 451.38,-73 451.38,-109"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="416" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">port block</text>
|
<text xml:space="preserve" text-anchor="middle" x="407" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">port block</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="416" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">21300–21309</text>
|
<text xml:space="preserve" text-anchor="middle" x="407" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">21300–21309</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- dirname->ports -->
|
<!-- dirname->ports -->
|
||||||
<g id="edge4" class="edge">
|
<g id="edge4" class="edge">
|
||||||
<title>dirname->ports</title>
|
<title>dirname->ports</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M324.94,-181.53C336.66,-170.19 350.54,-156.71 363,-144.5 372.11,-135.57 382.06,-125.74 390.85,-117.02"/>
|
<path fill="none" stroke="#4a5568" d="M318.87,-181.32C337.78,-162.15 363.29,-136.3 382,-117.34"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="393.15,-119.67 397.79,-110.14 388.22,-114.7 393.15,-119.67"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="384.47,-119.81 389,-110.24 379.49,-114.9 384.47,-119.81"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- reg -->
|
<!-- reg -->
|
||||||
<g id="node6" class="node">
|
<g id="node6" class="node">
|
||||||
<title>reg</title>
|
<title>reg</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="597.38,-109 478.62,-109 478.62,-73 597.38,-73 597.38,-109"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="588.38,-109 469.62,-109 469.62,-73 588.38,-73 588.38,-109"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="538" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">registry container</text>
|
<text xml:space="preserve" text-anchor="middle" x="529" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">registry container</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="538" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-registry</text>
|
<text xml:space="preserve" text-anchor="middle" x="529" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-registry</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- dirname->reg -->
|
<!-- dirname->reg -->
|
||||||
<g id="edge5" class="edge">
|
<g id="edge5" class="edge">
|
||||||
<title>dirname->reg</title>
|
<title>dirname->reg</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M356.87,-190.63C390.74,-179.74 433.49,-163.98 469,-144.5 483.3,-136.65 497.86,-126.04 509.9,-116.41"/>
|
<path fill="none" stroke="#4a5568" d="M350.86,-190.3C383.87,-179.35 425.43,-163.64 460,-144.5 474.27,-136.6 488.83,-125.98 500.87,-116.36"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="511.88,-119.31 517.39,-110.26 507.44,-113.9 511.88,-119.31"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="502.85,-119.26 508.36,-110.21 498.41,-113.84 502.85,-119.26"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- cluster -->
|
<!-- cluster -->
|
||||||
<g id="node11" class="node">
|
<g id="node11" class="node">
|
||||||
<title>cluster</title>
|
<title>cluster</title>
|
||||||
<polygon fill="#1a1a3a" stroke="#1e2a4a" points="469.81,-10.54 469.81,-25.46 438.29,-36 393.71,-36 362.19,-25.46 362.19,-10.54 393.71,0 438.29,0 469.81,-10.54"/>
|
<polygon fill="#1a1a3a" stroke="#1e2a4a" points="460.81,-10.54 460.81,-25.46 429.29,-36 384.71,-36 353.19,-25.46 353.19,-10.54 384.71,0 429.29,0 460.81,-10.54"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="416" y="-14.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">kind cluster</text>
|
<text xml:space="preserve" text-anchor="middle" x="407" y="-14.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">kind cluster</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- cname->cluster -->
|
<!-- cname->cluster -->
|
||||||
<g id="edge9" class="edge">
|
<g id="edge9" class="edge">
|
||||||
<title>cname->cluster</title>
|
<title>cname->cluster</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M93.06,-72.62C99.54,-69.72 106.38,-67.02 113,-65 192.78,-40.7 288.45,-28.91 350.65,-23.42"/>
|
<path fill="none" stroke="#4a5568" d="M93.35,-72.51C99.75,-69.67 106.48,-67 113,-65 189.55,-41.49 281.19,-29.58 341.58,-23.85"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="350.69,-26.93 360.36,-22.6 350.1,-19.96 350.69,-26.93"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="341.71,-27.35 351.35,-22.96 341.07,-20.38 341.71,-27.35"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- ports->cluster -->
|
<!-- ports->cluster -->
|
||||||
<g id="edge10" class="edge">
|
<g id="edge10" class="edge">
|
||||||
<title>ports->cluster</title>
|
<title>ports->cluster</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M416,-72.81C416,-65.23 416,-56.1 416,-47.54"/>
|
<path fill="none" stroke="#4a5568" d="M407,-72.81C407,-65.23 407,-56.1 407,-47.54"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="419.5,-47.54 416,-37.54 412.5,-47.54 419.5,-47.54"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="410.5,-47.54 407,-37.54 403.5,-47.54 410.5,-47.54"/>
|
||||||
</g>
|
</g>
|
||||||
<!-- versions -->
|
<!-- versions -->
|
||||||
<g id="node7" class="node">
|
<g id="node7" class="node">
|
||||||
<title>versions</title>
|
<title>versions</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="759.38,-401.83 652.62,-401.83 652.62,-365.83 759.38,-365.83 759.38,-401.83"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="750.38,-401.83 643.62,-401.83 643.62,-365.83 750.38,-365.83 750.38,-401.83"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="706" y="-386.88" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">versions.env</text>
|
<text xml:space="preserve" text-anchor="middle" x="697" y="-386.88" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">versions.env</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="706" y="-373.38" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">pinned toolchain</text>
|
<text xml:space="preserve" text-anchor="middle" x="697" y="-373.38" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">pinned toolchain</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- profile -->
|
<!-- profile -->
|
||||||
<g id="node8" class="node">
|
<g id="node8" class="node">
|
||||||
<title>profile</title>
|
<title>profile</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="790.5,-318.58 621.5,-318.58 621.5,-282.58 790.5,-282.58 790.5,-318.58"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="781.5,-318.58 612.5,-318.58 612.5,-282.58 781.5,-282.58 781.5,-318.58"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="706" y="-303.63" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">env.d/<profile>.env</text>
|
<text xml:space="preserve" text-anchor="middle" x="697" y="-303.63" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">env.d/<profile>.env</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="706" y="-290.13" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">nodes · CNI · audit · addons</text>
|
<text xml:space="preserve" text-anchor="middle" x="697" y="-290.13" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">nodes · CNI · audit · addons</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- versions->profile -->
|
<!-- versions->profile -->
|
||||||
<g id="edge6" class="edge">
|
<g id="edge6" class="edge">
|
||||||
<title>versions->profile</title>
|
<title>versions->profile</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M706,-365.59C706,-355.32 706,-342.03 706,-330.21"/>
|
<path fill="none" stroke="#4a5568" d="M697,-365.59C697,-355.32 697,-342.03 697,-330.21"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="709.5,-330.58 706,-320.58 702.5,-330.58 709.5,-330.58"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="700.5,-330.58 697,-320.58 693.5,-330.58 700.5,-330.58"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="737.5" y="-339.28" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
<text xml:space="preserve" text-anchor="middle" x="728.5" y="-339.28" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- localenv -->
|
<!-- localenv -->
|
||||||
<g id="node9" class="node">
|
<g id="node9" class="node">
|
||||||
<title>localenv</title>
|
<title>localenv</title>
|
||||||
<polygon fill="#121829" stroke="#1e2a4a" points="761.88,-226.54 646.12,-226.54 646.12,-190.54 761.88,-190.54 761.88,-226.54"/>
|
<polygon fill="#121829" stroke="#1e2a4a" points="752.88,-226.54 637.12,-226.54 637.12,-190.54 752.88,-190.54 752.88,-226.54"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="704" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">ctrl/.env</text>
|
<text xml:space="preserve" text-anchor="middle" x="695" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">ctrl/.env</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="704" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">secrets, overrides</text>
|
<text xml:space="preserve" text-anchor="middle" x="695" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">secrets, overrides</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- profile->localenv -->
|
<!-- profile->localenv -->
|
||||||
<g id="edge7" class="edge">
|
<g id="edge7" class="edge">
|
||||||
<title>profile->localenv</title>
|
<title>profile->localenv</title>
|
||||||
<path fill="none" stroke="#4a5568" d="M705.61,-282.22C705.34,-269.76 704.96,-252.69 704.64,-238.23"/>
|
<path fill="none" stroke="#4a5568" d="M696.61,-282.22C696.34,-269.76 695.96,-252.69 695.64,-238.23"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="708.14,-238.28 704.42,-228.36 701.14,-238.43 708.14,-238.28"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="699.14,-238.28 695.42,-228.36 692.14,-238.43 699.14,-238.28"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="736.68" y="-256.03" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
<text xml:space="preserve" text-anchor="middle" x="727.68" y="-256.03" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- shell -->
|
<!-- shell -->
|
||||||
<g id="node10" class="node">
|
<g id="node10" class="node">
|
||||||
<title>shell</title>
|
<title>shell</title>
|
||||||
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="772.38,-109 623.62,-109 623.62,-73 772.38,-73 772.38,-109"/>
|
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="763.38,-109 614.62,-109 614.62,-73 763.38,-73 763.38,-109"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="698" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">the environment</text>
|
<text xml:space="preserve" text-anchor="middle" x="689" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">the environment</text>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="698" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">PROFILE=client make …</text>
|
<text xml:space="preserve" text-anchor="middle" x="689" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">PROFILE=client make …</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- localenv->shell -->
|
<!-- localenv->shell -->
|
||||||
<g id="edge8" class="edge">
|
<g id="edge8" class="edge">
|
||||||
<title>localenv->shell</title>
|
<title>localenv->shell</title>
|
||||||
<path fill="none" stroke="#00c853" d="M703.11,-190.49C702.16,-172.16 700.63,-142.72 699.49,-120.79"/>
|
<path fill="none" stroke="#00c853" d="M694.11,-190.49C693.16,-172.16 691.63,-142.72 690.49,-120.79"/>
|
||||||
<polygon fill="#00c853" stroke="#00c853" points="703,-120.81 698.99,-111.01 696.01,-121.18 703,-120.81"/>
|
<polygon fill="#00c853" stroke="#00c853" points="694,-120.81 689.99,-111.01 687.01,-121.18 694,-120.81"/>
|
||||||
<text xml:space="preserve" text-anchor="middle" x="733.21" y="-155.2" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
<text xml:space="preserve" text-anchor="middle" x="724.21" y="-155.2" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||||
</g>
|
</g>
|
||||||
<!-- shell->cluster -->
|
<!-- shell->cluster -->
|
||||||
<g id="edge11" class="edge">
|
<g id="edge11" class="edge">
|
||||||
<title>shell->cluster</title>
|
<title>shell->cluster</title>
|
||||||
<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M637.29,-72.6C627.83,-69.99 618.17,-67.38 609,-65 562.72,-52.99 509.89,-40.48 471.22,-31.54"/>
|
<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M628.29,-72.6C618.83,-69.99 609.17,-67.38 600,-65 553.72,-52.99 500.89,-40.48 462.22,-31.54"/>
|
||||||
<polygon fill="#4a5568" stroke="#4a5568" points="472.2,-28.18 461.67,-29.35 470.63,-35 472.2,-28.18"/>
|
<polygon fill="#4a5568" stroke="#4a5568" points="463.2,-28.18 452.67,-29.35 461.63,-35 463.2,-28.18"/>
|
||||||
</g>
|
</g>
|
||||||
</g>
|
</g>
|
||||||
</svg>
|
</svg>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
@@ -265,11 +265,11 @@
|
|||||||
<h3>The only prerequisite</h3>
|
<h3>The only prerequisite</h3>
|
||||||
<p><b>Docker.</b> No curl, no jq, no python, no apt repositories to configure.</p>
|
<p><b>Docker.</b> No curl, no jq, no python, no apt repositories to configure.</p>
|
||||||
<pre><code><span class="c"># then, in the environment directory:</span>
|
<pre><code><span class="c"># then, in the environment directory:</span>
|
||||||
make station <span class="c"># is this workstation ready? reports, never fixes</span>
|
make check <span class="c"># is this machine ready? reports, never fixes</span>
|
||||||
make deps <span class="c"># install the pinned toolchain</span>
|
make deps <span class="c"># install the pinned toolchain</span>
|
||||||
make cluster up <span class="c"># build the cluster for the active profile</span>
|
make cluster up <span class="c"># build the cluster for the active profile</span>
|
||||||
</code></pre>
|
</code></pre>
|
||||||
<p>Read <code>make station</code> before <code>make deps</code>. It never changes
|
<p>Read <code>make check</code> before <code>make deps</code>. It never changes
|
||||||
anything — it prints what it found and, at the end, the steps it cannot perform
|
anything — it prints what it found and, at the end, the steps it cannot perform
|
||||||
for you.</p>
|
for you.</p>
|
||||||
</div>
|
</div>
|
||||||
@@ -280,13 +280,13 @@ make cluster up <span class="c"># build the cluster for the active profile</spa
|
|||||||
<p class="lede">Start to finish, in order, with what each one actually does.</p>
|
<p class="lede">Start to finish, in order, with what each one actually does.</p>
|
||||||
<div class="prose">
|
<div class="prose">
|
||||||
|
|
||||||
<h3>1 · make station</h3>
|
<h3>1 · make check</h3>
|
||||||
<p>Asks whether this workstation is ready. It <b>changes nothing</b> — it
|
<p>Asks whether this machine is ready. It <b>changes nothing</b> — it
|
||||||
reports what it found and, at the end, the things only a human can do
|
reports what it found and, at the end, the things only a human can do
|
||||||
(anything needing <code>sudo</code>, or a Windows-side restart). Read it
|
(anything needing <code>sudo</code>, or a Windows-side restart). Read it
|
||||||
before installing anything; it is faster than discovering the same problems
|
before installing anything; it is faster than discovering the same problems
|
||||||
one failure at a time.</p>
|
one failure at a time.</p>
|
||||||
<pre><code>make station</code></pre>
|
<pre><code>make check</code></pre>
|
||||||
|
|
||||||
<h3>2 · make setup</h3>
|
<h3>2 · make setup</h3>
|
||||||
<p>Does the preparation that can be automated: installs the pinned
|
<p>Does the preparation that can be automated: installs the pinned
|
||||||
@@ -381,8 +381,8 @@ make setup core <span class="c"># same distinction, via setup</span>
|
|||||||
wants only Docker.</p>
|
wants only Docker.</p>
|
||||||
|
|
||||||
<h3>Air-gapped</h3>
|
<h3>Air-gapped</h3>
|
||||||
<pre><code>make wizard full <span class="c"># bakes every binary into the image</span>
|
<pre><code>make deps-image full <span class="c"># bakes every binary into the image</span>
|
||||||
docker save …-wizard:full | gzip > rig.tgz
|
docker save …-deps:full | gzip > rig.tgz
|
||||||
<span class="c"># carry that one file in, then:</span>
|
<span class="c"># carry that one file in, then:</span>
|
||||||
docker load < rig.tgz && make cluster up PROFILE=offline
|
docker load < rig.tgz && make cluster up PROFILE=offline
|
||||||
</code></pre>
|
</code></pre>
|
||||||
@@ -485,7 +485,7 @@ docker load < rig.tgz && make cluster up PROFILE=offline
|
|||||||
registry is usually behind an internal CA, and trust has to reach
|
registry is usually behind an internal CA, and trust has to reach
|
||||||
<b>three</b> places: the host Docker daemon, every cluster node's containerd
|
<b>three</b> places: the host Docker daemon, every cluster node's containerd
|
||||||
(nodes do <i>not</i> inherit host trust), and any in-cluster client. Set
|
(nodes do <i>not</i> inherit host trust), and any in-cluster client. Set
|
||||||
<code>REGISTRY_CA_FILE</code> and <code>make station</code> reports which is
|
<code>REGISTRY_CA_FILE</code> and <code>make check</code> reports which is
|
||||||
still missing. The symptom otherwise is an opaque
|
still missing. The symptom otherwise is an opaque
|
||||||
<code>x509: certificate signed by unknown authority</code>.</p></div>
|
<code>x509: certificate signed by unknown authority</code>.</p></div>
|
||||||
|
|
||||||
@@ -528,11 +528,11 @@ docker load < rig.tgz && make cluster up PROFILE=offline
|
|||||||
<h3>Tilt stops noticing file changes</h3>
|
<h3>Tilt stops noticing file changes</h3>
|
||||||
<p>Almost always <code>inotify</code> limits, and it fails <i>silently</i> —
|
<p>Almost always <code>inotify</code> limits, and it fails <i>silently</i> —
|
||||||
nothing errors, changes just stop being picked up. Defaults on WSL are far too
|
nothing errors, changes just stop being picked up. Defaults on WSL are far too
|
||||||
low. <code>make station</code> reports it and prints the fix.</p>
|
low. <code>make check</code> reports it and prints the fix.</p>
|
||||||
|
|
||||||
<h3>Cluster creation dies halfway with a port error</h3>
|
<h3>Cluster creation dies halfway with a port error</h3>
|
||||||
<p>Docker reports <code>failed to bind host port … address already in use</code>
|
<p>Docker reports <code>failed to bind host port … address already in use</code>
|
||||||
partway through creating the cluster. Run <code>make station</code> first — it
|
partway through creating the cluster. Run <code>make check</code> first — it
|
||||||
checks every port in this environment's block before anything is built.</p>
|
checks every port in this environment's block before anything is built.</p>
|
||||||
|
|
||||||
<h3>Every node stays NotReady</h3>
|
<h3>Every node stays NotReady</h3>
|
||||||
|
|||||||
9
rig/sample-rig/.gitignore
vendored
9
rig/sample-rig/.gitignore
vendored
@@ -1,9 +0,0 @@
|
|||||||
# NOTE: generated/ is deliberately NOT ignored. The artifact IS the deliverable —
|
|
||||||
# the whole point is a folder you copy, apply and boot without generating
|
|
||||||
# anything first. Regenerate it with `make manifest` after editing bundle.json
|
|
||||||
# or app/serve.py, and commit the result.
|
|
||||||
#
|
|
||||||
# (This differs from rig's ctrl/k8s/generated, which is a local build artifact.)
|
|
||||||
|
|
||||||
__pycache__/
|
|
||||||
*.pyc
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
# Thin control Makefile — one target per ctrl/ script, subcommand as an
|
|
||||||
# argument. Same shape as rig's, for the same reason: the logic lives in the
|
|
||||||
# script, never here.
|
|
||||||
#
|
|
||||||
# make up -> ctrl/bundle.sh up
|
|
||||||
# make bundle down
|
|
||||||
#
|
|
||||||
# This directory is a BUNDLE, not an installer. It needs a cluster, which rig
|
|
||||||
# owns:
|
|
||||||
#
|
|
||||||
# cd .. && make cluster up # kind cluster for this environment
|
|
||||||
# make up # then deploy this bundle into it
|
|
||||||
#
|
|
||||||
# Start with: make up
|
|
||||||
|
|
||||||
ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
|
|
||||||
ifneq ($(ARGS),)
|
|
||||||
$(eval $(ARGS):;@:)
|
|
||||||
endif
|
|
||||||
|
|
||||||
.DEFAULT_GOAL := help
|
|
||||||
.PHONY: help bundle manifest up down status url list dev
|
|
||||||
|
|
||||||
help: ## list targets
|
|
||||||
@grep -hE '^[a-z]+:.*?##' $(MAKEFILE_LIST) | sed 's/:.*##/\t/' | expand -t16
|
|
||||||
|
|
||||||
bundle: ## the bundle [manifest|up|down|status|url|list|dev]
|
|
||||||
bash ctrl/bundle.sh $(or $(ARGS),status)
|
|
||||||
|
|
||||||
# Shorthands for the ones used constantly.
|
|
||||||
manifest: ## regenerate generated/<slug>.yaml — no cluster needed
|
|
||||||
bash ctrl/bundle.sh manifest
|
|
||||||
|
|
||||||
up: ## deploy this rig (installs MetalLB if absent)
|
|
||||||
bash ctrl/bundle.sh up
|
|
||||||
|
|
||||||
down: ## remove this rig (leaves cluster, MetalLB, siblings)
|
|
||||||
bash ctrl/bundle.sh down
|
|
||||||
|
|
||||||
status: ## what is deployed for this rig
|
|
||||||
bash ctrl/bundle.sh status
|
|
||||||
|
|
||||||
url: ## the address MetalLB assigned
|
|
||||||
bash ctrl/bundle.sh url
|
|
||||||
|
|
||||||
list: ## every rig in this cluster, with addresses
|
|
||||||
bash ctrl/bundle.sh list
|
|
||||||
|
|
||||||
dev: ## run the UI locally with vite — no cluster needed
|
|
||||||
bash ctrl/bundle.sh dev
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
# sample-rig
|
|
||||||
|
|
||||||
A minimal, non-sensitive bundle that proves an installation works and shows what
|
|
||||||
shipped. Copy it, rename it, and you have another rig.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make manifest # generate the artifact — no cluster, no kubectl needed
|
|
||||||
make up # deploy it into the local cluster
|
|
||||||
make list # every rig in this cluster, with addresses
|
|
||||||
make dev # run the UI locally with vite, no cluster at all
|
|
||||||
```
|
|
||||||
|
|
||||||
`make up` prints an address. Open it and the page says **IT WORKS**, then lists
|
|
||||||
the tools and rigs in the bundle.
|
|
||||||
|
|
||||||
## What it is for
|
|
||||||
|
|
||||||
Three jobs, in the order you hit them:
|
|
||||||
|
|
||||||
1. **Prove the install.** kind is there, a cluster exists, MetalLB hands out an
|
|
||||||
address, a `type: LoadBalancer` Service actually resolves, and a pod serves.
|
|
||||||
If all of that works, the environment is sound.
|
|
||||||
2. **Say what shipped.** The page renders [`bundle.json`](bundle.json) —
|
|
||||||
standalone tools and rigs, flat, with none of soleprint's internal hierarchy.
|
|
||||||
Editing that file is the only step needed to change the listing.
|
|
||||||
3. **Stand in for the real thing.** Nothing here is sensitive. The real
|
|
||||||
architecture connects separately, against a setup already known to work.
|
|
||||||
|
|
||||||
## The UI is a complement, not the product
|
|
||||||
|
|
||||||
`rig-ui/` is just a vite app. It complements a rig; a rig is complete and useful
|
|
||||||
without it, and nothing depends on it being there. It is deliberately **not**
|
|
||||||
generated by kind or tilt — you copy the folder into a rig after that rig is
|
|
||||||
pulled, and apply one manifest:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl apply -n <namespace> -f rig-ui/k8s.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
That file is the whole integration: one Pod running `npm run dev` on
|
|
||||||
`node:22-alpine`, one Service. A bare Pod rather than a Deployment because this
|
|
||||||
is a dev-loop convenience, not a workload to keep alive.
|
|
||||||
|
|
||||||
The app and `bundle.json` arrive as a ConfigMap, so nothing is baked into an
|
|
||||||
image and editing the bundle is the entire update cycle. The container runs
|
|
||||||
`npm install` at start, which needs egress to a registry — on a locked-down
|
|
||||||
cluster point npm at the internal one, or bake an image instead. Nothing else
|
|
||||||
changes if you do.
|
|
||||||
|
|
||||||
## One artifact, two destinations
|
|
||||||
|
|
||||||
`ctrl/manifest.py` emits `generated/<slug>.yaml` — namespace, the app and
|
|
||||||
bundle embedded in a ConfigMap, Pod, Service. It is self-contained and applies
|
|
||||||
unmodified anywhere:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl apply -f generated/sample-rig.yaml # local kind, or an external cluster
|
|
||||||
```
|
|
||||||
|
|
||||||
`make up` applies **that same file**. There is no separate local path, so what
|
|
||||||
works here cannot quietly differ from what is applied elsewhere.
|
|
||||||
|
|
||||||
This is what `type: LoadBalancer` buys. MetalLB answers it on kind; the AWS load
|
|
||||||
balancer controller answers it on EKS. NodePort would not survive the trip — it
|
|
||||||
is a single cluster-wide port range, so two rigs would have to negotiate numbers.
|
|
||||||
|
|
||||||
**VPC-agnostic on purpose.** The target is EKS, but the Service carries no
|
|
||||||
annotations — no `aws-load-balancer-subnets`, no security groups, no `-scheme`,
|
|
||||||
no `-type: nlb`. Each of those encodes a specific network layout, and one of them
|
|
||||||
appearing here would pin the artifact to the account and VPC it was written
|
|
||||||
against, which is precisely what stops it also working on kind. Subnet discovery
|
|
||||||
is the cluster's business: EKS resolves it from the tags its own subnets carry.
|
|
||||||
|
|
||||||
That leaves one thing genuinely environment-specific — internal versus
|
|
||||||
internet-facing. A bare `LoadBalancer` provisions internet-facing, which a
|
|
||||||
regulated account will usually refuse, and should. That belongs in a
|
|
||||||
per-environment overlay applied on top, never inlined into this artifact.
|
|
||||||
|
|
||||||
**MetalLB only — no ingress-nginx.** Its controller supports a narrow window of
|
|
||||||
Kubernetes versions, so depending on it constrains which k8s a rig can be built
|
|
||||||
with. That undercuts running trailing-edge control planes to model a legacy
|
|
||||||
estate, which is the reason `versions.env` pins v1_33..v1_36. MetalLB carries no
|
|
||||||
such constraint, so reachability costs nothing in version coverage.
|
|
||||||
|
|
||||||
## Several rigs, one cluster
|
|
||||||
|
|
||||||
Identity follows the **folder name**, the same rule rig uses for cluster
|
|
||||||
identity. The namespace is the folder; resource names are generic, and names only
|
|
||||||
have to be unique within a namespace.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r sample-rig corporate-rig
|
|
||||||
cd corporate-rig && make up # its own namespace, its own address
|
|
||||||
```
|
|
||||||
|
|
||||||
No edits, no collisions, both in the same local cluster. `make list` shows them
|
|
||||||
together. `make down` removes only this one — siblings, MetalLB and the cluster
|
|
||||||
are left alone.
|
|
||||||
|
|
||||||
Client rigs are gitignored (`*-rig/`, with `sample-rig/` the deliberate
|
|
||||||
exception): a rig's k8s files spell out a real architecture, and that is exactly
|
|
||||||
what must not land in this repo.
|
|
||||||
|
|
||||||
## Staging workstations
|
|
||||||
|
|
||||||
`ctrl/manifest.py` is stdlib-only on purpose: it runs on a bare machine before
|
|
||||||
anything is installed. The toolchain itself is rig's job — `make deps` installs
|
|
||||||
the pinned kind and tilt binaries, which is what makes a staging AWS workspace
|
|
||||||
reachable from the same commands as a laptop.
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```
|
|
||||||
sample-rig/
|
|
||||||
├── Makefile # thin — one target per ctrl/ script
|
|
||||||
├── bundle.json # what shipped; the UI renders THIS
|
|
||||||
├── rig-ui/ # the vite app — optional, copied into a rig to enable it
|
|
||||||
│ ├── k8s.yaml # how to plug it in: one Pod, one Service
|
|
||||||
│ ├── index.html
|
|
||||||
│ ├── package.json
|
|
||||||
│ ├── vite.config.js
|
|
||||||
│ └── src/{main.js,style.css}
|
|
||||||
├── ctrl/
|
|
||||||
│ ├── manifest.py # emits the artifact
|
|
||||||
│ └── bundle.sh # generate / deploy / inspect
|
|
||||||
└── generated/ # the artifact — committed, this is the deliverable
|
|
||||||
```
|
|
||||||
|
|
||||||
Editing `bundle.json` or anything in `rig-ui/` means re-running `make manifest`.
|
|
||||||
The ConfigMap carries a checksum of everything embedded, so a stale deployment is
|
|
||||||
visible rather than silent.
|
|
||||||
|
|
||||||
## Not built, but not foreclosed
|
|
||||||
|
|
||||||
Everything derives from `bundle.json` plus a target namespace. A Pulumi or
|
|
||||||
Terraform emitter would sit beside `ctrl/manifest.py` consuming the same inputs;
|
|
||||||
nothing above it assumes the artifact is YAML.
|
|
||||||
|
|
||||||
Licence terms for the compiled UI component belong in the soleprint-generated
|
|
||||||
bundle, not here — this sample carries no proprietary component.
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
{
|
|
||||||
"_comment": "What this bundle contains. Single source of truth — the landing page renders THIS file, so adding an entry here is the only edit needed. Deliberately FLAT: standalone tools and rigs, with none of soleprint's internal hierarchy (no artery/atlas/station layering). Nothing here is sensitive; the real architecture connects separately.",
|
|
||||||
"bundle": {
|
|
||||||
"name": "sample-rig",
|
|
||||||
"description": "Non-sensitive sample bundle. Proves the kind install works and shows what ships.",
|
|
||||||
"sensitive": false
|
|
||||||
},
|
|
||||||
"tools": [
|
|
||||||
{
|
|
||||||
"name": "modelgen",
|
|
||||||
"summary": "Generate models from config",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "datagen",
|
|
||||||
"summary": "Generate test data from rig-owned generators",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "graphgen",
|
|
||||||
"summary": "Generate navigable model graphs",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "tester",
|
|
||||||
"summary": "HTTP contract test runner — one suite, any environment",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "databrowse",
|
|
||||||
"summary": "SQL data browser",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "sbwrapper",
|
|
||||||
"summary": "Sandbox wrapper",
|
|
||||||
"standalone": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"rigs": [
|
|
||||||
{
|
|
||||||
"name": "sample-rig",
|
|
||||||
"summary": "This bundle — a minimal, copyable environment",
|
|
||||||
"active": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"next": [
|
|
||||||
"Point MANIFESTS_DIR at the real manifests to connect the actual architecture.",
|
|
||||||
"Real k8s files are versioned separately and are not part of this bundle."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
{
|
|
||||||
"_comment": "MOCKED cluster state. Nothing here is read from a live cluster — it exists so the UI can be shown when there is no cluster at all (a locked-down machine, a laptop with no memory to spare, a demo where kind will not start). The page labels it as mocked; a demo that looks live but is not is worse than one that says so. When a real cluster is present the same shapes come from kubectl.",
|
|
||||||
|
|
||||||
"mocked": true,
|
|
||||||
|
|
||||||
"cluster": {
|
|
||||||
"name": "sample-rig",
|
|
||||||
"context": "kind-sample-rig",
|
|
||||||
"provider": "kind",
|
|
||||||
"profile": "minimal",
|
|
||||||
"k8s": "v1.36.1",
|
|
||||||
"nodes": 1
|
|
||||||
},
|
|
||||||
|
|
||||||
"workloads": [
|
|
||||||
{
|
|
||||||
"name": "rig-ui",
|
|
||||||
"summary": "Pod · node:22-alpine · vite on :5173",
|
|
||||||
"state": "Running"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "metallb-system/controller",
|
|
||||||
"summary": "Deployment · assigns LoadBalancer addresses",
|
|
||||||
"state": "Running"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "metallb-system/speaker",
|
|
||||||
"summary": "DaemonSet · answers ARP in layer 2 mode",
|
|
||||||
"state": "Running"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
|
|
||||||
"services": [
|
|
||||||
{
|
|
||||||
"name": "rig-ui",
|
|
||||||
"summary": "LoadBalancer · 80 -> 5173 · no annotations, so it resolves on kind and on EKS alike",
|
|
||||||
"state": "172.18.255.200"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,223 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# The rig bundle: generate it, deploy it, tear it down, find it.
|
|
||||||
#
|
|
||||||
# Usage: bundle.sh manifest | up | down | status | url | list | dev
|
|
||||||
#
|
|
||||||
# What `up` proves, in order: kind installed and a cluster exists, MetalLB can
|
|
||||||
# hand out an address, a Service of type LoadBalancer actually resolves, and a
|
|
||||||
# pod serves the bundle listing. If all of that works the installation is sound,
|
|
||||||
# and the only thing missing is the real architecture.
|
|
||||||
#
|
|
||||||
# ONE ARTIFACT
|
|
||||||
# `up` applies generated/<slug>.yaml — the same self-contained file you would
|
|
||||||
# hand to an external cluster. There is no separate local path, so what works
|
|
||||||
# here cannot quietly differ from the master deployment applied elsewhere.
|
|
||||||
#
|
|
||||||
# ONE CLUSTER, SEVERAL RIGS
|
|
||||||
# Identity follows the FOLDER NAME, exactly as rig's cluster identity does. This
|
|
||||||
# directory deploys into a namespace named after itself, so copying it to
|
|
||||||
# corporate-rig/ yields a second rig in the SAME local cluster with no edits and
|
|
||||||
# no collisions — different namespace, its own MetalLB address. `list` shows all
|
|
||||||
# of them. The cluster itself is rig's business; this only ever owns a namespace.
|
|
||||||
#
|
|
||||||
# MetalLB is installed by calling rig's own addon script rather than
|
|
||||||
# reimplementing it — deriving the pool from the kind Docker network is the
|
|
||||||
# fiddly part and there should be exactly one copy of it.
|
|
||||||
set -euo pipefail
|
|
||||||
cd "$(dirname "$0")/.."
|
|
||||||
|
|
||||||
BUNDLE_ROOT="$(pwd)"
|
|
||||||
RIG_CTRL="$(cd .. && pwd)/ctrl"
|
|
||||||
|
|
||||||
# The containing folder's name, reduced to a DNS label (same rule as rig's
|
|
||||||
# default_cluster_name and ctrl/manifest.py, so all three agree on the slug).
|
|
||||||
slug() {
|
|
||||||
local n
|
|
||||||
n=$(basename "$BUNDLE_ROOT")
|
|
||||||
n=$(echo "$n" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')
|
|
||||||
n=$(echo "$n" | sed 's/^-*//; s/-*$//')
|
|
||||||
echo "${n:-rig-bundle}"
|
|
||||||
}
|
|
||||||
NS="$(slug)"
|
|
||||||
ARTIFACT="generated/${NS}.yaml"
|
|
||||||
|
|
||||||
# Resolved lazily, not at load time: `manifest` and `dev` deliberately work
|
|
||||||
# with no cluster and no kubectl at all, and a top-level check would break that.
|
|
||||||
#
|
|
||||||
# Follows whatever context rig's cluster.sh selected, so this bundle works in a
|
|
||||||
# copied-and-renamed environment without being told which cluster it is in.
|
|
||||||
init_kube() {
|
|
||||||
KUBECONTEXT="${KUBECONTEXT:-$(kubectl config current-context 2>/dev/null || true)}"
|
|
||||||
if [ -z "$KUBECONTEXT" ]; then
|
|
||||||
echo "no kubectl context — bring a cluster up first: (cd .. && make cluster up)" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
KCTX="kubectl --context ${KUBECONTEXT}"
|
|
||||||
K="kubectl --context ${KUBECONTEXT} --namespace ${NS}"
|
|
||||||
}
|
|
||||||
|
|
||||||
require_cluster() {
|
|
||||||
if ! $KCTX cluster-info >/dev/null 2>&1; then
|
|
||||||
echo "context '$KUBECONTEXT' does not reach a cluster" >&2
|
|
||||||
echo "bring one up: (cd .. && make cluster up)" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_metallb() {
|
|
||||||
if $KCTX get deployment -n metallb-system controller >/dev/null 2>&1; then
|
|
||||||
echo "metallb: present"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Only kind needs it. On a real cluster the cloud load balancer answers a
|
|
||||||
# `type: LoadBalancer` Service, and installing MetalLB there would be wrong.
|
|
||||||
case "$KUBECONTEXT" in
|
|
||||||
kind-*) ;;
|
|
||||||
*)
|
|
||||||
echo "metallb: skipped — '$KUBECONTEXT' is not a kind context"
|
|
||||||
echo " (a cloud load balancer answers LoadBalancer services there)"
|
|
||||||
return 0
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [ ! -f "$RIG_CTRL/addons/metallb.sh" ]; then
|
|
||||||
echo "metallb is not installed and rig's addon script was not found at" >&2
|
|
||||||
echo " $RIG_CTRL/addons/metallb.sh" >&2
|
|
||||||
echo "a Service of type LoadBalancer will sit at <pending> without it." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# rig's addons derive their target cluster from RIG'S OWN folder name via
|
|
||||||
# load_config, so left alone this bundle would install into `kind-rig` —
|
|
||||||
# a cluster that need not exist — while deploying everything else into the
|
|
||||||
# context actually selected. CLUSTER is in load_config's overridable set,
|
|
||||||
# so passing it here points the addon at the same cluster we are using.
|
|
||||||
local target="${KUBECONTEXT#kind-}"
|
|
||||||
echo "metallb: installing via rig's addon into '$target'"
|
|
||||||
CLUSTER="$target" bash "$RIG_CTRL/addons/metallb.sh"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Regenerate the artifact. No cluster and no kubectl required — this is the step
|
|
||||||
# a staging workstation runs before anything is installed.
|
|
||||||
manifest() {
|
|
||||||
mkdir -p generated
|
|
||||||
python3 ctrl/manifest.py "$NS" > "$ARTIFACT"
|
|
||||||
echo "wrote $ARTIFACT ($(wc -l < "$ARTIFACT") lines)"
|
|
||||||
echo " applies as-is anywhere: kubectl apply -f ${BUNDLE_ROOT}/${ARTIFACT}"
|
|
||||||
}
|
|
||||||
|
|
||||||
up() {
|
|
||||||
manifest
|
|
||||||
init_kube
|
|
||||||
require_cluster
|
|
||||||
ensure_metallb
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "applying '${NS}' to context '${KUBECONTEXT}'"
|
|
||||||
$KCTX apply -f "$ARTIFACT"
|
|
||||||
|
|
||||||
# `rollout status` does not work on a bare Pod — it only understands
|
|
||||||
# Deployments, StatefulSets and DaemonSets. Wait on the condition instead.
|
|
||||||
# This is the slow step: the container npm-installs before vite serves.
|
|
||||||
echo "waiting for the pod to be ready (npm install runs first)..."
|
|
||||||
$K wait --for=condition=Ready pod/rig-ui --timeout=300s
|
|
||||||
echo
|
|
||||||
url
|
|
||||||
}
|
|
||||||
|
|
||||||
down() {
|
|
||||||
init_kube
|
|
||||||
# Delete the namespace and everything in it goes with it. Scoped to THIS
|
|
||||||
# rig — a sibling rig in the same cluster is untouched.
|
|
||||||
$KCTX delete namespace "$NS" --ignore-not-found
|
|
||||||
echo "'${NS}' removed (cluster, metallb and any sibling rig are left alone)"
|
|
||||||
}
|
|
||||||
|
|
||||||
status() {
|
|
||||||
init_kube
|
|
||||||
require_cluster
|
|
||||||
if ! $KCTX get namespace "$NS" >/dev/null 2>&1; then
|
|
||||||
echo "'${NS}' is not deployed — run: make up"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
$K get pod,svc,configmap -o wide
|
|
||||||
}
|
|
||||||
|
|
||||||
# Every rig in this cluster, not just this one — the point of the namespace
|
|
||||||
# split is that several coexist, so there has to be a way to see them together.
|
|
||||||
list() {
|
|
||||||
init_kube
|
|
||||||
require_cluster
|
|
||||||
local names
|
|
||||||
names=$($KCTX get namespace -l rig.bundle/name \
|
|
||||||
-o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
|
|
||||||
if [ -z "$names" ]; then
|
|
||||||
echo "no rigs deployed in context '${KUBECONTEXT}'"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
printf "%-20s %-16s %s\n" RIG ADDRESS ""
|
|
||||||
local n ip
|
|
||||||
for n in $names; do
|
|
||||||
ip=$($KCTX -n "$n" get svc rig-ui \
|
|
||||||
-o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
|
|
||||||
printf "%-20s %-16s %s\n" "$n" "${ip:-<pending>}" \
|
|
||||||
"$([ "$n" = "$NS" ] && echo '<- this one')"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# The address MetalLB (or a cloud load balancer) assigned. <pending> here is the
|
|
||||||
# classic silent failure: everything reports healthy and nothing is reachable.
|
|
||||||
url() {
|
|
||||||
init_kube
|
|
||||||
local ip
|
|
||||||
ip=$($K get svc rig-ui \
|
|
||||||
-o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
|
|
||||||
if [ -z "$ip" ]; then
|
|
||||||
ip=$($K get svc rig-ui \
|
|
||||||
-o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)
|
|
||||||
fi
|
|
||||||
if [ -z "$ip" ]; then
|
|
||||||
echo "no external address yet — nothing has assigned one."
|
|
||||||
echo "on kind: kubectl --context $KUBECONTEXT -n metallb-system get pods"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
echo "IT WORKS -> http://${ip}/"
|
|
||||||
echo " bundle http://${ip}/bundle.json"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run the UI locally with no cluster at all — the fast way to iterate on
|
|
||||||
# bundle.json. Same vite command the pod runs, so what you see here is what
|
|
||||||
# gets served there.
|
|
||||||
dev() {
|
|
||||||
if ! command -v npm >/dev/null 2>&1; then
|
|
||||||
echo "npm not found — the UI needs node locally for this." >&2
|
|
||||||
echo "(in-cluster it runs on the node:22-alpine image instead)" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
# bundle.json lives one level up so it stays the rig's data rather than the
|
|
||||||
# app's; vite serves public/ at the root, which is where the app fetches it.
|
|
||||||
mkdir -p rig-ui/public
|
|
||||||
cp bundle.json rig-ui/public/bundle.json
|
|
||||||
|
|
||||||
# The mocked cluster is a DEMO asset and is deliberately not embedded in the
|
|
||||||
# deployed artifact — on a real rig the UI would then show canned values
|
|
||||||
# beside a live cluster, which is precisely the lie its banner warns about.
|
|
||||||
# It is served here, and in the static build for the public UI-only page.
|
|
||||||
cp cluster.mock.json rig-ui/public/cluster.mock.json
|
|
||||||
|
|
||||||
cd rig-ui
|
|
||||||
[ -d node_modules ] || npm install --no-audit --no-fund
|
|
||||||
VITE_RIG_NAME="$NS" npm run dev
|
|
||||||
}
|
|
||||||
|
|
||||||
case "${1:-status}" in
|
|
||||||
manifest) manifest ;;
|
|
||||||
up) up ;;
|
|
||||||
down) down ;;
|
|
||||||
status) status ;;
|
|
||||||
url) url ;;
|
|
||||||
list) list ;;
|
|
||||||
dev) dev ;;
|
|
||||||
*) echo "usage: $0 [manifest|up|down|status|url|list|dev]" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Emit the complete, self-contained deployment for this rig.
|
|
||||||
|
|
||||||
python3 ctrl/manifest.py [namespace] > generated/<slug>.yaml
|
|
||||||
|
|
||||||
The output is the ARTIFACT. It carries everything — namespace, the vite app and
|
|
||||||
bundle.json embedded in a ConfigMap, the Pod and the Service — so it applies
|
|
||||||
unmodified to any cluster:
|
|
||||||
|
|
||||||
kubectl apply -f generated/sample-rig.yaml
|
|
||||||
|
|
||||||
On kind, MetalLB answers the `type: LoadBalancer` Service. On a real external
|
|
||||||
cluster the cloud load balancer does. Same file, no edits, no branch — which is
|
|
||||||
the point: what runs locally is byte-identical to the deployment applied
|
|
||||||
elsewhere, so local success actually means something.
|
|
||||||
|
|
||||||
`ctrl/bundle.sh up` applies this same generated output rather than a separate
|
|
||||||
code path, so the local convenience wrapper can never drift from the artifact.
|
|
||||||
|
|
||||||
Stdlib only, deliberately: this must run on a bare staging workstation before
|
|
||||||
anything is installed, so it cannot depend on PyYAML or a template engine.
|
|
||||||
|
|
||||||
Open seam — not built: everything here derives from bundle.json plus a target
|
|
||||||
namespace. A Pulumi or Terraform emitter would sit beside this file consuming the
|
|
||||||
same inputs; nothing above it assumes the artifact is YAML.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import zlib
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
|
||||||
UI = ROOT / "rig-ui"
|
|
||||||
|
|
||||||
# Files embedded into the ConfigMap, mounted read-only at /src in the pod and
|
|
||||||
# copied into vite's layout at start (see rig-ui/k8s.yaml). Flat on purpose:
|
|
||||||
# ConfigMap keys cannot contain '/'.
|
|
||||||
EMBEDDED = {
|
|
||||||
"bundle.json": ROOT / "bundle.json",
|
|
||||||
"package.json": UI / "package.json",
|
|
||||||
"vite.config.js": UI / "vite.config.js",
|
|
||||||
"index.html": UI / "index.html",
|
|
||||||
"main.js": UI / "src" / "main.js",
|
|
||||||
"style.css": UI / "src" / "style.css",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def slug(name: str) -> str:
|
|
||||||
"""Reduce a folder name to a DNS label, matching ctrl/bundle.sh's rule."""
|
|
||||||
out = re.sub(r"[^a-z0-9-]", "-", name.lower()).strip("-")
|
|
||||||
return out or "rig-bundle"
|
|
||||||
|
|
||||||
|
|
||||||
def block(text: str, indent: int) -> str:
|
|
||||||
"""Indent a file's contents for a YAML literal block scalar.
|
|
||||||
|
|
||||||
Blank lines are emitted truly empty rather than as whitespace: trailing
|
|
||||||
spaces on an otherwise blank line are legal YAML but show up as diff noise
|
|
||||||
in a committed artifact.
|
|
||||||
"""
|
|
||||||
pad = " " * indent
|
|
||||||
return "\n".join(pad + line if line.strip() else "" for line in text.splitlines())
|
|
||||||
|
|
||||||
|
|
||||||
def checksum(parts: list[str]) -> str:
|
|
||||||
"""Stable content hash of everything embedded, stamped as a label.
|
|
||||||
|
|
||||||
A mounted ConfigMap updates in place without restarting anything, so without
|
|
||||||
a visible change nothing signals that the pod is serving stale content.
|
|
||||||
"""
|
|
||||||
return str(zlib.crc32("".join(parts).encode()) & 0xFFFFFFFF)
|
|
||||||
|
|
||||||
|
|
||||||
def build(namespace: str) -> str:
|
|
||||||
contents = {}
|
|
||||||
for key, path in EMBEDDED.items():
|
|
||||||
if not path.exists():
|
|
||||||
sys.exit(f"missing input: {path}")
|
|
||||||
contents[key] = path.read_text()
|
|
||||||
|
|
||||||
# Fail loudly here rather than shipping an artifact that renders an error.
|
|
||||||
try:
|
|
||||||
json.loads(contents["bundle.json"])
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
sys.exit(f"bundle.json is not valid JSON: {exc}")
|
|
||||||
|
|
||||||
app = (UI / "k8s.yaml").read_text()
|
|
||||||
app = app.replace("__RIG_NAME__", namespace)
|
|
||||||
|
|
||||||
data = "\n".join(
|
|
||||||
f" {key}: |\n{block(text, 4)}" for key, text in sorted(contents.items())
|
|
||||||
)
|
|
||||||
|
|
||||||
return f"""# GENERATED by ctrl/manifest.py — do not edit.
|
|
||||||
# Regenerate with: make manifest
|
|
||||||
#
|
|
||||||
# Self-contained: applies as-is to any cluster, local kind or external.
|
|
||||||
# kubectl apply -f this-file.yaml
|
|
||||||
#
|
|
||||||
# Namespace carries the identity, so several rigs coexist in one cluster.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
|
||||||
name: {namespace}
|
|
||||||
labels:
|
|
||||||
rig.bundle/name: {namespace}
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
namespace: {namespace}
|
|
||||||
labels:
|
|
||||||
rig.bundle/checksum: "{checksum(list(contents.values()))}"
|
|
||||||
data:
|
|
||||||
{data}
|
|
||||||
---
|
|
||||||
{_namespaced(app, namespace)}
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _namespaced(doc: str, namespace: str) -> str:
|
|
||||||
"""Add `namespace:` to each resource so the artifact applies without -n.
|
|
||||||
|
|
||||||
rig-ui/k8s.yaml omits it on purpose — applied by hand it should land in
|
|
||||||
whatever namespace you choose. Pinning it belongs to the generated artifact,
|
|
||||||
which has to be self-contained.
|
|
||||||
"""
|
|
||||||
return re.sub(
|
|
||||||
r"^(metadata:\n(?:[ \t]+.*\n)*?)([ \t]+)(name: rig-ui)$",
|
|
||||||
lambda m: f"{m.group(1)}{m.group(2)}{m.group(3)}\n{m.group(2)}namespace: {namespace}",
|
|
||||||
doc.strip(),
|
|
||||||
flags=re.MULTILINE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
target = sys.argv[1] if len(sys.argv) > 1 else slug(ROOT.name)
|
|
||||||
sys.stdout.write(build(target))
|
|
||||||
@@ -1,406 +0,0 @@
|
|||||||
# GENERATED by ctrl/manifest.py — do not edit.
|
|
||||||
# Regenerate with: make manifest
|
|
||||||
#
|
|
||||||
# Self-contained: applies as-is to any cluster, local kind or external.
|
|
||||||
# kubectl apply -f this-file.yaml
|
|
||||||
#
|
|
||||||
# Namespace carries the identity, so several rigs coexist in one cluster.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
|
||||||
name: sample-rig
|
|
||||||
labels:
|
|
||||||
rig.bundle/name: sample-rig
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
namespace: sample-rig
|
|
||||||
labels:
|
|
||||||
rig.bundle/checksum: "2074194964"
|
|
||||||
data:
|
|
||||||
bundle.json: |
|
|
||||||
{
|
|
||||||
"_comment": "What this bundle contains. Single source of truth — the landing page renders THIS file, so adding an entry here is the only edit needed. Deliberately FLAT: standalone tools and rigs, with none of soleprint's internal hierarchy (no artery/atlas/station layering). Nothing here is sensitive; the real architecture connects separately.",
|
|
||||||
"bundle": {
|
|
||||||
"name": "sample-rig",
|
|
||||||
"description": "Non-sensitive sample bundle. Proves the kind install works and shows what ships.",
|
|
||||||
"sensitive": false
|
|
||||||
},
|
|
||||||
"tools": [
|
|
||||||
{
|
|
||||||
"name": "modelgen",
|
|
||||||
"summary": "Generate models from config",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "datagen",
|
|
||||||
"summary": "Generate test data from rig-owned generators",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "graphgen",
|
|
||||||
"summary": "Generate navigable model graphs",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "tester",
|
|
||||||
"summary": "HTTP contract test runner — one suite, any environment",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "databrowse",
|
|
||||||
"summary": "SQL data browser",
|
|
||||||
"standalone": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "sbwrapper",
|
|
||||||
"summary": "Sandbox wrapper",
|
|
||||||
"standalone": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"rigs": [
|
|
||||||
{
|
|
||||||
"name": "sample-rig",
|
|
||||||
"summary": "This bundle — a minimal, copyable environment",
|
|
||||||
"active": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"next": [
|
|
||||||
"Point MANIFESTS_DIR at the real manifests to connect the actual architecture.",
|
|
||||||
"Real k8s files are versioned separately and are not part of this bundle."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
index.html: |
|
|
||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>IT WORKS</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<main id="app"></main>
|
|
||||||
<script type="module" src="/src/main.js"></script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
main.js: |
|
|
||||||
import "./style.css";
|
|
||||||
|
|
||||||
/* The IT WORKS page: renders bundle.json as the list of what shipped.
|
|
||||||
*
|
|
||||||
* Plain vite, no framework — this is a complement to the rig, not part of it,
|
|
||||||
* and it should stay small enough that nobody has to adopt a stack to read it.
|
|
||||||
*
|
|
||||||
* bundle.json is fetched at runtime rather than imported, so the same built app
|
|
||||||
* serves whatever rig it was copied into. Editing the ConfigMap changes the page
|
|
||||||
* without rebuilding.
|
|
||||||
*
|
|
||||||
* Styling is a handful of rules on purpose. The real visual identity lives in
|
|
||||||
* the soleprint UI package; nothing here should grow into a theme.
|
|
||||||
*/
|
|
||||||
|
|
||||||
const esc = (s) =>
|
|
||||||
String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
||||||
|
|
||||||
const tag = (text, on = false) =>
|
|
||||||
`<span class="tag${on ? " on" : ""}">${esc(text)}</span>`;
|
|
||||||
|
|
||||||
function items(list, activeKey) {
|
|
||||||
if (!list?.length) return `<li><span class="summary">nothing listed</span></li>`;
|
|
||||||
return list
|
|
||||||
.map((it) => {
|
|
||||||
const tags = [
|
|
||||||
it.standalone ? tag("standalone") : "",
|
|
||||||
it.state ? tag(it.state) : "",
|
|
||||||
activeKey && it[activeKey] ? tag("active", true) : "",
|
|
||||||
].join("");
|
|
||||||
return `<li><span class="name">${esc(it.name ?? "?")}</span>
|
|
||||||
<span class="summary">${esc(it.summary ?? "")}</span>${tags}</li>`;
|
|
||||||
})
|
|
||||||
.join("");
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Cluster state, when there is any to show.
|
|
||||||
*
|
|
||||||
* Fetched separately and allowed to fail: the bundle listing is the point, and a
|
|
||||||
* rig with no cluster reachable is a normal state, not an error. Renders nothing
|
|
||||||
* at all when absent.
|
|
||||||
*
|
|
||||||
* When the payload says `mocked`, say so loudly. This exists to demo the UI on a
|
|
||||||
* machine where kind will not run — and a demo that looks live but is not is
|
|
||||||
* worse than one that admits it. */
|
|
||||||
function clusterSection(c) {
|
|
||||||
if (!c) return "";
|
|
||||||
const m = c.cluster ?? {};
|
|
||||||
const banner = c.mocked
|
|
||||||
? `<p class="mock">mocked — no cluster was queried; these are canned values</p>`
|
|
||||||
: "";
|
|
||||||
const meta = [m.context, m.k8s, m.profile ? `profile ${m.profile}` : "",
|
|
||||||
m.nodes ? `${m.nodes} node${m.nodes > 1 ? "s" : ""}` : ""]
|
|
||||||
.filter(Boolean).join(" · ");
|
|
||||||
|
|
||||||
return `
|
|
||||||
<h2>Cluster${c.mocked ? " (mocked)" : ""}</h2>
|
|
||||||
${banner}
|
|
||||||
${meta ? `<p class="sub">${esc(meta)}</p>` : ""}
|
|
||||||
<ul>${items(c.workloads)}</ul>
|
|
||||||
<h2>Services (${c.services?.length ?? 0})</h2>
|
|
||||||
<ul>${items(c.services)}</ul>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function render(b, name, cluster) {
|
|
||||||
const meta = b.bundle ?? {};
|
|
||||||
const next = (b.next ?? []).map((n) => `<li>${esc(n)}</li>`).join("");
|
|
||||||
return `
|
|
||||||
<h1><span class="ok">IT WORKS</span> — ${esc(name || meta.name || "rig")}</h1>
|
|
||||||
<p class="sub">${esc(meta.description ?? "")}</p>
|
|
||||||
|
|
||||||
<h2>Tools (${b.tools?.length ?? 0})</h2>
|
|
||||||
<ul>${items(b.tools)}</ul>
|
|
||||||
|
|
||||||
<h2>Rigs (${b.rigs?.length ?? 0})</h2>
|
|
||||||
<ul>${items(b.rigs, "active")}</ul>
|
|
||||||
|
|
||||||
${clusterSection(cluster)}
|
|
||||||
|
|
||||||
${next ? `<div class="next"><ul>${next}</ul></div>` : ""}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const app = document.getElementById("app");
|
|
||||||
|
|
||||||
const json = (path, required) =>
|
|
||||||
fetch(path).then((r) => {
|
|
||||||
if (r.ok) return r.json();
|
|
||||||
if (required) throw new Error(`${path} -> HTTP ${r.status}`);
|
|
||||||
return null; // optional: absent is a normal state, not an error
|
|
||||||
}).catch((err) => {
|
|
||||||
if (required) throw err;
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
|
|
||||||
Promise.all([json("/bundle.json", true), json("/cluster.mock.json", false)])
|
|
||||||
// RIG_NAME is injected by vite from the pod env, so two rigs sharing a
|
|
||||||
// cluster are distinguishable even if a copied bundle.json kept its old name.
|
|
||||||
.then(([b, cluster]) => {
|
|
||||||
app.innerHTML = render(b, import.meta.env.VITE_RIG_NAME, cluster);
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
app.innerHTML = `<h1 class="err">bundle unavailable</h1>
|
|
||||||
<p class="sub">${esc(err.message)}</p>`;
|
|
||||||
});
|
|
||||||
package.json: |
|
|
||||||
{
|
|
||||||
"name": "rig-ui",
|
|
||||||
"private": true,
|
|
||||||
"version": "0.1.0",
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
|
||||||
"dev": "vite --host 0.0.0.0 --port 5173",
|
|
||||||
"build": "vite build",
|
|
||||||
"preview": "vite preview --host 0.0.0.0 --port 5173"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"vite": "^6"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
style.css: |
|
|
||||||
/* Minimal, self-contained. The real visual identity ships with the soleprint UI
|
|
||||||
package, which is a separate artifact — nothing here should grow into a theme. */
|
|
||||||
|
|
||||||
body {
|
|
||||||
margin: 0;
|
|
||||||
padding: 2.5rem 1.5rem;
|
|
||||||
background: #0d0d0f;
|
|
||||||
color: #e8e8f0;
|
|
||||||
font: 14px/1.6 ui-monospace, "JetBrains Mono", Menlo, monospace;
|
|
||||||
}
|
|
||||||
main { max-width: 52rem; margin: 0 auto; }
|
|
||||||
|
|
||||||
h1 { margin: 0; font-size: 1.6rem; letter-spacing: 0.02em; }
|
|
||||||
h1 .ok { color: #3ecf8e; }
|
|
||||||
h1.err { color: #f06565; }
|
|
||||||
.sub { color: #8888a0; margin: 0.35rem 0 2.25rem; }
|
|
||||||
|
|
||||||
h2 {
|
|
||||||
font-size: 0.75rem;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.12em;
|
|
||||||
color: #8888a0;
|
|
||||||
margin: 2rem 0 0.75rem;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
|
|
||||||
ul { list-style: none; margin: 0; padding: 0; }
|
|
||||||
li {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.75rem;
|
|
||||||
align-items: baseline;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
border: 1px solid #2e2e38;
|
|
||||||
border-radius: 6px;
|
|
||||||
margin-bottom: 0.4rem;
|
|
||||||
background: #16161a;
|
|
||||||
}
|
|
||||||
.name { font-weight: 600; min-width: 9rem; }
|
|
||||||
.summary { color: #8888a0; flex: 1; }
|
|
||||||
|
|
||||||
.tag {
|
|
||||||
font-size: 0.7rem;
|
|
||||||
padding: 0.1rem 0.45rem;
|
|
||||||
border-radius: 3px;
|
|
||||||
background: #26262f;
|
|
||||||
color: #8888a0;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
.tag.on { background: #3ecf8e; color: #0d0d0f; }
|
|
||||||
|
|
||||||
.next {
|
|
||||||
color: #555568;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
margin-top: 2.5rem;
|
|
||||||
border-top: 1px solid #2e2e38;
|
|
||||||
padding-top: 1rem;
|
|
||||||
}
|
|
||||||
.next li {
|
|
||||||
display: list-item;
|
|
||||||
border: 0;
|
|
||||||
background: none;
|
|
||||||
padding: 0.15rem 0;
|
|
||||||
margin: 0 0 0 1.1rem;
|
|
||||||
list-style: disc;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Mocked-data banner. Deliberately loud: this only appears when the cluster
|
|
||||||
payload is canned, and a demo that looks live but is not is worse than one
|
|
||||||
that says so. */
|
|
||||||
.mock {
|
|
||||||
margin: 0 0 0.75rem;
|
|
||||||
padding: 0.4rem 0.75rem;
|
|
||||||
border: 1px dashed #f5a623;
|
|
||||||
border-radius: 6px;
|
|
||||||
color: #f5a623;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
}
|
|
||||||
vite.config.js: |
|
|
||||||
import { defineConfig } from "vite";
|
|
||||||
|
|
||||||
/* Serves on 0.0.0.0 so the pod is reachable through the Service, and allows any
|
|
||||||
* Host header because the address is assigned at runtime (MetalLB locally, a
|
|
||||||
* cloud load balancer on EKS) and is never known at build time. */
|
|
||||||
export default defineConfig({
|
|
||||||
server: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true },
|
|
||||||
preview: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true },
|
|
||||||
});
|
|
||||||
---
|
|
||||||
# How to plug the UI into whatever k8s you generated. THIS IS THE WHOLE THING:
|
|
||||||
# one Pod running the vite app, one Service to reach it.
|
|
||||||
#
|
|
||||||
# Optional by design. The UI complements a rig; it is not part of the end
|
|
||||||
# product, and a rig is complete and useful without it. Apply this only when you
|
|
||||||
# want the listing:
|
|
||||||
#
|
|
||||||
# kubectl apply -n <your-namespace> -f rig-ui/k8s.yaml
|
|
||||||
#
|
|
||||||
# A bare Pod, not a Deployment — this is a dev-loop convenience, not a workload
|
|
||||||
# to keep alive. If it dies you re-apply it; nothing depends on it staying up.
|
|
||||||
#
|
|
||||||
# The app and bundle.json arrive as a ConfigMap named `rig-ui`, which
|
|
||||||
# ctrl/manifest.py generates from the folder. Nothing is baked into an image, so
|
|
||||||
# editing bundle.json and re-applying is the whole update cycle.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
namespace: sample-rig
|
|
||||||
labels:
|
|
||||||
app: rig-ui
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: vite
|
|
||||||
image: node:22-alpine
|
|
||||||
workingDir: /app
|
|
||||||
# npm install at start: no image to build and no registry to publish to,
|
|
||||||
# which is the point of a minimal plug-in. It needs egress to a registry —
|
|
||||||
# on a locked-down cluster point npm at the internal one, or bake an image
|
|
||||||
# instead. Nothing else here changes if you do.
|
|
||||||
command: ["sh", "-c"]
|
|
||||||
# A ConfigMap mounts flat (keys cannot contain '/'), so the files are
|
|
||||||
# placed into vite's expected layout here. bundle.json goes to public/
|
|
||||||
# because that is what vite serves at /bundle.json, which is where the
|
|
||||||
# app fetches it.
|
|
||||||
args:
|
|
||||||
- |
|
|
||||||
mkdir -p /app/src /app/public &&
|
|
||||||
cp /src/package.json /src/vite.config.js /src/index.html /app/ &&
|
|
||||||
cp /src/main.js /src/style.css /app/src/ &&
|
|
||||||
cp /src/bundle.json /app/public/ &&
|
|
||||||
npm install --no-audit --no-fund &&
|
|
||||||
npm run dev
|
|
||||||
env:
|
|
||||||
# Rendered in the heading so two rigs sharing a cluster stay
|
|
||||||
# distinguishable. Set from the namespace by ctrl/manifest.py.
|
|
||||||
- name: VITE_RIG_NAME
|
|
||||||
value: sample-rig
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 5173
|
|
||||||
volumeMounts:
|
|
||||||
# /src is read-only from the ConfigMap; the app is copied to a writable
|
|
||||||
# /app because npm install has to create node_modules.
|
|
||||||
- name: rig-ui
|
|
||||||
mountPath: /src
|
|
||||||
- name: app
|
|
||||||
mountPath: /app
|
|
||||||
readinessProbe:
|
|
||||||
httpGet: { path: /, port: 5173 }
|
|
||||||
# npm install decides how long this takes, and it is the slow part.
|
|
||||||
initialDelaySeconds: 15
|
|
||||||
periodSeconds: 5
|
|
||||||
failureThreshold: 30
|
|
||||||
resources:
|
|
||||||
requests: { memory: 128Mi, cpu: 50m }
|
|
||||||
limits: { memory: 512Mi }
|
|
||||||
volumes:
|
|
||||||
- name: rig-ui
|
|
||||||
configMap:
|
|
||||||
name: rig-ui
|
|
||||||
- name: app
|
|
||||||
emptyDir: {}
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
namespace: sample-rig
|
|
||||||
labels:
|
|
||||||
app: rig-ui
|
|
||||||
# No annotations, deliberately — see k8s/app.yaml. The target is EKS but this
|
|
||||||
# stays VPC-agnostic: no subnets, no security groups, no -scheme, no -type.
|
|
||||||
# A bare LoadBalancer is what lets one manifest work on kind and on EKS.
|
|
||||||
spec:
|
|
||||||
type: LoadBalancer
|
|
||||||
selector:
|
|
||||||
app: rig-ui
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: 80
|
|
||||||
targetPort: 5173
|
|
||||||
protocol: TCP
|
|
||||||
# Pinned, because a LoadBalancer Service also allocates a NodePort and
|
|
||||||
# this is the only address that works everywhere.
|
|
||||||
#
|
|
||||||
# On WSL the MetalLB address is on a docker bridge INSIDE the Linux VM,
|
|
||||||
# and Windows has no route to it — the page looks broken while the
|
|
||||||
# cluster is perfectly healthy. 30080 is what rig's `hostport` ingress
|
|
||||||
# mode publishes to the host, so this is reachable at
|
|
||||||
# localhost:$HTTP_PORT from a Windows browser with nothing configured.
|
|
||||||
#
|
|
||||||
# Costs nothing elsewhere: MetalLB still assigns an external IP on Linux,
|
|
||||||
# and on EKS the load balancer targets this NodePort anyway. One Service,
|
|
||||||
# no per-environment branch.
|
|
||||||
#
|
|
||||||
# A pinned NodePort is cluster-unique, so two rigs must live in separate
|
|
||||||
# clusters — which is how they are run anyway.
|
|
||||||
nodePort: 30080
|
|
||||||
3
rig/sample-rig/rig-ui/.gitignore
vendored
3
rig/sample-rig/rig-ui/.gitignore
vendored
@@ -1,3 +0,0 @@
|
|||||||
node_modules/
|
|
||||||
public/
|
|
||||||
dist/
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>IT WORKS</title>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<main id="app"></main>
|
|
||||||
<script type="module" src="/src/main.js"></script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
# How to plug the UI into whatever k8s you generated. THIS IS THE WHOLE THING:
|
|
||||||
# one Pod running the vite app, one Service to reach it.
|
|
||||||
#
|
|
||||||
# Optional by design. The UI complements a rig; it is not part of the end
|
|
||||||
# product, and a rig is complete and useful without it. Apply this only when you
|
|
||||||
# want the listing:
|
|
||||||
#
|
|
||||||
# kubectl apply -n <your-namespace> -f rig-ui/k8s.yaml
|
|
||||||
#
|
|
||||||
# A bare Pod, not a Deployment — this is a dev-loop convenience, not a workload
|
|
||||||
# to keep alive. If it dies you re-apply it; nothing depends on it staying up.
|
|
||||||
#
|
|
||||||
# The app and bundle.json arrive as a ConfigMap named `rig-ui`, which
|
|
||||||
# ctrl/manifest.py generates from the folder. Nothing is baked into an image, so
|
|
||||||
# editing bundle.json and re-applying is the whole update cycle.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
labels:
|
|
||||||
app: rig-ui
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: vite
|
|
||||||
image: node:22-alpine
|
|
||||||
workingDir: /app
|
|
||||||
# npm install at start: no image to build and no registry to publish to,
|
|
||||||
# which is the point of a minimal plug-in. It needs egress to a registry —
|
|
||||||
# on a locked-down cluster point npm at the internal one, or bake an image
|
|
||||||
# instead. Nothing else here changes if you do.
|
|
||||||
command: ["sh", "-c"]
|
|
||||||
# A ConfigMap mounts flat (keys cannot contain '/'), so the files are
|
|
||||||
# placed into vite's expected layout here. bundle.json goes to public/
|
|
||||||
# because that is what vite serves at /bundle.json, which is where the
|
|
||||||
# app fetches it.
|
|
||||||
args:
|
|
||||||
- |
|
|
||||||
mkdir -p /app/src /app/public &&
|
|
||||||
cp /src/package.json /src/vite.config.js /src/index.html /app/ &&
|
|
||||||
cp /src/main.js /src/style.css /app/src/ &&
|
|
||||||
cp /src/bundle.json /app/public/ &&
|
|
||||||
npm install --no-audit --no-fund &&
|
|
||||||
npm run dev
|
|
||||||
env:
|
|
||||||
# Rendered in the heading so two rigs sharing a cluster stay
|
|
||||||
# distinguishable. Set from the namespace by ctrl/manifest.py.
|
|
||||||
- name: VITE_RIG_NAME
|
|
||||||
value: __RIG_NAME__
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
containerPort: 5173
|
|
||||||
volumeMounts:
|
|
||||||
# /src is read-only from the ConfigMap; the app is copied to a writable
|
|
||||||
# /app because npm install has to create node_modules.
|
|
||||||
- name: rig-ui
|
|
||||||
mountPath: /src
|
|
||||||
- name: app
|
|
||||||
mountPath: /app
|
|
||||||
readinessProbe:
|
|
||||||
httpGet: { path: /, port: 5173 }
|
|
||||||
# npm install decides how long this takes, and it is the slow part.
|
|
||||||
initialDelaySeconds: 15
|
|
||||||
periodSeconds: 5
|
|
||||||
failureThreshold: 30
|
|
||||||
resources:
|
|
||||||
requests: { memory: 128Mi, cpu: 50m }
|
|
||||||
limits: { memory: 512Mi }
|
|
||||||
volumes:
|
|
||||||
- name: rig-ui
|
|
||||||
configMap:
|
|
||||||
name: rig-ui
|
|
||||||
- name: app
|
|
||||||
emptyDir: {}
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: rig-ui
|
|
||||||
labels:
|
|
||||||
app: rig-ui
|
|
||||||
# No annotations, deliberately — see k8s/app.yaml. The target is EKS but this
|
|
||||||
# stays VPC-agnostic: no subnets, no security groups, no -scheme, no -type.
|
|
||||||
# A bare LoadBalancer is what lets one manifest work on kind and on EKS.
|
|
||||||
spec:
|
|
||||||
type: LoadBalancer
|
|
||||||
selector:
|
|
||||||
app: rig-ui
|
|
||||||
ports:
|
|
||||||
- name: http
|
|
||||||
port: 80
|
|
||||||
targetPort: 5173
|
|
||||||
protocol: TCP
|
|
||||||
# Pinned, because a LoadBalancer Service also allocates a NodePort and
|
|
||||||
# this is the only address that works everywhere.
|
|
||||||
#
|
|
||||||
# On WSL the MetalLB address is on a docker bridge INSIDE the Linux VM,
|
|
||||||
# and Windows has no route to it — the page looks broken while the
|
|
||||||
# cluster is perfectly healthy. 30080 is what rig's `hostport` ingress
|
|
||||||
# mode publishes to the host, so this is reachable at
|
|
||||||
# localhost:$HTTP_PORT from a Windows browser with nothing configured.
|
|
||||||
#
|
|
||||||
# Costs nothing elsewhere: MetalLB still assigns an external IP on Linux,
|
|
||||||
# and on EKS the load balancer targets this NodePort anyway. One Service,
|
|
||||||
# no per-environment branch.
|
|
||||||
#
|
|
||||||
# A pinned NodePort is cluster-unique, so two rigs must live in separate
|
|
||||||
# clusters — which is how they are run anyway.
|
|
||||||
nodePort: 30080
|
|
||||||
1164
rig/sample-rig/rig-ui/package-lock.json
generated
1164
rig/sample-rig/rig-ui/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,14 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "rig-ui",
|
|
||||||
"private": true,
|
|
||||||
"version": "0.1.0",
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
|
||||||
"dev": "vite --host 0.0.0.0 --port 5173",
|
|
||||||
"build": "vite build",
|
|
||||||
"preview": "vite preview --host 0.0.0.0 --port 5173"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"vite": "^6"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
import "./style.css";
|
|
||||||
|
|
||||||
/* The IT WORKS page: renders bundle.json as the list of what shipped.
|
|
||||||
*
|
|
||||||
* Plain vite, no framework — this is a complement to the rig, not part of it,
|
|
||||||
* and it should stay small enough that nobody has to adopt a stack to read it.
|
|
||||||
*
|
|
||||||
* Laid out like soleprint's templated vein pages, because it does the same job:
|
|
||||||
* name each component, list what it exposes, show what comes back. Tool chrome
|
|
||||||
* and output are styled apart on purpose (see style.css) — that separation is
|
|
||||||
* what tells you whether you are reading the tool or its result.
|
|
||||||
*
|
|
||||||
* bundle.json is fetched at runtime rather than imported, so the same built app
|
|
||||||
* serves whatever rig it was copied into. Editing the ConfigMap changes the page
|
|
||||||
* without rebuilding.
|
|
||||||
*/
|
|
||||||
|
|
||||||
const esc = (s) =>
|
|
||||||
String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
||||||
|
|
||||||
const tag = (text, on = false) =>
|
|
||||||
`<span class="tag${on ? " on" : ""}">${esc(text)}</span>`;
|
|
||||||
|
|
||||||
/* Tool chrome: one bordered card per component. */
|
|
||||||
function components(list, activeKey) {
|
|
||||||
if (!list?.length)
|
|
||||||
return `<div class="component"><p>nothing listed</p></div>`;
|
|
||||||
return list
|
|
||||||
.map((it) => {
|
|
||||||
const tags = [
|
|
||||||
it.standalone ? tag("standalone") : "",
|
|
||||||
it.state ? tag(it.state) : "",
|
|
||||||
activeKey && it[activeKey] ? tag("active", true) : "",
|
|
||||||
].join("");
|
|
||||||
return `<div class="component">
|
|
||||||
<h4>${esc(it.name ?? "?")} ${tags}</h4>
|
|
||||||
<p>${esc(it.summary ?? "")}</p>
|
|
||||||
</div>`;
|
|
||||||
})
|
|
||||||
.join("");
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Endpoint rows: path on the left, what it returns on the right. */
|
|
||||||
function endpoints(list) {
|
|
||||||
return list
|
|
||||||
.map(
|
|
||||||
(e) => `<li><code>${esc(e.path)}</code>
|
|
||||||
<span class="desc">${esc(e.desc)}</span></li>`
|
|
||||||
)
|
|
||||||
.join("");
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Output: what the endpoint above actually returns, so the page demonstrates
|
|
||||||
itself rather than describing what a demonstration would look like. */
|
|
||||||
function example(bundle) {
|
|
||||||
const sample = {
|
|
||||||
bundle: bundle.bundle?.name,
|
|
||||||
tools: (bundle.tools ?? []).map((t) => t.name),
|
|
||||||
rigs: (bundle.rigs ?? []).map((r) => r.name),
|
|
||||||
};
|
|
||||||
return `<pre class="output">${esc(JSON.stringify(sample, null, 2))}</pre>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Cluster state, when there is any to show.
|
|
||||||
*
|
|
||||||
* Fetched separately and allowed to fail: the bundle listing is the point, and a
|
|
||||||
* rig with no cluster reachable is a normal state, not an error. Renders nothing
|
|
||||||
* at all when absent. When the payload says `mocked`, say so loudly. */
|
|
||||||
function clusterSection(c) {
|
|
||||||
if (!c) return "";
|
|
||||||
const m = c.cluster ?? {};
|
|
||||||
const meta = [m.context, m.k8s, m.profile && `profile ${m.profile}`,
|
|
||||||
m.nodes && `${m.nodes} node${m.nodes > 1 ? "s" : ""}`]
|
|
||||||
.filter(Boolean).join(" · ");
|
|
||||||
|
|
||||||
return `
|
|
||||||
<h2>Cluster${c.mocked ? " (mocked)" : ""}</h2>
|
|
||||||
${c.mocked ? `<p class="mock">mocked — no cluster was queried; these are canned values</p>` : ""}
|
|
||||||
${meta ? `<p class="tagline">${esc(meta)}</p>` : ""}
|
|
||||||
<div class="components">${components(c.workloads)}</div>
|
|
||||||
|
|
||||||
<h2>Services</h2>
|
|
||||||
<div class="components">${components(c.services)}</div>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function render(b, name, cluster) {
|
|
||||||
const meta = b.bundle ?? {};
|
|
||||||
const next = (b.next ?? []).map((n) => `<li>${esc(n)}</li>`).join("");
|
|
||||||
return `
|
|
||||||
<h1><span class="ok">IT WORKS</span> — ${esc(name || meta.name || "rig")}</h1>
|
|
||||||
<p class="tagline">${esc(meta.description ?? "")}</p>
|
|
||||||
|
|
||||||
<h2>Tools (${b.tools?.length ?? 0})</h2>
|
|
||||||
<div class="components">${components(b.tools)}</div>
|
|
||||||
|
|
||||||
<h2>Rigs (${b.rigs?.length ?? 0})</h2>
|
|
||||||
<div class="components">${components(b.rigs, "active")}</div>
|
|
||||||
|
|
||||||
<h2>Endpoints</h2>
|
|
||||||
<ul class="endpoints">${endpoints([
|
|
||||||
{ path: "/", desc: "this page" },
|
|
||||||
{ path: "/bundle.json", desc: "the manifest it renders" },
|
|
||||||
])}</ul>
|
|
||||||
|
|
||||||
<h2>Example — GET /bundle.json</h2>
|
|
||||||
${example(b)}
|
|
||||||
|
|
||||||
${clusterSection(cluster)}
|
|
||||||
|
|
||||||
${next ? `<div class="next"><ul>${next}</ul></div>` : ""}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const app = document.getElementById("app");
|
|
||||||
|
|
||||||
const json = (path, required) =>
|
|
||||||
fetch(path)
|
|
||||||
.then((r) => {
|
|
||||||
if (r.ok) return r.json();
|
|
||||||
if (required) throw new Error(`${path} -> HTTP ${r.status}`);
|
|
||||||
return null; // optional: absent is a normal state, not an error
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
if (required) throw err;
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
|
|
||||||
Promise.all([json("/bundle.json", true), json("/cluster.mock.json", false)])
|
|
||||||
// RIG_NAME is injected by vite from the pod env, so two rigs sharing a
|
|
||||||
// cluster are distinguishable even if a copied bundle.json kept its old name.
|
|
||||||
.then(([b, cluster]) => {
|
|
||||||
app.innerHTML = render(b, import.meta.env.VITE_RIG_NAME, cluster);
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
app.innerHTML = `<h1 class="err">bundle unavailable</h1>
|
|
||||||
<p class="tagline">${esc(err.message)}</p>`;
|
|
||||||
});
|
|
||||||
@@ -1,139 +0,0 @@
|
|||||||
/* Minimal and self-contained — no framework dependency.
|
|
||||||
*
|
|
||||||
* The visual language follows soleprint's templated vein pages, because this
|
|
||||||
* page does the same job: say what a component is, list what it exposes, and
|
|
||||||
* show what comes back. Two treatments, deliberately distinct:
|
|
||||||
*
|
|
||||||
* TOOL CHROME bordered cards on the darker background, accent-coloured
|
|
||||||
* titles, endpoint rows separated by rules.
|
|
||||||
* OUTPUT a lighter raised block, monospace, pre-wrap and selectable —
|
|
||||||
* it is data, not furniture, and should read as a payload.
|
|
||||||
*
|
|
||||||
* Keeping them apart matters more than either looks: it is what tells you at a
|
|
||||||
* glance whether you are reading the tool or the thing it produced. */
|
|
||||||
|
|
||||||
:root {
|
|
||||||
--bg: #0d0d0f;
|
|
||||||
--surface: #16161a;
|
|
||||||
--surface-raised: #1e1e24;
|
|
||||||
--border: #2e2e38;
|
|
||||||
--border-strong: #3d3d4a;
|
|
||||||
--text: #e8e8f0;
|
|
||||||
--muted: #8888a0;
|
|
||||||
--accent: #3ecf8e;
|
|
||||||
--accent-dim: #f5a623;
|
|
||||||
--mono: "JetBrains Mono", "Cascadia Mono", Consolas, ui-monospace, monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
body {
|
|
||||||
margin: 0;
|
|
||||||
padding: 2.5rem 1.5rem;
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text);
|
|
||||||
font: 14px/1.6 var(--mono);
|
|
||||||
}
|
|
||||||
main { max-width: 56rem; margin: 0 auto; }
|
|
||||||
|
|
||||||
h1 { margin: 0; font-size: 1.6rem; letter-spacing: 0.02em; }
|
|
||||||
h1 .ok { color: var(--accent); }
|
|
||||||
h1.err { color: #f06565; }
|
|
||||||
.tagline { color: var(--muted); margin: 0.35rem 0 2.25rem; }
|
|
||||||
|
|
||||||
h2 {
|
|
||||||
font-size: 0.75rem;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.12em;
|
|
||||||
color: var(--muted);
|
|
||||||
margin: 2.25rem 0 0.75rem;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── tool chrome ────────────────────────────────────────────────────────── */
|
|
||||||
|
|
||||||
.components {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
||||||
gap: 0.75rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.component {
|
|
||||||
background: var(--bg);
|
|
||||||
border: 1px solid var(--border-strong);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 0.75rem;
|
|
||||||
}
|
|
||||||
.component h4 {
|
|
||||||
margin: 0 0 0.25rem;
|
|
||||||
font-size: 0.95rem;
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
.component p {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: var(--muted);
|
|
||||||
}
|
|
||||||
|
|
||||||
.endpoints { list-style: none; margin: 0; padding: 0; }
|
|
||||||
.endpoints li {
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 0.5rem;
|
|
||||||
padding: 0.6rem 0;
|
|
||||||
border-bottom: 1px solid var(--border-strong);
|
|
||||||
}
|
|
||||||
.endpoints li:last-child { border-bottom: none; }
|
|
||||||
.endpoints code {
|
|
||||||
background: var(--surface);
|
|
||||||
color: var(--accent);
|
|
||||||
padding: 0.25rem 0.5rem;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
.endpoints .desc { color: var(--muted); font-size: 0.9rem; }
|
|
||||||
|
|
||||||
.tag {
|
|
||||||
font-size: 0.7rem;
|
|
||||||
padding: 0.1rem 0.45rem;
|
|
||||||
border-radius: 3px;
|
|
||||||
background: var(--border);
|
|
||||||
color: var(--muted);
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
.tag.on { background: var(--accent); color: var(--bg); }
|
|
||||||
|
|
||||||
/* ── output ─────────────────────────────────────────────────────────────── */
|
|
||||||
|
|
||||||
.output {
|
|
||||||
background: var(--surface);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1rem;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
white-space: pre-wrap;
|
|
||||||
word-break: break-word;
|
|
||||||
user-select: text;
|
|
||||||
color: var(--text);
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
.output .k { color: var(--accent); }
|
|
||||||
|
|
||||||
/* Mocked-data banner. Loud on purpose: it only appears when the payload is
|
|
||||||
canned, and a demo that looks live but is not is worse than one that says so. */
|
|
||||||
.mock {
|
|
||||||
margin: 0 0 0.75rem;
|
|
||||||
padding: 0.4rem 0.75rem;
|
|
||||||
border: 1px dashed var(--accent-dim);
|
|
||||||
border-radius: 6px;
|
|
||||||
color: var(--accent-dim);
|
|
||||||
font-size: 0.8rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.next {
|
|
||||||
color: #555568;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
margin-top: 2.5rem;
|
|
||||||
border-top: 1px solid var(--border);
|
|
||||||
padding-top: 1rem;
|
|
||||||
}
|
|
||||||
.next ul { margin: 0; padding: 0 0 0 1.1rem; }
|
|
||||||
.next li { list-style: disc; padding: 0.15rem 0; }
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
import { defineConfig } from "vite";
|
|
||||||
|
|
||||||
/* Serves on 0.0.0.0 so the pod is reachable through the Service, and allows any
|
|
||||||
* Host header because the address is assigned at runtime (MetalLB locally, a
|
|
||||||
* cloud load balancer on EKS) and is never known at build time. */
|
|
||||||
export default defineConfig({
|
|
||||||
server: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true },
|
|
||||||
preview: { host: "0.0.0.0", port: 5173, strictPort: true, allowedHosts: true },
|
|
||||||
});
|
|
||||||
37
rig/standalone/README.md
Normal file
37
rig/standalone/README.md
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# standalone — single files for a machine the full rig is not going to
|
||||||
|
|
||||||
|
Each script here does one of rig's jobs without the rest of the tree. Copy one
|
||||||
|
file onto a machine, run it, read the output. Nothing to clone, nothing to
|
||||||
|
install first.
|
||||||
|
|
||||||
|
| file | does | full-rig equivalent |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `rigdeps.sh` | installs kind, kubectl, tilt, ctlptl and jq at rig's pins, checksum-verified, no sudo | `make deps` (`ctrl/deps.sh`) |
|
||||||
|
| `rigmini.sh` | reports how much memory the machine *advertises* and what caps it; `push` measures what it will actually *survive* | `make mem`, and the memory section of `make check` |
|
||||||
|
|
||||||
|
**These are transitional.** Where the full rig is installed, use its own
|
||||||
|
targets instead; they read `ctrl/versions.env` and the profile, which these
|
||||||
|
cannot.
|
||||||
|
|
||||||
|
## Why single files
|
||||||
|
|
||||||
|
`rigdeps.sh` carries its pins inline, because `ctrl/versions.env` is not on the
|
||||||
|
machine it is for. That makes two copies of the same versions and checksums.
|
||||||
|
`make pins` compares them and fails on any difference — `ctrl/versions.env` is
|
||||||
|
the source of truth.
|
||||||
|
|
||||||
|
`rigmini.sh` exists because on a container or managed workspace `/proc/meminfo`
|
||||||
|
reports the *host's* memory while a cgroup cap kills processes at a fraction of
|
||||||
|
it. `status` reads the caps; `push` allocates until something stops it.
|
||||||
|
|
||||||
|
## Use
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash rigdeps.sh detect # report, change nothing
|
||||||
|
bash rigdeps.sh install dev # install into ~/.local/bin
|
||||||
|
bash rigmini.sh status # advertised memory and caps; safe
|
||||||
|
bash rigmini.sh push # allocates until it stops — not on a machine you need
|
||||||
|
```
|
||||||
|
|
||||||
|
`rigmini.sh push` deliberately consumes memory. Run `status` first, and only run
|
||||||
|
`push` somewhere it is acceptable for other processes to be squeezed.
|
||||||
574
rig/standalone/rigdeps.sh
Executable file
574
rig/standalone/rigdeps.sh
Executable file
@@ -0,0 +1,574 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Put kind, tilt and kubectl on a machine that has none of them.
|
||||||
|
#
|
||||||
|
# The single file companion to rigmini.sh, for the same reason: rig installs its
|
||||||
|
# toolchain from ctrl/deps.sh reading ctrl/versions.env, and neither of those is
|
||||||
|
# going to a fresh AWS WorkSpace. The pins live inline here instead.
|
||||||
|
#
|
||||||
|
# What it will not do, deliberately:
|
||||||
|
#
|
||||||
|
# * no sudo, no apt, no yum. It writes ONLY into $OUT_BIN (default
|
||||||
|
# ~/.local/bin). Everything needing root — installing Docker, joining the
|
||||||
|
# docker group, raising inotify limits — is REPORTED for you to decide on.
|
||||||
|
# That is what makes it safe to run on a machine that already works.
|
||||||
|
# * no unverified download. Every artifact is checked against a SHA256 taken
|
||||||
|
# from the publisher's own release list. A mismatch aborts.
|
||||||
|
# * no guessing at another architecture. See ARCHITECTURE below.
|
||||||
|
#
|
||||||
|
# Two tiers, because "install the toolchain" is not one decision:
|
||||||
|
#
|
||||||
|
# core kubectl, jq — talk to a cluster someone else runs. Nothing that
|
||||||
|
# creates one. The right answer on a managed or corporate machine.
|
||||||
|
# dev core plus kind, tilt and ctlptl — build clusters and hot-reload
|
||||||
|
# into them. The default, and what you want on a workspace of your own.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# rigdeps.sh detect report the host, change nothing
|
||||||
|
# rigdeps.sh list the pinned versions and where they come from
|
||||||
|
# rigdeps.sh install [core|dev] detect, download, verify, install, report
|
||||||
|
# rigdeps.sh fetch [core|dev] [--to DIR] download + verify only
|
||||||
|
# rigdeps.sh verify run what is installed and see if it works
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
|
||||||
|
|
||||||
|
# ── the pinned toolchain ───────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# ARCHITECTURE. These checksums are the upstream-published SHA256 of the
|
||||||
|
# **linux/amd64** artifact and of nothing else. An arm64 WorkSpace bundle needs
|
||||||
|
# a different binary with a different checksum, and this script refuses rather
|
||||||
|
# than reusing these — a checksum that is merely plausible is worse than none,
|
||||||
|
# because it turns a verified download into a ceremony.
|
||||||
|
#
|
||||||
|
# To bump a version, or to add arm64: take the checksum from the release's own
|
||||||
|
# published list, never from a download you did.
|
||||||
|
#
|
||||||
|
# curl -sSL https://github.com/<org>/<repo>/releases/download/<tag>/checksums.txt
|
||||||
|
#
|
||||||
|
# kubectl publishes its own instead, at <KUBECTL_URL>.sha256.
|
||||||
|
|
||||||
|
KIND_VERSION=v0.32.0
|
||||||
|
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||||
|
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
|
||||||
|
|
||||||
|
KUBECTL_VERSION=v1.36.3
|
||||||
|
KUBECTL_SHA256=ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336
|
||||||
|
KUBECTL_URL="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
|
||||||
|
|
||||||
|
TILT_VERSION=0.37.6
|
||||||
|
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
||||||
|
TILT_URL="https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz"
|
||||||
|
|
||||||
|
# ctlptl creates a kind cluster WITH a local registry wired in, which is what
|
||||||
|
# keeps images off docker.io — an unqualified image name resolves to
|
||||||
|
# docker.io/library/<name>, and there is nothing structural stopping a push there.
|
||||||
|
CTLPTL_VERSION=0.9.4
|
||||||
|
CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e
|
||||||
|
CTLPTL_URL="https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz"
|
||||||
|
|
||||||
|
# Upstream's static build. Debian's jq is linked against libjq/libonig, which is
|
||||||
|
# fine on Debian and not portable anywhere else.
|
||||||
|
JQ_VERSION=1.8.2
|
||||||
|
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
||||||
|
JQ_URL="https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
|
||||||
|
|
||||||
|
CORE_TOOLS="kubectl jq"
|
||||||
|
DEV_TOOLS="kind tilt ctlptl"
|
||||||
|
|
||||||
|
# No helm: every rig addon installs with `kubectl apply -f`, so nothing has ever
|
||||||
|
# invoked it. Add it the day something actually needs a chart.
|
||||||
|
|
||||||
|
# Collected as we go, printed by report_manual() at the very end. Anything that
|
||||||
|
# needs root or a decision lands here instead of being done.
|
||||||
|
MANUAL=()
|
||||||
|
|
||||||
|
# ── platform ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
Linux) ;;
|
||||||
|
*) echo "$(uname -s) is not Linux. These are linux binaries; nothing here" >&2
|
||||||
|
echo "would run even if it downloaded." >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
arch() {
|
||||||
|
case "$(uname -m)" in
|
||||||
|
x86_64|amd64) echo amd64 ;;
|
||||||
|
aarch64|arm64) echo arm64 ;;
|
||||||
|
*) uname -m ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# The pins above are amd64. Rather than download something that cannot execute
|
||||||
|
# and let it fail as "cannot execute binary file: Exec format error", say so
|
||||||
|
# here and hand over the commands that produce the right checksums.
|
||||||
|
require_amd64() {
|
||||||
|
local a; a=$(arch)
|
||||||
|
[ "$a" = "amd64" ] && return 0
|
||||||
|
cat >&2 <<EOF
|
||||||
|
This machine is ${a} ($(uname -m)); every pin in this script is linux/amd64.
|
||||||
|
|
||||||
|
Nothing here would run, so it does not download. To make an ${a} version, the
|
||||||
|
URLs need the ${a} artifact and the checksums need to come from each project's
|
||||||
|
own published list — not from these values, and not from a download you did:
|
||||||
|
|
||||||
|
curl -sSL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${a}/kubectl.sha256
|
||||||
|
curl -sSL https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt
|
||||||
|
|
||||||
|
Edit the pinned block at the top of this file with what those print.
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
|
# ── the tools this script itself needs ─────────────────────────────────────
|
||||||
|
|
||||||
|
# A fresh minimal image may genuinely have neither curl nor wget. Find out once,
|
||||||
|
# up front, rather than half way through the first download.
|
||||||
|
DL=""
|
||||||
|
pick_downloader() {
|
||||||
|
if command -v curl >/dev/null 2>&1; then DL=curl
|
||||||
|
elif command -v wget >/dev/null 2>&1; then DL=wget
|
||||||
|
else
|
||||||
|
echo "neither curl nor wget is installed, so nothing can be downloaded." >&2
|
||||||
|
echo "Install one first: $(pkg_install_cmd curl)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
download() {
|
||||||
|
local url="$1" out="$2"
|
||||||
|
case "$DL" in
|
||||||
|
curl) curl -fsSL --retry 3 -o "$out" "$url" ;;
|
||||||
|
wget) wget -q --tries=3 -O "$out" "$url" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# sha256sum is coreutils; shasum is the perl one that turns up on stripped
|
||||||
|
# images. Verification is not optional, so if neither exists that is fatal.
|
||||||
|
SHA=""
|
||||||
|
pick_sha() {
|
||||||
|
if command -v sha256sum >/dev/null 2>&1; then SHA=sha256sum
|
||||||
|
elif command -v shasum >/dev/null 2>&1; then SHA="shasum -a 256"
|
||||||
|
else
|
||||||
|
echo "no sha256sum and no shasum — downloads could not be verified." >&2
|
||||||
|
echo "Refusing to install unverified binaries." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── package manager, for the instructions only ─────────────────────────────
|
||||||
|
# This never runs a package manager. It names one so the reported action is
|
||||||
|
# something you can paste, on the distro you are actually on — an apt line on
|
||||||
|
# Amazon Linux 2 is a wrong answer dressed up as help.
|
||||||
|
|
||||||
|
pkg_install_cmd() {
|
||||||
|
local pkg="$1"
|
||||||
|
if command -v apt-get >/dev/null 2>&1; then echo "sudo apt-get update && sudo apt-get install -y $pkg"
|
||||||
|
elif command -v dnf >/dev/null 2>&1; then echo "sudo dnf install -y $pkg"
|
||||||
|
elif command -v yum >/dev/null 2>&1; then echo "sudo yum install -y $pkg"
|
||||||
|
elif command -v zypper >/dev/null 2>&1; then echo "sudo zypper install -y $pkg"
|
||||||
|
elif command -v apk >/dev/null 2>&1; then echo "sudo apk add $pkg"
|
||||||
|
else echo "install '$pkg' with this system's package manager"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_pkg() {
|
||||||
|
# Debian and Ubuntu call it docker.io; the RPM distros call it docker.
|
||||||
|
if command -v apt-get >/dev/null 2>&1; then echo docker.io; else echo docker; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── detect ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
detect() {
|
||||||
|
echo "host"
|
||||||
|
echo " kernel $(uname -r)"
|
||||||
|
echo " arch $(arch) ($(uname -m))"
|
||||||
|
[ -r /etc/os-release ] && \
|
||||||
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||||
|
if is_wsl; then echo " platform WSL"; else echo " platform native linux"; fi
|
||||||
|
|
||||||
|
local total_kb avail_kb
|
||||||
|
total_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo)
|
||||||
|
avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
|
||||||
|
printf " memory %d GB total, %d GB available\n" \
|
||||||
|
$((total_kb / 1024 / 1024)) $((avail_kb / 1024 / 1024))
|
||||||
|
if [ $((avail_kb / 1024 / 1024)) -lt 4 ]; then
|
||||||
|
echo " ! under 4 GB available — a cluster will struggle here."
|
||||||
|
echo " rigmini.sh says how much this box will actually give you."
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " install to $OUT_BIN"
|
||||||
|
detect_libc
|
||||||
|
detect_prereqs
|
||||||
|
detect_docker
|
||||||
|
detect_inotify
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# tilt is the one binary here that needs a recent glibc. MEASURED, not guessed:
|
||||||
|
# tilt 0.37.6 on Amazon Linux 2 (glibc 2.26) fails with
|
||||||
|
#
|
||||||
|
# /lib64/libc.so.6: version `GLIBC_2.34' not found (required by .../tilt)
|
||||||
|
#
|
||||||
|
# which names a symbol rather than the problem. Amazon Linux 2 is a stock
|
||||||
|
# WorkSpaces bundle, so this is the likely case, not an exotic one. Report the
|
||||||
|
# version now; `verify` catches the actual failure after installing.
|
||||||
|
detect_libc() {
|
||||||
|
local v=""
|
||||||
|
if command -v ldd >/dev/null 2>&1; then
|
||||||
|
v=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$' || true)
|
||||||
|
fi
|
||||||
|
if [ -z "$v" ]; then
|
||||||
|
echo " libc unknown (no ldd) — 'verify' is the real test"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " libc glibc $v"
|
||||||
|
if [ "$(printf '%s\n2.34\n' "$v" | sort -V | head -1)" != "2.34" ]; then
|
||||||
|
echo " ! older than glibc 2.34, which tilt needs. kubectl, kind, jq and"
|
||||||
|
echo " ctlptl are static or libc-only and work here; tilt will not start."
|
||||||
|
echo " Install the core tier, or run tilt from a container."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# What this script needs to do its own job. Reported here so `detect` answers
|
||||||
|
# "will install work?" instead of leaving you to find out one download in.
|
||||||
|
# Amazon Linux 2 ships without tar, which is exactly the surprise this catches.
|
||||||
|
detect_prereqs() {
|
||||||
|
local missing=""
|
||||||
|
if command -v curl >/dev/null 2>&1; then echo " download curl"
|
||||||
|
elif command -v wget >/dev/null 2>&1; then echo " download wget"
|
||||||
|
else echo " ! no curl and no wget — nothing can be downloaded"; missing+=" curl"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1; then
|
||||||
|
echo " checksums ok"
|
||||||
|
else
|
||||||
|
echo " ! no sha256sum or shasum — downloads could not be verified"
|
||||||
|
missing+=" coreutils"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v tar >/dev/null 2>&1 && command -v gzip >/dev/null 2>&1; then
|
||||||
|
echo " archives tar + gzip"
|
||||||
|
else
|
||||||
|
echo " ! no tar/gzip — tilt and ctlptl ship as tarballs, so the dev tier"
|
||||||
|
echo " cannot be unpacked. The core tier is two bare binaries and is fine."
|
||||||
|
missing+=" tar gzip"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$missing" ]; then
|
||||||
|
MANUAL+=("Install what this script needs to run at all:
|
||||||
|
$(pkg_install_cmd "${missing# }")")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_docker() {
|
||||||
|
# kind builds a cluster out of containers. Without a reachable daemon,
|
||||||
|
# everything here installs perfectly and then does nothing.
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
if [ -S /var/run/docker.sock ]; then
|
||||||
|
echo " docker socket present, no cli"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " ! docker not installed — kind has nothing to build a cluster in"
|
||||||
|
MANUAL+=("Install Docker. It is the one real prerequisite, and the only
|
||||||
|
thing here that needs root:
|
||||||
|
$(pkg_install_cmd "$(docker_pkg)")
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
sudo usermod -aG docker \"\$USER\"
|
||||||
|
then log out and back in, so the new group applies to your shell.")
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if docker info >/dev/null 2>&1; then
|
||||||
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null)"
|
||||||
|
local n
|
||||||
|
n=$(docker ps --filter "label=io.x-k8s.kind.cluster" --format '{{.Names}}' 2>/dev/null | wc -l)
|
||||||
|
# Must be an `if`, not `[ ] && echo`: as the last statement here the
|
||||||
|
# latter returns 1 when the count is zero, and `set -e` kills the
|
||||||
|
# caller. That is the fresh-machine case, where it does most harm.
|
||||||
|
if [ "$n" -gt 0 ]; then
|
||||||
|
echo " - $n kind node container(s) already running"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ! docker cli present but the daemon is unreachable"
|
||||||
|
MANUAL+=("Start Docker, or add yourself to the docker group:
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
sudo usermod -aG docker \"\$USER\" # then log out and back in")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# kind and tilt both watch large trees. Distro defaults are far too low and the
|
||||||
|
# failure mode is silent: tilt simply stops noticing that files changed.
|
||||||
|
detect_inotify() {
|
||||||
|
local w i
|
||||||
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||||
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||||
|
echo " inotify watches=$w instances=$i"
|
||||||
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||||
|
echo " ! low — tilt will silently stop seeing file changes"
|
||||||
|
MANUAL+=("Raise the inotify limits (needs root):
|
||||||
|
echo -e 'fs.inotify.max_user_watches=524288\\nfs.inotify.max_user_instances=512' \\
|
||||||
|
| sudo tee /etc/sysctl.d/99-rig.conf
|
||||||
|
sudo sysctl --system")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── fetch ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
verify_sha() {
|
||||||
|
local file="$1" want="$2" name="$3" got
|
||||||
|
got=$($SHA "$file" | awk '{print $1}')
|
||||||
|
if [ "$got" != "$want" ]; then
|
||||||
|
echo >&2
|
||||||
|
echo "CHECKSUM MISMATCH for $name — not installing it." >&2
|
||||||
|
echo " expected $want" >&2
|
||||||
|
echo " got $got" >&2
|
||||||
|
echo >&2
|
||||||
|
echo "Either the pin in this script is stale, or what arrived is not what" >&2
|
||||||
|
echo "the publisher released. Neither is worth guessing about." >&2
|
||||||
|
rm -f "$file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# fetch_bin <name> <url> <sha256> <dest-dir> — a bare binary
|
||||||
|
fetch_bin() {
|
||||||
|
local name="$1" url="$2" sha="$3" dest="$4"
|
||||||
|
local tmp="$dest/.$name.tmp"
|
||||||
|
printf ' %-8s ' "$name"
|
||||||
|
download "$url" "$tmp"
|
||||||
|
verify_sha "$tmp" "$sha" "$name"
|
||||||
|
mv "$tmp" "$dest/$name"
|
||||||
|
chmod +x "$dest/$name"
|
||||||
|
echo "ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
# fetch_tgz <name> <url> <sha256> <dest-dir> <path-inside> <strip>
|
||||||
|
# Archive layouts differ, so the caller says which. tilt and ctlptl both ship
|
||||||
|
# the binary at the archive root, hence strip=0.
|
||||||
|
fetch_tgz() {
|
||||||
|
local name="$1" url="$2" sha="$3" dest="$4" inner="$5" strip="$6"
|
||||||
|
local tmp="$dest/.$name.tgz"
|
||||||
|
printf ' %-8s ' "$name"
|
||||||
|
download "$url" "$tmp"
|
||||||
|
verify_sha "$tmp" "$sha" "$name"
|
||||||
|
# --no-same-owner: some archives ship as uid 1001, and extracting as root
|
||||||
|
# would otherwise restore an owner that is not you.
|
||||||
|
tar -xzf "$tmp" -C "$dest" --strip-components="$strip" --no-same-owner "$inner"
|
||||||
|
rm -f "$tmp"
|
||||||
|
chmod +x "$dest/$name"
|
||||||
|
echo "ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch() {
|
||||||
|
local dest="$OUT_BIN" tier="dev"
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--to) dest="${2:?--to needs a directory}"; shift 2 ;;
|
||||||
|
core|dev) tier="$1"; shift ;;
|
||||||
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
mkdir -p "$dest"
|
||||||
|
|
||||||
|
if ! command -v tar >/dev/null 2>&1 && [ "$tier" = "dev" ]; then
|
||||||
|
echo "tar is missing, and tilt and ctlptl ship as tarballs." >&2
|
||||||
|
echo " $(pkg_install_cmd tar)" >&2
|
||||||
|
echo "Or install the core tier, which is two bare binaries: $0 install core" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "fetching '$tier' into $dest (verifying every checksum)"
|
||||||
|
fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"
|
||||||
|
fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"
|
||||||
|
if [ "$tier" = "dev" ]; then
|
||||||
|
fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"
|
||||||
|
fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0
|
||||||
|
fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── verify ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
tier_tools() { [ "$1" = "core" ] && echo "$CORE_TOOLS" || echo "$CORE_TOOLS $DEV_TOOLS"; }
|
||||||
|
|
||||||
|
# Downloading a verified binary proves it is the right file, not that this
|
||||||
|
# machine can run it. On an old distro tilt fails here, with a linker error
|
||||||
|
# about a missing symbol, and finding that out now beats finding out during a
|
||||||
|
# first cluster build.
|
||||||
|
verify_tools() {
|
||||||
|
local tier="${1:-dev}" b bin out rc broke=0
|
||||||
|
echo "checking that each one actually runs"
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
bin="$OUT_BIN/$b"
|
||||||
|
if [ ! -x "$bin" ]; then
|
||||||
|
printf ' %-8s not installed\n' "$b"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
rc=0
|
||||||
|
case "$b" in
|
||||||
|
kubectl) out=$("$bin" version --client 2>&1 | head -1) || rc=$? ;;
|
||||||
|
jq) out=$("$bin" --version 2>&1 | head -1) || rc=$? ;;
|
||||||
|
*) out=$("$bin" version 2>&1 | head -1) || rc=$? ;;
|
||||||
|
esac
|
||||||
|
if [ "$rc" -eq 0 ]; then
|
||||||
|
printf ' %-8s %s\n' "$b" "$out"
|
||||||
|
else
|
||||||
|
printf ' ! %-6s does not run here: %s\n' "$b" "$out"
|
||||||
|
broke=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$broke" -eq 1 ]; then
|
||||||
|
echo
|
||||||
|
echo " A binary that downloads and verifies but will not start is almost"
|
||||||
|
echo " always this distro's libc being older than the release needs."
|
||||||
|
echo " 'detect' prints the glibc version. The core tier (kubectl + jq)"
|
||||||
|
echo " has no such dependency and will work regardless."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── install ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Installing into a directory early in PATH silently replaces whatever the
|
||||||
|
# machine was already using, which on a shared or corporate machine can break
|
||||||
|
# unrelated work — kubectl more than one minor away from its cluster is the
|
||||||
|
# common one. Say so; never decide it.
|
||||||
|
warn_shadowing() {
|
||||||
|
local b existing shadowed="" tier="${1:-dev}"
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) return 0 ;; # not on PATH, so nothing is being shadowed yet
|
||||||
|
esac
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
[ -x "$OUT_BIN/$b" ] || continue
|
||||||
|
existing=$(PATH=$(echo "$PATH" | tr ':' '\n' | grep -vx "$OUT_BIN" | paste -sd:) \
|
||||||
|
command -v "$b" 2>/dev/null || true)
|
||||||
|
[ -n "$existing" ] || continue
|
||||||
|
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||||
|
shadowed+=" $b $existing"$'\n'
|
||||||
|
done
|
||||||
|
[ -n "$shadowed" ] || return 0
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo " ! these were already installed elsewhere and are now shadowed:"
|
||||||
|
printf '%s' "$shadowed"
|
||||||
|
MANUAL+=("Decide which toolchain wins. To keep the previous one:
|
||||||
|
rm -f $(for b in $(tier_tools "$tier"); do printf '%s ' "$OUT_BIN/$b"; done)
|
||||||
|
Or install somewhere private instead:
|
||||||
|
OUT_BIN=\$PWD/bin $0 install")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
report_manual() {
|
||||||
|
echo
|
||||||
|
if [ ${#MANUAL[@]} -eq 0 ]; then
|
||||||
|
echo "nothing left to do by hand."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "host actions this cannot perform (${#MANUAL[@]}):"
|
||||||
|
echo
|
||||||
|
local n=1 m
|
||||||
|
for m in "${MANUAL[@]}"; do
|
||||||
|
echo " $n. $m"
|
||||||
|
echo
|
||||||
|
n=$((n + 1))
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
install() {
|
||||||
|
local tier="${1:-dev}"
|
||||||
|
detect
|
||||||
|
echo
|
||||||
|
fetch "$tier"
|
||||||
|
echo
|
||||||
|
verify_tools "$tier"
|
||||||
|
warn_shadowing "$tier"
|
||||||
|
|
||||||
|
if [ "$tier" = "core" ]; then
|
||||||
|
echo
|
||||||
|
echo " core tier: no kind, tilt or ctlptl. '$0 install dev' adds them."
|
||||||
|
fi
|
||||||
|
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||||
|
export PATH=\"${OUT_BIN}:\$PATH\"
|
||||||
|
then: source ~/.bashrc") ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
report_manual
|
||||||
|
|
||||||
|
if [ "$tier" = "dev" ]; then
|
||||||
|
echo "Once Docker is reachable and this is on PATH:"
|
||||||
|
echo
|
||||||
|
echo " kind create cluster --name scratch"
|
||||||
|
echo " kubectl cluster-info --context kind-scratch"
|
||||||
|
echo " kind delete cluster --name scratch"
|
||||||
|
echo
|
||||||
|
echo "That round trip is the real test that this machine can host a rig."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
list() {
|
||||||
|
echo "pinned, linux/amd64 only:"
|
||||||
|
printf ' %-8s %s\n' kubectl "$KUBECTL_VERSION"
|
||||||
|
printf ' %-8s %s\n' jq "$JQ_VERSION"
|
||||||
|
printf ' %-8s %s\n' kind "$KIND_VERSION"
|
||||||
|
printf ' %-8s %s\n' tilt "$TILT_VERSION"
|
||||||
|
printf ' %-8s %s\n' ctlptl "$CTLPTL_VERSION"
|
||||||
|
echo
|
||||||
|
echo " core = $CORE_TOOLS"
|
||||||
|
echo " dev = $CORE_TOOLS $DEV_TOOLS"
|
||||||
|
echo
|
||||||
|
echo "Checksums are pinned in the block at the top of this file. To bump one,"
|
||||||
|
echo "take the new checksum from the publisher's own release list — the header"
|
||||||
|
echo "comment has the exact commands."
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── main ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
|
||||||
|
case "${1:-install}" in
|
||||||
|
detect) detect; report_manual ;;
|
||||||
|
list) list ;;
|
||||||
|
verify) verify_tools "${2:-dev}" ;;
|
||||||
|
fetch) shift; require_amd64; pick_downloader; pick_sha; fetch "$@" ;;
|
||||||
|
install) shift; require_amd64; pick_downloader; pick_sha; install "${1:-dev}" ;;
|
||||||
|
*) echo "usage: $0 [detect|list|install|fetch|verify]" >&2
|
||||||
|
echo " install [core|dev] (default dev)" >&2
|
||||||
|
echo " fetch [core|dev] [--to DIR]" >&2
|
||||||
|
echo " OUT_BIN=<dir> overrides the install directory" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
635
rig/standalone/rigmini.sh
Executable file
635
rig/standalone/rigmini.sh
Executable file
@@ -0,0 +1,635 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# How much memory this box will actually give you before something dies.
|
||||||
|
#
|
||||||
|
# rig answers this for a machine it is installed on. This is the single file
|
||||||
|
# version, for a machine rig is not going to: paste it onto a fresh AWS
|
||||||
|
# WorkSpace, an EC2 box or a container, run it, and get the same numbers in the
|
||||||
|
# same order so two machines can be read side by side.
|
||||||
|
#
|
||||||
|
# There are two numbers and they are rarely the same. `status` reports what the
|
||||||
|
# machine ADVERTISES and what is quietly capping it. `push` finds what it will
|
||||||
|
# SURVIVE, by allocating until it stops.
|
||||||
|
#
|
||||||
|
# The gap between them is the whole reason this exists. Under WSL the cap lives
|
||||||
|
# in .wslconfig; in a container or a managed workspace it is a cgroup limit, and
|
||||||
|
# there /proc/meminfo reports the HOST's memory while the kernel kills you at a
|
||||||
|
# fraction of it. A script that only read MemTotal would confidently report 32 GB
|
||||||
|
# on a box that OOMs at 2.
|
||||||
|
#
|
||||||
|
# Reports and instructs. It never raises a limit, frees anything, writes a
|
||||||
|
# config or installs a package — on a machine you are still evaluating, a probe
|
||||||
|
# that changes what it is measuring is worse than no probe.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# rigmini.sh status what it has, what caps it
|
||||||
|
# rigmini.sh push [--to GB] [--to-oom] climb until it stops
|
||||||
|
# rigmini.sh all [--budget GB] both, then the verdict
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ── defaults ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push().
|
||||||
|
STEP_EXPLICIT=no # whether --step was given, which turns the scaling off.
|
||||||
|
TO_MB="" # --to: stop here regardless. Empty means no hard cap.
|
||||||
|
TO_OOM=no # --to-oom: opt in to running until the kernel intervenes.
|
||||||
|
BUDGET_GB=6 # what the rig data profile is assumed to want; see all().
|
||||||
|
BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below.
|
||||||
|
|
||||||
|
# ── platform ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Everything below reads /proc. Without it there is nothing to measure, and
|
||||||
|
# failing here beats printing a page of empty fields.
|
||||||
|
if [ ! -r /proc/meminfo ]; then
|
||||||
|
echo "no readable /proc/meminfo — this needs a Linux kernel." >&2
|
||||||
|
echo "On macOS or a BSD none of the numbers below exist." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
|
is_container() {
|
||||||
|
[ -f /.dockerenv ] && return 0
|
||||||
|
grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
platform() {
|
||||||
|
if is_wsl; then echo WSL
|
||||||
|
elif is_container; then echo container
|
||||||
|
else echo "native linux"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── reading memory ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
||||||
|
|
||||||
|
# MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on
|
||||||
|
# corporate images — need the estimate it replaced, which is worse but not wrong.
|
||||||
|
avail_meminfo_mb() {
|
||||||
|
if grep -q '^MemAvailable:' /proc/meminfo; then
|
||||||
|
mb MemAvailable
|
||||||
|
else
|
||||||
|
awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Where a cgroup records this cgroup's own limit and usage. Set once by
|
||||||
|
# find_cgroup, because every later reading needs both and hunting for the files
|
||||||
|
# on each call would be the slow part of the poll loop.
|
||||||
|
CG_MAX_FILE=""
|
||||||
|
CG_CUR_FILE=""
|
||||||
|
CG_VERSION=""
|
||||||
|
|
||||||
|
find_cgroup() {
|
||||||
|
local rel
|
||||||
|
|
||||||
|
# Inside a container the cgroup namespace makes the top of the tree BE the
|
||||||
|
# container's own cgroup, so the unqualified path is already the right one.
|
||||||
|
# On a host it is the root cgroup, which is never limited — hence the second
|
||||||
|
# attempt via /proc/self/cgroup, which names the slice this shell is in.
|
||||||
|
if [ -r /sys/fs/cgroup/memory.max ]; then
|
||||||
|
CG_VERSION=v2
|
||||||
|
CG_MAX_FILE=/sys/fs/cgroup/memory.max
|
||||||
|
CG_CUR_FILE=/sys/fs/cgroup/memory.current
|
||||||
|
elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
|
||||||
|
CG_VERSION=v1
|
||||||
|
CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes
|
||||||
|
CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes
|
||||||
|
fi
|
||||||
|
|
||||||
|
rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||||
|
if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then
|
||||||
|
CG_VERSION=v2
|
||||||
|
CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max"
|
||||||
|
CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||||
|
if [ -n "$rel" ] && [ "$rel" != "/" ] \
|
||||||
|
&& [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then
|
||||||
|
CG_VERSION=v1
|
||||||
|
CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes"
|
||||||
|
CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes"
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# The cap in MB, or "" when there is none worth reporting. v2 spells unlimited
|
||||||
|
# "max"; v1 spells it as a number near 2^63, which is why this compares against
|
||||||
|
# MemTotal rather than testing for a magic value — a "limit" above the machine's
|
||||||
|
# own memory is not a limit, however it is written.
|
||||||
|
cgroup_cap_mb() {
|
||||||
|
local raw cap
|
||||||
|
[ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; }
|
||||||
|
raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max)
|
||||||
|
[ "$raw" = "max" ] && { echo ""; return 0; }
|
||||||
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
cap=$((raw / 1024 / 1024))
|
||||||
|
[ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; }
|
||||||
|
echo "$cap"
|
||||||
|
}
|
||||||
|
|
||||||
|
cgroup_used_mb() {
|
||||||
|
local raw
|
||||||
|
[ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; }
|
||||||
|
raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "")
|
||||||
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
echo $((raw / 1024 / 1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
# ulimit -v is a per-process address-space cap. It stops YOU long before the box
|
||||||
|
# does, and because it is inherited from a login shell it is easy to hit without
|
||||||
|
# knowing it is set.
|
||||||
|
ulimit_v_mb() {
|
||||||
|
local v; v=$(ulimit -v 2>/dev/null || echo unlimited)
|
||||||
|
[ "$v" = "unlimited" ] && { echo ""; return 0; }
|
||||||
|
case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
echo $((v / 1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
# The number everything else is about: the lowest of the things that can stop
|
||||||
|
# you. Printed at the end of `status` and used as the sanity bound in `push`.
|
||||||
|
effective_ceiling_mb() {
|
||||||
|
local c; c=$(mb MemTotal)
|
||||||
|
local cap; cap=$(cgroup_cap_mb)
|
||||||
|
local ul; ul=$(ulimit_v_mb)
|
||||||
|
[ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap"
|
||||||
|
[ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul"
|
||||||
|
echo "$c"
|
||||||
|
}
|
||||||
|
|
||||||
|
# How much room is left RIGHT NOW, from whichever accounting actually governs.
|
||||||
|
# In a capped container /proc/meminfo describes the host and is worse than
|
||||||
|
# useless for this — it would report tens of gigabytes free on a box that is one
|
||||||
|
# allocation from being killed.
|
||||||
|
headroom_mb() {
|
||||||
|
local cap used
|
||||||
|
cap=$(cgroup_cap_mb)
|
||||||
|
used=$(cgroup_used_mb)
|
||||||
|
if [ -n "$cap" ] && [ -n "$used" ]; then
|
||||||
|
echo $(( cap - used ))
|
||||||
|
else
|
||||||
|
avail_meminfo_mb
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── status ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
||||||
|
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
||||||
|
# Windows, then fall back to whichever profile actually owns a config.
|
||||||
|
wslconfig_path() {
|
||||||
|
local profile winpath found
|
||||||
|
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
||||||
|
case "$profile" in
|
||||||
|
""|*%*) ;;
|
||||||
|
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
||||||
|
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
||||||
|
echo "$winpath/.wslconfig"; return 0
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||||
|
[ -n "$found" ] && echo "$found"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
hogs() {
|
||||||
|
echo " holding the most:"
|
||||||
|
ps -eo rss,comm --sort=-rss 2>/dev/null \
|
||||||
|
| awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
status() {
|
||||||
|
local total avail swap_total swap_free cap ul cur
|
||||||
|
|
||||||
|
echo "host"
|
||||||
|
echo " platform $(platform)"
|
||||||
|
echo " kernel $(uname -r)"
|
||||||
|
[ -r /etc/os-release ] && \
|
||||||
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||||
|
echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)"
|
||||||
|
|
||||||
|
# ── the caps first, because they decide what the totals below are worth ──
|
||||||
|
echo
|
||||||
|
echo "caps"
|
||||||
|
cap=$(cgroup_cap_mb)
|
||||||
|
if [ -n "$cap" ]; then
|
||||||
|
cur=$(cgroup_used_mb)
|
||||||
|
echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)"
|
||||||
|
echo " ! /proc/meminfo below describes the HOST, not this cgroup."
|
||||||
|
echo " $(mb MemTotal) MB total is not yours; ${cap} MB is."
|
||||||
|
elif [ -n "$CG_VERSION" ]; then
|
||||||
|
echo " cgroup none (${CG_VERSION} present, no memory limit set)"
|
||||||
|
else
|
||||||
|
echo " cgroup no memory controller found"
|
||||||
|
fi
|
||||||
|
|
||||||
|
ul=$(ulimit_v_mb)
|
||||||
|
if [ -n "$ul" ]; then
|
||||||
|
echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell"
|
||||||
|
echo " it stops this process long before the machine runs out"
|
||||||
|
else
|
||||||
|
echo " ulimit -v unlimited"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# overcommit_memory=0 is the default heuristic: a large allocation is
|
||||||
|
# granted on a guess, and the reckoning arrives later as an OOM kill rather
|
||||||
|
# than as a failed malloc. It is why `push` touches every page it asks for.
|
||||||
|
local om or_
|
||||||
|
om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?')
|
||||||
|
or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?')
|
||||||
|
case "$om" in
|
||||||
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;;
|
||||||
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;;
|
||||||
|
2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;;
|
||||||
|
*) echo " overcommit ${om}" ;;
|
||||||
|
esac
|
||||||
|
[ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for"
|
||||||
|
|
||||||
|
# ── what it says it has ──
|
||||||
|
total=$(mb MemTotal); avail=$(avail_meminfo_mb)
|
||||||
|
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
||||||
|
echo
|
||||||
|
echo "memory"
|
||||||
|
echo " total ${total} MB"
|
||||||
|
echo " available ${avail} MB"
|
||||||
|
echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||||
|
if [ "$swap_total" -eq 0 ]; then
|
||||||
|
echo " - no swap: this box has no cushion. It goes from fine to OOM-killed"
|
||||||
|
echo " with nothing in between, which is the abrupt failure you get in a VM."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB,
|
||||||
|
# and the resulting failure names neither shm nor the size.
|
||||||
|
if [ -d /dev/shm ]; then
|
||||||
|
local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}')
|
||||||
|
if [ -n "$shm" ]; then
|
||||||
|
if [ "$shm" -le 64 ]; then
|
||||||
|
echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB"
|
||||||
|
echo " is docker's default. Raise it with --shm-size when the cabinet fails."
|
||||||
|
else
|
||||||
|
echo " /dev/shm ${shm} MB"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "disk"
|
||||||
|
local d
|
||||||
|
for d in / /tmp /var/lib/docker; do
|
||||||
|
[ -d "$d" ] || continue
|
||||||
|
df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}'
|
||||||
|
done
|
||||||
|
|
||||||
|
# kind and Tilt both watch large trees, and the failure mode is silent:
|
||||||
|
# they simply stop noticing file changes. Cheap to report while we are here.
|
||||||
|
local w i
|
||||||
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||||
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||||
|
echo
|
||||||
|
echo "tooling"
|
||||||
|
echo " inotify watches=$w instances=$i"
|
||||||
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||||
|
echo " ! low — anything watching files will silently stop seeing changes"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
if [ -S /var/run/docker.sock ]; then
|
||||||
|
echo " docker socket present, no cli"
|
||||||
|
else
|
||||||
|
echo " docker not installed"
|
||||||
|
fi
|
||||||
|
elif docker info >/dev/null 2>&1; then
|
||||||
|
local n
|
||||||
|
n=$(docker ps -q 2>/dev/null | wc -l)
|
||||||
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running"
|
||||||
|
else
|
||||||
|
echo " ! docker cli present but the daemon is unreachable"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# WSL keeps its cap on the Windows side, in a file this shell can read but
|
||||||
|
# not usefully apply — the change costs a full VM restart. Report it, and
|
||||||
|
# report the commonest mistake, which is editing it and not restarting.
|
||||||
|
if is_wsl; then
|
||||||
|
local cfg conf
|
||||||
|
cfg=$(wslconfig_path)
|
||||||
|
echo
|
||||||
|
echo "wsl"
|
||||||
|
if [ -z "$cfg" ]; then
|
||||||
|
echo " ! cannot tell which Windows profile owns .wslconfig"
|
||||||
|
else
|
||||||
|
echo " config $cfg"
|
||||||
|
conf=$(sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$cfg" 2>/dev/null \
|
||||||
|
| tail -1 | tr -d '[:space:]')
|
||||||
|
if [ -n "$conf" ]; then
|
||||||
|
echo " configured $conf (booted ${total} MB)"
|
||||||
|
echo " - if those disagree the edit has not been applied."
|
||||||
|
echo " From a WINDOWS terminal: wsl --shutdown"
|
||||||
|
else
|
||||||
|
echo " configured no memory= set (WSL defaults to half the host RAM, or 8 GB,"
|
||||||
|
echo " whichever is less — which is where your Airflow ceiling comes from)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "effective ceiling $(effective_ceiling_mb) MB"
|
||||||
|
echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box"
|
||||||
|
echo " claims. 'push' measures what it will actually hand over."
|
||||||
|
|
||||||
|
[ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; }
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── push ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
STATE=""
|
||||||
|
CHILD=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then
|
||||||
|
kill -KILL "$CHILD" 2>/dev/null || true
|
||||||
|
wait "$CHILD" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
[ -n "$STATE" ] && rm -f "$STATE"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# The child allocates and stops itself; the parent only watches. That split is
|
||||||
|
# the point: under --to-oom the allocating process is expected to be killed, and
|
||||||
|
# something has to survive to say how far it got.
|
||||||
|
allocator() {
|
||||||
|
# Raise our own OOM score to the maximum so the kernel picks THIS process
|
||||||
|
# first. Raising needs no privilege (only lowering does). Without it, the
|
||||||
|
# kernel is free to choose your shell, your ssh session or dockerd — on a
|
||||||
|
# box you are still using, that is not an acceptable coin toss.
|
||||||
|
echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true
|
||||||
|
|
||||||
|
local arr=() held=0 i=0 rss swapped avail first_swap=0
|
||||||
|
local bytes=$((STEP_MB * 1024 * 1024))
|
||||||
|
local swap_used_start
|
||||||
|
swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) ))
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
# Written STRAIGHT INTO the array element. The obvious spelling —
|
||||||
|
# build one chunk and `arr+=("$chunk")` — costs three copies per step,
|
||||||
|
# not one: the template stays resident, expanding "$chunk" makes a
|
||||||
|
# temporary word, and the append makes the element. A 128 MB step then
|
||||||
|
# needs 384 MB transiently, and on a small box it is killed on the
|
||||||
|
# first append while reporting a third of the true ceiling.
|
||||||
|
#
|
||||||
|
# printf -v into a subscript also means every page is written, so it is
|
||||||
|
# resident rather than merely promised — the only kind of allocation
|
||||||
|
# that measures anything under heuristic overcommit.
|
||||||
|
printf -v "arr[$i]" '%*s' "$bytes" ''
|
||||||
|
i=$((i + 1)); held=$((held + STEP_MB))
|
||||||
|
|
||||||
|
rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0)
|
||||||
|
avail=$(headroom_mb)
|
||||||
|
swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start ))
|
||||||
|
[ "$swapped" -lt 0 ] && swapped=0
|
||||||
|
|
||||||
|
printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \
|
||||||
|
"$held" "$rss" "$avail" "$swapped"
|
||||||
|
printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE"
|
||||||
|
|
||||||
|
# Worth calling out separately from the ceiling: this is where the box
|
||||||
|
# stops being fast and starts being unusable, which for a scheduler is
|
||||||
|
# a different and earlier problem than being killed.
|
||||||
|
if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then
|
||||||
|
first_swap=$held
|
||||||
|
echo " - first swap page at ${held} MB — past here it works but crawls"
|
||||||
|
echo "swapat $held" >> "$STATE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then
|
||||||
|
echo "stop reached-the-cap" >> "$STATE"; return 0
|
||||||
|
fi
|
||||||
|
if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then
|
||||||
|
echo "stop floor" >> "$STATE"; return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
push() {
|
||||||
|
local total ceiling rc=0 last held rss swapat stop
|
||||||
|
total=$(mb MemTotal)
|
||||||
|
ceiling=$(effective_ceiling_mb)
|
||||||
|
|
||||||
|
# A step is worth about a sixty-fourth of the ceiling: enough resolution to
|
||||||
|
# find the edge, few enough lines to read, and small enough that the
|
||||||
|
# transient cost of one allocation never dominates a small box. A fixed
|
||||||
|
# size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB.
|
||||||
|
if [ "$STEP_EXPLICIT" = no ]; then
|
||||||
|
STEP_MB=$(( ceiling / 64 ))
|
||||||
|
[ "$STEP_MB" -lt 4 ] && STEP_MB=4
|
||||||
|
[ "$STEP_MB" -gt 256 ] && STEP_MB=256
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Stop with a cushion rather than riding it to the kill. How big a cushion
|
||||||
|
# depends on what it is protecting. Under a cgroup cap, running out kills
|
||||||
|
# only this container's own processes, so it need cover no more than the
|
||||||
|
# shell that prints the result — and a 512 MB cushion on a 1 GB box would
|
||||||
|
# halve the answer. On a host there is everything else to protect, and the
|
||||||
|
# OOM killer does not promise to pick the process that caused the problem.
|
||||||
|
if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi
|
||||||
|
[ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 ))
|
||||||
|
|
||||||
|
STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX")
|
||||||
|
trap cleanup EXIT
|
||||||
|
# INT kills the child and lets the summary below print anyway, so an
|
||||||
|
# impatient Ctrl-C still tells you how far it got — and, more importantly,
|
||||||
|
# still gives the memory back.
|
||||||
|
trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT
|
||||||
|
|
||||||
|
echo "push"
|
||||||
|
echo " step ${STEP_MB} MB per allocation, every page touched"
|
||||||
|
echo " ceiling ${ceiling} MB claimed"
|
||||||
|
if [ -n "$TO_MB" ]; then
|
||||||
|
echo " stopping at ${TO_MB} MB (--to)"
|
||||||
|
elif [ "$TO_OOM" = yes ]; then
|
||||||
|
echo " ! stopping only when the kernel stops it (--to-oom)"
|
||||||
|
echo " the allocating child is marked as the preferred OOM victim,"
|
||||||
|
echo " but nothing about an OOM kill is entirely polite. Not on a box"
|
||||||
|
echo " running anything you mind losing."
|
||||||
|
else
|
||||||
|
echo " stopping when headroom drops below ${FLOOR_MB} MB"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
|
allocator &
|
||||||
|
CHILD=$!
|
||||||
|
wait "$CHILD" || rc=$?
|
||||||
|
CHILD=""
|
||||||
|
trap - INT
|
||||||
|
|
||||||
|
last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true)
|
||||||
|
held=$(echo "$last" | awk '{print $1}')
|
||||||
|
rss=$(echo "$last" | awk '{print $2}')
|
||||||
|
swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||||
|
stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ -z "$held" ]; then
|
||||||
|
echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —"
|
||||||
|
echo " try a smaller --step, or check ulimit -v in 'status'."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " reached ${rss:-$held} MB resident"
|
||||||
|
[ -n "$swapat" ] && echo " swapping from ${swapat} MB"
|
||||||
|
|
||||||
|
case "$stop" in
|
||||||
|
reached-the-cap)
|
||||||
|
echo " outcome stopped at the --to cap, not at a limit."
|
||||||
|
echo " The box held ${TO_MB} MB without complaint; there is more." ;;
|
||||||
|
floor)
|
||||||
|
echo " outcome stopped with a cushion intact, by choice."
|
||||||
|
echo " The real ceiling is higher — --to-oom finds it, at the"
|
||||||
|
echo " cost of an actual OOM kill." ;;
|
||||||
|
interrupted)
|
||||||
|
echo " outcome interrupted at ${rss:-$held} MB — where you stopped it,"
|
||||||
|
echo " not where the box did." ;;
|
||||||
|
*)
|
||||||
|
# No stop line means the child did not decide to stop: it was ended.
|
||||||
|
if [ "$rc" -ge 128 ]; then
|
||||||
|
echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB."
|
||||||
|
elif [ "$rc" -ne 0 ]; then
|
||||||
|
echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})."
|
||||||
|
echo " bash could not get the next chunk — an honest malloc"
|
||||||
|
echo " failure rather than a kill. That is the strict-overcommit"
|
||||||
|
echo " or ulimit path."
|
||||||
|
else
|
||||||
|
echo " outcome ended at ${rss:-$held} MB."
|
||||||
|
fi
|
||||||
|
local ev
|
||||||
|
ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true)
|
||||||
|
if [ -n "$ev" ]; then
|
||||||
|
echo " kernel ${ev#*] }"
|
||||||
|
else
|
||||||
|
echo " - dmesg is unreadable here (dmesg_restrict, or no privilege),"
|
||||||
|
echo " so the kill cannot be confirmed from this side. The number stands."
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# The gap between the claim and the measurement is the finding — but only
|
||||||
|
# when the BOX chose where to stop. An empty $stop means the child was ended
|
||||||
|
# rather than deciding to end; anything else (--to, the floor) is a stop we
|
||||||
|
# asked for, and flagging those as short of the ceiling would put a warning
|
||||||
|
# on every deliberately small run.
|
||||||
|
local got="${rss:-$held}"
|
||||||
|
echo
|
||||||
|
if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then
|
||||||
|
echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it."
|
||||||
|
echo " Something is taking the difference. 'status' names the candidates:"
|
||||||
|
echo " a cgroup cap, ulimit -v, or memory already resident."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── all ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
all() {
|
||||||
|
status
|
||||||
|
echo
|
||||||
|
echo "────────────────────────────────────────────────────────────"
|
||||||
|
echo
|
||||||
|
push
|
||||||
|
|
||||||
|
local got budget_mb ceiling
|
||||||
|
budget_mb=$(( BUDGET_GB * 1024 ))
|
||||||
|
ceiling=$(effective_ceiling_mb)
|
||||||
|
got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true)
|
||||||
|
[ -n "$got" ] || got=0
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "verdict"
|
||||||
|
echo " budget ${BUDGET_GB} GB for kind + postgres + redis + airflow"
|
||||||
|
# Only worth explaining while it is still a guess. Once --budget is given
|
||||||
|
# the number came from somewhere better than this reasoning, and repeating
|
||||||
|
# the derivation would describe a figure that is no longer in use.
|
||||||
|
if [ "$BUDGET_EXPLICIT" = no ]; then
|
||||||
|
echo " - that is 2 GB per kind node, which is rig's own figure, plus about"
|
||||||
|
echo " 4 GB for the three cabinets. THE 4 GB IS AN ESTIMATE, not something"
|
||||||
|
echo " measured. Re-run with --budget once you have watched the real thing."
|
||||||
|
fi
|
||||||
|
echo " measured ${got} MB handed over"
|
||||||
|
|
||||||
|
if [ "$got" -ge "$budget_mb" ]; then
|
||||||
|
echo " fits, with $(( got - budget_mb )) MB spare."
|
||||||
|
if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then
|
||||||
|
echo " - under 30% spare is thin for a scheduler. Airflow's memory use"
|
||||||
|
echo " is spiky, and the spikes are what get killed."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ! short by $(( budget_mb - got )) MB."
|
||||||
|
if [ "$ceiling" -ge "$budget_mb" ]; then
|
||||||
|
echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it."
|
||||||
|
echo " Free something, or read the caps section again."
|
||||||
|
else
|
||||||
|
echo " The box does not have it to give. A bigger bundle, or a smaller"
|
||||||
|
echo " profile: PROFILE=minimal drops the cabinets entirely."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── main ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
parse_flags() {
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;;
|
||||||
|
--to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;;
|
||||||
|
--step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;;
|
||||||
|
--to-oom) TO_OOM=yes; shift ;;
|
||||||
|
--budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;;
|
||||||
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then
|
||||||
|
echo "--to and --to-oom contradict each other: one stops early, the other" >&2
|
||||||
|
echo "refuses to stop at all. Pick one." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
find_cgroup
|
||||||
|
|
||||||
|
cmd="${1:-status}"
|
||||||
|
[ $# -gt 0 ] && shift
|
||||||
|
|
||||||
|
case "$cmd" in
|
||||||
|
status) parse_flags "$@"; status ;;
|
||||||
|
push) parse_flags "$@"; push ;;
|
||||||
|
all) parse_flags "$@"; all ;;
|
||||||
|
*) echo "usage: $0 [status|push|all]" >&2
|
||||||
|
echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2
|
||||||
|
echo " all [--budget GB]" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Render per-room manifests for deploy into the shared `spr` kind cluster.
|
"""Render per-room manifests for deploy into the shared `spr` kind cluster.
|
||||||
|
|
||||||
The `spr` cluster itself is created via `ctrl/kind-up.sh` at the repo root
|
The `spr` cluster itself is created by `make cluster up` at the repo root,
|
||||||
(one cluster, all rooms). Each room becomes a namespace inside it.
|
which hands spr's shape to rig to build (one cluster, all rooms). Each room becomes a namespace inside it.
|
||||||
|
|
||||||
Called from build.py when a room opts in to k8s output. Emits:
|
Called from build.py when a room opts in to k8s output. Emits:
|
||||||
|
|
||||||
|
|||||||
@@ -395,21 +395,21 @@ resources:
|
|||||||
|
|
||||||
|
|
||||||
# ─── Lifecycle scripts ──────────────────────────────────────────────
|
# ─── Lifecycle scripts ──────────────────────────────────────────────
|
||||||
# These target the shared `spr` kind cluster (created via repo-root
|
# These target the shared `spr` kind cluster (created by `make cluster up`
|
||||||
# ctrl/kind-up.sh). Each room owns a namespace inside that cluster.
|
# at the repo root, which builds it with rig). Each room owns a namespace inside that cluster.
|
||||||
|
|
||||||
def k8s_up_sh(*, room: str, cluster: str, nodeport: int) -> str:
|
def k8s_up_sh(*, room: str, cluster: str, nodeport: int) -> str:
|
||||||
return f"""\
|
return f"""\
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Apply the "{room}" room into the shared `{cluster}` kind cluster.
|
# Apply the "{room}" room into the shared `{cluster}` kind cluster.
|
||||||
# (Run repo-root ctrl/kind-up.sh first if the cluster doesn't exist.)
|
# (Run 'make cluster up' at the repo root first if the cluster doesn't exist.)
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)"
|
||||||
K8S_DIR="$SCRIPT_DIR/k8s"
|
K8S_DIR="$SCRIPT_DIR/k8s"
|
||||||
|
|
||||||
if ! kind get clusters 2>/dev/null | grep -q '^{cluster}$'; then
|
if ! kind get clusters 2>/dev/null | grep -q '^{cluster}$'; then
|
||||||
echo "Kind cluster '{cluster}' not found — run ctrl/kind-up.sh from the repo root first."
|
echo "Kind cluster '{cluster}' not found — run 'make cluster up' from the repo root first."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -432,7 +432,7 @@ def k8s_down_sh(*, room: str, cluster: str) -> str:
|
|||||||
return f"""\
|
return f"""\
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Remove the "{room}" namespace from the shared `{cluster}` cluster.
|
# Remove the "{room}" namespace from the shared `{cluster}` cluster.
|
||||||
# Leaves the cluster itself running (use repo-root ctrl/kind-down.sh to drop everything).
|
# Leaves the cluster itself running (use 'make cluster down' at the repo root to drop everything).
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
CTX="kind-{cluster}"
|
CTX="kind-{cluster}"
|
||||||
|
|||||||
@@ -577,7 +577,7 @@ def station_index(request: Request):
|
|||||||
# not the server. Route registration raises more than ImportError — FastAPI
|
# not the server. Route registration raises more than ImportError — FastAPI
|
||||||
# turns a missing optional dependency into a RuntimeError at decoration time —
|
# turns a missing optional dependency into a RuntimeError at decoration time —
|
||||||
# and catching only ImportError meant one such tool took the whole app down.
|
# and catching only ImportError meant one such tool took the whole app down.
|
||||||
for _tool in ("tester", "graphgen", "datagen", "shuntgen"):
|
for _tool in ("tester", "graphgen", "datagen", "shuntgen", "histgen"):
|
||||||
try:
|
try:
|
||||||
_module = importlib.import_module(f"station.tools.{_tool}.api")
|
_module = importlib.import_module(f"station.tools.{_tool}.api")
|
||||||
app.include_router(_module.router, prefix="/station")
|
app.include_router(_module.router, prefix="/station")
|
||||||
|
|||||||
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# Local job: which repos this machine distills is not a fact about the tool, and
|
||||||
|
# the list names whatever those repos are. Copy distill-example.json to
|
||||||
|
# distill.json and edit that; it stays here.
|
||||||
|
distill.json
|
||||||
26
soleprint/station/tools/distill/distill-example.json
Normal file
26
soleprint/station/tools/distill/distill-example.json
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that. Everything that shapes the run is at the top; 'repos' is just a list of paths. Preview any change with: ./distill.sh list",
|
||||||
|
|
||||||
|
"_command": "tree = a directory per repo. digest = one .md per repo, flattened into a single readable file. both = each repo as a directory AND a .md. list = write nothing, just report what would be kept.",
|
||||||
|
"command": "list",
|
||||||
|
|
||||||
|
"out": "distilled",
|
||||||
|
|
||||||
|
"_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base (it gets a _PARTIAL.md saying so).",
|
||||||
|
"branch_mode": "full",
|
||||||
|
"diff_base": "main",
|
||||||
|
|
||||||
|
"_filters": "Applied to every repo. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, images, minified, maps). max_bytes skips anything larger and names it in MANIFEST.md.",
|
||||||
|
"exclude": [],
|
||||||
|
"include": [],
|
||||||
|
"all": false,
|
||||||
|
"max_bytes": null,
|
||||||
|
"skip_unchanged": true,
|
||||||
|
"prune": true,
|
||||||
|
|
||||||
|
"_repos": "'path' is absolute, or a name resolved under --root. 'branches' is optional — leave it out for the working tree as it stands, uncommitted changes included. A path may appear more than once.",
|
||||||
|
"repos": [
|
||||||
|
{ "path": "/path/to/some-repo" },
|
||||||
|
{ "path": "/path/to/another-repo", "branches": ["origin/main", "origin/feature/example"] }
|
||||||
|
]
|
||||||
|
}
|
||||||
1639
soleprint/station/tools/distill/distill.sh
Executable file
1639
soleprint/station/tools/distill/distill.sh
Executable file
File diff suppressed because it is too large
Load Diff
75
soleprint/station/tools/distill/explode.md
Normal file
75
soleprint/station/tools/distill/explode.md
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
# explode — one file back into the tree it describes
|
||||||
|
|
||||||
|
The other half of `distill.sh`. Together they are a round trip:
|
||||||
|
|
||||||
|
```
|
||||||
|
distill.sh digest -> one file -> paste into a chat -> the reply -> explode.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
`distill` flattens repos into a single readable file so they fit somewhere that
|
||||||
|
only takes text. `explode` takes the answer and writes it back onto disk. Neither
|
||||||
|
is much use without the other.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./explode.sh --list reply.md # what is in there; writes nothing
|
||||||
|
./explode.sh -o ./restored reply.md # write the tree
|
||||||
|
./explode.sh -o ./restored --force x.md # overwrite what is already there
|
||||||
|
./explode.sh --contract > contract.txt # the format to hand to the model
|
||||||
|
./explode.sh --selftest # check this copy against known input
|
||||||
|
```
|
||||||
|
|
||||||
|
## Ask for `@@`, and attach it
|
||||||
|
|
||||||
|
`--contract` prints the output format to give whatever writes the reply. **Attach
|
||||||
|
that file; do not paste it into the message.** A chat box renders markdown before
|
||||||
|
the model sees it, and `===` alone under a line of text is setext syntax for a
|
||||||
|
heading — so a pasted spec gets rendered as a title and the model is told nothing.
|
||||||
|
`---` is worse, `##` is a heading, backticks open a fence. `@@` means nothing in
|
||||||
|
markdown, which is exactly why it is the marker to ask for.
|
||||||
|
|
||||||
|
Keeping the wording in `--contract` rather than in a note somewhere means what you
|
||||||
|
ask for cannot drift from what the parser accepts.
|
||||||
|
|
||||||
|
## Layouts
|
||||||
|
|
||||||
|
Four shapes are recognised, picked automatically; `--format` overrides the guess.
|
||||||
|
|
||||||
|
| shape | when |
|
||||||
|
| --- | --- |
|
||||||
|
| `@@ FILE: path` … `@@ END` | **ask for this** — explicit, and invisible to markdown |
|
||||||
|
| `=== FILE: path` … `=== END` | the same thing, still read; do not ask for it |
|
||||||
|
| `=== path` marker | a marker line, then the file until the next one |
|
||||||
|
| `## path` + fenced block | `distill.sh`'s own digest |
|
||||||
|
|
||||||
|
Explicit open and close is worth insisting on: a writer emitting plain three-backtick
|
||||||
|
fences silently truncates any file that itself contains a fence — every README with a
|
||||||
|
shell example — because the nested fence looks exactly like the closing one.
|
||||||
|
|
||||||
|
## One reply, several projects
|
||||||
|
|
||||||
|
A thread usually touches more than one repo, and produces one file regardless. The
|
||||||
|
contract asks for paths that begin with the project name, so point `-o` at the
|
||||||
|
directory those projects sit in and each file lands in its own worktree:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./explode.sh -o ~/wdir ~/Downloads/reply.md
|
||||||
|
```
|
||||||
|
|
||||||
|
No mapping table to maintain, and a new project needs no change here.
|
||||||
|
|
||||||
|
## Refusals
|
||||||
|
|
||||||
|
Paths come out of a text file, so they are untrusted. Anything absolute or reaching
|
||||||
|
upward with `..` is refused and **nothing** is written — the check runs over the whole
|
||||||
|
input before the first file is created. An unclosed block is refused too, rather than
|
||||||
|
writing the file short. Existing files are never overwritten without `--force`.
|
||||||
|
|
||||||
|
A digest whose files `distill` **clipped** to fit a token budget is refused for the
|
||||||
|
same reason: the document holds only their head and tail, and a truncated file that
|
||||||
|
reads complete is the failure the whole format exists to prevent. The tree copy beside
|
||||||
|
the digest has them whole — take them from there.
|
||||||
|
|
||||||
|
Two limits worth knowing: a file whose last line had no trailing newline comes back
|
||||||
|
with one, and in the bare `=== path` layout a line starting with `=== ` inside a file's
|
||||||
|
own content cannot be told from a real marker. The digest and `@@` layouts have no such
|
||||||
|
ambiguity.
|
||||||
466
soleprint/station/tools/distill/explode.sh
Executable file
466
soleprint/station/tools/distill/explode.sh
Executable file
@@ -0,0 +1,466 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Explode one file back into the tree of files it describes.
|
||||||
|
#
|
||||||
|
# The inverse of distill.sh's digest: something hands you a single text
|
||||||
|
# file with many files inside it, each introduced by its path, and you want the
|
||||||
|
# directory back.
|
||||||
|
#
|
||||||
|
# Three layouts are understood, picked automatically. Prefer the first if you
|
||||||
|
# control what writes the file:
|
||||||
|
#
|
||||||
|
# @@ FILE: pkg/models/domain.py explicit open and close. Nothing has to be
|
||||||
|
# <the file> counted or inferred, and a block that is
|
||||||
|
# @@ END never closed is an error rather than a
|
||||||
|
# file quietly missing its tail.
|
||||||
|
#
|
||||||
|
# === FILE: pkg/models/domain.py the same thing with '===' instead of '@@'.
|
||||||
|
# <the file> Still read, but do not ask for it: see the
|
||||||
|
# === END note on markdown below.
|
||||||
|
#
|
||||||
|
# === ./pkg/models/domain.py a marker line, then the file, until the
|
||||||
|
# <the file> next marker or the end
|
||||||
|
#
|
||||||
|
# ## pkg/models/domain.py distill.sh's own digest: a heading, then
|
||||||
|
# ```python a fenced block. The fence may be longer
|
||||||
|
# <the file> than three backticks, and the closing one
|
||||||
|
# ``` has to match it exactly.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# explode.sh [opts] FILE
|
||||||
|
#
|
||||||
|
# Options:
|
||||||
|
# -o DEST where to write the tree (default: the current directory)
|
||||||
|
# --list print what the file contains and write nothing
|
||||||
|
# -n same as --list
|
||||||
|
# --force overwrite files that already exist
|
||||||
|
# --format F fenced | marker | digest | auto (default: auto)
|
||||||
|
# --contract print the output format to hand to whatever generates the file
|
||||||
|
# --selftest check this copy of the script against known input and exit
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
# explode.sh --list bundle.txt
|
||||||
|
# explode.sh -o ./restored bundle.txt
|
||||||
|
# explode.sh -o ./restored --force repo.md
|
||||||
|
# explode.sh --contract > /tmp/contract.txt # attach this, do not paste it
|
||||||
|
# explode.sh -o ~/wdir ~/Downloads/reply.md # one reply, every project
|
||||||
|
#
|
||||||
|
# Why the explicit form is worth asking for: a writer that emits plain three-
|
||||||
|
# backtick fences truncates any file that itself contains a fence — every README
|
||||||
|
# with a shell example — and does it silently, because the nested fence looks
|
||||||
|
# exactly like the closing one. distill.sh avoids that by making its fences
|
||||||
|
# longer than anything inside the file, but nothing else will bother.
|
||||||
|
#
|
||||||
|
# Ask for '@@', not '==='. A chat box renders markdown before the model sees the
|
||||||
|
# message, and '===' alone on a line directly under text is setext syntax for a
|
||||||
|
# level-one heading — so the format spec you paste gets swallowed and rendered as
|
||||||
|
# a title, and the model is told nothing. '---' is worse (heading AND horizontal
|
||||||
|
# rule), '##' is a heading, backticks open a fence. '@@' has no meaning in
|
||||||
|
# markdown at all, which is the whole reason to use it. Both are parsed here, so
|
||||||
|
# nothing already written stops working.
|
||||||
|
#
|
||||||
|
# The other half of that: put the spec in an attached file rather than the
|
||||||
|
# message body. Attachments are not rendered. '--contract' prints the exact text
|
||||||
|
# to attach, so the wording cannot drift from the parser that reads the reply
|
||||||
|
# back.
|
||||||
|
#
|
||||||
|
# One reply, several projects. A thread produces one file however many repos it
|
||||||
|
# touched, so --contract asks for paths that start with the project name. Point
|
||||||
|
# -o at the directory those projects are siblings in and each file lands in its
|
||||||
|
# own worktree — no table to maintain, and a new worktree needs no change here.
|
||||||
|
#
|
||||||
|
# Two limits worth knowing. A file whose last line has no trailing newline comes
|
||||||
|
# back with one: the digest has to put a newline before the closing fence, so the
|
||||||
|
# distinction is not in the input to recover. And in marker layout a line
|
||||||
|
# starting with "=== " inside a file's own content is indistinguishable from a
|
||||||
|
# real marker — there are no fences to say otherwise. The digest layout has no
|
||||||
|
# such ambiguity, which is the reason to prefer it when something else is
|
||||||
|
# generating the file.
|
||||||
|
#
|
||||||
|
# Paths come out of a text file, so they are treated as untrusted: anything
|
||||||
|
# absolute, or reaching upward with .., is refused and nothing is written. A
|
||||||
|
# file that describes /etc/cron.d/x is not a file you want to expand blindly.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SELF="$(basename "$0")"
|
||||||
|
usage() { awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"; }
|
||||||
|
die() { echo "$SELF: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
DEST="."
|
||||||
|
LIST=""
|
||||||
|
FORCE=""
|
||||||
|
FORMAT="auto"
|
||||||
|
SRC=""
|
||||||
|
SELFTEST=""
|
||||||
|
CONTRACT=""
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
-o) shift; DEST="${1:-}" ;;
|
||||||
|
--list|-n) LIST=1 ;;
|
||||||
|
--force) FORCE=1 ;;
|
||||||
|
--format) shift; FORMAT="${1:-}" ;;
|
||||||
|
--contract) CONTRACT=1 ;;
|
||||||
|
--selftest) SELFTEST=1 ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
-*) die "unknown option: $1" ;;
|
||||||
|
*) [ -z "$SRC" ] && SRC="$1" || die "one input file at a time (got '$SRC' and '$1')" ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── the contract ───────────────────────────────────────────────────────────
|
||||||
|
# One copy of the wording, printed rather than remembered, so what you ask for
|
||||||
|
# and what this parses cannot drift apart. Attach it; do not paste it into the
|
||||||
|
# message body, where markdown gets a say first.
|
||||||
|
contract() {
|
||||||
|
cat <<'CONTRACT'
|
||||||
|
OUTPUT FORMAT
|
||||||
|
|
||||||
|
Return every file you changed or created in full, one after another, using
|
||||||
|
exactly this shape and nothing else:
|
||||||
|
|
||||||
|
@@ FILE: <project>/relative/path/to/file.py
|
||||||
|
<the complete contents of the file>
|
||||||
|
@@ END
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- One @@ FILE: line per file, and a matching @@ END line after its last line.
|
||||||
|
- Start every path with the project it belongs to, spelled exactly as the
|
||||||
|
heading of the document it came from, then the path relative to that
|
||||||
|
project's root. One reply covers every project we touched; the prefix is
|
||||||
|
the only thing that says which file goes where, so it is never optional
|
||||||
|
and never abbreviated.
|
||||||
|
- No leading ./ or /.
|
||||||
|
- Between @@ FILE: and @@ END, emit the file verbatim. Do not wrap it in
|
||||||
|
markdown fences, do not add line numbers, do not elide anything as
|
||||||
|
"unchanged" or "...". A partial file is worse than no file.
|
||||||
|
- Anything you want to say to me goes outside the blocks, before the first
|
||||||
|
@@ FILE: or after the last @@ END. Text between blocks is ignored.
|
||||||
|
- Return whole files only. No diffs, no patches, no hunks.
|
||||||
|
- If a file's own content happens to contain a line starting with @@, say so
|
||||||
|
in your prose so I know to check that block by hand.
|
||||||
|
CONTRACT
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ -n "$CONTRACT" ]; then contract; exit 0; fi
|
||||||
|
|
||||||
|
# ── self-test ──────────────────────────────────────────────────────────────
|
||||||
|
# So a copy of this script on another machine can be checked without any real
|
||||||
|
# input, and without asking whether it is the version that knows a given format.
|
||||||
|
# Every case here is one that has actually gone wrong.
|
||||||
|
selftest() {
|
||||||
|
local t rc=0 got want
|
||||||
|
t="$(mktemp -d)"; trap 'rm -rf "$t"' RETURN
|
||||||
|
|
||||||
|
check() { # name, expected, actual
|
||||||
|
if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1"
|
||||||
|
else printf ' FAIL %s\n expected: %s\n got: %s\n' "$1" "$2" "$3"; rc=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# A file containing a fence and a stray === line: the two things that break
|
||||||
|
# naive parsers.
|
||||||
|
cat > "$t/a.txt" <<'FIXTURE'
|
||||||
|
Here is the code.
|
||||||
|
|
||||||
|
=== FILE: pkg/core/client.py
|
||||||
|
class Client:
|
||||||
|
pass
|
||||||
|
=== END
|
||||||
|
|
||||||
|
=== FILE: README.md
|
||||||
|
# proj
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install proj
|
||||||
|
```
|
||||||
|
|
||||||
|
A line that says === not a marker
|
||||||
|
=== END
|
||||||
|
FIXTURE
|
||||||
|
"$0" -o "$t/a" "$t/a.txt" >/dev/null 2>&1 || true
|
||||||
|
check "fenced: file count" "2" "$(find "$t/a" -type f 2>/dev/null | wc -l)"
|
||||||
|
check "fenced: nested fences" "2" "$(grep -c '```' "$t/a/README.md" 2>/dev/null || echo 0)"
|
||||||
|
check "fenced: === in content" "1" "$(grep -c 'not a marker' "$t/a/README.md" 2>/dev/null || echo 0)"
|
||||||
|
check "fenced: subdirectory" "class Client:" "$(head -1 "$t/a/pkg/core/client.py" 2>/dev/null)"
|
||||||
|
check "fenced: prose skipped" "0" "$(find "$t/a" -name 'Here*' 2>/dev/null | wc -l)"
|
||||||
|
|
||||||
|
# An unclosed block must be refused, not written short.
|
||||||
|
printf '=== FILE: a.py\nx = 1\n=== END\n\n=== FILE: b.py\ny = 2\n' > "$t/b.txt"
|
||||||
|
"$0" -o "$t/b" "$t/b.txt" >/dev/null 2>&1 || true
|
||||||
|
check "unterminated: refused" "1" "$([ -e "$t/b" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
# Paths out of a text file are untrusted.
|
||||||
|
printf '=== FILE: ../escape.py\nx\n=== END\n' > "$t/c.txt"
|
||||||
|
"$0" -o "$t/c" "$t/c.txt" >/dev/null 2>&1 || true
|
||||||
|
check "traversal: refused" "1" "$([ -e "$t/c" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
# The stale-copy failure: explicit format read by the marker parser.
|
||||||
|
"$0" --format marker -o "$t/d" "$t/a.txt" >/dev/null 2>&1 || true
|
||||||
|
check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
# The other two layouts still work.
|
||||||
|
# The @@ markers, which are the ones to ask a chat model for.
|
||||||
|
printf '@@ FILE: pkg/a.py\nx = 1\n@@ END\n@@ FILE: b.md\n# t\n\n```sh\nls\n```\n@@ END\n' > "$t/g.txt"
|
||||||
|
"$0" -o "$t/g" "$t/g.txt" >/dev/null 2>&1 || true
|
||||||
|
check "at-markers: file count" "2" "$(find "$t/g" -type f 2>/dev/null | wc -l)"
|
||||||
|
check "at-markers: fenced body" "2" "$(grep -c '```' "$t/g/b.md" 2>/dev/null || echo 0)"
|
||||||
|
printf '@@ FILE: a.py\nx = 1\n' > "$t/h.txt"
|
||||||
|
"$0" -o "$t/h" "$t/h.txt" >/dev/null 2>&1 || true
|
||||||
|
check "at-markers: unterminated" "1" "$([ -e "$t/h" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
# One reply, several projects: the prefix is just the first directory.
|
||||||
|
printf '@@ FILE: projA/a.py\nx = 1\n@@ END\n@@ FILE: projB/deep/b.py\ny = 2\n@@ END\n' > "$t/i.txt"
|
||||||
|
"$0" -o "$t/i" "$t/i.txt" >/dev/null 2>&1 || true
|
||||||
|
check "multi-project: first" "x = 1" "$(cat "$t/i/projA/a.py" 2>/dev/null)"
|
||||||
|
check "multi-project: nested" "y = 2" "$(cat "$t/i/projB/deep/b.py" 2>/dev/null)"
|
||||||
|
|
||||||
|
printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt"
|
||||||
|
"$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true
|
||||||
|
check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)"
|
||||||
|
|
||||||
|
printf '# d\n\n## x/y.py\n\n```python\nz = 1\n```\n' > "$t/f.txt"
|
||||||
|
"$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true
|
||||||
|
check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)"
|
||||||
|
|
||||||
|
# What distill actually writes: a metadata line between the heading and the
|
||||||
|
# fence, and prose sections whose heading is followed by no fence at all.
|
||||||
|
printf '# d\n\n## Tree\n\nx/\n y.py\n\n## x/y.py\n\n_1 lines · 6 bytes_\n\n```python\nz = 1\n```\n' > "$t/j.txt"
|
||||||
|
"$0" -o "$t/j" "$t/j.txt" >/dev/null 2>&1 || true
|
||||||
|
check "digest: metadata line" "z = 1" "$(cat "$t/j/x/y.py" 2>/dev/null)"
|
||||||
|
check "digest: prose skipped" "1" "$(find "$t/j" -type f 2>/dev/null | wc -l)"
|
||||||
|
|
||||||
|
# A clipped file is head and tail only. Writing it would truncate the real
|
||||||
|
# one, so the whole run is refused.
|
||||||
|
printf '# d\n\n## x/y.py\n\n_900 lines · 60000 bytes · CLIPPED — head and tail only_\n\n```python\nz = 1\n```\n' > "$t/k.txt"
|
||||||
|
"$0" -o "$t/k" "$t/k.txt" >/dev/null 2>&1 || true
|
||||||
|
check "digest: clipped refused" "1" "$([ -e "$t/k" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current"
|
||||||
|
else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2
|
||||||
|
fi
|
||||||
|
return "$rc"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ -n "$SELFTEST" ]; then selftest; exit $?; fi
|
||||||
|
|
||||||
|
[ -n "$SRC" ] || { usage >&2; exit 1; }
|
||||||
|
[ -f "$SRC" ] || die "no such file: $SRC"
|
||||||
|
case "$FORMAT" in fenced|marker|digest|auto) ;; *) die "--format must be fenced, marker, digest or auto" ;; esac
|
||||||
|
|
||||||
|
# Which layout is it? Count the two shapes and take the commoner one, rather
|
||||||
|
# than trusting the first line that happens to match: a digest of a repo full of
|
||||||
|
# markdown will contain plenty of '=== ' inside its own fenced content, and a
|
||||||
|
# marker file can quote a '## ' heading just as easily.
|
||||||
|
if [ "$FORMAT" = auto ]; then
|
||||||
|
n_fenced=$(grep -cE '^(===|@@) +FILE: +[^ ]' "$SRC" || true)
|
||||||
|
n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true)
|
||||||
|
n_marker=$((n_marker - n_fenced - $(grep -cE '^(===|@@) +END[ \t]*$' "$SRC" || true)))
|
||||||
|
[ "$n_marker" -lt 0 ] && n_marker=0
|
||||||
|
n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true)
|
||||||
|
if [ "$n_fenced" -gt 0 ]; then
|
||||||
|
FORMAT=fenced
|
||||||
|
elif [ "$n_marker" -eq 0 ] && [ "$n_digest" -eq 0 ]; then
|
||||||
|
die "found no '=== FILE:' blocks, no '=== path' markers and no '## path' headings in $SRC"
|
||||||
|
elif [ "$n_marker" -ge "$n_digest" ]; then FORMAT=marker
|
||||||
|
else FORMAT=digest
|
||||||
|
fi
|
||||||
|
echo "format: $FORMAT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── the parser ─────────────────────────────────────────────────────────────
|
||||||
|
# One awk, two modes. In list mode it prints "path<TAB>lines"; otherwise it
|
||||||
|
# writes each file under DEST. Reading the whole thing in awk rather than a
|
||||||
|
# bash read-loop matters once the input is a few megabytes.
|
||||||
|
#
|
||||||
|
# In digest mode a heading only opens a file if a fence follows it. distill.sh
|
||||||
|
# writes '## Tree' and '## Binary files ...' sections that are prose, and
|
||||||
|
# treating those as files would scatter junk through the output.
|
||||||
|
parse() {
|
||||||
|
awk -v dest="$DEST" -v mode="$1" -v fmt="$FORMAT" '
|
||||||
|
function flush() {
|
||||||
|
if (path != "") {
|
||||||
|
if (mode == "list") { printf "%s\t%d\n", path, n }
|
||||||
|
path = ""
|
||||||
|
}
|
||||||
|
n = 0
|
||||||
|
}
|
||||||
|
function clean(p) {
|
||||||
|
sub(/^\.\//, "", p)
|
||||||
|
sub(/[ \t\r]+$/, "", p)
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
function unsafe(p) {
|
||||||
|
return (p == "" || p ~ /^\// || p ~ /^[A-Za-z]:/ || p ~ /(^|\/)\.\.(\/|$)/)
|
||||||
|
}
|
||||||
|
function open_file(p) {
|
||||||
|
path = p
|
||||||
|
n = 0
|
||||||
|
if (mode == "write") {
|
||||||
|
out = dest "/" path
|
||||||
|
d = out; sub(/\/[^\/]*$/, "", d)
|
||||||
|
system("mkdir -p \"" d "\"")
|
||||||
|
printf "" > out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function emit(line) {
|
||||||
|
n++
|
||||||
|
if (mode == "write") print line >> (dest "/" path)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Explicit open/close. The whole point is that nothing is inferred:
|
||||||
|
# content is content until the END line, whatever it looks like.
|
||||||
|
fmt == "fenced" && path == "" && /^(===|@@) +FILE: +/ {
|
||||||
|
p = substr($0, index($0, "FILE:") + 5)
|
||||||
|
sub(/^[ \t]+/, "", p)
|
||||||
|
p = clean(p)
|
||||||
|
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
||||||
|
open_file(p)
|
||||||
|
next
|
||||||
|
}
|
||||||
|
fmt == "fenced" && path != "" && /^(===|@@) +END[ \t]*$/ { flush(); next }
|
||||||
|
fmt == "fenced" && path == "" { next } # anything between blocks is prose
|
||||||
|
|
||||||
|
fmt == "marker" && /^=== +/ {
|
||||||
|
flush()
|
||||||
|
p = clean(substr($0, index($0, " ") + 1))
|
||||||
|
# "FILE: ./x.py" and "END" are not paths, they are the explicit
|
||||||
|
# format being read by the wrong parser. Left alone this writes a
|
||||||
|
# directory literally called "FILE: ." and a file called "END",
|
||||||
|
# which is what an out-of-date copy of this script did once.
|
||||||
|
if (p ~ /^FILE:/ || p == "END") { print "WRONGFMT\t" p; bad = 1; next }
|
||||||
|
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
||||||
|
open_file(p)
|
||||||
|
next
|
||||||
|
}
|
||||||
|
|
||||||
|
# Only when NOT already inside a file. The content of any markdown file
|
||||||
|
# in the input is full of "## " headings, and the fence is the only
|
||||||
|
# thing that says which ones are structure and which are text. Once a
|
||||||
|
# file is open, the matching close fence is the sole way out.
|
||||||
|
fmt == "digest" && path == "" && /^## +/ {
|
||||||
|
flush()
|
||||||
|
pending = clean(substr($0, 4))
|
||||||
|
expect = 1
|
||||||
|
next
|
||||||
|
}
|
||||||
|
fmt == "digest" && expect == 1 {
|
||||||
|
if ($0 ~ /^[ \t]*$/) next # blank line between the two
|
||||||
|
# distill.sh puts an italic "N lines, B bytes" line under each heading,
|
||||||
|
# so the fence is no longer the next thing after it. Step over that
|
||||||
|
# line rather than reading it as prose — without this, every file in
|
||||||
|
# a current digest is skipped and the whole document looks empty.
|
||||||
|
if ($0 ~ /^_.*_[ \t]*$/) {
|
||||||
|
# Unless it says the file was clipped. A clipped body is head
|
||||||
|
# and tail with a marker in between; writing it out would
|
||||||
|
# replace a real file with a truncated one that reads complete,
|
||||||
|
# which is the exact failure the unterminated check exists for.
|
||||||
|
if ($0 ~ /CLIPPED/) {
|
||||||
|
print "CLIPPED\t" pending
|
||||||
|
bad = 1; expect = 0; pending = ""
|
||||||
|
next
|
||||||
|
}
|
||||||
|
next
|
||||||
|
}
|
||||||
|
if ($0 ~ /^`{3,}/) { # a fence: this is a file
|
||||||
|
match($0, /^`+/)
|
||||||
|
fence = substr($0, 1, RLENGTH)
|
||||||
|
expect = 0
|
||||||
|
if (unsafe(pending)) { print "UNSAFE\t" pending; bad = 1; next }
|
||||||
|
open_file(pending)
|
||||||
|
next
|
||||||
|
}
|
||||||
|
expect = 0 # prose section, not a file
|
||||||
|
pending = ""
|
||||||
|
next
|
||||||
|
}
|
||||||
|
fmt == "digest" && path != "" && $0 == fence { flush(); next }
|
||||||
|
|
||||||
|
|
||||||
|
{ if (path != "") emit($0) }
|
||||||
|
|
||||||
|
END {
|
||||||
|
if (fmt == "fenced" && path != "") {
|
||||||
|
print "UNTERMINATED\t" path
|
||||||
|
bad = 1
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
exit (bad ? 3 : 0)
|
||||||
|
}
|
||||||
|
' "$SRC"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Validate before writing anything: a refusal after half the tree is on disk is
|
||||||
|
# not a refusal.
|
||||||
|
# awk exits non-zero when it found something wrong; that is the signal, not a
|
||||||
|
# crash, so let it through and report it properly below.
|
||||||
|
scan="$(parse list || true)"
|
||||||
|
|
||||||
|
refused="$(printf '%s\n' "$scan" | grep '^UNSAFE' || true)"
|
||||||
|
if [ -n "$refused" ]; then
|
||||||
|
echo "$SELF: refusing — these paths escape the destination:" >&2
|
||||||
|
printf '%s\n' "$refused" | sed 's/^UNSAFE\t/ /' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# A block that never closed means the input is malformed, or a file contained
|
||||||
|
# the END line. Either way the tail is missing, and a truncated source file that
|
||||||
|
# looks complete is the failure this format exists to prevent.
|
||||||
|
wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)"
|
||||||
|
if [ -n "$wrongfmt" ]; then
|
||||||
|
echo "$SELF: this file uses 'FILE: path' / 'END' markers, but it was read as" >&2
|
||||||
|
echo "the plain marker format, which would create a directory called 'FILE: .'" >&2
|
||||||
|
echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)"
|
||||||
|
if [ -n "$unterminated" ]; then
|
||||||
|
echo "$SELF: refusing — this block was never closed with '@@ END' or '=== END':" >&2
|
||||||
|
printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2
|
||||||
|
echo "the file it describes would be silently truncated" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# distill clips the largest files to fit a budget, and says so. The tree copy
|
||||||
|
# beside the digest has them whole, so the fix is to take them from there — not
|
||||||
|
# to write out the head and tail under the real name.
|
||||||
|
clipped="$(printf '%s\n' "$scan" | grep '^CLIPPED' || true)"
|
||||||
|
if [ -n "$clipped" ]; then
|
||||||
|
echo "$SELF: refusing — distill clipped these, so the digest has only their" >&2
|
||||||
|
echo "head and tail:" >&2
|
||||||
|
printf '%s\n' "$clipped" | sed 's/^CLIPPED\t/ /' >&2
|
||||||
|
echo "take them from the tree copy instead; writing these would truncate them" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT|CLIPPED)' || true)"
|
||||||
|
[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)"
|
||||||
|
count=$(printf '%s\n' "$listing" | grep -c . )
|
||||||
|
|
||||||
|
if [ -n "$LIST" ]; then
|
||||||
|
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %-60s %5d lines\n", $1, $2 }'
|
||||||
|
echo "$count files"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Existing files are someone's work until proven otherwise.
|
||||||
|
if [ -z "$FORCE" ]; then
|
||||||
|
clashes=""
|
||||||
|
while IFS=$'\t' read -r p _; do
|
||||||
|
[ -e "$DEST/$p" ] && clashes="$clashes $p"$'\n'
|
||||||
|
done <<< "$listing"
|
||||||
|
if [ -n "$clashes" ]; then
|
||||||
|
echo "$SELF: these already exist under $DEST:" >&2
|
||||||
|
printf '%s' "$clashes" >&2
|
||||||
|
echo "re-run with --force to overwrite" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$DEST"
|
||||||
|
parse write >/dev/null
|
||||||
|
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %s\n", $1 }'
|
||||||
|
echo "wrote $count files to $DEST"
|
||||||
5
soleprint/station/tools/histgen/.gitignore
vendored
Normal file
5
soleprint/station/tools/histgen/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# Local settings: whose repo this machine points at is not a fact about the
|
||||||
|
# tool. Copy the folder, run `make init-config`, and the answer stays here.
|
||||||
|
histgen.json
|
||||||
|
def
|
||||||
|
__pycache__/
|
||||||
137
soleprint/station/tools/histgen/Makefile
Normal file
137
soleprint/station/tools/histgen/Makefile
Normal file
@@ -0,0 +1,137 @@
|
|||||||
|
# histgen — one target per verb.
|
||||||
|
#
|
||||||
|
# The folder is meant to be copied out of soleprint and used on its own, so
|
||||||
|
# everything here is derived from where this file sits rather than written down:
|
||||||
|
# copy the directory anywhere, `cd` into it, and `make` works. Renaming it works
|
||||||
|
# too, since the package name comes from the directory.
|
||||||
|
#
|
||||||
|
# Two directories, and the whole tool hangs off the difference:
|
||||||
|
#
|
||||||
|
# SOURCE the tree to read. Read-only, always. Nothing is written into it.
|
||||||
|
# OUT the plan, the briefs, and OUT/<name>/ — a copy of the source with
|
||||||
|
# the designed history committed into it.
|
||||||
|
#
|
||||||
|
# make check prove it works, on its own fixture
|
||||||
|
# make copy SOURCE=~/work/x OUT=~/out just the files, no repo, no keys
|
||||||
|
# make run SOURCE=~/work/x OUT=~/out scan + plan + brief
|
||||||
|
# make list the commits, to confirm
|
||||||
|
# make commands copy the files, hand back git commands
|
||||||
|
# make export ...or have it commit them for you
|
||||||
|
#
|
||||||
|
# Set them once and the verbs take no arguments:
|
||||||
|
#
|
||||||
|
# make init-config SOURCE=... OUT=...
|
||||||
|
# make config what everything resolves to
|
||||||
|
# make status what is in OUT, and what is left
|
||||||
|
#
|
||||||
|
# The logic lives in the Python, never here. Each target is one invocation.
|
||||||
|
|
||||||
|
HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
|
||||||
|
PKG := $(notdir $(HERE))
|
||||||
|
PARENT := $(patsubst %/,%,$(dir $(HERE)))
|
||||||
|
PY ?= python3
|
||||||
|
PREFIX ?= $(HOME)/.local
|
||||||
|
|
||||||
|
# Run the package from its parent, which is what `python -m` needs and what
|
||||||
|
# lets this work without installing anything.
|
||||||
|
HISTGEN := PYTHONPATH=$(PARENT) $(PY) -m $(PKG)
|
||||||
|
|
||||||
|
# Extra flags for the verb being run: make plan REPO=x ARGS=--max-files=12
|
||||||
|
ARGS ?=
|
||||||
|
|
||||||
|
# Left empty, these say nothing and the config file decides. Passing REPO= or
|
||||||
|
# OUT= on the command line overrides it, which is the precedence the tool
|
||||||
|
# already applies — the Makefile just has to not invent a default of its own.
|
||||||
|
SOURCE ?=
|
||||||
|
OUT ?=
|
||||||
|
CONFIG ?=
|
||||||
|
|
||||||
|
WHERE := $(if $(SOURCE),--source $(SOURCE)) $(if $(OUT),--out $(OUT)) \
|
||||||
|
$(if $(CONFIG),--config $(CONFIG))
|
||||||
|
|
||||||
|
.PHONY: help check run copy scan plan list brief export keep commands dry-run verify status \
|
||||||
|
against-history clean install uninstall doctor config init-config
|
||||||
|
|
||||||
|
help: ## List every target
|
||||||
|
@echo "histgen — seed a clean, logical history into a repo"
|
||||||
|
@echo
|
||||||
|
@grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \
|
||||||
|
| awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-16s\033[0m %s\n", $$1, $$2}'
|
||||||
|
@echo
|
||||||
|
@echo " SOURCE=/path/to/tree what to read (read-only, never written to)"
|
||||||
|
@echo " OUT=/path/to/out what to write (plan, briefs, and OUT/<name>/)"
|
||||||
|
@echo " ARGS=... extra flags, e.g. ARGS=\"--max-files 25\""
|
||||||
|
@echo
|
||||||
|
@echo " Both can live in histgen.json instead: make init-config SOURCE=.. OUT=.."
|
||||||
|
|
||||||
|
check: ## Prove the whole pipeline works, needing no repo and nothing installed
|
||||||
|
@$(PY) $(HERE)/selftest.py
|
||||||
|
|
||||||
|
config: ## Show what repo, out and max-files resolve to
|
||||||
|
@$(HISTGEN) config $(WHERE)
|
||||||
|
|
||||||
|
init-config: ## Write a starter histgen.json beside the tool
|
||||||
|
@$(HISTGEN) config --init $(WHERE)
|
||||||
|
|
||||||
|
doctor: ## Report whether this machine can run it
|
||||||
|
@printf 'python : '; $(PY) --version 2>&1 || echo MISSING
|
||||||
|
@printf 'git : '; git --version 2>&1 || echo MISSING
|
||||||
|
@printf 'package: %s (from %s)\n' '$(PKG)' '$(PARENT)'
|
||||||
|
@$(HISTGEN) --help >/dev/null 2>&1 \
|
||||||
|
&& echo 'import : ok' || echo 'import : FAILED — is the folder intact?'
|
||||||
|
|
||||||
|
run: ## scan + plan + brief, everything before the messages are needed
|
||||||
|
@$(HISTGEN) run $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
list: ## Print the commits, to confirm before exporting
|
||||||
|
@$(HISTGEN) list $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
status: ## What is in OUT, what state it is in, and what is left to do
|
||||||
|
@$(HISTGEN) status $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
copy: ## Copy the files out: no .git, nothing ignored, no keys, no build output
|
||||||
|
@$(HISTGEN) copy $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
scan: ## Read the source and cache what was read
|
||||||
|
@$(HISTGEN) scan $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
plan: ## Order the files and cut them into commits
|
||||||
|
@$(HISTGEN) plan $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
brief: ## Write one brief per commit, for the messages
|
||||||
|
@$(HISTGEN) brief $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
against-history: ## Report how an existing history compares. Reads only
|
||||||
|
@$(HISTGEN) plan $(WHERE) --against-history $(ARGS)
|
||||||
|
|
||||||
|
dry-run: ## Write the export as a reviewable regen.sh instead of running it
|
||||||
|
@$(HISTGEN) export $(WHERE) --dry-run $(ARGS)
|
||||||
|
|
||||||
|
export: ## Copy the source into OUT and commit the history. Resumes if interrupted
|
||||||
|
@$(HISTGEN) export $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
keep: ## Export, carrying an existing history over onto its own branch
|
||||||
|
@$(HISTGEN) export $(WHERE) --keep-history $(ARGS)
|
||||||
|
|
||||||
|
commands: ## Copy the files, create no repo, print the git commands to run yourself
|
||||||
|
@$(HISTGEN) export $(WHERE) --commands $(ARGS)
|
||||||
|
|
||||||
|
verify: ## Nothing left untracked, and the exported tree matches the source
|
||||||
|
@$(HISTGEN) verify $(WHERE) $(ARGS)
|
||||||
|
|
||||||
|
clean: ## Delete the whole OUT directory. The source is not touched
|
||||||
|
@d=$$($(HISTGEN) config $(WHERE) | awk '/^out /{print $$2}'); \
|
||||||
|
test -n "$$d" -a "$$d" != "(unset)" || { echo "Error: no OUT set." >&2; exit 1; }; \
|
||||||
|
rm -rf "$$d" && echo "Removed $$d. The source was never written to."
|
||||||
|
|
||||||
|
install: ## Put a `histgen` command on PATH, pointing back at this folder
|
||||||
|
@mkdir -p $(PREFIX)/bin
|
||||||
|
@printf '#!/bin/sh\n# Generated by histgen'"'"'s Makefile; points at the folder it was run from.\nPYTHONPATH="%s" exec "%s" -m %s "$$@"\n' \
|
||||||
|
'$(PARENT)' '$(shell command -v $(PY))' '$(PKG)' > $(PREFIX)/bin/histgen
|
||||||
|
@chmod +x $(PREFIX)/bin/histgen
|
||||||
|
@echo "Installed $(PREFIX)/bin/histgen -> $(HERE)"
|
||||||
|
@case ":$$PATH:" in *":$(PREFIX)/bin:"*) ;; \
|
||||||
|
*) echo "Note: $(PREFIX)/bin is not on PATH." ;; esac
|
||||||
|
|
||||||
|
uninstall: ## Remove that command
|
||||||
|
@rm -f $(PREFIX)/bin/histgen && echo "Removed $(PREFIX)/bin/histgen"
|
||||||
545
soleprint/station/tools/histgen/README.md
Normal file
545
soleprint/station/tools/histgen/README.md
Normal file
@@ -0,0 +1,545 @@
|
|||||||
|
# histgen
|
||||||
|
|
||||||
|
Seeds a clean, logical history into a repo — reading one directory and writing
|
||||||
|
another, so the tree it reads is never touched.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
histgen copy # just the files: no .git, nothing ignored, no keys
|
||||||
|
histgen run # scan the source, plan the commits, write the briefs
|
||||||
|
histgen list # the commits, to confirm
|
||||||
|
histgen commands # copy the files, hand back the git add/commit commands
|
||||||
|
histgen export # ...or have it do the committing itself
|
||||||
|
```
|
||||||
|
|
||||||
|
Two directories, and the whole tool hangs off the difference:
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **source** | the tree to read. Opened read-only, always. Not a commit, not a `.git`, not a state file is written into it — it can be a checkout you do not own or a read-only mount. |
|
||||||
|
| **out** | everything produced. The index, the plan, the briefs, and `out/<name>/` — a copy of the source with the designed history committed into it. |
|
||||||
|
|
||||||
|
That separation is what makes the history safe to argue with. It is an argument
|
||||||
|
you will have more than once, and every attempt is a directory you can delete
|
||||||
|
rather than a repo you have to put back.
|
||||||
|
|
||||||
|
Stdlib only, no network, no API key. Also readable in the browser at
|
||||||
|
`/station/tools/histgen/`, which shows a plan and never writes one.
|
||||||
|
|
||||||
|
## Copying it out
|
||||||
|
|
||||||
|
Copy the folder anywhere, `cd` into it, and use the Makefile. It derives the
|
||||||
|
package name and path from where it sits, so the directory can be renamed and
|
||||||
|
still work, and nothing has to be installed.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp -r histgen ~/tools/ && cd ~/tools/histgen
|
||||||
|
|
||||||
|
make check # prove it works, on its own fixture
|
||||||
|
make doctor # what this machine has
|
||||||
|
make init-config SOURCE=~/work/x OUT=~/out # set both once
|
||||||
|
make run # scan + plan + brief
|
||||||
|
make list # the commits, to confirm
|
||||||
|
make export # write them into ~/out/x
|
||||||
|
make help # every target
|
||||||
|
```
|
||||||
|
|
||||||
|
One target per verb, plus `dry-run`, `against-history`, `verify` and `clean`.
|
||||||
|
Extra flags go in `ARGS`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make plan REPO=/path/to/repo ARGS="--max-files 25"
|
||||||
|
```
|
||||||
|
|
||||||
|
`make install` drops a `histgen` command in `~/.local/bin` pointing back at the
|
||||||
|
folder, if you would rather not `cd` into it.
|
||||||
|
|
||||||
|
## Just the files, without the repo
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make copy SOURCE=~/code/myproject OUT=~/clean
|
||||||
|
```
|
||||||
|
|
||||||
|
Gives you `~/clean/myproject` holding what the project actually is — no `.git`,
|
||||||
|
nothing gitignored, nothing a build regenerates, and nothing that looks like a
|
||||||
|
key. No history is planned and nothing is committed; this is the plain utility
|
||||||
|
underneath the rest.
|
||||||
|
|
||||||
|
```
|
||||||
|
14 files tracked, 7 to copy, 7 left behind.
|
||||||
|
|
||||||
|
secret — looks like a key or a credential (2):
|
||||||
|
.env
|
||||||
|
certs/server.key
|
||||||
|
|
||||||
|
ignored — tracked, but the ignore rules say they should not be (1):
|
||||||
|
data/raw/dump.sql
|
||||||
|
|
||||||
|
derived — a build regenerates these (3):
|
||||||
|
dist/bundle.js.map
|
||||||
|
dist/bundle.min.js
|
||||||
|
package-lock.json
|
||||||
|
|
||||||
|
oversize — larger than --max-bytes (1):
|
||||||
|
data/big.csv
|
||||||
|
|
||||||
|
Copied to /home/you/clean/myproject
|
||||||
|
no .git — the source has one and it was not copied
|
||||||
|
what was left behind: /home/you/clean/copied.md
|
||||||
|
```
|
||||||
|
|
||||||
|
**Every drop is named**, on screen and in `copied.md`. A file quietly missing
|
||||||
|
from a copy is the same class of failure as a file quietly missing from a
|
||||||
|
history, one directory earlier.
|
||||||
|
|
||||||
|
The same filter runs on the history path. `scan` drops secrets and
|
||||||
|
ignored-but-tracked files before they ever reach a plan, and says so:
|
||||||
|
|
||||||
|
```
|
||||||
|
Scanned 5 files: 5 read, 0 reused from cache.
|
||||||
|
3 left out of the history:
|
||||||
|
.env (looks like a key or a credential)
|
||||||
|
certs/tls.key (looks like a key or a credential)
|
||||||
|
data/dump.sql (tracked, but the ignore rules say they should not be)
|
||||||
|
```
|
||||||
|
|
||||||
|
so `make commands` and `make export` cannot commit a key that `make copy` would
|
||||||
|
have left behind. `make status` keeps saying it afterwards, and
|
||||||
|
`--keep-secrets` turns it off.
|
||||||
|
|
||||||
|
**One deliberate difference between the two.** `copy` also drops what a build
|
||||||
|
regenerates — lockfiles, maps, minified output — because a snapshot is for
|
||||||
|
reading. `scan` keeps them, because a lockfile is content in a repo somebody is
|
||||||
|
going to use. `copy --all` keeps them too.
|
||||||
|
|
||||||
|
### What gets left behind, and why
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **secret** | `.env`, `*.pem`, `*.key`, `id_rsa`, `.netrc`, `credentials.json`, `service-account*.json`, `.ssh/` |
|
||||||
|
| **ignored** | tracked *despite* the repo's own ignore rules — someone ran `git add -f` once |
|
||||||
|
| **derived** | lockfiles, `*.map`, `*.min.js`, `*.pyc`, `*.so`, `__pycache__/`, `node_modules/` |
|
||||||
|
| **oversize** | whatever `--max-bytes` says |
|
||||||
|
|
||||||
|
The derived list is ported from `ppl/ctrl/distill.sh`, including the lesson in
|
||||||
|
its comments: **the line is derived-vs-content, not text-vs-binary.** Images,
|
||||||
|
fonts, spreadsheets and PDFs are content and are kept — none of them can be
|
||||||
|
regenerated from what is left, which is the only thing that makes a file safe to
|
||||||
|
drop. That distinction was wrong in distill once and cost real files.
|
||||||
|
|
||||||
|
**Lookalikes are kept on purpose.** `.env.example` is the documented way to say
|
||||||
|
what the real one needs, and dropping it takes the documentation with the
|
||||||
|
secret. Same for `server.key.pub` — a public key is not a private one.
|
||||||
|
|
||||||
|
The **ignored** case is the one worth reading. `git add -f` is not always a
|
||||||
|
mistake, so these are named rather than assumed either way; but a dump or a
|
||||||
|
credentials file that went in once and was never noticed since looks exactly
|
||||||
|
like this, and the repo is already contradicting itself about them.
|
||||||
|
|
||||||
|
### Knobs
|
||||||
|
|
||||||
|
`--keep-secrets`, `--exclude` and `--include` work on `copy`, `scan` and `run`
|
||||||
|
alike, and can live in `histgen.json`. The rest are `copy`'s own.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make copy ARGS="--dry-run" # report only, write nothing
|
||||||
|
make copy ARGS="--max-bytes 100000" # leave anything bigger
|
||||||
|
make copy ARGS="--exclude '*.csv' --exclude data/"
|
||||||
|
make copy ARGS="--include package-lock.json" # keep it, whatever the filters say
|
||||||
|
make copy ARGS="--all --keep-secrets" # turn the two filters off
|
||||||
|
```
|
||||||
|
|
||||||
|
`--exclude` follows distill's rule: a pattern with no `/` matches basenames at
|
||||||
|
any depth. `--include` is checked first and wins outright, so one file can be
|
||||||
|
rescued without turning a whole filter off.
|
||||||
|
|
||||||
|
To then plan a history from the cleaned tree, point a fresh run at it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make run SOURCE=~/clean/myproject OUT=~/history
|
||||||
|
```
|
||||||
|
|
||||||
|
## Recipe: a copy with a clean history
|
||||||
|
|
||||||
|
The common case. You have a repo, you want the same files somewhere new with a
|
||||||
|
history that reads like the thing was built on purpose, and you do not care what
|
||||||
|
the old history said.
|
||||||
|
|
||||||
|
Nothing is written to the original. The old history is simply not carried over —
|
||||||
|
that is the default, and `--keep-history` is the opt-in for when you do want it.
|
||||||
|
|
||||||
|
**`OUT` is the parent directory, not the repo.** The copy keeps the source's own
|
||||||
|
name underneath it. `OUT` does not have to exist yet; it is created on the first
|
||||||
|
command.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ~/tools/histgen # wherever you copied the folder
|
||||||
|
|
||||||
|
make init-config SOURCE=~/code/myproject OUT=~/clean
|
||||||
|
make config # check both paths before anything runs
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
source /home/you/code/myproject read-only
|
||||||
|
out /home/you/clean
|
||||||
|
exported to /home/you/clean/myproject <- the copy ends up here
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1. Plan it
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make run # reads the source, groups the files, writes a brief per commit
|
||||||
|
make list # the proposed commits, in order
|
||||||
|
```
|
||||||
|
|
||||||
|
`make list` is the thing to look at. Every commit is marked `*` until it has a
|
||||||
|
message. If a commit holds two unrelated ideas, move a path between groups in
|
||||||
|
`~/clean/plan.json` and run `make plan && make list` again — the grouping is a
|
||||||
|
proposal, and re-planning keeps every message whose group still holds the same
|
||||||
|
files.
|
||||||
|
|
||||||
|
### 2. Write the messages
|
||||||
|
|
||||||
|
`~/clean/briefs/` has one markdown file per commit, holding each file's opening
|
||||||
|
comment. Read them and write a `title` and `body` into each group in
|
||||||
|
`~/clean/plan.json`.
|
||||||
|
|
||||||
|
This is the part worth doing properly: the briefs carry the reasoning already in
|
||||||
|
the code, which is what makes a message worth reading. A message reconstructed
|
||||||
|
from the diff just restates the diff.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make list # again — the titles you wrote now show instead of the * marks
|
||||||
|
```
|
||||||
|
|
||||||
|
To see the shape end to end before writing any of them, use
|
||||||
|
`ARGS=--allow-untitled` in the next step; the commits get their group name as a
|
||||||
|
subject, which is fine for a throwaway pass and not fine for anything you keep.
|
||||||
|
|
||||||
|
### 3. Get the commands, and run them yourself
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make commands
|
||||||
|
```
|
||||||
|
|
||||||
|
This copies the planned files into `~/clean/myproject` and **creates no repo** —
|
||||||
|
no `git init`, no `.git`. What comes back is the list, also saved to
|
||||||
|
`~/clean/commands.sh`:
|
||||||
|
|
||||||
|
```
|
||||||
|
cd /home/you/clean/myproject
|
||||||
|
git init
|
||||||
|
|
||||||
|
# 01 Repo skeleton: ignore rules and line-endings policy
|
||||||
|
git add -- .gitattributes .gitignore
|
||||||
|
git commit -F /home/you/clean/messages/01-skeleton.txt
|
||||||
|
|
||||||
|
# 02 Pin the toolchain in one manifest
|
||||||
|
git add -- versions.env
|
||||||
|
git commit -F /home/you/clean/messages/02-versions.txt
|
||||||
|
|
||||||
|
...
|
||||||
|
|
||||||
|
# Worth running afterwards. The first says no file was silently
|
||||||
|
# missed; the second says the result is byte-identical to the source.
|
||||||
|
git status --porcelain
|
||||||
|
git rev-parse HEAD^{tree} # expect 3132703a817922f9f83bafa0e86e6bdf002ce8cb
|
||||||
|
```
|
||||||
|
|
||||||
|
`git init` is the first line of the list rather than something already done: a
|
||||||
|
repo that appeared without you asking is exactly what someone reaching for this
|
||||||
|
mode does not want. Read the list, edit it, reorder it, run it a line at a time.
|
||||||
|
|
||||||
|
Messages go in files rather than `-m` because bodies are multi-line, and the
|
||||||
|
body is where the *why* lives. Edit the message files directly if you want to
|
||||||
|
reword something — nothing has been committed yet.
|
||||||
|
|
||||||
|
The last two commands are worth running when you are done. `git status
|
||||||
|
--porcelain` printing nothing means no file was silently missed, which is the
|
||||||
|
failure this whole exercise exists to prevent. The tree hash matching means the
|
||||||
|
result is byte-identical to the source.
|
||||||
|
|
||||||
|
Only the files in the plan are copied: not the source's `.git`, not anything
|
||||||
|
gitignored. If `~/clean/myproject` already contains a repo, this refuses rather
|
||||||
|
than handing you commands that would commit into it.
|
||||||
|
|
||||||
|
### Or let it do the committing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make dry-run # optional: writes ~/clean/regen.sh, a script that does everything
|
||||||
|
make export # copy and commit, checking both guards itself
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
18 commits on main in /home/you/clean/myproject. Checking:
|
||||||
|
nothing left untracked: ok
|
||||||
|
tree matches source (3132703a8179): ok
|
||||||
|
```
|
||||||
|
|
||||||
|
Three modes, and the difference is who does what:
|
||||||
|
|
||||||
|
| | copies the files | makes the repo | commits |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `make commands` | yes | no — you run `git init` | you |
|
||||||
|
| `make dry-run` | no — writes a script that would | in the script | in the script |
|
||||||
|
| `make export` | yes | yes | yes, and checks both guards |
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ~/clean/myproject
|
||||||
|
git log --oneline
|
||||||
|
```
|
||||||
|
|
||||||
|
### If something goes wrong
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make status # says which of the four states out is in, and what to do next
|
||||||
|
```
|
||||||
|
|
||||||
|
- Interrupted partway? Run `make export` again — it continues from the commit
|
||||||
|
after the last one recorded, rather than starting over or refusing. (This
|
||||||
|
applies to `make export`; with `make commands` the repo is yours, so a
|
||||||
|
half-finished run is yours to continue from the list.)
|
||||||
|
- Changed the plan after exporting? `make status` says so; `make export
|
||||||
|
ARGS=--force` discards the copy and redoes it.
|
||||||
|
- Want to start completely fresh? `make clean` deletes the whole `OUT`
|
||||||
|
directory. The source is not touched, so there is nothing to put back.
|
||||||
|
|
||||||
|
### Handing this to someone else
|
||||||
|
|
||||||
|
Everything above needs the folder, `python3` and `git` — nothing installed, no
|
||||||
|
network, no API key. Copy the directory, then:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd histgen && make check # proves the whole pipeline on a fixture it builds
|
||||||
|
make help # every target
|
||||||
|
```
|
||||||
|
|
||||||
|
## What is already in out
|
||||||
|
|
||||||
|
`export` writes, so it starts by working out what it is writing into. Four
|
||||||
|
states, and they are genuinely different — `histgen status` prints which one:
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **absent** | nothing there yet. Copy the tree, init, commit. |
|
||||||
|
| **unfinished** | commits this tool made, and a record of where it stopped. Something interrupted the run. **Continue from the group after the last one recorded.** |
|
||||||
|
| **foreign** | commits this tool did *not* make. That history is someone's, so nothing is rewritten, moved or deleted: the designed account goes on its own orphan branch and the existing branch is left exactly where it was. |
|
||||||
|
| **stale** | the plan changed, or the copy moved underneath us. Refuse, and say which of the two it was. `--force` discards and starts over. |
|
||||||
|
|
||||||
|
Telling **unfinished** from **foreign** is the whole reason `progress.json`
|
||||||
|
exists. Without it both read as "there are commits here", and the tool either
|
||||||
|
destroys work it should have kept or refuses to finish work it started — which
|
||||||
|
is exactly what it used to do.
|
||||||
|
|
||||||
|
```
|
||||||
|
$ histgen status
|
||||||
|
source /home/mariano/wdir/rdir/adapter
|
||||||
|
out /home/mariano/histories/adapter
|
||||||
|
census 91 files
|
||||||
|
plan 24 commits, 6 without a message
|
||||||
|
export unfinished — 18 group(s) committed by this tool, 6 to go
|
||||||
|
committed: 1-18
|
||||||
|
remaining: 19-24
|
||||||
|
|
||||||
|
Run `export` again; it continues from where it stopped.
|
||||||
|
```
|
||||||
|
|
||||||
|
The record is written after **each** commit, not at the end — the point is to
|
||||||
|
survive the run not reaching the end. It stores the plan's fingerprint (the
|
||||||
|
groups and their paths, never the messages, so rewording commit 20 does not
|
||||||
|
invalidate the nineteen already made) and each commit's sha, which must still
|
||||||
|
be where the branch tip is or the copy has moved and it says so.
|
||||||
|
|
||||||
|
## Keeping a history that already exists
|
||||||
|
|
||||||
|
```bash
|
||||||
|
histgen export --keep-history
|
||||||
|
```
|
||||||
|
|
||||||
|
Carries the source's `.git` into the copy and commits the designed account to an
|
||||||
|
orphan branch, leaving the original branch pointing exactly where it did. Two
|
||||||
|
tiers, which is what `all/ctrl/handover.sh` has been saying all along:
|
||||||
|
|
||||||
|
```
|
||||||
|
main o-o-o-o "updates 33.1 139" (untouched)
|
||||||
|
designed-history o-o-o-o-o-o-o-o the designed account (no shared parent)
|
||||||
|
```
|
||||||
|
|
||||||
|
Both are present in `out/<name>/`; which one to publish is a decision for later
|
||||||
|
and by hand. Nothing is rewritten and the source is not touched either way.
|
||||||
|
|
||||||
|
## Settings
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make init-config SOURCE=~/work/thing OUT=~/histories/thing
|
||||||
|
make config # what everything resolves to, and where it came from
|
||||||
|
make run # no arguments
|
||||||
|
```
|
||||||
|
|
||||||
|
Writes `histgen.json` beside the tool — the arrangement `ppl/ctrl/distill.sh`
|
||||||
|
already uses, where the JSON next to the script is picked up when nothing else
|
||||||
|
says otherwise.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"source": "~/work/thing",
|
||||||
|
"out": "~/histories/thing",
|
||||||
|
"max_files": null,
|
||||||
|
"keep_history": false,
|
||||||
|
"branch": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Precedence, most specific first:
|
||||||
|
|
||||||
|
```
|
||||||
|
the command line -> --config FILE -> histgen.json beside the tool -> defaults
|
||||||
|
```
|
||||||
|
|
||||||
|
so a config sets a starting point and never wins an argument with a flag typed
|
||||||
|
deliberately. A mistyped key is refused rather than ignored, because a setting
|
||||||
|
plainly written in the file and silently not applied is a bad thing to debug.
|
||||||
|
`out` inside `source` is refused too — the source is read-only by design, and
|
||||||
|
`out` would end up in its own census.
|
||||||
|
|
||||||
|
`histgen.json` is gitignored by the folder's own `.gitignore`: which tree this
|
||||||
|
machine points at is not a fact about the tool.
|
||||||
|
|
||||||
|
## The verbs
|
||||||
|
|
||||||
|
Each writes one file under the repo's `.histgen/`, so the step before it is
|
||||||
|
never repeated.
|
||||||
|
|
||||||
|
| | | |
|
||||||
|
|---|---|---|
|
||||||
|
| `scan` | `out/index.json` | what is in the source, cached by content hash |
|
||||||
|
| `plan` | `out/plan.json` | the order, cut into commits |
|
||||||
|
| `brief` | `out/briefs/*.md` | one pack per commit, for the messages |
|
||||||
|
| `copy` | `out/<name>/` | the files alone, no repo — needs no plan |
|
||||||
|
| `list` | — | the commits, printed to confirm |
|
||||||
|
| `export` | `out/<name>/` | the copy and its history, with both guards |
|
||||||
|
| `status` | — | which of the four states `out` is in |
|
||||||
|
| `verify` | — | the guards, on their own |
|
||||||
|
|
||||||
|
`plan.json` is the seam. Everything above it is analysis that can be recomputed
|
||||||
|
from the tree; everything below is git commands. That split is the whole design:
|
||||||
|
the expensive half is a model reading code, and it should run once.
|
||||||
|
|
||||||
|
## Where the messages come from
|
||||||
|
|
||||||
|
`brief` writes a markdown pack per commit holding each file's **opening
|
||||||
|
comment** — not the file. An agent reads the packs and writes `title` and
|
||||||
|
`body` back into `plan.json`.
|
||||||
|
|
||||||
|
That is deliberate. A commit message reconstructed from a diff restates the
|
||||||
|
diff, and the thing worth recording was never in the diff: it was in the comment
|
||||||
|
explaining why the ignore rules exist before the code they exclude, or why a
|
||||||
|
port offset has to mean the same thing in two different projects. Handing over
|
||||||
|
the reasoning that is already written down produces a message worth reading;
|
||||||
|
handing over the diff produces `Update files`.
|
||||||
|
|
||||||
|
Keeping the model outside the tool is also what keeps the tool offline, keeps
|
||||||
|
every message editable before a single commit exists, and keeps the cost of a
|
||||||
|
500-file repo to the comments rather than the code.
|
||||||
|
|
||||||
|
## The order
|
||||||
|
|
||||||
|
Role first, references second.
|
||||||
|
|
||||||
|
```
|
||||||
|
skeleton (.gitignore) -> README -> version pins -> config layer -> profiles
|
||||||
|
-> templates -> the things that source them -> front door (Makefile) LATE
|
||||||
|
-> the bootstrap account LAST
|
||||||
|
```
|
||||||
|
|
||||||
|
The front door is late because it only dispatches; the bootstrap account is last
|
||||||
|
because it narrates everything above it. References refine within that, so a
|
||||||
|
config lands before the script that sources it.
|
||||||
|
|
||||||
|
**References never override roles.** A reference in code is a dependency; a
|
||||||
|
reference in a comment is a footnote. `.gitignore` names `ctrl/wizard.sh` to say
|
||||||
|
the opposite of "I need this", and a README names every file in the repo.
|
||||||
|
Counting those as edges commits the ignore rules after the code they exclude —
|
||||||
|
consistent, and unreadable. So refs are taken from non-comment lines only, and
|
||||||
|
narrative files (`.gitignore`, README, docs, BOOTSTRAP) contribute no outgoing
|
||||||
|
edges at all.
|
||||||
|
|
||||||
|
## The grouping is a proposal
|
||||||
|
|
||||||
|
One coherent idea per commit, not one directory per commit. What holds a group
|
||||||
|
together is that its files name each other; a hub and the directory named after
|
||||||
|
it (`addons.sh` and `addons/`) always travel together, because committing a
|
||||||
|
loader without the things it loads produces a commit that cannot run.
|
||||||
|
|
||||||
|
Where it cannot know — five scripts in one directory that never mention each
|
||||||
|
other are five ideas or one, and nothing in the text says which — it guesses and
|
||||||
|
says so. **Moving a path from one group to another in `plan.json` is the
|
||||||
|
expected way to use this**, and `plan` re-run afterwards keeps every message
|
||||||
|
whose group still holds the same files.
|
||||||
|
|
||||||
|
`--max-files` sets the cap. The default is about a twentieth of the tree with a
|
||||||
|
floor of eight, which lands near how these repos were actually built — rig plans
|
||||||
|
18 against a real 18, spr 77 against a real 78. Raising it gives fewer, larger
|
||||||
|
commits; lowering it gives more.
|
||||||
|
|
||||||
|
## The two guards
|
||||||
|
|
||||||
|
`export` refuses a plan whose paths are missing, duplicated, or do not cover the
|
||||||
|
source — before it writes anything. After the last commit it checks both:
|
||||||
|
|
||||||
|
1. **nothing left untracked**, with nothing exempt. The state lives in `out`
|
||||||
|
and the copy lives inside it, so there is genuinely nothing of ours in the
|
||||||
|
tree being checked. A file silently missed is the failure this whole tool
|
||||||
|
exists to prevent. It is quiet at the time and surfaces much later, when
|
||||||
|
something does not build on a fresh clone and the history offers no clue
|
||||||
|
which commit should have carried it.
|
||||||
|
2. **the exported tree still matches the source**, by tree hash. Every path
|
||||||
|
committed is not the same claim as the same tree: a stale index, a path in
|
||||||
|
two groups, or a file edited mid-plan all pass the first check and fail this
|
||||||
|
one.
|
||||||
|
|
||||||
|
`--dry-run` writes `out/regen.sh` and `out/messages/` instead — ordinary git
|
||||||
|
commands that copy the files and make the commits, both guards included,
|
||||||
|
reviewable before anything runs.
|
||||||
|
|
||||||
|
## Reporting on a history that already exists
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m station.tools.histgen plan /path/to/repo --against-history
|
||||||
|
```
|
||||||
|
|
||||||
|
Maps each of the source's existing commits onto the group holding most of the files it touched,
|
||||||
|
then reports what agrees and what does not:
|
||||||
|
|
||||||
|
```
|
||||||
|
= 01 skeleton a127b1d matched one commit
|
||||||
|
~ 03 ctrl split across 2 one idea, committed piecemeal
|
||||||
|
+ 04 ctrl-lib no commit never landed as its own change
|
||||||
|
! 4 commit(s) land earlier in the proposed order than work already done
|
||||||
|
? 31 commit(s) carry no usable account of the change ("updates 33.1 139")
|
||||||
|
```
|
||||||
|
|
||||||
|
It reads and prints. It never rewrites: published history is someone else's
|
||||||
|
clone.
|
||||||
|
|
||||||
|
## Verified against
|
||||||
|
|
||||||
|
`rig`'s 18-commit history, which was built by hand and is what this reproduces.
|
||||||
|
Run over the same 64 files, histgen plans 18 commits; the profiles, the cluster
|
||||||
|
templates, the addons hub, the k8s manifests, the Makefile, `sample-rig` and
|
||||||
|
`BOOTSTRAP.md` all land as their own commits in the same places. Replaying it
|
||||||
|
produces a tree hash identical to the one rig ships.
|
||||||
|
|
||||||
|
Where it differs is where the difference is semantic: rig splits its wizard,
|
||||||
|
host checks, cluster lifecycle and registry into four commits, and nothing in
|
||||||
|
those four files' text says they are four things.
|
||||||
|
|
||||||
|
## The CLI shape
|
||||||
|
|
||||||
|
`cli.py` is a shared scaffold — subcommand registration, one spelling for
|
||||||
|
`--source/-o/-n/--force/--dry-run`, `Error: … -> stderr -> exit 1` as the only
|
||||||
|
exit path, deferred heavy imports so `--help` stays instant, and
|
||||||
|
`refuse_to_clobber`. It exists because every tool here grew its own slightly
|
||||||
|
different copy of the same three things.
|
||||||
|
|
||||||
|
histgen is its first user. Nothing else was rewritten to use it: a scaffold
|
||||||
|
earns adoption by being there when the next tool is written.
|
||||||
27
soleprint/station/tools/histgen/__init__.py
Normal file
27
soleprint/station/tools/histgen/__init__.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
"""
|
||||||
|
Histgen — seed a clean, logical history into a repo.
|
||||||
|
|
||||||
|
The general case is a tree with no git at all: read the code, then commit it in
|
||||||
|
parts, ordered so each commit stands on what came before. Done by hand it means
|
||||||
|
uploading everything and asking a model; done twice it stops being worth the
|
||||||
|
time.
|
||||||
|
|
||||||
|
The expensive half (reading the code) and the mechanical half (applying a plan)
|
||||||
|
are split on purpose, and `plan.json` is the seam. Everything before it is
|
||||||
|
analysis and can be cached; everything after it is git commands that fail fast.
|
||||||
|
|
||||||
|
python -m station.tools.histgen scan /path/to/repo
|
||||||
|
python -m station.tools.histgen plan /path/to/repo
|
||||||
|
python -m station.tools.histgen brief /path/to/repo
|
||||||
|
# an agent reads briefs/ and writes title+body back into plan.json
|
||||||
|
python -m station.tools.histgen apply /path/to/repo --dry-run
|
||||||
|
python -m station.tools.histgen verify /path/to/repo
|
||||||
|
|
||||||
|
Stdlib only, no network. The directory can be copied out of soleprint and run
|
||||||
|
on its own — a repo that needs a history seeded is, by definition, not one that
|
||||||
|
already has this framework on its path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
|
|
||||||
|
__all__ = ["census", "order", "brief", "snapshot", "export"]
|
||||||
318
soleprint/station/tools/histgen/__main__.py
Normal file
318
soleprint/station/tools/histgen/__main__.py
Normal file
@@ -0,0 +1,318 @@
|
|||||||
|
"""
|
||||||
|
Histgen CLI — seed a clean, logical history into a repo.
|
||||||
|
|
||||||
|
python -m station.tools.histgen run --source ~/work/thing --out ~/out
|
||||||
|
python -m station.tools.histgen list # the commits, to confirm
|
||||||
|
python -m station.tools.histgen export # write them into out/thing
|
||||||
|
|
||||||
|
Two directories and one rule: the source is read-only, everything is written
|
||||||
|
under out. Set both once with `config --init` and the verbs take no arguments.
|
||||||
|
|
||||||
|
Run from the soleprint/ directory so `station.tools...` resolves, or copy the
|
||||||
|
folder out and use its Makefile.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .cli import Tool, fail
|
||||||
|
|
||||||
|
VERBS = (
|
||||||
|
("copy", "Copy the files out, with no .git and nothing private."),
|
||||||
|
("scan", "Read the source and cache what was read."),
|
||||||
|
("plan", "Order the files and cut them into commits."),
|
||||||
|
("list", "Print the commits, to confirm before exporting."),
|
||||||
|
("brief", "Write one brief per commit, for the messages."),
|
||||||
|
("export", "Copy the source into out and commit the history."),
|
||||||
|
("verify", "Nothing left untracked, and the tree still matches."),
|
||||||
|
("status", "What is in the out directory, and what is left to do."),
|
||||||
|
("run", "scan + plan + brief."),
|
||||||
|
("config", "Show the resolved settings, or write a starter file."),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _settings(args, need_out=True):
|
||||||
|
"""
|
||||||
|
Where to read and where to write, with the command line on top.
|
||||||
|
|
||||||
|
Resolved once per invocation and passed down, rather than each module
|
||||||
|
working it out again — two places deciding where things live is how `scan`
|
||||||
|
and `plan` end up disagreeing about it.
|
||||||
|
"""
|
||||||
|
from . import config
|
||||||
|
s = config.resolve(args, getattr(args, "config", None))
|
||||||
|
|
||||||
|
if not s["source"]:
|
||||||
|
fail("No source given.",
|
||||||
|
f"Pass --source, or set it in {config.default_path()} "
|
||||||
|
"(see `histgen config --init`).")
|
||||||
|
if not s["source"].is_dir():
|
||||||
|
fail(f"Not a directory: {s['source']}")
|
||||||
|
|
||||||
|
if need_out and not s["out"]:
|
||||||
|
fail("No out directory given.",
|
||||||
|
"Pass --out, or set it in the config. It is where the plan and "
|
||||||
|
"the exported repo go; the source is never written to.")
|
||||||
|
if s["out"]:
|
||||||
|
# The source must stay clean, and an out inside it would be scanned as
|
||||||
|
# part of the tree it describes on the very next run.
|
||||||
|
try:
|
||||||
|
if s["out"] == s["source"] or s["out"].is_relative_to(s["source"]):
|
||||||
|
fail("out is inside source.",
|
||||||
|
"The source is read-only by design, and out would end up "
|
||||||
|
"in its own census. Put out somewhere else.")
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _index(out):
|
||||||
|
"""The census, which must already exist — scanning is its own verb."""
|
||||||
|
from . import census
|
||||||
|
index = census.load_index(out)
|
||||||
|
if not index.get("files"):
|
||||||
|
fail(f"No census at {census.state_path(out)}.", "Run `scan` first.")
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_copy(args):
|
||||||
|
"""The plain utility: files out, repo and private things left behind."""
|
||||||
|
from . import snapshot
|
||||||
|
s = _settings(args)
|
||||||
|
snapshot.take(s["source"], s["out"],
|
||||||
|
keep_noise=args.all, keep_secrets=s["keep_secrets"],
|
||||||
|
max_bytes=args.max_bytes, exclude=s["exclude"],
|
||||||
|
include=s["include"], force=args.force, dry_run=args.dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_scan(args):
|
||||||
|
from . import census
|
||||||
|
s = _settings(args)
|
||||||
|
census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"],
|
||||||
|
exclude=s["exclude"], include=s["include"])
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_plan(args):
|
||||||
|
from . import order
|
||||||
|
s = _settings(args)
|
||||||
|
plan = order.build_plan(_index(s["out"]), s["out"], max_files=s["max_files"])
|
||||||
|
if args.against_history:
|
||||||
|
from . import history
|
||||||
|
print()
|
||||||
|
history.compare(s["source"], plan)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_list(args):
|
||||||
|
"""
|
||||||
|
Print the commits as a numbered list, which is the thing to confirm.
|
||||||
|
|
||||||
|
Deliberately the plainest output here: a number, a title, and the files
|
||||||
|
under it. Deciding whether a commit is one idea is done by reading it, and
|
||||||
|
anything else on the line is in the way.
|
||||||
|
"""
|
||||||
|
from . import export as exporter
|
||||||
|
s = _settings(args)
|
||||||
|
plan = exporter.load_plan(s["out"])
|
||||||
|
files = _index(s["out"])["files"] if args.roles else {}
|
||||||
|
for g in plan["groups"]:
|
||||||
|
title = (g.get("title") or "").strip()
|
||||||
|
mark = " " if title else "*"
|
||||||
|
print(f"{mark}{g['n']:3}. {title or g['slug'] + ' (no message yet)'}")
|
||||||
|
for p in g["paths"]:
|
||||||
|
role = f" [{files[p]['role']}]" if args.roles and p in files else ""
|
||||||
|
print(f" {p}{role}")
|
||||||
|
untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||||
|
print(f"\n{len(plan['groups'])} commits, "
|
||||||
|
f"{sum(len(g['paths']) for g in plan['groups'])} files.")
|
||||||
|
if untitled:
|
||||||
|
print(f"* {len(untitled)} still without a message — read "
|
||||||
|
f"{s['out']}/briefs/ and write them into {s['out']}/plan.json.")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_brief(args):
|
||||||
|
from . import brief, order
|
||||||
|
s = _settings(args)
|
||||||
|
if not order.plan_path(s["out"]).exists():
|
||||||
|
fail(f"No plan at {order.plan_path(s['out'])}.", "Run `plan` first.")
|
||||||
|
brief.write_briefs(s["out"])
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_export(args):
|
||||||
|
from . import export as exporter
|
||||||
|
s = _settings(args)
|
||||||
|
exporter.export(s["source"], s["out"], dry_run=args.dry_run,
|
||||||
|
commands=args.commands, allow_untitled=args.allow_untitled,
|
||||||
|
keep_history=s["keep_history"], branch=s["branch"],
|
||||||
|
force=args.force)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_verify(args):
|
||||||
|
from . import export as exporter
|
||||||
|
s = _settings(args)
|
||||||
|
plan = exporter.load_plan(s["out"])
|
||||||
|
planned = [p for g in plan["groups"] for p in g["paths"]]
|
||||||
|
copy = exporter.repo_dir(s["source"], s["out"])
|
||||||
|
if not copy.is_dir():
|
||||||
|
fail(f"Nothing exported at {copy}.", "Run `export` first.")
|
||||||
|
print("Checking:", flush=True)
|
||||||
|
if not exporter.verify(copy, exporter.source_tree_hash(s["source"], planned)):
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status(args):
|
||||||
|
"""What is in out, what state it is in, and what is left."""
|
||||||
|
from . import census, export as exporter, order
|
||||||
|
s = _settings(args)
|
||||||
|
|
||||||
|
index = census.load_index(s["out"])
|
||||||
|
plan = None
|
||||||
|
if order.plan_path(s["out"]).exists():
|
||||||
|
plan = exporter.load_plan(s["out"])
|
||||||
|
|
||||||
|
print(f"source {s['source']}")
|
||||||
|
print(f"out {s['out']}")
|
||||||
|
if index.get("files"):
|
||||||
|
left = index.get("left_out", [])
|
||||||
|
print(f"census {len(index['files'])} files"
|
||||||
|
+ (f", {len(left)} left out" if left else ""))
|
||||||
|
for item in left:
|
||||||
|
print(f" left out: {item['path']} ({item['why']})")
|
||||||
|
else:
|
||||||
|
print("census none — run `scan`")
|
||||||
|
if plan:
|
||||||
|
untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||||
|
print(f"plan {len(plan['groups'])} commits"
|
||||||
|
+ (f", {len(untitled)} without a message" if untitled else
|
||||||
|
", all messages written"))
|
||||||
|
else:
|
||||||
|
print("plan none — run `plan`")
|
||||||
|
|
||||||
|
report = exporter.inspect(s["source"], s["out"], plan)
|
||||||
|
print(f"export {report['state']} — {report['detail']}")
|
||||||
|
if report["done"]:
|
||||||
|
print(f" committed: {_ranges(report['done'])}")
|
||||||
|
if report["remaining"]:
|
||||||
|
print(f" remaining: {_ranges(report['remaining'])}")
|
||||||
|
|
||||||
|
advice = {
|
||||||
|
"absent": "Run `export`.",
|
||||||
|
"unfinished": "Run `export` again; it continues from where it stopped.",
|
||||||
|
"foreign": "Run `export`; the designed history goes on its own branch "
|
||||||
|
"and nothing existing is touched.",
|
||||||
|
"stale": "Run `export --force` to discard and redo, or point --out elsewhere.",
|
||||||
|
"complete": "Nothing to do.",
|
||||||
|
}
|
||||||
|
print(f"\n{advice.get(report['state'], '')}")
|
||||||
|
|
||||||
|
|
||||||
|
def _ranges(numbers):
|
||||||
|
"""[1,2,3,7,8] -> '1-3, 7-8'. A list of eighteen numbers is unreadable."""
|
||||||
|
if not numbers:
|
||||||
|
return "none"
|
||||||
|
out, start, previous = [], numbers[0], numbers[0]
|
||||||
|
for n in numbers[1:] + [None]:
|
||||||
|
if n == previous + 1:
|
||||||
|
previous = n
|
||||||
|
continue
|
||||||
|
out.append(str(start) if start == previous else f"{start}-{previous}")
|
||||||
|
start = previous = n
|
||||||
|
return ", ".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_run(args):
|
||||||
|
from . import brief, census, order
|
||||||
|
s = _settings(args)
|
||||||
|
index = census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"],
|
||||||
|
exclude=s["exclude"], include=s["include"])
|
||||||
|
order.build_plan(index, s["out"], max_files=s["max_files"])
|
||||||
|
brief.write_briefs(s["out"])
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_config(args):
|
||||||
|
from . import config
|
||||||
|
if args.init:
|
||||||
|
path = config.write_template(
|
||||||
|
Path(args.config).expanduser() if args.config else config.default_path(),
|
||||||
|
source=args.source, out=args.out)
|
||||||
|
print(f"Wrote {path}. Edit \"source\" and \"out\".")
|
||||||
|
return
|
||||||
|
s = config.resolve(args, args.config)
|
||||||
|
print(f"config {s['config_path'] or '(none; using defaults)'}")
|
||||||
|
print(f"source {s['source'] or '(unset)'} read-only")
|
||||||
|
print(f"out {s['out'] or '(unset)'}")
|
||||||
|
if s["source"] and s["out"]:
|
||||||
|
from .export import repo_dir
|
||||||
|
print(f"exported to {repo_dir(s['source'], s['out'])}")
|
||||||
|
print(f"max-files {s['max_files'] or '(scales with the source)'}")
|
||||||
|
print(f"keep-history {s['keep_history']}")
|
||||||
|
print(f"keep-secrets {s['keep_secrets']}"
|
||||||
|
+ ("" if s["keep_secrets"] else " keys and credentials are left out"))
|
||||||
|
if s["exclude"]:
|
||||||
|
print(f"exclude {', '.join(s['exclude'])}")
|
||||||
|
if s["include"]:
|
||||||
|
print(f"include {', '.join(s['include'])}")
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
tool = Tool("histgen", __doc__, package=__package__)
|
||||||
|
handlers = {name: globals()[f"cmd_{name}"] for name, _ in VERBS}
|
||||||
|
|
||||||
|
for verb, help_text in VERBS:
|
||||||
|
tool.command(verb, handlers[verb], help_text)
|
||||||
|
tool.argument(verb, "--source", "-s", default=None, metavar="DIR",
|
||||||
|
help="The tree to read. Never written to.")
|
||||||
|
tool.argument(verb, "--out", "-o", default=None, metavar="DIR",
|
||||||
|
help="Where the plan and the exported repo go.")
|
||||||
|
tool.argument(verb, "--config", "-c", default=None, metavar="FILE",
|
||||||
|
help="Settings file, instead of histgen.json beside the tool.")
|
||||||
|
|
||||||
|
for verb in ("plan", "run"):
|
||||||
|
tool.argument(verb, "--max-files", type=int, default=None, metavar="N",
|
||||||
|
help="Files per commit before a group is cut. Default "
|
||||||
|
"scales with the source (about N/20, floor 8).")
|
||||||
|
tool.argument("plan", "--against-history", action="store_true",
|
||||||
|
help="Also report how the source's existing history compares.")
|
||||||
|
tool.argument("list", "--roles", action="store_true",
|
||||||
|
help="Show each file's detected role.")
|
||||||
|
|
||||||
|
# Whatever reads the source can filter it, so the same three answers hold
|
||||||
|
# for a snapshot and for a history. Keeping them on `copy` alone was how a
|
||||||
|
# tracked key stayed out of one and went straight into the other.
|
||||||
|
for verb in ("copy", "scan", "run"):
|
||||||
|
tool.argument(verb, "--keep-secrets", action="store_true",
|
||||||
|
help="Keep files that look like keys or credentials.")
|
||||||
|
tool.argument(verb, "--exclude", action="append", default=[], metavar="GLOB",
|
||||||
|
help="Leave these out. Repeatable; a pattern with no / "
|
||||||
|
"matches basenames at any depth.")
|
||||||
|
tool.argument(verb, "--include", action="append", default=[], metavar="GLOB",
|
||||||
|
help="Keep these whatever the filters say. Repeatable.")
|
||||||
|
|
||||||
|
tool.common("copy", "dry_run")
|
||||||
|
tool.argument("copy", "--all", action="store_true",
|
||||||
|
help="Keep what a build regenerates too: lockfiles, maps, "
|
||||||
|
"minified and compiled output.")
|
||||||
|
tool.argument("copy", "--max-bytes", type=int, default=None, metavar="N",
|
||||||
|
help="Leave behind anything larger.")
|
||||||
|
tool.argument("copy", "--force", action="store_true",
|
||||||
|
help="Write into a destination that is not empty.")
|
||||||
|
tool.common("export", "dry_run")
|
||||||
|
tool.argument("export", "--commands", action="store_true",
|
||||||
|
help="Copy the files, create no repo, and print the git add "
|
||||||
|
"and git commit commands to run yourself.")
|
||||||
|
tool.argument("export", "--allow-untitled", action="store_true",
|
||||||
|
help="Commit groups whose message was never written.")
|
||||||
|
tool.argument("export", "--keep-history", action="store_true",
|
||||||
|
help="Carry the source's existing history into the copy, and "
|
||||||
|
"put the designed one on its own branch.")
|
||||||
|
tool.argument("export", "--branch", default=None, metavar="NAME",
|
||||||
|
help="Branch for the designed history when one is kept.")
|
||||||
|
tool.argument("export", "--force", action="store_true",
|
||||||
|
help="Discard an out directory that no longer matches the plan.")
|
||||||
|
tool.argument("config", "--init", action="store_true",
|
||||||
|
help="Write a starter config file rather than reading one.")
|
||||||
|
|
||||||
|
tool.run(argv)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
98
soleprint/station/tools/histgen/api.py
Normal file
98
soleprint/station/tools/histgen/api.py
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
"""
|
||||||
|
Histgen's HTTP surface.
|
||||||
|
|
||||||
|
Read-only on purpose. The CLI seeds histories; this shows what a plan looks
|
||||||
|
like before anyone runs it, because the interesting failure — a group that
|
||||||
|
holds two unrelated ideas — is one you see by reading, not by testing.
|
||||||
|
|
||||||
|
Nothing here writes commits. A browser tab is the wrong place to decide that a
|
||||||
|
repository's history is about to be rebuilt.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from fastapi.responses import HTMLResponse
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/tools/histgen", tags=["histgen"])
|
||||||
|
|
||||||
|
SPR_ROOT = Path(__file__).parents[3]
|
||||||
|
HERE = Path(__file__).parent
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Pages
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@router.get("/", response_class=HTMLResponse)
|
||||||
|
def index():
|
||||||
|
template = HERE / "templates" / "index.html"
|
||||||
|
if template.exists():
|
||||||
|
return template.read_text()
|
||||||
|
return "<h1>histgen</h1>"
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/health")
|
||||||
|
def health():
|
||||||
|
return {"status": "ok", "tool": "histgen"}
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# API
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _resolve(out: str) -> Path:
|
||||||
|
"""
|
||||||
|
An out directory, kept inside the tree this instance was built from.
|
||||||
|
|
||||||
|
The parameter is a filesystem path from a query string, so it is the one
|
||||||
|
input here worth distrusting: without the containment check, `../..` reads
|
||||||
|
any directory the server can.
|
||||||
|
"""
|
||||||
|
target = Path(out).resolve() if Path(out).is_absolute() else (SPR_ROOT.parent / out).resolve()
|
||||||
|
root = SPR_ROOT.parent.resolve()
|
||||||
|
if root not in target.parents and target != root:
|
||||||
|
raise HTTPException(400, f"Outside the tree: {out}")
|
||||||
|
if not target.is_dir():
|
||||||
|
raise HTTPException(404, f"Not a directory: {out}")
|
||||||
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/plan")
|
||||||
|
def get_plan(out: str):
|
||||||
|
"""The plan as it stands, with each group's files and message."""
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .order import plan_path
|
||||||
|
|
||||||
|
path = plan_path(_resolve(out))
|
||||||
|
if not path.exists():
|
||||||
|
raise HTTPException(404, "No plan yet. Run `histgen run`.")
|
||||||
|
plan = json.loads(path.read_text())
|
||||||
|
return {
|
||||||
|
"groups": plan["groups"],
|
||||||
|
"commits": len(plan["groups"]),
|
||||||
|
"files": sum(len(g["paths"]) for g in plan["groups"]),
|
||||||
|
"untitled": [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/census")
|
||||||
|
def get_census(out: str):
|
||||||
|
"""What the scan found, without the per-file detail."""
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
from .census import load_index
|
||||||
|
|
||||||
|
index = load_index(_resolve(out))
|
||||||
|
if not index.get("files"):
|
||||||
|
raise HTTPException(404, "No census yet. Run `scan` first.")
|
||||||
|
files = index["files"]
|
||||||
|
return {
|
||||||
|
"files": len(files),
|
||||||
|
"roles": dict(Counter(f["role"] for f in files.values())),
|
||||||
|
"edges": sum(len(f["refs"]) for f in files.values()),
|
||||||
|
}
|
||||||
111
soleprint/station/tools/histgen/brief.py
Normal file
111
soleprint/station/tools/histgen/brief.py
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
"""
|
||||||
|
One pack per commit, for whoever writes the message.
|
||||||
|
|
||||||
|
This is the interface to the expensive reader, and it is a directory of
|
||||||
|
markdown rather than a network call. The tool does not know how to reach a
|
||||||
|
model and does not want to: an agent already in this repo reads the briefs and
|
||||||
|
writes titles and bodies back into plan.json, which keeps the messages editable
|
||||||
|
before a single commit exists and keeps an API key out of a tool that otherwise
|
||||||
|
runs offline.
|
||||||
|
|
||||||
|
What travels is the reasoning already in the code — each file's opening
|
||||||
|
comment — and not the file. That is both what makes the pack cheap and what
|
||||||
|
makes the message right: a commit message reconstructed from a diff restates
|
||||||
|
the diff, and the thing worth recording was never in the diff. It was in the
|
||||||
|
comment explaining why the port offsets have to match another project's, or why
|
||||||
|
the ignore rules exist before the code they exclude.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .census import INDEX_FILE, state_dir
|
||||||
|
from .order import plan_path
|
||||||
|
|
||||||
|
BRIEF_DIR = "briefs"
|
||||||
|
|
||||||
|
WHY_CHARS = 700 # an opening comment past this is an essay; the head carries it
|
||||||
|
MAX_LISTED = 40
|
||||||
|
|
||||||
|
|
||||||
|
HEADER = """# {n:02d} — {slug}
|
||||||
|
|
||||||
|
**{count} file(s), commit {n} of {total}.**
|
||||||
|
|
||||||
|
Write a title and a body for this commit, then put them in
|
||||||
|
`{plan}` under group {n} as `"title"` and `"body"`.
|
||||||
|
|
||||||
|
- The title says what this commit establishes, in the repo's own words.
|
||||||
|
- The body carries the **why** — take it from the reasoning already in the
|
||||||
|
comments below. Do not restate the diff; the diff is already in the commit.
|
||||||
|
- If a file below does not belong in this commit, move its path to another
|
||||||
|
group in plan.json. The grouping is a proposal.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_why(text):
|
||||||
|
text = (text or "").strip()
|
||||||
|
if not text:
|
||||||
|
return "_(no opening comment)_"
|
||||||
|
if len(text) > WHY_CHARS:
|
||||||
|
text = text[:WHY_CHARS].rsplit("\n", 1)[0] + "\n…"
|
||||||
|
return "\n".join("> " + line if line.strip() else ">" for line in text.split("\n"))
|
||||||
|
|
||||||
|
|
||||||
|
def write_briefs(out, quiet=False):
|
||||||
|
state = state_dir(out)
|
||||||
|
plan = json.loads(plan_path(out).read_text())
|
||||||
|
index = json.loads((state / INDEX_FILE).read_text())
|
||||||
|
files = index["files"]
|
||||||
|
groups = plan["groups"]
|
||||||
|
|
||||||
|
# Which commit each path lands in, so a dependency can be named by the
|
||||||
|
# commit that introduced it rather than by a bare path. "stands on 04" is
|
||||||
|
# the sentence the ordering exists to make true.
|
||||||
|
landed = {p: g["n"] for g in groups for p in g["paths"]}
|
||||||
|
|
||||||
|
out_dir = state / BRIEF_DIR
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for stale in out_dir.glob("*.md"):
|
||||||
|
stale.unlink()
|
||||||
|
|
||||||
|
for g in groups:
|
||||||
|
lines = [HEADER.format(n=g["n"], slug=g["slug"], count=len(g["paths"]),
|
||||||
|
total=len(groups), plan=plan_path(out))]
|
||||||
|
|
||||||
|
earlier, later = {}, set()
|
||||||
|
for p in g["paths"]:
|
||||||
|
for dep in files.get(p, {}).get("refs", []):
|
||||||
|
n = landed.get(dep)
|
||||||
|
if n is None or dep in g["paths"]:
|
||||||
|
continue
|
||||||
|
(earlier.setdefault(n, set()).add(dep) if n < g["n"] else later.add(dep))
|
||||||
|
|
||||||
|
if earlier:
|
||||||
|
lines.append("\n## Stands on\n")
|
||||||
|
for n in sorted(earlier):
|
||||||
|
names = ", ".join(f"`{d}`" for d in sorted(earlier[n])[:MAX_LISTED])
|
||||||
|
lines.append(f"- commit {n:02d}: {names}")
|
||||||
|
if later:
|
||||||
|
# Worth stating plainly rather than hiding: it is the one thing a
|
||||||
|
# reader of the finished history would notice and the tool cannot
|
||||||
|
# fix, because the fix is a judgement about which comes first.
|
||||||
|
names = ", ".join(f"`{d}`" for d in sorted(later)[:MAX_LISTED])
|
||||||
|
lines.append("\n## Forward references (this commit names things not yet committed)\n")
|
||||||
|
lines.append(f"- {names}")
|
||||||
|
|
||||||
|
lines.append("\n## Files\n")
|
||||||
|
for p in g["paths"]:
|
||||||
|
e = files.get(p, {})
|
||||||
|
meta = f"{e.get('role', '?')}, {e.get('lines', 0)} lines"
|
||||||
|
if e.get("binary"):
|
||||||
|
meta += ", binary"
|
||||||
|
lines.append(f"\n### `{p}`\n\n_{meta}_\n")
|
||||||
|
lines.append(_fmt_why(e.get("why")))
|
||||||
|
|
||||||
|
path = out_dir / f"{g['n']:02d}-{g['slug']}.md"
|
||||||
|
path.write_text("\n".join(lines) + "\n")
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
print(f"Wrote {len(groups)} briefs -> {out_dir}")
|
||||||
|
print(f"Read them, then write title and body into {plan_path(out)}.")
|
||||||
|
return out_dir
|
||||||
456
soleprint/station/tools/histgen/census.py
Normal file
456
soleprint/station/tools/histgen/census.py
Normal file
@@ -0,0 +1,456 @@
|
|||||||
|
"""
|
||||||
|
What is in the tree, and what each file says about itself.
|
||||||
|
|
||||||
|
This is the expensive half. Walking is cheap; reading is not, so what gets read
|
||||||
|
is cached by content hash and a second run costs only the files that changed.
|
||||||
|
The cache lives beside the output, in the repo's own .histgen/, because a
|
||||||
|
destination that carries its own state cannot be confused with another one's.
|
||||||
|
|
||||||
|
Two things are extracted from every file, and both are used twice:
|
||||||
|
|
||||||
|
the opening comment why the file exists, in the author's words. `order`
|
||||||
|
does not read it; `brief` hands it to whoever writes
|
||||||
|
the commit message, because that reasoning is the
|
||||||
|
message. A commit that restates its own diff is noise.
|
||||||
|
|
||||||
|
path references which other files this one names. `order` turns them
|
||||||
|
into edges, so a config lands before the script that
|
||||||
|
sources it. `brief` reports them as the seam between
|
||||||
|
one commit and the last.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
STATE_DIR = ".histgen"
|
||||||
|
INDEX_FILE = "index.json"
|
||||||
|
|
||||||
|
# Read caps. A file's opening comment is at the top by definition, and nothing
|
||||||
|
# below the cap has ever been the reason a file exists. The byte cap is what
|
||||||
|
# keeps a vendored 2 MB bundle from being tokenised for no reason.
|
||||||
|
HEAD_LINES = 60
|
||||||
|
MAX_BYTES = 400_000
|
||||||
|
|
||||||
|
|
||||||
|
# ── the file set ───────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# Never parse .gitignore. It has negations, directory semantics, precedence
|
||||||
|
# across nested files and a global excludes file, and a hand-rolled parser that
|
||||||
|
# gets 95% of that right is worse than none: it is wrong silently, on exactly
|
||||||
|
# the files someone took care to exclude. Ask git, which is always installed
|
||||||
|
# here because the output of this tool is a git repository.
|
||||||
|
|
||||||
|
def _git(args, **kw):
|
||||||
|
return subprocess.run(["git", *args], capture_output=True, text=True, **kw)
|
||||||
|
|
||||||
|
|
||||||
|
def is_git(path: Path) -> bool:
|
||||||
|
r = _git(["-C", str(path), "rev-parse", "--git-dir"])
|
||||||
|
return r.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
def file_set(source: Path):
|
||||||
|
"""
|
||||||
|
The paths a history would contain, relative to the source, sorted.
|
||||||
|
|
||||||
|
A source with git is asked what it tracks — the same question handover.sh
|
||||||
|
asks, and for the same reason: a hand-maintained list drifts, and the drift
|
||||||
|
shows up as a file that silently never got committed.
|
||||||
|
|
||||||
|
A source with NO git is the general case, and the interesting one. Rather
|
||||||
|
than reimplementing the ignore rules, git is pointed at the tree with its
|
||||||
|
own directory kept in a temporary path: `ls-files -o --exclude-standard`
|
||||||
|
then means untracked-and-not-ignored, which is exactly the candidate set.
|
||||||
|
Nothing is written inside the tree, so a dry run leaves no .git behind to
|
||||||
|
explain later.
|
||||||
|
"""
|
||||||
|
if is_git(source):
|
||||||
|
listed = _git(["-C", str(source), "ls-files", "-z"]).stdout
|
||||||
|
paths = [p for p in listed.split("\0") if p]
|
||||||
|
# A tracked file that has been deleted but not committed is still in
|
||||||
|
# ls-files. It would abort `apply` partway through with a missing path,
|
||||||
|
# so drop it here and let `verify` be the thing that complains.
|
||||||
|
return sorted(p for p in paths if (source / p).is_file())
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-git-") as tmp:
|
||||||
|
env = dict(os.environ, GIT_DIR=str(Path(tmp) / "git"), GIT_WORK_TREE=str(source))
|
||||||
|
subprocess.run(["git", "init", "-q"], env=env, capture_output=True)
|
||||||
|
listed = subprocess.run(
|
||||||
|
["git", "ls-files", "-o", "--exclude-standard", "-z"],
|
||||||
|
env=env, capture_output=True, text=True,
|
||||||
|
).stdout
|
||||||
|
return sorted(p for p in listed.split("\0")
|
||||||
|
if p and (source / p).is_file())
|
||||||
|
|
||||||
|
|
||||||
|
# ── roles ──────────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# A file's role is what decides where it lands when nothing references it, and
|
||||||
|
# most files reference nothing. The ranks are the ordering heuristic itself,
|
||||||
|
# read bottom-up off rig's log: ignore rules and README first, then the config
|
||||||
|
# layer, then the things that source it, the front door late because it only
|
||||||
|
# dispatches, and the bootstrap account last because it narrates the rest.
|
||||||
|
|
||||||
|
ROLE_RANK = {
|
||||||
|
"skeleton": 0, # .gitignore, .gitattributes — the rules before the files
|
||||||
|
"readme": 10, # what this is, and the one prerequisite
|
||||||
|
"pin": 20, # versions, dependency manifests
|
||||||
|
"config": 30, # the layer everything else reads
|
||||||
|
"profile": 40, # named variants of that config
|
||||||
|
"template": 50, # shapes rendered later
|
||||||
|
"source": 60, # the work
|
||||||
|
"test": 70,
|
||||||
|
"asset": 76,
|
||||||
|
"lock": 78, # generated from a pin; never interesting, never first
|
||||||
|
"doc": 80,
|
||||||
|
"frontdoor": 90, # Makefile, Tiltfile — a dispatcher, so it comes after
|
||||||
|
"bootstrap": 100, # BOOTSTRAP/INSTALL — the account of everything above
|
||||||
|
}
|
||||||
|
|
||||||
|
_SKELETON = {".gitignore", ".gitattributes", ".editorconfig", ".dockerignore",
|
||||||
|
"license", "license.md", "license.txt", "copying", "notice"}
|
||||||
|
_FRONTDOOR = {"makefile", "gnumakefile", "tiltfile", "justfile", "taskfile.yml",
|
||||||
|
"dockerfile", "docker-compose.yml", "docker-compose.yaml"}
|
||||||
|
_LOCK = {"package-lock.json", "poetry.lock", "pnpm-lock.yaml", "yarn.lock",
|
||||||
|
"cargo.lock", "go.sum", "composer.lock", "gemfile.lock", "uv.lock"}
|
||||||
|
_PIN = {"requirements.txt", "pyproject.toml", "package.json", "go.mod",
|
||||||
|
"cargo.toml", "gemfile", "versions.env", "setup.py", "setup.cfg"}
|
||||||
|
_ASSET_EXT = {".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".webp", ".pdf",
|
||||||
|
".woff", ".woff2", ".ttf", ".eot", ".mp4", ".zip", ".ods", ".xlsx",
|
||||||
|
".bundle", ".so", ".dylib", ".dll", ".wasm"}
|
||||||
|
_DOC_EXT = {".md", ".rst", ".txt", ".adoc"}
|
||||||
|
|
||||||
|
|
||||||
|
def ignored_but_tracked(source: Path, paths):
|
||||||
|
"""
|
||||||
|
Files git tracks that the ignore rules say it should not.
|
||||||
|
|
||||||
|
`git add -f` is how they get there, and it is not always a mistake — a
|
||||||
|
built artifact committed on purpose looks exactly like this. But so does a
|
||||||
|
dump, a credentials file or a data directory that someone forced in once
|
||||||
|
and nobody noticed since, and those are the ones worth catching.
|
||||||
|
|
||||||
|
`--no-index` is the whole trick: without it check-ignore stays quiet about
|
||||||
|
anything already tracked, which is precisely the set being asked about.
|
||||||
|
"""
|
||||||
|
if not paths or not is_git(source):
|
||||||
|
return set()
|
||||||
|
r = subprocess.run(
|
||||||
|
["git", "-C", str(source), "check-ignore", "--no-index", "--stdin", "-z"],
|
||||||
|
input="\0".join(paths) + "\0", text=True, capture_output=True)
|
||||||
|
# 0 = some matched, 1 = none matched, anything else is a real failure and
|
||||||
|
# not a reason to refuse to copy.
|
||||||
|
if r.returncode not in (0, 1):
|
||||||
|
return set()
|
||||||
|
return {p for p in r.stdout.split("\0") if p}
|
||||||
|
|
||||||
|
|
||||||
|
def role_of(path: str) -> str:
|
||||||
|
p = Path(path)
|
||||||
|
name, low = p.name, p.name.lower()
|
||||||
|
parts = [s.lower() for s in p.parts]
|
||||||
|
stem = p.stem.lower()
|
||||||
|
|
||||||
|
if low in _SKELETON:
|
||||||
|
return "skeleton"
|
||||||
|
if low in _LOCK:
|
||||||
|
return "lock"
|
||||||
|
if low in _FRONTDOOR or low.startswith("dockerfile"):
|
||||||
|
return "frontdoor"
|
||||||
|
if stem in ("bootstrap", "install", "installing", "getting-started", "quickstart"):
|
||||||
|
return "bootstrap"
|
||||||
|
if stem == "readme":
|
||||||
|
# Only the repo's own README opens the history. A README inside a
|
||||||
|
# subdirectory documents that subdirectory and travels with it.
|
||||||
|
return "readme" if len(p.parts) == 1 else "doc"
|
||||||
|
if low in _PIN or low.endswith(".lock"):
|
||||||
|
return "lock" if low.endswith(".lock") else "pin"
|
||||||
|
if p.suffix.lower() in _ASSET_EXT:
|
||||||
|
return "asset"
|
||||||
|
if "test" in parts or "tests" in parts or stem.startswith("test_") or stem.endswith("_test"):
|
||||||
|
return "test"
|
||||||
|
if p.suffix in (".tpl", ".tmpl", ".j2", ".mustache") or low.endswith((".yaml.tpl", ".tmpl")):
|
||||||
|
return "template"
|
||||||
|
if "templates" in parts:
|
||||||
|
return "template"
|
||||||
|
# A profile is a named variant sitting in a directory of siblings: env.d/,
|
||||||
|
# profiles/, overlays/. The directory is the signal, not the extension.
|
||||||
|
if any(d in parts for d in ("env.d", "profiles", "environments")):
|
||||||
|
return "profile"
|
||||||
|
if stem in ("config", "settings", "conf", "defaults") or low in (".env.example", "env.example"):
|
||||||
|
return "config"
|
||||||
|
if low.endswith(".env") or low.endswith(".env.example"):
|
||||||
|
return "profile"
|
||||||
|
if p.suffix.lower() in _DOC_EXT or "docs" in parts or "doc" in parts:
|
||||||
|
return "doc"
|
||||||
|
return "source"
|
||||||
|
|
||||||
|
|
||||||
|
# ── what a file says about itself ──────────────────────────────────────────
|
||||||
|
|
||||||
|
_COMMENT = {
|
||||||
|
"#": (".sh", ".bash", ".py", ".yaml", ".yml", ".toml", ".env", ".cfg", ".conf", ".tf", ""),
|
||||||
|
"//": (".js", ".ts", ".jsx", ".tsx", ".go", ".java", ".c", ".h", ".cpp", ".rs", ".scala"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def opening_comment(text: str, path: str) -> str:
|
||||||
|
"""
|
||||||
|
The comment block at the top of the file, or the module docstring.
|
||||||
|
|
||||||
|
The shebang and any editor modeline are skipped — they are not prose. The
|
||||||
|
block ends at the first line that is not a comment, which is what makes it
|
||||||
|
the file's own statement of purpose rather than a running commentary.
|
||||||
|
"""
|
||||||
|
lines = text.split("\n")[:HEAD_LINES]
|
||||||
|
i = 0
|
||||||
|
while i < len(lines) and (
|
||||||
|
lines[i].startswith("#!") or not lines[i].strip()
|
||||||
|
or lines[i].lstrip().startswith(("# -*-", "# vim:", "# shellcheck"))
|
||||||
|
):
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
# A docstring: take it whole, it is the same statement in another syntax.
|
||||||
|
rest = "\n".join(lines[i:]).lstrip()
|
||||||
|
for quote in ('"""', "'''"):
|
||||||
|
if rest.startswith(quote):
|
||||||
|
end = rest.find(quote, len(quote))
|
||||||
|
if end != -1:
|
||||||
|
return rest[len(quote):end].strip()
|
||||||
|
|
||||||
|
suffix = Path(path).suffix.lower()
|
||||||
|
markers = [m for m, exts in _COMMENT.items() if suffix in exts] or ["#"]
|
||||||
|
block = []
|
||||||
|
for line in lines[i:]:
|
||||||
|
stripped = line.strip()
|
||||||
|
if not any(stripped.startswith(m) for m in markers):
|
||||||
|
break
|
||||||
|
for m in markers:
|
||||||
|
if stripped.startswith(m):
|
||||||
|
block.append(stripped[len(m):].strip())
|
||||||
|
break
|
||||||
|
return "\n".join(block).strip()
|
||||||
|
|
||||||
|
|
||||||
|
_TOKEN = re.compile(r"[A-Za-z0-9_./+-]{4,}")
|
||||||
|
|
||||||
|
# What opens a comment, by language. Used to tell a reference apart from a
|
||||||
|
# mention, which is the difference between an edge and a footnote.
|
||||||
|
_COMMENT_PREFIX = ("#", "//", "--", ";", "*", "/*")
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_comment(line: str) -> str:
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith(_COMMENT_PREFIX):
|
||||||
|
return ""
|
||||||
|
# A trailing comment on a real line: keep the code, drop the aside.
|
||||||
|
for marker in (" #", " //"):
|
||||||
|
cut = line.find(marker)
|
||||||
|
if cut != -1:
|
||||||
|
line = line[:cut]
|
||||||
|
return line
|
||||||
|
|
||||||
|
|
||||||
|
def referenced_paths(text: str, path: str, by_path, by_base):
|
||||||
|
"""
|
||||||
|
Which other files in this repo this one names, split by how it names them.
|
||||||
|
|
||||||
|
Deliberately textual rather than per-language. A shell `source
|
||||||
|
"$DIR/lib/config.sh"`, a Makefile's `ctrl/cluster.sh`, a kustomization's
|
||||||
|
`- namespace.yaml` and a Dockerfile's `COPY run.py .` are all the same fact
|
||||||
|
— this file needs that one — and four parsers would find it four ways and
|
||||||
|
disagree at the edges.
|
||||||
|
|
||||||
|
The split matters more than the matching does. A reference in code is a
|
||||||
|
dependency: config.sh has to exist before the script that sources it. A
|
||||||
|
reference in a comment is a footnote — .gitignore names `ctrl/wizard.sh`
|
||||||
|
to say the opposite of "I need this", and README names every file in the
|
||||||
|
repo. Counting those as edges puts the ignore rules after the code they
|
||||||
|
exclude, which is exactly backwards. So prose informs the brief and never
|
||||||
|
the order.
|
||||||
|
|
||||||
|
Bare filenames only count when they are unique in the repo, and only with
|
||||||
|
their extension. Without both, every `config.py` in a tree of them becomes
|
||||||
|
an edge to all the others and the ordering collapses into one cycle.
|
||||||
|
"""
|
||||||
|
def hits(blob):
|
||||||
|
found = set()
|
||||||
|
for token in set(_TOKEN.findall(blob)):
|
||||||
|
token = token.strip("./")
|
||||||
|
if not token:
|
||||||
|
continue
|
||||||
|
if token in by_path:
|
||||||
|
found.update(by_path[token])
|
||||||
|
continue
|
||||||
|
# A bare name is only a reference if it carries an extension and
|
||||||
|
# names exactly one file. "cluster" is a word; "cluster.sh" is not.
|
||||||
|
if "." in token:
|
||||||
|
candidates = by_base.get(token)
|
||||||
|
if candidates and len(candidates) == 1:
|
||||||
|
found.update(candidates)
|
||||||
|
found.discard(path)
|
||||||
|
return found
|
||||||
|
|
||||||
|
lines = text.split("\n")
|
||||||
|
code = "\n".join(_strip_comment(l) for l in lines)
|
||||||
|
strong = hits(code)
|
||||||
|
mentions = hits(text) - strong
|
||||||
|
return sorted(strong), sorted(mentions)
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(full: Path):
|
||||||
|
"""Text, or None if this is not text. Size is checked before reading."""
|
||||||
|
try:
|
||||||
|
if full.stat().st_size > MAX_BYTES:
|
||||||
|
return None
|
||||||
|
raw = full.read_bytes()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
if b"\0" in raw[:8000]:
|
||||||
|
return None
|
||||||
|
return raw.decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
# ── the index ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _digest(full: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with full.open("rb") as fh:
|
||||||
|
for chunk in iter(lambda: fh.read(65536), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def state_dir(out) -> Path:
|
||||||
|
"""
|
||||||
|
Where this run's index, plan, briefs and messages live.
|
||||||
|
|
||||||
|
Always `out`, never the source. The source is opened read-only and nothing
|
||||||
|
is written into it — which is what removed a whole class of special cases
|
||||||
|
that used to be here: skipping the state directory during its own census,
|
||||||
|
exempting it from `git status`, and writing a .git/info/exclude entry to
|
||||||
|
keep it quiet. None of that has anywhere to happen now.
|
||||||
|
"""
|
||||||
|
return Path(out)
|
||||||
|
|
||||||
|
|
||||||
|
def state_path(out) -> Path:
|
||||||
|
return state_dir(out) / INDEX_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def current_file_set(source: Path, index) -> set:
|
||||||
|
"""
|
||||||
|
What the source holds right now, filtered the way the census was.
|
||||||
|
|
||||||
|
Recomputed rather than read back, because the guard it feeds exists to
|
||||||
|
catch a file added after the scan. Reading the stored list would answer the
|
||||||
|
easy question — "did the plan cover what we saw?" — instead of the one
|
||||||
|
worth asking, which is "does the plan cover what is there?".
|
||||||
|
"""
|
||||||
|
from .sift import sift
|
||||||
|
settings = (index or {}).get("filter", {})
|
||||||
|
tracked = file_set(source)
|
||||||
|
kept, _ = sift(source, tracked,
|
||||||
|
keep_noise=True,
|
||||||
|
keep_secrets=settings.get("keep_secrets", False),
|
||||||
|
exclude=settings.get("exclude", ()),
|
||||||
|
include=settings.get("include", ()),
|
||||||
|
ignored=ignored_but_tracked(source, tracked))
|
||||||
|
return set(kept)
|
||||||
|
|
||||||
|
|
||||||
|
def load_index(out) -> dict:
|
||||||
|
p = state_path(out)
|
||||||
|
if not p.exists():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return json.loads(p.read_text())
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
# A half-written cache is a cache miss, not a crash. Rebuilding costs
|
||||||
|
# one run; refusing to start costs an explanation.
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def scan(source: Path, out, keep_secrets=False, exclude=(), include=(),
|
||||||
|
quiet=False) -> dict:
|
||||||
|
"""
|
||||||
|
Census the tree, reusing everything whose content has not changed.
|
||||||
|
|
||||||
|
The reuse is per file and keyed on content, not on mtime: a checkout, a
|
||||||
|
branch switch or a `touch` all move mtimes without changing a byte, and
|
||||||
|
re-reading the whole tree because git rewrote it is the cost this exists to
|
||||||
|
avoid.
|
||||||
|
"""
|
||||||
|
from .sift import REASONS, sift
|
||||||
|
|
||||||
|
# The filter runs here, not at export time, so a key never reaches a plan
|
||||||
|
# in the first place. Secrets and files the repo's own ignore rules
|
||||||
|
# contradict are dropped; what a build regenerates is NOT — a lockfile is
|
||||||
|
# content in a repo somebody is going to use, however little it says.
|
||||||
|
# `copy` is the one that drops those, because a snapshot is for reading.
|
||||||
|
tracked = file_set(source)
|
||||||
|
paths, left_out = sift(source, tracked, keep_noise=True,
|
||||||
|
keep_secrets=keep_secrets, exclude=exclude,
|
||||||
|
include=include,
|
||||||
|
ignored=ignored_but_tracked(source, tracked))
|
||||||
|
previous = load_index(out).get("files", {})
|
||||||
|
|
||||||
|
by_path, by_base = {}, {}
|
||||||
|
for p in paths:
|
||||||
|
by_path.setdefault(p, []).append(p)
|
||||||
|
by_base.setdefault(Path(p).name, []).append(p)
|
||||||
|
|
||||||
|
files, reused = {}, 0
|
||||||
|
for rel in paths:
|
||||||
|
full = source / rel
|
||||||
|
try:
|
||||||
|
digest = _digest(full)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
old = previous.get(rel)
|
||||||
|
if old and old.get("hash") == digest and "mentions" in old:
|
||||||
|
files[rel] = old
|
||||||
|
reused += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
text = read_text(full)
|
||||||
|
refs, mentions = ([], []) if text is None else referenced_paths(text, rel, by_path, by_base)
|
||||||
|
entry = {
|
||||||
|
"hash": digest,
|
||||||
|
"size": full.stat().st_size,
|
||||||
|
"role": role_of(rel),
|
||||||
|
"binary": text is None,
|
||||||
|
"why": "" if text is None else opening_comment(text, rel),
|
||||||
|
"refs": refs, # in code: a dependency, and an edge
|
||||||
|
"mentions": mentions, # in prose: context for the brief, never an edge
|
||||||
|
}
|
||||||
|
entry["lines"] = 0 if text is None else text.count("\n") + 1
|
||||||
|
files[rel] = entry
|
||||||
|
|
||||||
|
# The filter settings travel with the index so the guards can re-derive
|
||||||
|
# the same set later. Without them `check_plan` has to choose between
|
||||||
|
# trusting a stale list and flagging every filtered file as missing.
|
||||||
|
index = {"version": 1, "source": str(source), "files": files,
|
||||||
|
"filter": {"keep_secrets": bool(keep_secrets),
|
||||||
|
"exclude": list(exclude), "include": list(include)},
|
||||||
|
"left_out": [{"path": p, "why": w} for p, w in left_out]}
|
||||||
|
destination = state_path(out)
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination.write_text(json.dumps(index, indent=2, sort_keys=True))
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
fresh = len(files) - reused
|
||||||
|
print(f"Scanned {len(files)} files: {fresh} read, {reused} reused from cache.")
|
||||||
|
if left_out:
|
||||||
|
# Never silent. A key that was tracked is a thing to know about,
|
||||||
|
# and it stays true after the file stops travelling.
|
||||||
|
print(f" {len(left_out)} left out of the history:")
|
||||||
|
for rel, why in left_out:
|
||||||
|
print(f" {rel} ({REASONS[why]})")
|
||||||
|
print(f" -> {destination}")
|
||||||
|
return index
|
||||||
132
soleprint/station/tools/histgen/cli.py
Normal file
132
soleprint/station/tools/histgen/cli.py
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
"""
|
||||||
|
The shape a station tool's command line has.
|
||||||
|
|
||||||
|
Every tool here grew its own copy of the same three things: argparse subcommands
|
||||||
|
wired to `cmd_*` functions, a flag vocabulary that is nearly but not quite the
|
||||||
|
same between tools, and an error convention. shuntgen spells `-s` as --spec in
|
||||||
|
one subcommand and --source in another; modelgen calls the same idea --source
|
||||||
|
everywhere; tester has neither. The differences are accidents, not decisions.
|
||||||
|
|
||||||
|
This is that shared shape, factored out. histgen is the first user. Nothing else
|
||||||
|
is rewritten to use it — a scaffold earns adoption by being there when the next
|
||||||
|
tool is written, not by a flag-day.
|
||||||
|
|
||||||
|
from .cli import Tool, fail
|
||||||
|
|
||||||
|
tool = Tool("histgen", __doc__)
|
||||||
|
tool.command("scan", cmd_scan, "Read the repo and cache what was read.")
|
||||||
|
tool.argument("scan", "repo", help="The tree to read.")
|
||||||
|
tool.run()
|
||||||
|
|
||||||
|
Conventions it encodes, so they stop being re-decided:
|
||||||
|
|
||||||
|
--source/-s what to read --output/-o where to write
|
||||||
|
--name/-n what to call it --force/-f write anyway
|
||||||
|
--dry-run print, do not do
|
||||||
|
|
||||||
|
Progress goes to stdout as plain print(). Errors go to stderr prefixed
|
||||||
|
'Error: ' and exit 1 — never a traceback, which tells a user nothing they
|
||||||
|
can act on. `fail()` is the only exit path.
|
||||||
|
|
||||||
|
Heavy imports live inside the cmd_* function, never at module top, so
|
||||||
|
`--help` stays instant and an optional dependency only costs the one
|
||||||
|
subcommand that needs it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# The flags worth having exactly one spelling of. A tool adds its own on top;
|
||||||
|
# it does not redefine these.
|
||||||
|
COMMON = {
|
||||||
|
"source": dict(flags=("--source", "-s"), help="What to read."),
|
||||||
|
"output": dict(flags=("--output", "-o"), help="Where to write."),
|
||||||
|
"name": dict(flags=("--name", "-n"), help="What to call it."),
|
||||||
|
"force": dict(flags=("--force", "-f"), action="store_true",
|
||||||
|
help="Write even if the destination is occupied."),
|
||||||
|
"dry_run": dict(flags=("--dry-run",), action="store_true",
|
||||||
|
help="Print what would happen; change nothing."),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fail(message, hint=None):
|
||||||
|
"""The only way out on error: a line a user can act on, never a traceback."""
|
||||||
|
print(f"Error: {message}", file=sys.stderr)
|
||||||
|
if hint:
|
||||||
|
print(f" {hint}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def refuse_to_clobber(path, force, marker, what):
|
||||||
|
"""
|
||||||
|
Regenerating is fine; overwriting something we did not write is not.
|
||||||
|
|
||||||
|
Lifted from shuntgen, which refuses a non-empty output directory unless it
|
||||||
|
carries the file its own generator leaves behind. The check is cheap and the
|
||||||
|
failure it prevents — silently eating a directory someone hand-wrote — is
|
||||||
|
not recoverable from.
|
||||||
|
"""
|
||||||
|
if force or not path.exists():
|
||||||
|
return
|
||||||
|
if not any(path.iterdir()):
|
||||||
|
return
|
||||||
|
if (path / marker).exists():
|
||||||
|
return
|
||||||
|
fail(f"{path} already exists and was not written by {what}.",
|
||||||
|
"Pick another path, or pass --force to write into it anyway.")
|
||||||
|
|
||||||
|
|
||||||
|
def _prog(package, name):
|
||||||
|
"""
|
||||||
|
How this tool was actually invoked, for the usage line.
|
||||||
|
|
||||||
|
The folder is meant to be copied out and run on its own, so a usage line
|
||||||
|
hardcoding `python -m station.tools.histgen` is wrong the moment it is —
|
||||||
|
it names a path that does not exist on the machine reading it.
|
||||||
|
"""
|
||||||
|
return f"python -m {package or name}"
|
||||||
|
|
||||||
|
|
||||||
|
class Tool:
|
||||||
|
"""A tool's whole command line: subcommands, shared flags, one exit path."""
|
||||||
|
|
||||||
|
def __init__(self, name, description, package=None):
|
||||||
|
self.name = name
|
||||||
|
self.parser = argparse.ArgumentParser(
|
||||||
|
prog=_prog(package, name),
|
||||||
|
description=(description or "").strip().split("\n\n")[0],
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
)
|
||||||
|
self._subparsers = self.parser.add_subparsers(dest="command", required=True)
|
||||||
|
self._commands = {}
|
||||||
|
|
||||||
|
def command(self, verb, func, help_text):
|
||||||
|
"""Register a subcommand. `func` takes parsed args and returns None."""
|
||||||
|
sub = self._subparsers.add_parser(verb, help=help_text, description=help_text)
|
||||||
|
sub.set_defaults(func=func)
|
||||||
|
self._commands[verb] = sub
|
||||||
|
return sub
|
||||||
|
|
||||||
|
def argument(self, verb, *args, **kwargs):
|
||||||
|
"""Add a positional or flag to one subcommand."""
|
||||||
|
self._commands[verb].add_argument(*args, **kwargs)
|
||||||
|
|
||||||
|
def common(self, verb, *names, **overrides):
|
||||||
|
"""Add shared flags by name, so their spelling is decided in one place."""
|
||||||
|
for key in names:
|
||||||
|
spec = dict(COMMON[key])
|
||||||
|
flags = spec.pop("flags")
|
||||||
|
spec.update(overrides.get(key, {}))
|
||||||
|
self._commands[verb].add_argument(*flags, **spec)
|
||||||
|
|
||||||
|
def run(self, argv=None):
|
||||||
|
args = self.parser.parse_args(argv)
|
||||||
|
try:
|
||||||
|
args.func(args)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
# Ctrl-C is a decision, not a crash. Say so and leave quietly.
|
||||||
|
print("\nInterrupted.", file=sys.stderr)
|
||||||
|
sys.exit(130)
|
||||||
|
except BrokenPipeError:
|
||||||
|
# `... | head` closes the pipe early; that is the caller's business.
|
||||||
|
sys.exit(0)
|
||||||
153
soleprint/station/tools/histgen/config.py
Normal file
153
soleprint/station/tools/histgen/config.py
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
"""
|
||||||
|
Where to read from and where to write to.
|
||||||
|
|
||||||
|
Two directories, and the whole tool hangs off the difference between them:
|
||||||
|
|
||||||
|
source the tree to read. Opened read-only, always. Nothing is written
|
||||||
|
into it, ever — not a commit, not a .git, not a state file. It
|
||||||
|
can be a checkout you do not own or a read-only mount.
|
||||||
|
|
||||||
|
out everything this produces. The index, the plan, the briefs, and
|
||||||
|
`out/<name>/` — a copy of the source with the designed history
|
||||||
|
committed into it. Delete the directory and you have lost
|
||||||
|
nothing but time.
|
||||||
|
|
||||||
|
The same repo gets scanned, planned and re-planned a dozen times while its
|
||||||
|
grouping is argued with, and passing the pair of paths to every one of five
|
||||||
|
verbs gets old. So they can live in a file instead — the arrangement
|
||||||
|
`ppl/ctrl/distill.sh` already uses, where the JSON beside the script is picked
|
||||||
|
up when nothing else says otherwise.
|
||||||
|
|
||||||
|
{
|
||||||
|
"source": "~/work/some-project",
|
||||||
|
"out": "~/histories/some-project",
|
||||||
|
"max_files": null
|
||||||
|
}
|
||||||
|
|
||||||
|
That separation is what makes the thing safe to experiment with. The history
|
||||||
|
is an argument you will have more than once, and every attempt is a directory
|
||||||
|
you can throw away rather than a repo you have to put back.
|
||||||
|
|
||||||
|
Precedence is the usual one, most specific first:
|
||||||
|
|
||||||
|
the command line -> --config FILE -> histgen.json beside the tool
|
||||||
|
-> the defaults
|
||||||
|
|
||||||
|
so a config file sets a starting point and never wins an argument with a flag
|
||||||
|
that was typed deliberately.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
CONFIG_NAME = "histgen.json"
|
||||||
|
HERE = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
KEYS = ("source", "out", "max_files", "keep_history", "branch",
|
||||||
|
"keep_secrets", "exclude", "include")
|
||||||
|
|
||||||
|
# `repo` was what `source` used to be called, back when the tool committed
|
||||||
|
# into the tree it read. Accepted rather than rejected, because a config file
|
||||||
|
# written last week should not be an error message.
|
||||||
|
ALIASES = {"repo": "source"}
|
||||||
|
|
||||||
|
TEMPLATE = {
|
||||||
|
"source": None,
|
||||||
|
"out": None,
|
||||||
|
"max_files": None,
|
||||||
|
"keep_history": False,
|
||||||
|
"branch": None,
|
||||||
|
"keep_secrets": False,
|
||||||
|
"exclude": [],
|
||||||
|
"include": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def default_path() -> Path:
|
||||||
|
"""The config beside the tool, used when nothing else is named."""
|
||||||
|
return HERE / CONFIG_NAME
|
||||||
|
|
||||||
|
|
||||||
|
def find(explicit=None):
|
||||||
|
"""The config file to read, or None. An explicit one that is missing is an error."""
|
||||||
|
if explicit:
|
||||||
|
path = Path(explicit).expanduser()
|
||||||
|
if not path.is_file():
|
||||||
|
from .cli import fail
|
||||||
|
fail(f"No such config file: {path}")
|
||||||
|
return path
|
||||||
|
beside = default_path()
|
||||||
|
return beside if beside.is_file() else None
|
||||||
|
|
||||||
|
|
||||||
|
def load(explicit=None) -> dict:
|
||||||
|
"""Read the config, or return the defaults. Unknown keys are an error."""
|
||||||
|
settings = dict(TEMPLATE)
|
||||||
|
path = find(explicit)
|
||||||
|
if not path:
|
||||||
|
return settings
|
||||||
|
|
||||||
|
from .cli import fail
|
||||||
|
try:
|
||||||
|
raw = json.loads(path.read_text())
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
fail(f"{path} is not valid JSON: {e}")
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
fail(f"{path} should hold an object, not a {type(raw).__name__}.")
|
||||||
|
|
||||||
|
# A typo in a key would otherwise be silent, and the symptom — the tool
|
||||||
|
# ignoring a setting that is plainly written in the file — is a bad one to
|
||||||
|
# debug. Naming the valid keys costs one line.
|
||||||
|
raw = {ALIASES.get(k, k): v for k, v in raw.items()}
|
||||||
|
unknown = sorted(set(raw) - set(KEYS))
|
||||||
|
if unknown:
|
||||||
|
fail(f"{path}: unknown key(s) {', '.join(unknown)}.",
|
||||||
|
f"Known keys: {', '.join(KEYS)}.")
|
||||||
|
|
||||||
|
settings.update({k: v for k, v in raw.items() if v is not None})
|
||||||
|
settings["_path"] = str(path)
|
||||||
|
return settings
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(args, explicit=None):
|
||||||
|
"""
|
||||||
|
Fold the config under the command line and hand back what to actually use.
|
||||||
|
|
||||||
|
Paths are expanded and made absolute here rather than at each use, so
|
||||||
|
everything downstream compares like with like — a `~` that survived into a
|
||||||
|
path comparison is a bug that only shows up on someone else's machine.
|
||||||
|
"""
|
||||||
|
settings = load(explicit)
|
||||||
|
|
||||||
|
source = getattr(args, "source", None) or settings.get("source")
|
||||||
|
out = getattr(args, "out", None) or settings.get("out")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"source": Path(source).expanduser().resolve() if source else None,
|
||||||
|
"out": Path(out).expanduser().resolve() if out else None,
|
||||||
|
"max_files": getattr(args, "max_files", None) or settings.get("max_files"),
|
||||||
|
"keep_history": (getattr(args, "keep_history", False)
|
||||||
|
or settings.get("keep_history", False)),
|
||||||
|
"branch": getattr(args, "branch", None) or settings.get("branch"),
|
||||||
|
"keep_secrets": (getattr(args, "keep_secrets", False)
|
||||||
|
or settings.get("keep_secrets", False)),
|
||||||
|
"exclude": list(getattr(args, "exclude", None) or [])
|
||||||
|
+ list(settings.get("exclude") or []),
|
||||||
|
"include": list(getattr(args, "include", None) or [])
|
||||||
|
+ list(settings.get("include") or []),
|
||||||
|
"config_path": settings.get("_path"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_template(path: Path, source=None, out=None) -> Path:
|
||||||
|
"""Write a starter config, never over one that already exists."""
|
||||||
|
from .cli import fail
|
||||||
|
if path.exists():
|
||||||
|
fail(f"{path} already exists.", "Edit it, or name another path.")
|
||||||
|
body = dict(TEMPLATE)
|
||||||
|
if source:
|
||||||
|
body["source"] = str(source)
|
||||||
|
if out:
|
||||||
|
body["out"] = str(out)
|
||||||
|
path.write_text(json.dumps(body, indent=2) + "\n")
|
||||||
|
return path
|
||||||
613
soleprint/station/tools/histgen/export.py
Normal file
613
soleprint/station/tools/histgen/export.py
Normal file
@@ -0,0 +1,613 @@
|
|||||||
|
"""
|
||||||
|
Materialise the source into the out directory and commit the designed history.
|
||||||
|
|
||||||
|
The source is never touched. What gets committed is a copy, made here, and the
|
||||||
|
copy is the only thing that ends up with a history — so an attempt that goes
|
||||||
|
wrong costs a `rm -rf` rather than an afternoon putting a real checkout back.
|
||||||
|
|
||||||
|
Everything up to this point is analysis and can be recomputed. This part writes,
|
||||||
|
so it starts by working out what it is writing into. Four states, and they are
|
||||||
|
genuinely different:
|
||||||
|
|
||||||
|
absent nothing there yet. Copy the tree, init, commit.
|
||||||
|
|
||||||
|
unfinished a copy is there with commits this tool made and a record of
|
||||||
|
where it stopped. Something interrupted the run — a signal, a
|
||||||
|
full disk, a hook that refused. Continue from the group after
|
||||||
|
the last one recorded.
|
||||||
|
|
||||||
|
foreign a copy is there with commits this tool did not make. That is
|
||||||
|
history someone else is entitled to, so nothing is rewritten,
|
||||||
|
moved or deleted: the designed account is committed to its own
|
||||||
|
orphan branch and the existing branch is left exactly as it
|
||||||
|
was. Two tiers, which is what `all/ctrl/handover.sh` has been
|
||||||
|
saying all along.
|
||||||
|
|
||||||
|
stale a copy is there that does not match the plan any more. Refuse,
|
||||||
|
and say which of the two moved.
|
||||||
|
|
||||||
|
Telling the second apart from the third is the whole reason progress.json
|
||||||
|
exists. Without it both read as "there are commits here", and the tool either
|
||||||
|
destroys work it should have kept or refuses to finish work it started.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .census import current_file_set, is_git, load_index, state_dir
|
||||||
|
from .cli import fail
|
||||||
|
from .order import plan_path
|
||||||
|
|
||||||
|
MESSAGE_DIR = "messages"
|
||||||
|
SCRIPT_FILE = "regen.sh"
|
||||||
|
PROGRESS_FILE = "progress.json"
|
||||||
|
|
||||||
|
# Where the designed account goes when the copy already carries a history that
|
||||||
|
# is not ours. A name, not a number, because it has to mean something in a
|
||||||
|
# branch list six months from now.
|
||||||
|
DEFAULT_BRANCH = "designed-history"
|
||||||
|
|
||||||
|
|
||||||
|
# ── where the copy lives ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def repo_dir(source: Path, out: Path) -> Path:
|
||||||
|
"""
|
||||||
|
The copy, named after the source.
|
||||||
|
|
||||||
|
Named rather than called `repo/`, because this directory gets `cd`-ed into,
|
||||||
|
pushed from and looked at in a file manager, and "adapter" answers a
|
||||||
|
question there that "repo" does not.
|
||||||
|
"""
|
||||||
|
return Path(out) / source.name
|
||||||
|
|
||||||
|
|
||||||
|
def progress_path(out) -> Path:
|
||||||
|
return state_dir(out) / PROGRESS_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def plan_fingerprint(plan) -> str:
|
||||||
|
"""
|
||||||
|
Identifies the plan a history was built from, by its groups and their paths.
|
||||||
|
|
||||||
|
Messages are deliberately not in it. Rewording a commit that has not been
|
||||||
|
made yet must not invalidate the twelve that have — that is the normal way
|
||||||
|
this tool gets used, one group's message at a time.
|
||||||
|
"""
|
||||||
|
shape = [[g["n"], sorted(g["paths"])] for g in plan["groups"]]
|
||||||
|
return hashlib.sha256(json.dumps(shape, sort_keys=True).encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def load_progress(out) -> dict:
|
||||||
|
p = progress_path(out)
|
||||||
|
if not p.exists():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return json.loads(p.read_text())
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
# Unreadable progress means we cannot prove which commits are ours, and
|
||||||
|
# guessing is exactly the thing this file exists to avoid.
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def save_progress(out, data) -> None:
|
||||||
|
progress_path(out).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
progress_path(out).write_text(json.dumps(data, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
# ── git ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _git(repo: Path, *args, check=True):
|
||||||
|
r = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True)
|
||||||
|
if check and r.returncode != 0:
|
||||||
|
fail(f"git {' '.join(args[:2])} failed: {r.stderr.strip() or r.stdout.strip()}")
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def _head(repo: Path):
|
||||||
|
r = _git(repo, "rev-parse", "HEAD", check=False)
|
||||||
|
return r.stdout.strip() if r.returncode == 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _commit_count(repo: Path) -> int:
|
||||||
|
r = _git(repo, "rev-list", "--count", "HEAD", check=False)
|
||||||
|
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── what state is the out directory in ─────────────────────────────────────
|
||||||
|
|
||||||
|
def inspect(source: Path, out: Path, plan=None):
|
||||||
|
"""
|
||||||
|
Read the out directory and say what is there. Writes nothing.
|
||||||
|
|
||||||
|
`status` prints this; `export` branches on it. One function so the two can
|
||||||
|
never disagree about what they are looking at, which they would within a
|
||||||
|
week of being written separately.
|
||||||
|
"""
|
||||||
|
copy = repo_dir(source, out)
|
||||||
|
progress = load_progress(out)
|
||||||
|
report = {
|
||||||
|
"copy": copy,
|
||||||
|
"exists": copy.is_dir(),
|
||||||
|
"git": copy.is_dir() and is_git(copy),
|
||||||
|
"commits": 0,
|
||||||
|
"state": "absent",
|
||||||
|
"done": [],
|
||||||
|
"remaining": [],
|
||||||
|
"branch": None,
|
||||||
|
"detail": "",
|
||||||
|
}
|
||||||
|
if not report["exists"]:
|
||||||
|
report["detail"] = "nothing exported yet"
|
||||||
|
if plan:
|
||||||
|
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||||
|
return report
|
||||||
|
|
||||||
|
if not report["git"]:
|
||||||
|
report["state"] = "stale"
|
||||||
|
report["detail"] = ("a directory is there but it is not a git repo — "
|
||||||
|
"an export that died before `git init`")
|
||||||
|
return report
|
||||||
|
|
||||||
|
report["commits"] = _commit_count(copy)
|
||||||
|
report["branch"] = _git(copy, "rev-parse", "--abbrev-ref", "HEAD",
|
||||||
|
check=False).stdout.strip() or None
|
||||||
|
|
||||||
|
if report["commits"] == 0:
|
||||||
|
report["state"] = "absent"
|
||||||
|
report["detail"] = "a repo with no commits"
|
||||||
|
if plan:
|
||||||
|
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||||
|
return report
|
||||||
|
|
||||||
|
ours = progress.get("commits", [])
|
||||||
|
head = _head(copy)
|
||||||
|
|
||||||
|
if not ours:
|
||||||
|
report["state"] = "foreign"
|
||||||
|
report["detail"] = (f"{report['commits']} commit(s) this tool did not make")
|
||||||
|
if plan:
|
||||||
|
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||||
|
return report
|
||||||
|
|
||||||
|
if plan and progress.get("plan") != plan_fingerprint(plan):
|
||||||
|
report["state"] = "stale"
|
||||||
|
report["detail"] = ("the plan changed after this history was started — "
|
||||||
|
"the groups are not the ones these commits were made from")
|
||||||
|
report["done"] = [c["n"] for c in ours]
|
||||||
|
return report
|
||||||
|
|
||||||
|
# The record has to still describe reality. If the branch moved underneath
|
||||||
|
# us — a rebase, a reset, an amend — continuing would build on something
|
||||||
|
# other than what was recorded, and quietly.
|
||||||
|
if head != ours[-1]["sha"]:
|
||||||
|
report["state"] = "stale"
|
||||||
|
report["detail"] = ("the copy has moved since this tool last wrote to it "
|
||||||
|
f"(expected {ours[-1]['sha'][:9]}, found "
|
||||||
|
f"{(head or '-')[:9]})")
|
||||||
|
report["done"] = [c["n"] for c in ours]
|
||||||
|
return report
|
||||||
|
|
||||||
|
report["done"] = [c["n"] for c in ours]
|
||||||
|
if plan:
|
||||||
|
report["remaining"] = [g["n"] for g in plan["groups"]
|
||||||
|
if g["n"] not in set(report["done"])]
|
||||||
|
report["state"] = "complete" if plan and not report["remaining"] else "unfinished"
|
||||||
|
report["detail"] = (f"{len(report['done'])} group(s) committed by this tool"
|
||||||
|
+ (f", {len(report['remaining'])} to go" if report["remaining"] else ""))
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
# ── checks ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def load_plan(out):
|
||||||
|
p = plan_path(out)
|
||||||
|
if not p.exists():
|
||||||
|
fail(f"No plan at {p}.", "Run `scan` then `plan` first.")
|
||||||
|
try:
|
||||||
|
return json.loads(p.read_text())
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
# Hand-editing plan.json is the expected workflow, so a trailing comma
|
||||||
|
# is a normal event and deserves a line number rather than a traceback.
|
||||||
|
fail(f"{p} is not valid JSON: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def check_plan(source: Path, out, plan, require_messages=True):
|
||||||
|
"""
|
||||||
|
Refuse a plan that could not produce the tree it claims to.
|
||||||
|
|
||||||
|
What the plan is measured against is the source as it stands *now*, with
|
||||||
|
the same filter the census used. Not the stored list: a file added after
|
||||||
|
the scan is exactly what this is here to catch, and a stored list cannot
|
||||||
|
see it. Not the unfiltered source either, or every deliberately dropped key
|
||||||
|
comes back as a file no group covers.
|
||||||
|
"""
|
||||||
|
planned, dupes = [], []
|
||||||
|
for g in plan["groups"]:
|
||||||
|
for p in g["paths"]:
|
||||||
|
(dupes if p in planned else planned).append(p)
|
||||||
|
|
||||||
|
problems = []
|
||||||
|
if dupes:
|
||||||
|
problems.append(f"{len(dupes)} path(s) appear in more than one group: "
|
||||||
|
+ ", ".join(sorted(set(dupes))[:5]))
|
||||||
|
|
||||||
|
missing = [p for p in planned if not (source / p).is_file()]
|
||||||
|
if missing:
|
||||||
|
problems.append(f"{len(missing)} planned path(s) are not in the source: "
|
||||||
|
+ ", ".join(missing[:5]))
|
||||||
|
|
||||||
|
present = current_file_set(source, load_index(out))
|
||||||
|
unplanned = sorted(present - set(planned))
|
||||||
|
if unplanned:
|
||||||
|
problems.append(f"{len(unplanned)} file(s) are in the source but in no group: "
|
||||||
|
+ ", ".join(unplanned[:5])
|
||||||
|
+ "\n Re-run `scan` and `plan` if the source changed since.")
|
||||||
|
|
||||||
|
empty = [g["n"] for g in plan["groups"] if not g["paths"]]
|
||||||
|
if empty:
|
||||||
|
problems.append(f"group(s) {empty} have no paths")
|
||||||
|
|
||||||
|
if require_messages:
|
||||||
|
unwritten = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||||
|
if unwritten:
|
||||||
|
problems.append(
|
||||||
|
f"{len(unwritten)} group(s) have no title: "
|
||||||
|
+ ", ".join(str(n) for n in unwritten[:8])
|
||||||
|
+ f"\n Read {state_dir(out) / 'briefs'}/ and fill them in, "
|
||||||
|
"or pass --allow-untitled.")
|
||||||
|
|
||||||
|
if problems:
|
||||||
|
for p in problems:
|
||||||
|
print(f"Error: {p}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return planned
|
||||||
|
|
||||||
|
|
||||||
|
def _message(group):
|
||||||
|
title = (group.get("title") or "").strip() or f"{group['slug']} ({len(group['paths'])} files)"
|
||||||
|
body = (group.get("body") or "").strip()
|
||||||
|
return f"{title}\n\n{body}\n" if body else f"{title}\n"
|
||||||
|
|
||||||
|
|
||||||
|
def source_tree_hash(source: Path, paths):
|
||||||
|
"""
|
||||||
|
The tree hash the source files would produce, without committing anything.
|
||||||
|
|
||||||
|
Runs against a temporary index and, when the source has no git, a temporary
|
||||||
|
git directory too. The source's own index is never touched: someone running
|
||||||
|
this mid-edit must not lose their staging area to a verification step.
|
||||||
|
"""
|
||||||
|
import tempfile
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-idx-") as tmp:
|
||||||
|
env = dict(os.environ, GIT_INDEX_FILE=str(Path(tmp) / "index"))
|
||||||
|
if not (source / ".git").exists():
|
||||||
|
env["GIT_DIR"] = str(Path(tmp) / "git")
|
||||||
|
env["GIT_WORK_TREE"] = str(source)
|
||||||
|
subprocess.run(["git", "init", "-q"], env=env, capture_output=True)
|
||||||
|
proc = subprocess.run(
|
||||||
|
["git", "-C", str(source), "update-index", "--add", "--stdin"],
|
||||||
|
input="\n".join(paths) + "\n", text=True, capture_output=True, env=env)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
return None
|
||||||
|
r = subprocess.run(["git", "-C", str(source), "write-tree"],
|
||||||
|
capture_output=True, text=True, env=env)
|
||||||
|
return r.stdout.strip() if r.returncode == 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
# ── the guards ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def verify(copy: Path, expected_tree=None, quiet=False):
|
||||||
|
"""
|
||||||
|
Nothing left untracked, and the tree still matches the source.
|
||||||
|
|
||||||
|
Nothing is exempt from the first check. The state directory lives in `out`
|
||||||
|
and the copy lives inside it, so there is genuinely nothing of ours in the
|
||||||
|
tree being checked — which is stricter than the version that had to forgive
|
||||||
|
its own scaffolding.
|
||||||
|
"""
|
||||||
|
ok = True
|
||||||
|
status = _git(copy, "status", "--porcelain").stdout.strip()
|
||||||
|
if status:
|
||||||
|
ok = False
|
||||||
|
print("Error: the tree is not clean — these never made it into a commit:",
|
||||||
|
file=sys.stderr)
|
||||||
|
for line in status.split("\n")[:20]:
|
||||||
|
print(f" {line}", file=sys.stderr)
|
||||||
|
if len(status.split("\n")) > 20:
|
||||||
|
print(f" ... and {len(status.split(chr(10))) - 20} more", file=sys.stderr)
|
||||||
|
elif not quiet:
|
||||||
|
print(" nothing left untracked: ok")
|
||||||
|
|
||||||
|
if expected_tree:
|
||||||
|
head = _git(copy, "rev-parse", "HEAD^{tree}", check=False).stdout.strip()
|
||||||
|
if head != expected_tree:
|
||||||
|
ok = False
|
||||||
|
print(f"Error: the exported tree does not match the source.\n"
|
||||||
|
f" source {expected_tree}\n HEAD {head}", file=sys.stderr)
|
||||||
|
elif not quiet:
|
||||||
|
print(f" tree matches source ({head[:12]}): ok")
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
|
# ── materialising the copy ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def materialise(source: Path, copy: Path, paths, keep_history=False, quiet=False):
|
||||||
|
"""
|
||||||
|
Put the planned files into the copy, and nothing else.
|
||||||
|
|
||||||
|
Copied file by file from the plan rather than with `cp -r`, because the
|
||||||
|
plan is the definition of what belongs in the history: anything gitignored,
|
||||||
|
anything untracked, and the source's own .git are all things the source has
|
||||||
|
and the export must not.
|
||||||
|
|
||||||
|
`keep_history` is the exception, and the only reason the source's .git ever
|
||||||
|
comes across: it is what lets an existing history be carried into the copy
|
||||||
|
so the designed account can sit beside it instead of replacing it.
|
||||||
|
"""
|
||||||
|
copy.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
if keep_history and (source / ".git").is_dir() and not (copy / ".git").exists():
|
||||||
|
shutil.copytree(source / ".git", copy / ".git", symlinks=True)
|
||||||
|
# A copied .git still points its index at files that are about to be
|
||||||
|
# rewritten underneath it; reset so status reflects the copy, not the
|
||||||
|
# source's staging area at the moment it was cloned.
|
||||||
|
_git(copy, "reset", "-q", check=False)
|
||||||
|
|
||||||
|
for rel in paths:
|
||||||
|
target = copy / rel
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(source / rel, target)
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
print(f" copied {len(paths)} files -> {copy}")
|
||||||
|
|
||||||
|
|
||||||
|
def _start_branch(copy: Path, report, branch, quiet=False):
|
||||||
|
"""
|
||||||
|
Decide which ref the designed history goes on, and get there.
|
||||||
|
|
||||||
|
A foreign history is not ours to move, so the designed account starts from
|
||||||
|
an orphan — no parent, nothing shared — and the branch that was there keeps
|
||||||
|
pointing exactly where it did.
|
||||||
|
"""
|
||||||
|
if report["state"] != "foreign":
|
||||||
|
return None
|
||||||
|
name = branch or DEFAULT_BRANCH
|
||||||
|
if _git(copy, "rev-parse", "--verify", name, check=False).returncode == 0:
|
||||||
|
fail(f"Branch '{name}' already exists in {copy}.",
|
||||||
|
"Pick another with --branch, or delete it if it was a failed attempt.")
|
||||||
|
kept = report["branch"] or "the existing branch"
|
||||||
|
if not quiet:
|
||||||
|
print(f" {report['commits']} existing commit(s) on {kept}: kept, untouched")
|
||||||
|
print(f" the designed history goes on a new orphan branch '{name}'")
|
||||||
|
_git(copy, "checkout", "-q", "--orphan", name)
|
||||||
|
# --orphan keeps the index, which would make the first designed commit
|
||||||
|
# carry every file the old branch had staged.
|
||||||
|
_git(copy, "rm", "-rq", "--cached", ".", check=False)
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def export(source: Path, out, dry_run=False, commands=False, allow_untitled=False,
|
||||||
|
keep_history=False, branch=None, force=False, quiet=False):
|
||||||
|
plan = load_plan(out)
|
||||||
|
planned = check_plan(source, out, plan, require_messages=not allow_untitled)
|
||||||
|
expected = source_tree_hash(source, planned)
|
||||||
|
if not expected:
|
||||||
|
fail("Could not compute the source tree hash.",
|
||||||
|
"Without it the export cannot be checked, and an unchecked export "
|
||||||
|
"is the thing this refuses to produce.")
|
||||||
|
|
||||||
|
copy = repo_dir(source, out)
|
||||||
|
report = inspect(source, out, plan)
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return _emit_script(source, out, plan, planned, expected, report,
|
||||||
|
keep_history, branch, quiet)
|
||||||
|
|
||||||
|
if commands:
|
||||||
|
return _emit_commands(source, out, plan, planned, expected, force, quiet)
|
||||||
|
|
||||||
|
if report["state"] == "stale" and not force:
|
||||||
|
fail(f"{copy}: {report['detail']}.",
|
||||||
|
"Pass --force to discard what is there and export again, or point "
|
||||||
|
"--out somewhere else to keep it.")
|
||||||
|
if report["state"] == "complete":
|
||||||
|
print(f"Already exported: {len(report['done'])} groups committed in {copy}.")
|
||||||
|
print("Nothing to do. Re-plan, or use --force to start over.")
|
||||||
|
return True
|
||||||
|
|
||||||
|
if report["state"] == "stale" and force:
|
||||||
|
if not quiet:
|
||||||
|
print(f" discarding {copy}")
|
||||||
|
shutil.rmtree(copy)
|
||||||
|
save_progress(out, {})
|
||||||
|
report = inspect(source, out, plan)
|
||||||
|
|
||||||
|
progress = load_progress(out)
|
||||||
|
resuming = report["state"] == "unfinished"
|
||||||
|
|
||||||
|
if resuming:
|
||||||
|
done = set(report["done"])
|
||||||
|
if not quiet:
|
||||||
|
print(f"Resuming: {len(done)} of {len(plan['groups'])} groups already "
|
||||||
|
f"committed in {copy}.")
|
||||||
|
# The files are already there from the interrupted run, but a source
|
||||||
|
# edited since would otherwise be silently ignored.
|
||||||
|
materialise(source, copy, planned, quiet=quiet)
|
||||||
|
else:
|
||||||
|
done = set()
|
||||||
|
materialise(source, copy, planned, keep_history=keep_history, quiet=quiet)
|
||||||
|
if not is_git(copy):
|
||||||
|
_git(copy, "init", "-q")
|
||||||
|
# Re-read the copy. --keep-history has only just put a history into it,
|
||||||
|
# so the state worked out before the directory existed cannot have seen
|
||||||
|
# it — and acting on the stale answer commits the designed account on
|
||||||
|
# top of the history it was supposed to sit beside.
|
||||||
|
report = inspect(source, out, plan)
|
||||||
|
active = _start_branch(copy, report, branch, quiet)
|
||||||
|
progress = {"plan": plan_fingerprint(plan), "source": str(source),
|
||||||
|
"branch": active, "commits": []}
|
||||||
|
save_progress(out, progress)
|
||||||
|
|
||||||
|
for g in plan["groups"]:
|
||||||
|
if g["n"] in done:
|
||||||
|
continue
|
||||||
|
_git(copy, "add", "--", *g["paths"])
|
||||||
|
msg = copy / ".git" / "HISTGEN_MSG"
|
||||||
|
msg.write_text(_message(g))
|
||||||
|
_git(copy, "commit", "-q", "-F", str(msg))
|
||||||
|
msg.unlink(missing_ok=True)
|
||||||
|
# Recorded after each commit, not at the end. The whole point is to
|
||||||
|
# survive the run not reaching the end.
|
||||||
|
progress.setdefault("commits", []).append({"n": g["n"], "sha": _head(copy)})
|
||||||
|
progress["plan"] = plan_fingerprint(plan)
|
||||||
|
save_progress(out, progress)
|
||||||
|
if not quiet:
|
||||||
|
print(f" {g['n']:02d} {_message(g).splitlines()[0]}")
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
# The repo's own count, not the plan's. With a kept history the two
|
||||||
|
# differ, and the number a reader wants is what is actually in there.
|
||||||
|
where = _git(copy, "rev-parse", "--abbrev-ref", "HEAD",
|
||||||
|
check=False).stdout.strip()
|
||||||
|
print(f"\n{_commit_count(copy)} commits on {where} in {copy}. "
|
||||||
|
"Checking:", flush=True)
|
||||||
|
if not verify(copy, expected, quiet=quiet):
|
||||||
|
sys.exit(1)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_script(source, out, plan, planned, expected, report,
|
||||||
|
keep_history, branch, quiet):
|
||||||
|
"""
|
||||||
|
Write the export as a shell script instead of running it.
|
||||||
|
|
||||||
|
Reviewing plain git commands before they run is worth more here than
|
||||||
|
anywhere else: this is the one operation whose mistakes are baked into
|
||||||
|
every commit that follows.
|
||||||
|
"""
|
||||||
|
state = state_dir(out)
|
||||||
|
copy = repo_dir(source, out)
|
||||||
|
msg_dir = state / MESSAGE_DIR
|
||||||
|
msg_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for stale in msg_dir.glob("*.txt"):
|
||||||
|
stale.unlink()
|
||||||
|
|
||||||
|
q = shlex.quote
|
||||||
|
lines = [
|
||||||
|
"#!/usr/bin/env bash",
|
||||||
|
"# Generated by histgen. Review, then run from anywhere.",
|
||||||
|
"set -euo pipefail",
|
||||||
|
"",
|
||||||
|
f"SOURCE={q(str(source))}",
|
||||||
|
f"COPY={q(str(copy))}",
|
||||||
|
"",
|
||||||
|
'mkdir -p "$COPY"',
|
||||||
|
]
|
||||||
|
if keep_history and (source / ".git").is_dir():
|
||||||
|
lines.append('test -d "$COPY/.git" || cp -a "$SOURCE/.git" "$COPY/.git"')
|
||||||
|
lines += [
|
||||||
|
"# Only the planned files: not the source's .git, not anything ignored.",
|
||||||
|
'while IFS= read -r f; do mkdir -p "$COPY/$(dirname "$f")"; '
|
||||||
|
'cp -p "$SOURCE/$f" "$COPY/$f"; done <<\'PATHS\'',
|
||||||
|
*planned,
|
||||||
|
"PATHS",
|
||||||
|
"",
|
||||||
|
'cd "$COPY"',
|
||||||
|
"test -d .git || git init -q",
|
||||||
|
]
|
||||||
|
if report["state"] == "foreign":
|
||||||
|
name = branch or DEFAULT_BRANCH
|
||||||
|
lines += [f"# {report['commits']} existing commit(s) stay where they are.",
|
||||||
|
f"git checkout -q --orphan {q(name)}",
|
||||||
|
"git rm -rq --cached . || true", ""]
|
||||||
|
|
||||||
|
for g in plan["groups"]:
|
||||||
|
name = f"{g['n']:02d}-{g['slug']}.txt"
|
||||||
|
(msg_dir / name).write_text(_message(g))
|
||||||
|
lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}",
|
||||||
|
"git add -- " + " ".join(q(p) for p in g["paths"]),
|
||||||
|
f"git commit -q -F {q(str(msg_dir / name))}",
|
||||||
|
""]
|
||||||
|
|
||||||
|
lines += [
|
||||||
|
"# The two guards. A history that fails these is not worth keeping.",
|
||||||
|
'test -z "$(git status --porcelain)" || '
|
||||||
|
'{ echo "FAILED: files left untracked" >&2; exit 1; }',
|
||||||
|
f'test "$(git rev-parse HEAD^{{tree}})" = "{expected}" || '
|
||||||
|
'{ echo "FAILED: exported tree does not match source" >&2; exit 1; }',
|
||||||
|
'echo "ok: $(git rev-list --count HEAD) commits, tree verified"',
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
script = state / SCRIPT_FILE
|
||||||
|
script.write_text("\n".join(lines))
|
||||||
|
script.chmod(0o755)
|
||||||
|
if not quiet:
|
||||||
|
print(f"Wrote {len(plan['groups'])} commits as commands -> {script}")
|
||||||
|
print(f" messages -> {msg_dir}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_commands(source: Path, out, plan, planned, expected, force, quiet):
|
||||||
|
"""
|
||||||
|
Copy the files, create no repo, and print the commands to make the history.
|
||||||
|
|
||||||
|
The other two modes each decide something for you: `export` runs the whole
|
||||||
|
thing, `--dry-run` writes a script that would. This one does the half that
|
||||||
|
is tedious and gets the other half out of the way — the copy is made, and
|
||||||
|
what comes back is a list you read, edit and run yourself.
|
||||||
|
|
||||||
|
Nothing here creates a .git. `git init` is the first line of the list rather
|
||||||
|
than something already done, because a repo that appeared without you asking
|
||||||
|
is exactly what someone reaching for this mode does not want.
|
||||||
|
"""
|
||||||
|
copy = repo_dir(source, out)
|
||||||
|
|
||||||
|
# Asked for no repo, so an existing one is a contradiction worth stopping
|
||||||
|
# for: the commands below would commit into it rather than into a fresh one.
|
||||||
|
if (copy / ".git").exists() and not force:
|
||||||
|
fail(f"{copy} already contains a git repo.",
|
||||||
|
"This mode creates none and the commands assume none. Delete it, "
|
||||||
|
"point --out elsewhere, or pass --force to copy the files in anyway.")
|
||||||
|
|
||||||
|
materialise(source, copy, planned, quiet=quiet)
|
||||||
|
|
||||||
|
state = state_dir(out)
|
||||||
|
msg_dir = state / MESSAGE_DIR
|
||||||
|
msg_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for stale in msg_dir.glob("*.txt"):
|
||||||
|
stale.unlink()
|
||||||
|
|
||||||
|
q = shlex.quote
|
||||||
|
lines = [f"cd {q(str(copy))}", "git init", ""]
|
||||||
|
for g in plan["groups"]:
|
||||||
|
name = f"{g['n']:02d}-{g['slug']}.txt"
|
||||||
|
(msg_dir / name).write_text(_message(g))
|
||||||
|
lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}",
|
||||||
|
"git add -- " + " ".join(q(p) for p in g["paths"]),
|
||||||
|
f"git commit -F {q(str(msg_dir / name))}",
|
||||||
|
""]
|
||||||
|
|
||||||
|
lines += [
|
||||||
|
"# Worth running afterwards. The first says no file was silently",
|
||||||
|
"# missed; the second says the result is byte-identical to the source.",
|
||||||
|
"git status --porcelain",
|
||||||
|
f"git rev-parse HEAD^{{tree}} # expect {expected}",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
listing = "\n".join(lines)
|
||||||
|
(state / "commands.sh").write_text(listing)
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
print(f" messages -> {msg_dir}")
|
||||||
|
print(f" this list -> {state / 'commands.sh'}\n")
|
||||||
|
print(listing)
|
||||||
|
return True
|
||||||
116
soleprint/station/tools/histgen/history.py
Normal file
116
soleprint/station/tools/histgen/history.py
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
"""
|
||||||
|
What the history already there says, next to what the plan proposes.
|
||||||
|
|
||||||
|
Reads and reports. It never rewrites: published history is someone else's
|
||||||
|
clone, and the useful output here is an argument about ordering, not a
|
||||||
|
force-push.
|
||||||
|
|
||||||
|
Most repos this gets pointed at are in the checkpoint tier — "updates 33.1
|
||||||
|
139", "working state", "debugging" — where the honest finding is that the log
|
||||||
|
records when work was saved and nothing about how the thing is built. That is
|
||||||
|
worth printing plainly, because it is the case for keeping a second, designed
|
||||||
|
history rather than trying to repair this one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
# Subjects that carry no information about what changed. Matching is on the
|
||||||
|
# whole subject, lowercased, with trailing numbers dropped — "updates 33.1 139"
|
||||||
|
# and "updates 33.1 84" are the same non-statement.
|
||||||
|
NOISE = {"update", "updates", "wip", "fix", "fixes", "changes", "some changes",
|
||||||
|
"working state", "debugging", "init commit", "initial commit", "misc",
|
||||||
|
"checkpoint", "save", "final", "for final test", "cleanup", "tmp"}
|
||||||
|
|
||||||
|
|
||||||
|
def _subject_is_noise(subject):
|
||||||
|
s = subject.strip().lower().rstrip("0123456789. ")
|
||||||
|
return s in NOISE or not s
|
||||||
|
|
||||||
|
|
||||||
|
def read_history(repo):
|
||||||
|
"""[(sha, subject, [paths])] oldest first, or [] if there is no history."""
|
||||||
|
r = subprocess.run(
|
||||||
|
["git", "-C", str(repo), "log", "--reverse", "--name-only",
|
||||||
|
"--format=%x00%h%x1f%s"],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
if r.returncode != 0:
|
||||||
|
return []
|
||||||
|
commits = []
|
||||||
|
for chunk in r.stdout.split("\0"):
|
||||||
|
if not chunk.strip():
|
||||||
|
continue
|
||||||
|
head, _, rest = chunk.partition("\n")
|
||||||
|
sha, _, subject = head.partition("\x1f")
|
||||||
|
paths = [l for l in rest.split("\n") if l.strip()]
|
||||||
|
commits.append((sha, subject, paths))
|
||||||
|
return commits
|
||||||
|
|
||||||
|
|
||||||
|
def compare(source, plan, quiet=False):
|
||||||
|
"""Print how the existing history lines up with the proposed one."""
|
||||||
|
commits = read_history(source)
|
||||||
|
groups = plan["groups"]
|
||||||
|
if not commits:
|
||||||
|
print(f"{len(groups)} groups proposed; the source has no history to compare.")
|
||||||
|
return {"commits": 0, "groups": len(groups)}
|
||||||
|
|
||||||
|
where = {p: g["n"] for g in groups for p in g["paths"]}
|
||||||
|
|
||||||
|
# A commit maps to the group holding most of the files it touched. Files
|
||||||
|
# that no longer exist are dropped rather than counted against it — a
|
||||||
|
# commit that deleted something is not disagreeing about order.
|
||||||
|
mapped, noise, touches = {}, [], defaultdict(list)
|
||||||
|
for sha, subject, paths in commits:
|
||||||
|
hits = [where[p] for p in paths if p in where]
|
||||||
|
if not hits:
|
||||||
|
noise.append((sha, subject, "touches nothing that still exists"))
|
||||||
|
continue
|
||||||
|
best = max(set(hits), key=lambda n: (hits.count(n), -n))
|
||||||
|
spread = len(set(hits))
|
||||||
|
mapped[sha] = (best, subject, spread, len(hits))
|
||||||
|
touches[best].append(sha)
|
||||||
|
if _subject_is_noise(subject):
|
||||||
|
noise.append((sha, subject, f"says nothing; touches {spread} group(s)"))
|
||||||
|
|
||||||
|
print(f"{len(groups)} groups proposed, {len(commits)} existing commits.\n")
|
||||||
|
|
||||||
|
# Order disagreement: walking the real history, does the group number ever
|
||||||
|
# go backwards? That is the concrete "this was built in a different order".
|
||||||
|
seen_max, inversions = 0, []
|
||||||
|
for sha, subject, _ in commits:
|
||||||
|
if sha not in mapped:
|
||||||
|
continue
|
||||||
|
n = mapped[sha][0]
|
||||||
|
if n < seen_max:
|
||||||
|
inversions.append((sha, n, seen_max, subject))
|
||||||
|
seen_max = max(seen_max, n)
|
||||||
|
|
||||||
|
for g in groups:
|
||||||
|
shas = touches.get(g["n"], [])
|
||||||
|
title = g.get("title") or g["slug"]
|
||||||
|
if not shas:
|
||||||
|
mark, note = "+", "no existing commit builds this"
|
||||||
|
elif len(shas) == 1:
|
||||||
|
mark, note = "=", f"{shas[0]}"
|
||||||
|
else:
|
||||||
|
mark, note = "~", f"split across {len(shas)} commits ({', '.join(shas[:4])})"
|
||||||
|
print(f" {mark} {g['n']:02d} {title[:44]:46} {note}")
|
||||||
|
|
||||||
|
if inversions:
|
||||||
|
print(f"\n ! {len(inversions)} commit(s) land earlier in the proposed order "
|
||||||
|
f"than work already done:")
|
||||||
|
for sha, n, high, subject in inversions[:10]:
|
||||||
|
print(f" {sha} group {n:02d} after group {high:02d} {subject[:44]}")
|
||||||
|
|
||||||
|
if noise:
|
||||||
|
print(f"\n ? {len(noise)} commit(s) carry no usable account of the change:")
|
||||||
|
for sha, subject, why in noise[:10]:
|
||||||
|
print(f" {sha} {subject[:44]:46} {why}")
|
||||||
|
if len(noise) > 10:
|
||||||
|
print(f" ... and {len(noise) - 10} more")
|
||||||
|
|
||||||
|
print("\n = matched one commit ~ split + not in history "
|
||||||
|
"! out of order ? uninformative")
|
||||||
|
return {"commits": len(commits), "groups": len(groups),
|
||||||
|
"inversions": len(inversions), "noise": len(noise)}
|
||||||
450
soleprint/station/tools/histgen/order.py
Normal file
450
soleprint/station/tools/histgen/order.py
Normal file
@@ -0,0 +1,450 @@
|
|||||||
|
"""
|
||||||
|
The order the files go in, and where one commit stops and the next begins.
|
||||||
|
|
||||||
|
Two decisions, and they are not the same decision. Order answers "what can be
|
||||||
|
understood before what"; grouping answers "what is one idea". Getting the first
|
||||||
|
right and the second wrong gives you 64 correct commits nobody wants to read.
|
||||||
|
|
||||||
|
The order is role first, references second. Roles carry the heuristic — ignore
|
||||||
|
rules and README, then the config layer, then the things that source it, the
|
||||||
|
front door late because it only dispatches, the bootstrap account last because
|
||||||
|
it narrates everything above it. References refine within that, so a config
|
||||||
|
lands before the script that sources it.
|
||||||
|
|
||||||
|
References never override roles. A reference in code is a dependency, but roles
|
||||||
|
already encode dependencies that no reference states: nothing in the repo
|
||||||
|
*refers to* .gitignore, and the README is named by nothing while naming
|
||||||
|
everything. Letting edges win produces the ignore rules committed after the
|
||||||
|
code they exclude — technically consistent, and unreadable.
|
||||||
|
|
||||||
|
Nothing here is authoritative. plan.json is a file, and moving a path from one
|
||||||
|
group to another is the expected way to use it: this gets the shape right so
|
||||||
|
the argument is about two or three groups, not sixty-four paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .census import ROLE_RANK, state_dir
|
||||||
|
|
||||||
|
PLAN_FILE = "plan.json"
|
||||||
|
|
||||||
|
# Roles that name other files without depending on them. Their outgoing edges
|
||||||
|
# are dropped: a README mentioning every script in the tree is a table of
|
||||||
|
# contents, not a build order.
|
||||||
|
NARRATIVE = {"skeleton", "readme", "doc", "bootstrap", "asset", "lock"}
|
||||||
|
|
||||||
|
# A directory that carries its own README, Makefile or package manifest is a
|
||||||
|
# project in its own right. It is committed whole and late — it stands on the
|
||||||
|
# repo around it, so it cannot be read before it. rig's sample-rig is the case:
|
||||||
|
# sixteen files, one idea, and it calls rig's own addon script rather than
|
||||||
|
# reimplementing it.
|
||||||
|
# A README is NOT one of these. Any directory worth having explains itself, and
|
||||||
|
# ctrl/k8s/README.md documenting four manifests does not make them a project —
|
||||||
|
# it made them sort after the Makefile, which is where this rule came from. What
|
||||||
|
# marks a project is something that builds or resolves it.
|
||||||
|
SUBPROJECT_MARKERS = {"makefile", "package.json", "pyproject.toml",
|
||||||
|
"go.mod", "cargo.toml", "gemfile", "build.gradle"}
|
||||||
|
SUBPROJECT_RANK = 95 # after the front door, before the bootstrap account
|
||||||
|
|
||||||
|
# How many files one commit may hold before it stops being one idea. Soft: a
|
||||||
|
# subproject and a hub with its satellites are exempt, because splitting those
|
||||||
|
# produces a commit that does not build.
|
||||||
|
DEFAULT_MAX_FILES = 8
|
||||||
|
|
||||||
|
# Roles whose files earn their way into a commit by referring to each other,
|
||||||
|
# rather than by sitting in the same directory.
|
||||||
|
CODE_ROLES = {"source", "test", "frontdoor"}
|
||||||
|
|
||||||
|
|
||||||
|
def _subprojects(paths):
|
||||||
|
"""Directories that are their own project -> the files beneath them."""
|
||||||
|
marked = set()
|
||||||
|
for p in paths:
|
||||||
|
parent = str(Path(p).parent)
|
||||||
|
if parent not in (".", "") and Path(p).name.lower() in SUBPROJECT_MARKERS:
|
||||||
|
marked.add(parent)
|
||||||
|
# A subproject inside a subproject belongs to the outer one; one commit,
|
||||||
|
# not two nested ones.
|
||||||
|
roots = {d for d in marked
|
||||||
|
if not any(d != o and d.startswith(o + "/") for o in marked)}
|
||||||
|
owned = {}
|
||||||
|
for p in paths:
|
||||||
|
for root in roots:
|
||||||
|
if p == root or p.startswith(root + "/"):
|
||||||
|
owned[p] = root
|
||||||
|
break
|
||||||
|
return owned
|
||||||
|
|
||||||
|
|
||||||
|
# Roles that are one idea when they sit side by side. A diagram and the source
|
||||||
|
# it renders from belong in the same commit; so do a pin and the config that
|
||||||
|
# reads it. Grouping only — the ordering still keeps them apart.
|
||||||
|
GROUP_TIER = {"asset": "doc", "lock": "pin", "pin": "pin", "config": "pin"}
|
||||||
|
|
||||||
|
|
||||||
|
def _cluster(path, owner, dirs=()):
|
||||||
|
"""
|
||||||
|
The directory a file is grouped under: its subproject, else its own parent.
|
||||||
|
|
||||||
|
The parent, not the top-level directory. Under `ctrl` everything in a tree
|
||||||
|
this shape lands in one bucket — eleven scripts, four profiles and a k8s
|
||||||
|
tree — and the split has to be reconstructed afterwards from references
|
||||||
|
that were never going to describe it.
|
||||||
|
"""
|
||||||
|
if owner:
|
||||||
|
return owner
|
||||||
|
# `ctrl/addons.sh` belongs with `ctrl/addons/`, not with its own siblings.
|
||||||
|
# Clustering is what decides which files are even considered together, so a
|
||||||
|
# hub parted from its satellites here can never be rejoined later.
|
||||||
|
hub = _hub_of(path)
|
||||||
|
if hub and hub in dirs:
|
||||||
|
return hub
|
||||||
|
return str(Path(path).parent)
|
||||||
|
|
||||||
|
|
||||||
|
def adaptive_cap(count, requested=None):
|
||||||
|
"""
|
||||||
|
How many files one commit may hold, for a repo this size.
|
||||||
|
|
||||||
|
A fixed cap does not survive the range. rig is 64 files and wants commits
|
||||||
|
of three or four; spr is 507 and at a flat eight plans a hundred and
|
||||||
|
forty-three, which is the same failure as one commit from the other end.
|
||||||
|
|
||||||
|
A twentieth of the tree, floored at the default, lands close to what these
|
||||||
|
repos were actually built as: rig plans 18 against a real 18, spr 77
|
||||||
|
against a real 78. It is a starting point, not a claim — --max-files
|
||||||
|
overrides it and plan.json is editable either way.
|
||||||
|
"""
|
||||||
|
if requested:
|
||||||
|
return requested
|
||||||
|
return max(DEFAULT_MAX_FILES, -(-count // 20))
|
||||||
|
|
||||||
|
|
||||||
|
def order_files(index, max_files=None):
|
||||||
|
"""Return an ordered list of groups, each a list of repo-relative paths."""
|
||||||
|
files = index["files"]
|
||||||
|
paths = sorted(files)
|
||||||
|
max_files = adaptive_cap(len(paths), max_files)
|
||||||
|
owner = _subprojects(paths)
|
||||||
|
|
||||||
|
def rank(p):
|
||||||
|
return SUBPROJECT_RANK if p in owner else ROLE_RANK.get(files[p]["role"], 60)
|
||||||
|
|
||||||
|
# ── edges ──────────────────────────────────────────────────────────────
|
||||||
|
# An edge b -> a means "b must come after a". Only real dependencies count,
|
||||||
|
# and only between files whose roles do not already disagree: a reference
|
||||||
|
# pointing backwards up the role order is a mention the extractor could not
|
||||||
|
# tell from a dependency, and honouring it inverts the tier.
|
||||||
|
after = defaultdict(set)
|
||||||
|
for p in paths:
|
||||||
|
if files[p]["role"] in NARRATIVE or p in owner:
|
||||||
|
continue
|
||||||
|
for dep in files[p]["refs"]:
|
||||||
|
if dep in files and dep != p and rank(dep) <= rank(p):
|
||||||
|
after[p].add(dep)
|
||||||
|
|
||||||
|
# ── ordering ───────────────────────────────────────────────────────────
|
||||||
|
# Kahn's algorithm, taking the lowest (rank, path) that is ready. Ties are
|
||||||
|
# broken by path so two runs on the same tree produce the same history.
|
||||||
|
blockers = {p: set(after[p]) for p in paths}
|
||||||
|
dependents = defaultdict(set)
|
||||||
|
for p, deps in blockers.items():
|
||||||
|
for d in deps:
|
||||||
|
dependents[d].add(p)
|
||||||
|
|
||||||
|
ready = sorted((p for p in paths if not blockers[p]), key=lambda p: (rank(p), p))
|
||||||
|
ordered = []
|
||||||
|
while ready:
|
||||||
|
p = ready.pop(0)
|
||||||
|
ordered.append(p)
|
||||||
|
for d in sorted(dependents[p]):
|
||||||
|
blockers[d].discard(p)
|
||||||
|
if not blockers[d]:
|
||||||
|
ready.append(d)
|
||||||
|
ready.sort(key=lambda q: (rank(q), q))
|
||||||
|
|
||||||
|
# A cycle leaves files unplaced. Two shell scripts that source each other is
|
||||||
|
# a real thing and not an error here, so append them in role order rather
|
||||||
|
# than refusing to produce a plan at all.
|
||||||
|
if len(ordered) < len(paths):
|
||||||
|
ordered += sorted(set(paths) - set(ordered), key=lambda p: (rank(p), p))
|
||||||
|
|
||||||
|
return _coalesce(_group(ordered, files, owner, after, max_files), owner, max_files)
|
||||||
|
|
||||||
|
|
||||||
|
def _common_dir(paths):
|
||||||
|
"""The deepest directory every path in the group sits under."""
|
||||||
|
parts = list(Path(paths[0]).parent.parts)
|
||||||
|
for p in paths[1:]:
|
||||||
|
other = Path(p).parent.parts
|
||||||
|
keep = []
|
||||||
|
for a, b in zip(parts, other):
|
||||||
|
if a != b:
|
||||||
|
break
|
||||||
|
keep.append(a)
|
||||||
|
parts = keep
|
||||||
|
return tuple(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _coalesce(groups, owner, max_files):
|
||||||
|
"""
|
||||||
|
Merge neighbouring groups that are really one idea in one place.
|
||||||
|
|
||||||
|
Cutting on directory is right for a shallow tree and wrong for a deep one:
|
||||||
|
a directory holding a single file is not an idea, and a repo of five
|
||||||
|
hundred files has a lot of them. Left alone spr planned three hundred and
|
||||||
|
thirty-seven commits, which is the same failure as one commit, from the
|
||||||
|
other end.
|
||||||
|
|
||||||
|
Only neighbours already adjacent in the order merge, only when they share
|
||||||
|
a top-level directory, only under the cap, and only when BOTH are small.
|
||||||
|
|
||||||
|
Both, not either. Letting a lone file join whatever it happened to sit next
|
||||||
|
to put `dockerhost.sh` inside the addons commit — a coherent group of seven
|
||||||
|
with an eighth file that has nothing to do with it. A run of scattered
|
||||||
|
singletons is fragmentation and should close up; a group that already says
|
||||||
|
something should not absorb a stray because it had room.
|
||||||
|
|
||||||
|
What counts as small scales with the cap, so raising --max-files actually
|
||||||
|
buys fewer commits. Gated at a flat two it did not: on a 500-file repo the
|
||||||
|
cap could be raised from 8 to 40 and the count moved by four, because
|
||||||
|
nothing was ever allowed to merge. Asking for bigger commits should widen
|
||||||
|
what is considered fragmentation, not just what is allowed to survive.
|
||||||
|
"""
|
||||||
|
loose = max(2, max_files // 4)
|
||||||
|
out = []
|
||||||
|
for group in groups:
|
||||||
|
if not out:
|
||||||
|
out.append(group)
|
||||||
|
continue
|
||||||
|
previous = out[-1]
|
||||||
|
if (len(previous) + len(group) <= max_files
|
||||||
|
and max(len(previous), len(group)) <= loose
|
||||||
|
and not any(p in owner for p in previous + group)
|
||||||
|
and _shares_ancestor(_common_dir(previous), _common_dir(group))):
|
||||||
|
out[-1] = previous + group
|
||||||
|
else:
|
||||||
|
out.append(group)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _shares_ancestor(a, b):
|
||||||
|
"""Both at the root, or under a common top-level directory."""
|
||||||
|
if not a and not b:
|
||||||
|
return True
|
||||||
|
return bool(a) and bool(b) and a[0] == b[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _group(ordered, files, owner, after, max_files):
|
||||||
|
"""
|
||||||
|
Cut the ordered list into commits.
|
||||||
|
|
||||||
|
One coherent idea per commit, not one directory per commit. What holds a
|
||||||
|
group together is that its files refer to each other — ports.sh and the
|
||||||
|
hosts template it renders, a kustomization and the manifests it lists. What
|
||||||
|
separates two groups in the same directory is that neither names the other.
|
||||||
|
|
||||||
|
A hub and the directory named after it (addons.sh and addons/) travel
|
||||||
|
together whatever their references say: committing the loader without the
|
||||||
|
things it loads produces a commit that cannot run.
|
||||||
|
"""
|
||||||
|
dirs = {str(Path(p).parent) for p in ordered}
|
||||||
|
groups, buf = [], []
|
||||||
|
seen_cluster = seen_role = None
|
||||||
|
|
||||||
|
def flush():
|
||||||
|
nonlocal buf
|
||||||
|
if buf:
|
||||||
|
groups.append(buf)
|
||||||
|
buf = []
|
||||||
|
|
||||||
|
for path in ordered:
|
||||||
|
own = owner.get(path)
|
||||||
|
cluster = _cluster(path, own, dirs)
|
||||||
|
role = "subproject" if own else GROUP_TIER.get(
|
||||||
|
files[path]["role"], files[path]["role"])
|
||||||
|
if cluster != seen_cluster or role != seen_role:
|
||||||
|
flush()
|
||||||
|
seen_cluster, seen_role = cluster, role
|
||||||
|
buf.append(path)
|
||||||
|
flush()
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for group in groups:
|
||||||
|
# A subproject is one commit no matter how many files it holds.
|
||||||
|
if any(p in owner for p in group):
|
||||||
|
out.append(group)
|
||||||
|
continue
|
||||||
|
out.extend(_split(group, files, after, max_files))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _hub_of(path):
|
||||||
|
"""`ctrl/addons.sh` is the hub of `ctrl/addons/`; returns that directory."""
|
||||||
|
p = Path(path)
|
||||||
|
return str(p.parent / p.stem) if p.suffix else None
|
||||||
|
|
||||||
|
|
||||||
|
def _pure_hub(members, hubs):
|
||||||
|
"""True when the group is exactly one hub and things inside its directory."""
|
||||||
|
for hub, owner_file in hubs.items():
|
||||||
|
if owner_file in members and all(
|
||||||
|
m == owner_file or m.startswith(hub + "/") for m in members
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _split(group, files, after, max_files):
|
||||||
|
"""
|
||||||
|
Break one cluster into commits.
|
||||||
|
|
||||||
|
A run of siblings of the same kind in the same directory is left alone —
|
||||||
|
four profile files under env.d/ are one idea, and nothing in them refers to
|
||||||
|
anything, so splitting on references turns them into four commits saying
|
||||||
|
the same thing four times. References are only asked about a group that is
|
||||||
|
already too big or already spans directories.
|
||||||
|
|
||||||
|
Past that: connected components, because what holds a commit together is
|
||||||
|
that its files name each other. A hub and the directory named after it
|
||||||
|
(addons.sh and addons/) survive the cap, since committing the loader
|
||||||
|
without the things it loads produces a commit that cannot run. Anything
|
||||||
|
else over the cap gets its hub peeled off into its own commit and the
|
||||||
|
remainder reconsidered — a cut at a named seam rather than at a count.
|
||||||
|
"""
|
||||||
|
if len(group) <= 1:
|
||||||
|
return [group]
|
||||||
|
|
||||||
|
# ...but only for roles where sitting side by side IS the relationship.
|
||||||
|
# Four env.d profiles are one idea. Seven scripts that happen to share a
|
||||||
|
# directory are seven ideas, and `ctrl/` is full of them, so code always
|
||||||
|
# gets asked about its references.
|
||||||
|
parents = {str(Path(p).parent) for p in group}
|
||||||
|
kinds = {files[p]["role"] for p in group}
|
||||||
|
if len(parents) == 1 and not (kinds & CODE_ROLES):
|
||||||
|
if len(group) <= max_files:
|
||||||
|
return [group]
|
||||||
|
# Over the cap and still one kind in one directory: cut it into runs.
|
||||||
|
# Nothing here refers to anything, so asking references to find the
|
||||||
|
# seam yields one commit per file — thirteen commits each saying "a
|
||||||
|
# project note", which is worse than an admitted arbitrary cut.
|
||||||
|
return [group[i:i + max_files] for i in range(0, len(group), max_files)]
|
||||||
|
|
||||||
|
index = {p: i for i, p in enumerate(group)}
|
||||||
|
parent = list(range(len(group)))
|
||||||
|
|
||||||
|
def find(i):
|
||||||
|
while parent[i] != i:
|
||||||
|
parent[i] = parent[parent[i]]
|
||||||
|
i = parent[i]
|
||||||
|
return i
|
||||||
|
|
||||||
|
def union(a, b):
|
||||||
|
ra, rb = find(a), find(b)
|
||||||
|
if ra != rb:
|
||||||
|
parent[max(ra, rb)] = min(ra, rb)
|
||||||
|
|
||||||
|
hubs = {}
|
||||||
|
for p in group:
|
||||||
|
hub = _hub_of(p)
|
||||||
|
if hub:
|
||||||
|
hubs[hub] = p
|
||||||
|
for p in group:
|
||||||
|
for hub, owner_file in hubs.items():
|
||||||
|
if p.startswith(hub + "/"):
|
||||||
|
union(index[p], index[owner_file])
|
||||||
|
for dep in after.get(p, ()):
|
||||||
|
if dep in index:
|
||||||
|
union(index[p], index[dep])
|
||||||
|
|
||||||
|
components = defaultdict(list)
|
||||||
|
for p in group:
|
||||||
|
components[find(index[p])].append(p)
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for key in sorted(components):
|
||||||
|
members = components[key]
|
||||||
|
while len(members) > max_files and not _pure_hub(members, hubs):
|
||||||
|
degree = {m: sum(1 for o in members if m in after.get(o, ())) for m in members}
|
||||||
|
hub = max(degree, key=lambda m: (degree[m], m))
|
||||||
|
if degree[hub] == 0:
|
||||||
|
# Nothing holds this together and nothing names anything: an
|
||||||
|
# arbitrary cut is the honest answer, so cut on the order we
|
||||||
|
# already have rather than inventing a reason.
|
||||||
|
out.extend([members[i:i + max_files]
|
||||||
|
for i in range(0, len(members), max_files)])
|
||||||
|
members = []
|
||||||
|
break
|
||||||
|
members.remove(hub)
|
||||||
|
out.append([hub])
|
||||||
|
if members:
|
||||||
|
out.append(members)
|
||||||
|
return [g for g in out if g]
|
||||||
|
|
||||||
|
|
||||||
|
# ── the plan ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def plan_path(out) -> Path:
|
||||||
|
return state_dir(out) / PLAN_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def build_plan(index, out, max_files=None, quiet=False):
|
||||||
|
"""
|
||||||
|
Write plan.json: ordered groups of paths with empty message slots.
|
||||||
|
|
||||||
|
This is the seam. Everything above is analysis and can be recomputed from
|
||||||
|
the tree; everything below is git commands. An existing plan's messages are
|
||||||
|
carried over when the group's paths still match exactly, so re-planning
|
||||||
|
after editing three files does not throw away sixty written messages.
|
||||||
|
"""
|
||||||
|
groups = order_files(index, max_files)
|
||||||
|
|
||||||
|
written = {}
|
||||||
|
existing = plan_path(out)
|
||||||
|
if existing.exists():
|
||||||
|
try:
|
||||||
|
for g in json.loads(existing.read_text()).get("groups", []):
|
||||||
|
if g.get("title"):
|
||||||
|
written[tuple(sorted(g["paths"]))] = (g.get("title"), g.get("body", ""))
|
||||||
|
except (OSError, json.JSONDecodeError, KeyError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
document = {"version": 1, "groups": []}
|
||||||
|
for i, paths in enumerate(groups, 1):
|
||||||
|
title, body = written.get(tuple(sorted(paths)), ("", ""))
|
||||||
|
document["groups"].append({
|
||||||
|
"n": i,
|
||||||
|
"slug": _slug(paths, index),
|
||||||
|
"paths": paths,
|
||||||
|
"title": title,
|
||||||
|
"body": body,
|
||||||
|
})
|
||||||
|
|
||||||
|
destination = plan_path(out)
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination.write_text(json.dumps(document, indent=2))
|
||||||
|
if not quiet:
|
||||||
|
kept = sum(1 for g in document["groups"] if g["title"])
|
||||||
|
print(f"Planned {len(groups)} commits over "
|
||||||
|
f"{sum(len(g) for g in groups)} files"
|
||||||
|
+ (f", {kept} messages carried over" if kept else "") + ".")
|
||||||
|
print(f" -> {destination}")
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def _slug(paths, index):
|
||||||
|
"""A stable handle for a group, for filenames and for talking about it."""
|
||||||
|
common = Path(paths[0]).parent
|
||||||
|
for p in paths[1:]:
|
||||||
|
parts = []
|
||||||
|
for a, b in zip(common.parts, Path(p).parent.parts):
|
||||||
|
if a != b:
|
||||||
|
break
|
||||||
|
parts.append(a)
|
||||||
|
common = Path(*parts) if parts else Path(".")
|
||||||
|
base = str(common).strip("./").replace("/", "-")
|
||||||
|
if not base:
|
||||||
|
base = Path(paths[0]).stem if len(paths) == 1 else index["files"][paths[0]]["role"]
|
||||||
|
return base.lower().replace("_", "-").replace(".", "")[:40] or "root"
|
||||||
413
soleprint/station/tools/histgen/selftest.py
Normal file
413
soleprint/station/tools/histgen/selftest.py
Normal file
@@ -0,0 +1,413 @@
|
|||||||
|
"""
|
||||||
|
Prove the whole pipeline on a tree this builds itself.
|
||||||
|
|
||||||
|
`make check` after copying the folder somewhere new, with no repo to point at
|
||||||
|
and nothing installed. It builds a small tree with the shapes that matter —
|
||||||
|
ignore rules, a README, a pin, a config that sources it, a directory of
|
||||||
|
profiles, a hub with satellites, a front door, an ignored directory — runs all
|
||||||
|
five verbs over it, and asserts what has to be true.
|
||||||
|
|
||||||
|
The guards get the same treatment as the happy path. A check that only ever
|
||||||
|
proves things work would have missed the one real bug found while writing this:
|
||||||
|
the tree-hash guard returned None on exactly the trees it was written for, and
|
||||||
|
reported success anyway.
|
||||||
|
|
||||||
|
python3 selftest.py # or: make check
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
TREE = {
|
||||||
|
".gitignore": "# Generated output; regenerate rather than commit.\nbuild/\n*.log\n",
|
||||||
|
".gitattributes": "# LF everywhere: a CRLF checkout breaks the shebang.\n* text=auto eol=lf\n",
|
||||||
|
"README.md": "# demo\n\nWhat this is, and the one prerequisite.\n",
|
||||||
|
"ctrl/versions.env": "# Pinned toolchain — one manifest, one answer.\nKUBECTL=1.31.0\n",
|
||||||
|
"ctrl/lib/config.sh": "# Shared config loading. Sourced, never executed.\n"
|
||||||
|
'. "$(dirname "$0")/../versions.env"\n',
|
||||||
|
"ctrl/env.d/minimal.env": "# The smallest profile that still starts.\nADDONS=\n",
|
||||||
|
"ctrl/env.d/full.env": "# Everything on, for a demo machine.\nADDONS=redis\n",
|
||||||
|
"ctrl/addons.sh": "# Install the addons the profile asked for.\n"
|
||||||
|
"# Adding one is adding a file, not editing a dispatcher.\n"
|
||||||
|
'for a in ctrl/addons/*.sh; do sh "$a"; done\n',
|
||||||
|
"ctrl/addons/redis.sh": "# Redis — the broker half, nothing else uses it.\necho redis\n",
|
||||||
|
"ctrl/addons/postgres.sh": "# Postgres — the metadata store.\necho postgres\n",
|
||||||
|
"Makefile": "# Thin front door: one target per ctrl/ script.\nup:\n\tsh ctrl/addons.sh\n",
|
||||||
|
"BOOTSTRAP.md": "# From a bare machine to something running.\n",
|
||||||
|
"build/generated.txt": "this is ignored and must never be committed",
|
||||||
|
"noisy.log": "also ignored",
|
||||||
|
}
|
||||||
|
|
||||||
|
IGNORED = {"build/generated.txt", "noisy.log"}
|
||||||
|
|
||||||
|
# Things a copy should leave behind, and the two lookalikes it must not. Added
|
||||||
|
# to the tree only for the `copy` checks, and force-added so the ignored-but-
|
||||||
|
# tracked case is real rather than described.
|
||||||
|
SIFTABLE = {
|
||||||
|
"package-lock.json": "lockfile contents",
|
||||||
|
"dist/app.min.js": "minified",
|
||||||
|
".env": "API_KEY=real-secret-value",
|
||||||
|
".env.example": "API_KEY=",
|
||||||
|
"certs/server.key": "-----BEGIN PRIVATE KEY-----",
|
||||||
|
"certs/server.key.pub": "ssh-rsa AAAA",
|
||||||
|
"assets/logo.png": "PNG",
|
||||||
|
"build/forced.bin": "ignored yet tracked",
|
||||||
|
}
|
||||||
|
DROPPED = {"package-lock.json", "dist/app.min.js", ".env",
|
||||||
|
"certs/server.key", "build/forced.bin"}
|
||||||
|
KEPT_LOOKALIKES = {".env.example", "certs/server.key.pub", "assets/logo.png"}
|
||||||
|
|
||||||
|
|
||||||
|
def run(pkg, parent, *args):
|
||||||
|
return subprocess.run([sys.executable, "-m", pkg, *args],
|
||||||
|
capture_output=True, text=True,
|
||||||
|
env={"PYTHONPATH": str(parent), "PATH": "/usr/bin:/bin",
|
||||||
|
"HOME": str(Path.home())})
|
||||||
|
|
||||||
|
|
||||||
|
def build(root: Path):
|
||||||
|
for rel, body in TREE.items():
|
||||||
|
path = root / rel
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(body)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
here = Path(__file__).resolve().parent
|
||||||
|
pkg, parent = here.name, here.parent
|
||||||
|
failures = []
|
||||||
|
|
||||||
|
def check(label, condition, detail=""):
|
||||||
|
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
||||||
|
if not condition:
|
||||||
|
failures.append(f"{label}{': ' + detail if detail else ''}")
|
||||||
|
|
||||||
|
def hg(*args):
|
||||||
|
return run(pkg, parent, *args)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-selftest-") as tmp:
|
||||||
|
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||||
|
build(source)
|
||||||
|
W = ["--source", str(source), "--out", str(out)]
|
||||||
|
copy = out / "demo"
|
||||||
|
|
||||||
|
r = hg("run", *W)
|
||||||
|
check("run: scan + plan + brief", r.returncode == 0, r.stderr.strip())
|
||||||
|
if r.returncode != 0:
|
||||||
|
print(r.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
index = json.loads((out / "index.json").read_text())
|
||||||
|
plan = json.loads((out / "plan.json").read_text())
|
||||||
|
planned = [p for g in plan["groups"] for p in g["paths"]]
|
||||||
|
|
||||||
|
check("the ignore rules were honoured", not (set(planned) & IGNORED),
|
||||||
|
f"ignored files got planned: {sorted(set(planned) & IGNORED)}")
|
||||||
|
check("every other file is in exactly one group",
|
||||||
|
sorted(planned) == sorted(set(TREE) - IGNORED) and len(planned) == len(set(planned)))
|
||||||
|
check("the ignore rules are committed first",
|
||||||
|
plan["groups"][0]["paths"][0].startswith(".git"))
|
||||||
|
check("the front door is not", "Makefile" not in plan["groups"][0]["paths"])
|
||||||
|
check("the hub travels with its satellites",
|
||||||
|
any({"ctrl/addons.sh", "ctrl/addons/redis.sh", "ctrl/addons/postgres.sh"}
|
||||||
|
<= set(g["paths"]) for g in plan["groups"]))
|
||||||
|
check("a comment is not mistaken for a dependency",
|
||||||
|
index["files"][".gitignore"]["refs"] == [])
|
||||||
|
check("a real dependency is found",
|
||||||
|
"ctrl/versions.env" in index["files"]["ctrl/lib/config.sh"]["refs"])
|
||||||
|
check("the reasoning was extracted for the message",
|
||||||
|
"not editing a dispatcher" in index["files"]["ctrl/addons.sh"]["why"])
|
||||||
|
check("a brief exists per commit",
|
||||||
|
len(list((out / "briefs").glob("*.md"))) == len(plan["groups"]))
|
||||||
|
check("list prints one line per commit",
|
||||||
|
all(f"{g['n']:3}." in hg("list", *W).stdout for g in plan["groups"]))
|
||||||
|
|
||||||
|
# The guards refuse before they approve.
|
||||||
|
check("export refuses a plan with no messages",
|
||||||
|
"no title" in hg("export", *W).stderr)
|
||||||
|
(source / "appeared-late.sh").write_text("# added after planning\n")
|
||||||
|
r = hg("export", *W, "--allow-untitled")
|
||||||
|
check("export refuses a file no group covers",
|
||||||
|
r.returncode != 0 and "no group" in r.stderr)
|
||||||
|
(source / "appeared-late.sh").unlink()
|
||||||
|
|
||||||
|
# --- commands: copy the files, make no repo, hand back the list ---
|
||||||
|
# Its own out directory, so it needs its own plan in it: the plan is a
|
||||||
|
# fact about an out directory, not about the source.
|
||||||
|
hands = Path(str(out) + "-byhand")
|
||||||
|
H = ["--source", str(source), "--out", str(hands)]
|
||||||
|
hg("run", *H)
|
||||||
|
r = hg("export", *H, "--allow-untitled", "--commands")
|
||||||
|
check("commands: succeeds", r.returncode == 0, r.stderr.strip())
|
||||||
|
made = hands / "demo"
|
||||||
|
check("commands: the files were copied", (made / "README.md").is_file())
|
||||||
|
check("commands: NO repo was created", not (made / ".git").exists())
|
||||||
|
check("commands: gitignored files did not come across",
|
||||||
|
not any((made / i).exists() for i in IGNORED))
|
||||||
|
check("commands: git init is the first thing offered, not done for you",
|
||||||
|
"git init" in r.stdout)
|
||||||
|
check("commands: one add and one commit per group",
|
||||||
|
r.stdout.count("git add -- ") == len(plan["groups"])
|
||||||
|
and r.stdout.count("git commit -F ") == len(plan["groups"]))
|
||||||
|
check("commands: the list is saved too", (hands / "commands.sh").is_file())
|
||||||
|
|
||||||
|
# The list has to actually work, which is the only claim that matters.
|
||||||
|
subprocess.run(["bash", str(hands / "commands.sh")], capture_output=True,
|
||||||
|
env={**os.environ, "GIT_AUTHOR_NAME": "t",
|
||||||
|
"GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
|
||||||
|
"GIT_COMMITTER_EMAIL": "t@t"})
|
||||||
|
check("commands: running the list builds the history",
|
||||||
|
subprocess.run(["git", "-C", str(made), "rev-list", "--count", "HEAD"],
|
||||||
|
capture_output=True, text=True).stdout.strip()
|
||||||
|
== str(len(plan["groups"])))
|
||||||
|
check("commands: it leaves nothing untracked",
|
||||||
|
not subprocess.run(["git", "-C", str(made), "status", "--porcelain"],
|
||||||
|
capture_output=True, text=True).stdout.strip())
|
||||||
|
expected = [l for l in r.stdout.split("\n") if "expect " in l][0].split("expect ")[1].strip()
|
||||||
|
check("commands: the tree matches the hash it told you to expect",
|
||||||
|
subprocess.run(["git", "-C", str(made), "rev-parse", "HEAD^{tree}"],
|
||||||
|
capture_output=True, text=True).stdout.strip() == expected)
|
||||||
|
check("commands: refuses when a repo is already there",
|
||||||
|
hg("export", *H, "--allow-untitled", "--commands").returncode != 0)
|
||||||
|
|
||||||
|
# --- absent -> exported ---
|
||||||
|
check("status says absent before anything is exported",
|
||||||
|
"absent" in hg("status", *W).stdout)
|
||||||
|
r = hg("export", *W, "--allow-untitled")
|
||||||
|
check("export writes the history", r.returncode == 0, r.stderr.strip())
|
||||||
|
check("both guards ran",
|
||||||
|
"nothing left untracked: ok" in r.stdout and "tree matches source" in r.stdout)
|
||||||
|
check("the copy is named after the source", copy.is_dir())
|
||||||
|
check("one commit per group",
|
||||||
|
subprocess.run(["git", "-C", str(copy), "rev-list", "--count", "HEAD"],
|
||||||
|
capture_output=True, text=True).stdout.strip()
|
||||||
|
== str(len(plan["groups"])))
|
||||||
|
check("the ignored files never entered the history",
|
||||||
|
not (set(subprocess.run(["git", "-C", str(copy), "ls-files"],
|
||||||
|
capture_output=True, text=True).stdout.split()) & IGNORED))
|
||||||
|
|
||||||
|
# The invariant the whole design rests on.
|
||||||
|
check("THE SOURCE WAS NEVER WRITTEN TO",
|
||||||
|
not (source / ".git").exists() and not (source / ".histgen").exists()
|
||||||
|
and sorted(p.relative_to(source).as_posix()
|
||||||
|
for p in source.rglob("*") if p.is_file()) == sorted(TREE))
|
||||||
|
|
||||||
|
check("a finished export says so and stops",
|
||||||
|
"Nothing to do" in hg("export", *W, "--allow-untitled").stdout)
|
||||||
|
check("verify passes on its own", hg("verify", *W).returncode == 0)
|
||||||
|
|
||||||
|
# --- unfinished: interrupt, then resume ---
|
||||||
|
progress = json.loads((out / "progress.json").read_text())
|
||||||
|
# Half of them, whatever the fixture happens to plan. Hardcoding four
|
||||||
|
# made this pass vacuously the moment the fixture planned exactly four:
|
||||||
|
# nothing was left to resume, so "complete" was the honest answer and
|
||||||
|
# the resume path was never entered.
|
||||||
|
keep = progress["commits"][:max(1, len(progress["commits"]) // 2)]
|
||||||
|
subprocess.run(["git", "-C", str(copy), "reset", "-q", "--hard", keep[-1]["sha"]])
|
||||||
|
progress["commits"] = keep
|
||||||
|
(out / "progress.json").write_text(json.dumps(progress))
|
||||||
|
|
||||||
|
check("status spots a half-finished history",
|
||||||
|
"unfinished" in hg("status", *W).stdout)
|
||||||
|
r = hg("export", *W, "--allow-untitled")
|
||||||
|
check("export resumes rather than restarting",
|
||||||
|
r.returncode == 0 and "Resuming" in r.stdout, r.stdout + r.stderr)
|
||||||
|
check("resuming did not redo the commits already made",
|
||||||
|
f"{len(keep)} of {len(plan['groups'])}" in r.stdout, r.stdout)
|
||||||
|
check("the resumed history is complete and verified",
|
||||||
|
"tree matches source" in r.stdout)
|
||||||
|
|
||||||
|
# --- stale: the plan moved ---
|
||||||
|
plan["groups"][1]["paths"].append(plan["groups"][2]["paths"].pop())
|
||||||
|
(out / "plan.json").write_text(json.dumps(plan))
|
||||||
|
check("status spots a plan that no longer matches",
|
||||||
|
"stale" in hg("status", *W).stdout)
|
||||||
|
check("export refuses a stale export",
|
||||||
|
hg("export", *W, "--allow-untitled").returncode != 0)
|
||||||
|
check("--force starts over",
|
||||||
|
hg("export", *W, "--allow-untitled", "--force").returncode == 0)
|
||||||
|
|
||||||
|
# --- foreign: a history that has to be kept ---
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-selftest-keep-") as tmp:
|
||||||
|
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||||
|
build(source)
|
||||||
|
for cmd in (["init", "-q"], ["add", "-A"], ["commit", "-qm", "init commit"]):
|
||||||
|
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True,
|
||||||
|
env={**os.environ, "GIT_AUTHOR_NAME": "t",
|
||||||
|
"GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
|
||||||
|
"GIT_COMMITTER_EMAIL": "t@t"})
|
||||||
|
before = subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"],
|
||||||
|
capture_output=True, text=True).stdout.strip()
|
||||||
|
W = ["--source", str(source), "--out", str(out)]
|
||||||
|
copy = out / "demo"
|
||||||
|
|
||||||
|
hg("run", *W)
|
||||||
|
r = hg("export", *W, "--allow-untitled", "--keep-history")
|
||||||
|
check("keep: export succeeds", r.returncode == 0, r.stderr.strip())
|
||||||
|
check("keep: the old history is reported as kept", "kept, untouched" in r.stdout)
|
||||||
|
|
||||||
|
branches = subprocess.run(["git", "-C", str(copy), "branch", "--format=%(refname:short)"],
|
||||||
|
capture_output=True, text=True).stdout.split()
|
||||||
|
check("keep: the designed history is on its own branch",
|
||||||
|
"designed-history" in branches and len(branches) >= 2, str(branches))
|
||||||
|
main = [b for b in branches if b != "designed-history"][0]
|
||||||
|
check("keep: the old branch still points where it did",
|
||||||
|
subprocess.run(["git", "-C", str(copy), "rev-parse", main],
|
||||||
|
capture_output=True, text=True).stdout.strip() == before)
|
||||||
|
check("keep: the two histories share no commit",
|
||||||
|
subprocess.run(["git", "-C", str(copy), "merge-base", main, "designed-history"],
|
||||||
|
capture_output=True, text=True).returncode != 0)
|
||||||
|
check("keep: the source's own history is untouched",
|
||||||
|
subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"],
|
||||||
|
capture_output=True, text=True).stdout.strip() == before)
|
||||||
|
|
||||||
|
# --- the filter has to hold on the HISTORY path, not just on copy ---
|
||||||
|
# This is the case that was wrong: `copy` left a tracked key behind while
|
||||||
|
# `scan` walked straight past it, so the key stayed out of the snapshot and
|
||||||
|
# went into the commits.
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-selftest-secret-") as tmp:
|
||||||
|
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||||
|
build(source)
|
||||||
|
for rel, body in SIFTABLE.items():
|
||||||
|
(source / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
(source / rel).write_text(body)
|
||||||
|
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||||
|
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||||
|
for cmd in (["init", "-q"], ["add", "-A", "-f", "."], ["commit", "-qm", "init"]):
|
||||||
|
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env)
|
||||||
|
|
||||||
|
W = ["--source", str(source), "--out", str(out)]
|
||||||
|
r = hg("run", *W)
|
||||||
|
check("filter: scan says what it left out", "left out of the history" in r.stdout)
|
||||||
|
|
||||||
|
index = json.loads((out / "index.json").read_text())
|
||||||
|
plan = json.loads((out / "plan.json").read_text())
|
||||||
|
planned = {p for g in plan["groups"] for p in g["paths"]}
|
||||||
|
secrets = {".env", "certs/server.key"}
|
||||||
|
check("filter: no secret reached the census", not (set(index["files"]) & secrets))
|
||||||
|
check("filter: no secret reached the plan", not (planned & secrets))
|
||||||
|
check("filter: the ignored-but-tracked file did not either",
|
||||||
|
"build/forced.bin" not in planned)
|
||||||
|
check("filter: the lookalikes are still in the plan",
|
||||||
|
{".env.example", "certs/server.key.pub"} <= planned)
|
||||||
|
check("filter: a lockfile IS kept — a repo wants its lockfile",
|
||||||
|
"package-lock.json" in planned)
|
||||||
|
|
||||||
|
r = hg("export", *W, "--allow-untitled", "--commands")
|
||||||
|
# Exact tokens, not substrings: ".env" is inside ".env.example", so a
|
||||||
|
# substring test reports a leak every time the lookalike is kept —
|
||||||
|
# which is exactly the behaviour that is wanted.
|
||||||
|
added = {tok for line in r.stdout.split("\n") if line.startswith("git add -- ")
|
||||||
|
for tok in line[len("git add -- "):].split()}
|
||||||
|
check("filter: no secret reached the commands",
|
||||||
|
not (added & secrets), str(sorted(added & secrets)))
|
||||||
|
check("filter: the commands cover exactly the planned files",
|
||||||
|
added == planned, str(sorted(added ^ planned)))
|
||||||
|
check("filter: no secret reached the copy",
|
||||||
|
not any((out / "demo" / x).exists() for x in secrets))
|
||||||
|
|
||||||
|
out2 = Path(tmp) / "out2"
|
||||||
|
hg("run", "--source", str(source), "--out", str(out2))
|
||||||
|
r = hg("export", "--source", str(source), "--out", str(out2), "--allow-untitled")
|
||||||
|
check("filter: the guards still pass on the filtered set",
|
||||||
|
"tree matches source" in r.stdout, r.stdout + r.stderr)
|
||||||
|
tracked = subprocess.run(["git", "-C", str(out2 / "demo"), "ls-files"],
|
||||||
|
capture_output=True, text=True).stdout.split()
|
||||||
|
check("filter: the committed history holds no secret",
|
||||||
|
not (set(tracked) & secrets), str(tracked))
|
||||||
|
|
||||||
|
out3 = Path(tmp) / "out3"
|
||||||
|
hg("run", "--source", str(source), "--out", str(out3), "--keep-secrets")
|
||||||
|
index3 = json.loads((out3 / "index.json").read_text())
|
||||||
|
check("filter: --keep-secrets brings them back",
|
||||||
|
secrets <= set(index3["files"]))
|
||||||
|
|
||||||
|
# --- copy: the plain utility, no history involved ---
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-selftest-copy-") as tmp:
|
||||||
|
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||||
|
build(source)
|
||||||
|
for rel, body in SIFTABLE.items():
|
||||||
|
(source / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
(source / rel).write_text(body)
|
||||||
|
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||||
|
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||||
|
for cmd in (["init", "-q"], ["add", "-A", "-f", "."],
|
||||||
|
["commit", "-qm", "init"]):
|
||||||
|
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env)
|
||||||
|
|
||||||
|
C = ["--source", str(source), "--out", str(out)]
|
||||||
|
r = hg("copy", *C)
|
||||||
|
check("copy: succeeds", r.returncode == 0, r.stderr.strip())
|
||||||
|
made = out / "demo"
|
||||||
|
check("copy: no .git came across", not (made / ".git").exists())
|
||||||
|
check("copy: nothing gitignored came across",
|
||||||
|
not any((made / i).exists() for i in IGNORED))
|
||||||
|
for rel in sorted(DROPPED):
|
||||||
|
check(f"copy: left behind {rel}", not (made / rel).exists())
|
||||||
|
for rel in sorted(KEPT_LOOKALIKES):
|
||||||
|
check(f"copy: kept {rel}", (made / rel).is_file())
|
||||||
|
check("copy: every drop is named in the output",
|
||||||
|
all(rel in r.stdout for rel in DROPPED), r.stdout)
|
||||||
|
check("copy: a manifest records what was left behind",
|
||||||
|
(out / "copied.md").is_file()
|
||||||
|
and all(rel in (out / "copied.md").read_text() for rel in DROPPED))
|
||||||
|
check("copy: refuses a destination that is not empty",
|
||||||
|
hg("copy", *C).returncode != 0)
|
||||||
|
|
||||||
|
preview = Path(tmp) / "preview"
|
||||||
|
r = hg("copy", "--source", str(source), "--out", str(preview), "--dry-run")
|
||||||
|
check("copy: --dry-run writes nothing", not preview.exists() and r.returncode == 0)
|
||||||
|
|
||||||
|
full = Path(tmp) / "full"
|
||||||
|
r = hg("copy", "--source", str(source), "--out", str(full),
|
||||||
|
"--all", "--keep-secrets")
|
||||||
|
check("copy: --all --keep-secrets keeps what it says",
|
||||||
|
(full / "demo" / "package-lock.json").is_file()
|
||||||
|
and (full / "demo" / ".env").is_file())
|
||||||
|
|
||||||
|
picky = Path(tmp) / "picky"
|
||||||
|
r = hg("copy", "--source", str(source), "--out", str(picky),
|
||||||
|
"--exclude", "*.png", "--include", "package-lock.json")
|
||||||
|
check("copy: --exclude drops by glob at any depth",
|
||||||
|
not (picky / "demo" / "assets" / "logo.png").exists())
|
||||||
|
check("copy: --include overrides the filters",
|
||||||
|
(picky / "demo" / "package-lock.json").is_file())
|
||||||
|
|
||||||
|
# --- config ---
|
||||||
|
with tempfile.TemporaryDirectory(prefix="histgen-selftest-cfg-") as tmp:
|
||||||
|
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||||
|
build(source)
|
||||||
|
cfg = Path(tmp) / "settings.json"
|
||||||
|
cfg.write_text(json.dumps({"source": str(source), "out": str(out)}))
|
||||||
|
r = hg("config", "--config", str(cfg))
|
||||||
|
check("config: a file supplies source and out",
|
||||||
|
r.returncode == 0 and str(out) in r.stdout, r.stderr.strip())
|
||||||
|
check("config: the command line wins",
|
||||||
|
"/tmp/override" in hg("config", "--config", str(cfg),
|
||||||
|
"--out", "/tmp/override").stdout)
|
||||||
|
cfg.write_text(json.dumps({"source": str(source), "outp": "typo"}))
|
||||||
|
check("config: a mistyped key is refused",
|
||||||
|
"unknown key" in hg("config", "--config", str(cfg)).stderr)
|
||||||
|
cfg.write_text(json.dumps({"repo": str(source), "out": str(out)}))
|
||||||
|
check("config: the old 'repo' key still works",
|
||||||
|
hg("config", "--config", str(cfg)).returncode == 0)
|
||||||
|
check("out inside source is refused",
|
||||||
|
"inside source" in hg("status", "--source", str(source),
|
||||||
|
"--out", str(source / "sub")).stderr)
|
||||||
|
|
||||||
|
print()
|
||||||
|
if failures:
|
||||||
|
print(f"{len(failures)} check(s) failed:")
|
||||||
|
for f in failures:
|
||||||
|
print(f" - {f}")
|
||||||
|
return 1
|
||||||
|
print("all checks passed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
142
soleprint/station/tools/histgen/sift.py
Normal file
142
soleprint/station/tools/histgen/sift.py
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
"""
|
||||||
|
What to leave behind when copying a repo out of itself.
|
||||||
|
|
||||||
|
Three kinds of file get dropped, and they are dropped for three different
|
||||||
|
reasons. Keeping them apart matters, because only one of the three is safe to
|
||||||
|
decide silently.
|
||||||
|
|
||||||
|
derived something a build regenerates. Lockfiles, source maps, minified
|
||||||
|
output, compiled objects, cache directories. The list is ported
|
||||||
|
from `ppl/ctrl/distill.sh`, including the lesson written into its
|
||||||
|
comments: the line is **derived-vs-content, not text-vs-binary**.
|
||||||
|
That distinction was wrong there once and cost real files — a
|
||||||
|
logo, a font the site loads, a downloadable PDF — because none of
|
||||||
|
those can be regenerated from what is left, which is the only
|
||||||
|
thing that makes a file safe to drop. So images, fonts and
|
||||||
|
spreadsheets are content and are kept.
|
||||||
|
|
||||||
|
secret a private key, a credential store, an .env holding real values.
|
||||||
|
distill does not do this; .gitignore usually has, and where it has
|
||||||
|
not, the file is tracked and travels. That is not hypothetical —
|
||||||
|
soleprint's own notes record an API key that was tracked in a
|
||||||
|
tool's .env, and untracking it did not unpublish it.
|
||||||
|
|
||||||
|
ignored tracked, and yet matched by the repo's own ignore rules. Someone
|
||||||
|
ran `git add -f` once. Sometimes deliberate — a built artifact
|
||||||
|
committed on purpose — and sometimes a dump or a credentials file
|
||||||
|
that went in and was never noticed again. Reported by name either
|
||||||
|
way, because the repo is already contradicting itself about them.
|
||||||
|
|
||||||
|
oversize whatever --max-bytes says. Size is its own worry and gets its own
|
||||||
|
knob rather than being smuggled in as a guess about kind.
|
||||||
|
|
||||||
|
Every drop is reported. A file quietly missing from a copy is the same class of
|
||||||
|
failure as a file quietly missing from a history, and this tool exists because
|
||||||
|
that class of failure is expensive.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# ── derived ────────────────────────────────────────────────────────────────
|
||||||
|
# One list, one place to edit. `--all` turns it off wholesale.
|
||||||
|
NOISE = re.compile(
|
||||||
|
r'(^|/)(package-lock\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|yarn\.lock'
|
||||||
|
r'|bun\.lock|bun\.lockb|uv\.lock|poetry\.lock|Pipfile\.lock|Cargo\.lock'
|
||||||
|
r'|composer\.lock|Gemfile\.lock|go\.sum|\.DS_Store|Thumbs\.db)$'
|
||||||
|
r'|\.(map|min\.js|min\.css)$'
|
||||||
|
r'|\.(pyc|pyo|pyd|class|o|obj|a|lib|so|dylib|dll|wasm|pack|idx)$'
|
||||||
|
r'|(^|/)(__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|node_modules)/'
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── secret ─────────────────────────────────────────────────────────────────
|
||||||
|
# Deliberately narrow. A pattern that catches a real key once a year and a
|
||||||
|
# needed file once a week gets turned off, and then it catches nothing.
|
||||||
|
SECRET = re.compile(
|
||||||
|
r'(^|/)\.env(\.[A-Za-z0-9_-]+)?$'
|
||||||
|
r'|(^|/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)$'
|
||||||
|
r'|\.(pem|key|p12|pfx|jks|keystore|ppk|asc|gpg)$'
|
||||||
|
r'|(^|/)(\.netrc|\.npmrc|\.pypirc|\.htpasswd|\.dockercfg)$'
|
||||||
|
r'|(^|/)\.ssh/'
|
||||||
|
r'|(^|/)credentials(\.json|\.yaml|\.yml)?$'
|
||||||
|
r'|(^|/)service-account[-_A-Za-z0-9]*\.json$'
|
||||||
|
)
|
||||||
|
|
||||||
|
# The exceptions matter more than the rule. A committed .env.example is the
|
||||||
|
# documented way to say what the real one needs, and dropping it takes the
|
||||||
|
# documentation with the secret.
|
||||||
|
SECRET_OK = re.compile(
|
||||||
|
r'\.(example|sample|template|dist|tmpl)$'
|
||||||
|
r'|(^|/)\.env\.(example|sample|template)$'
|
||||||
|
r'|\.pub$'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_derived(rel: str) -> bool:
|
||||||
|
return bool(NOISE.search(rel))
|
||||||
|
|
||||||
|
|
||||||
|
def is_secret(rel: str) -> bool:
|
||||||
|
return bool(SECRET.search(rel)) and not SECRET_OK.search(rel)
|
||||||
|
|
||||||
|
|
||||||
|
def matches(rel: str, patterns) -> bool:
|
||||||
|
"""
|
||||||
|
Glob match, with distill's rule: a pattern holding no `/` also matches
|
||||||
|
basenames at any depth, so `--exclude '*.csv'` means what it looks like.
|
||||||
|
"""
|
||||||
|
name = Path(rel).name
|
||||||
|
for pattern in patterns or ():
|
||||||
|
if fnmatch.fnmatch(rel, pattern):
|
||||||
|
return True
|
||||||
|
if "/" not in pattern and fnmatch.fnmatch(name, pattern):
|
||||||
|
return True
|
||||||
|
if pattern.endswith("/") and (rel + "/").startswith(pattern.lstrip("/")):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def sift(source: Path, paths, keep_noise=False, keep_secrets=False,
|
||||||
|
max_bytes=None, exclude=(), include=(), ignored=()):
|
||||||
|
"""
|
||||||
|
Split the file list into what to copy and what to leave, with reasons.
|
||||||
|
|
||||||
|
`include` is checked first and wins outright: it is the way to say "yes, I
|
||||||
|
do want that lockfile" without turning the whole filter off.
|
||||||
|
"""
|
||||||
|
kept, dropped = [], []
|
||||||
|
for rel in paths:
|
||||||
|
if include and matches(rel, include):
|
||||||
|
kept.append(rel)
|
||||||
|
continue
|
||||||
|
if exclude and matches(rel, exclude):
|
||||||
|
dropped.append((rel, "excluded"))
|
||||||
|
continue
|
||||||
|
if not keep_secrets and is_secret(rel):
|
||||||
|
dropped.append((rel, "secret"))
|
||||||
|
continue
|
||||||
|
if rel in ignored:
|
||||||
|
dropped.append((rel, "ignored"))
|
||||||
|
continue
|
||||||
|
if not keep_noise and is_derived(rel):
|
||||||
|
dropped.append((rel, "derived"))
|
||||||
|
continue
|
||||||
|
if max_bytes:
|
||||||
|
try:
|
||||||
|
if (source / rel).stat().st_size > max_bytes:
|
||||||
|
dropped.append((rel, "oversize"))
|
||||||
|
continue
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
kept.append(rel)
|
||||||
|
return kept, dropped
|
||||||
|
|
||||||
|
|
||||||
|
REASONS = {
|
||||||
|
"ignored": "tracked, but the ignore rules say they should not be",
|
||||||
|
"derived": "a build regenerates these",
|
||||||
|
"secret": "looks like a key or a credential",
|
||||||
|
"oversize": "larger than --max-bytes",
|
||||||
|
"excluded": "matched --exclude",
|
||||||
|
}
|
||||||
121
soleprint/station/tools/histgen/snapshot.py
Normal file
121
soleprint/station/tools/histgen/snapshot.py
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
"""
|
||||||
|
Copy a repo's files out of it, without the repo.
|
||||||
|
|
||||||
|
Named snapshot.py and not copy.py, which is what it was for about ten minutes.
|
||||||
|
A module called `copy` beside the code shadows the standard library's, and the
|
||||||
|
directory lands on sys.path whenever anything is run from inside it — so
|
||||||
|
`dataclasses` imported this file instead, and every command died on an import
|
||||||
|
error before parsing a single argument. The verb is still `copy`; the file
|
||||||
|
cannot be.
|
||||||
|
|
||||||
|
The plain utility underneath everything else: point it at a tree, get a folder
|
||||||
|
holding what the project actually is — no `.git`, nothing gitignored, nothing a
|
||||||
|
build regenerates, and nothing that looks like a key.
|
||||||
|
|
||||||
|
It is the thing to reach for when the history is not the point. Handing a
|
||||||
|
snapshot to someone, feeding a tree to something that should not see the
|
||||||
|
history, or getting a clean starting tree before planning one.
|
||||||
|
|
||||||
|
What is dropped is reported and written to a manifest, never assumed. A file
|
||||||
|
missing from a copy without a line saying so is the same failure this whole
|
||||||
|
tool exists to prevent, one directory earlier.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .census import file_set, ignored_but_tracked, is_git, state_dir
|
||||||
|
from .cli import fail
|
||||||
|
from .sift import REASONS, sift
|
||||||
|
|
||||||
|
MANIFEST = "copied.md"
|
||||||
|
|
||||||
|
|
||||||
|
def destination(source: Path, out: Path) -> Path:
|
||||||
|
"""`out/<name>`, so the folder keeps the name the thing already had."""
|
||||||
|
return Path(out) / source.name
|
||||||
|
|
||||||
|
|
||||||
|
def take(source: Path, out, keep_noise=False, keep_secrets=False, max_bytes=None,
|
||||||
|
exclude=(), include=(), force=False, dry_run=False, quiet=False):
|
||||||
|
dest = destination(source, out)
|
||||||
|
|
||||||
|
if dest.exists() and any(dest.iterdir()) and not force and not dry_run:
|
||||||
|
fail(f"{dest} already exists and is not empty.",
|
||||||
|
"Pass --force to write into it anyway, or point --out elsewhere.")
|
||||||
|
|
||||||
|
paths = file_set(source)
|
||||||
|
kept, dropped = sift(source, paths, keep_noise=keep_noise,
|
||||||
|
keep_secrets=keep_secrets, max_bytes=max_bytes,
|
||||||
|
exclude=exclude, include=include,
|
||||||
|
ignored=ignored_but_tracked(source, paths))
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
print(f"{len(paths)} files tracked, {len(kept)} to copy, {len(dropped)} left behind.")
|
||||||
|
by_reason = {}
|
||||||
|
for rel, why in dropped:
|
||||||
|
by_reason.setdefault(why, []).append(rel)
|
||||||
|
for why in ("secret", "ignored", "derived", "oversize", "excluded"):
|
||||||
|
hits = by_reason.get(why)
|
||||||
|
if not hits:
|
||||||
|
continue
|
||||||
|
# Secrets are listed in full however many there are. The others are
|
||||||
|
# bulk and a count is enough; a key that got dropped is a thing you
|
||||||
|
# want to see the name of, because it means it was tracked.
|
||||||
|
shown = hits if why in ("secret", "ignored") else hits[:5]
|
||||||
|
print(f"\n {why} — {REASONS[why]} ({len(hits)}):")
|
||||||
|
for rel in shown:
|
||||||
|
print(f" {rel}")
|
||||||
|
if len(hits) > len(shown):
|
||||||
|
print(f" ... and {len(hits) - len(shown)} more")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
if not quiet:
|
||||||
|
print(f"\nNothing written. Would copy to {dest}.")
|
||||||
|
return {"kept": kept, "dropped": dropped, "dest": dest}
|
||||||
|
|
||||||
|
dest.mkdir(parents=True, exist_ok=True)
|
||||||
|
for rel in kept:
|
||||||
|
target = dest / rel
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(source / rel, target)
|
||||||
|
|
||||||
|
_manifest(source, out, dest, paths, kept, dropped)
|
||||||
|
|
||||||
|
if not quiet:
|
||||||
|
print(f"\nCopied to {dest}")
|
||||||
|
print(f" no .git — {'the source has one and it was not copied'
|
||||||
|
if is_git(source) else 'the source has none either'}")
|
||||||
|
print(f" what was left behind: {state_dir(out) / MANIFEST}")
|
||||||
|
return {"kept": kept, "dropped": dropped, "dest": dest}
|
||||||
|
|
||||||
|
|
||||||
|
def _manifest(source, out, dest, paths, kept, dropped):
|
||||||
|
"""A record of the decision, beside the copy rather than inside it."""
|
||||||
|
lines = [
|
||||||
|
f"# Copied from `{source}`", "",
|
||||||
|
f"- source: `{source}`",
|
||||||
|
f"- copy: `{dest}`",
|
||||||
|
f"- {len(paths)} files tracked, {len(kept)} copied, {len(dropped)} left behind",
|
||||||
|
"",
|
||||||
|
"No `.git` was copied. The file list is what git tracks, so nothing "
|
||||||
|
"untracked or ignored came across — except where a file was tracked "
|
||||||
|
"*despite* the ignore rules, which is listed below rather than assumed.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
by_reason = {}
|
||||||
|
for rel, why in dropped:
|
||||||
|
by_reason.setdefault(why, []).append(rel)
|
||||||
|
for why in ("secret", "ignored", "derived", "oversize", "excluded"):
|
||||||
|
hits = by_reason.get(why)
|
||||||
|
if not hits:
|
||||||
|
continue
|
||||||
|
lines += [f"## {why} — {REASONS[why]}", ""]
|
||||||
|
lines += [f"- `{rel}`" for rel in hits]
|
||||||
|
lines.append("")
|
||||||
|
if not dropped:
|
||||||
|
lines += ["Nothing was left behind.", ""]
|
||||||
|
|
||||||
|
path = state_dir(out) / MANIFEST
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text("\n".join(lines))
|
||||||
77
soleprint/station/tools/histgen/templates/index.html
Normal file
77
soleprint/station/tools/histgen/templates/index.html
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>histgen — station</title>
|
||||||
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
<style>
|
||||||
|
body { font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
background: var(--surface-0, #101418); color: var(--text-0, #d8dee4);
|
||||||
|
margin: 0; padding: 2rem; line-height: 1.55; }
|
||||||
|
h1 { margin: 0 0 .25rem; font-size: 1.4rem; }
|
||||||
|
p.lede { margin: 0 0 1.5rem; color: var(--text-1, #8b98a5); }
|
||||||
|
form { display: flex; gap: .5rem; margin-bottom: 1.5rem; }
|
||||||
|
input, button { font: inherit; padding: .45rem .7rem;
|
||||||
|
background: var(--surface-1, #161c22); color: inherit;
|
||||||
|
border: 1px solid var(--border, #2a333d); border-radius: 4px; }
|
||||||
|
button { cursor: pointer; }
|
||||||
|
.group { border: 1px solid var(--border, #2a333d); border-radius: 4px;
|
||||||
|
padding: .6rem .9rem; margin-bottom: .5rem;
|
||||||
|
background: var(--surface-1, #161c22); }
|
||||||
|
.n { color: var(--accent, #4fb3a6); }
|
||||||
|
.title { font-weight: 600; }
|
||||||
|
.untitled { color: var(--text-1, #8b98a5); font-style: italic; }
|
||||||
|
.paths { margin: .4rem 0 0; padding-left: 1.1rem; color: var(--text-1, #8b98a5);
|
||||||
|
font-size: .87rem; }
|
||||||
|
#summary { color: var(--text-1, #8b98a5); margin-bottom: 1rem; }
|
||||||
|
.err { color: #e08a5c; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>histgen</h1>
|
||||||
|
<p class="lede">A proposed history, before it is a history. Read-only —
|
||||||
|
<code>python -m station.tools.histgen apply <repo></code> is what commits.</p>
|
||||||
|
|
||||||
|
<form onsubmit="load(event)">
|
||||||
|
<input id="repo" placeholder="path to a repo, e.g. rig" size="40" autofocus>
|
||||||
|
<button type="submit">Show plan</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div id="summary"></div>
|
||||||
|
<div id="groups"></div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
async function load(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
const repo = document.getElementById('repo').value.trim();
|
||||||
|
const summary = document.getElementById('summary');
|
||||||
|
const groups = document.getElementById('groups');
|
||||||
|
groups.innerHTML = ''; summary.textContent = 'Loading…';
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/station/tools/histgen/api/plan?repo=${encodeURIComponent(repo)}`);
|
||||||
|
const data = await res.json();
|
||||||
|
if (!res.ok) throw new Error(data.detail || res.statusText);
|
||||||
|
summary.textContent =
|
||||||
|
`${data.commits} commits over ${data.files} files` +
|
||||||
|
(data.untitled.length ? ` — ${data.untitled.length} still without a message` : '');
|
||||||
|
for (const g of data.groups) {
|
||||||
|
const el = document.createElement('div');
|
||||||
|
el.className = 'group';
|
||||||
|
const title = g.title
|
||||||
|
? `<span class="title">${escapeHtml(g.title)}</span>`
|
||||||
|
: `<span class="untitled">${escapeHtml(g.slug)} — no message yet</span>`;
|
||||||
|
el.innerHTML = `<span class="n">${String(g.n).padStart(2,'0')}</span> ${title}
|
||||||
|
<ul class="paths">${g.paths.map(p => `<li>${escapeHtml(p)}</li>`).join('')}</ul>`;
|
||||||
|
groups.appendChild(el);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
summary.innerHTML = `<span class="err">${escapeHtml(e.message)}</span>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function escapeHtml(s) {
|
||||||
|
return String(s).replace(/[&<>"']/g, c =>
|
||||||
|
({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in New Issue
Block a user