Compare commits
7 Commits
2d9bc9289c
...
49a9f8ee57
| Author | SHA1 | Date | |
|---|---|---|---|
| 49a9f8ee57 | |||
| 966f8fc821 | |||
| 83b6cbebe3 | |||
| a65c92257d | |||
| 78595f1bd9 | |||
| 9bcd439266 | |||
| 74f03566f1 |
27
.gitattributes
vendored
Normal file
27
.gitattributes
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
# Copied verbatim from rig/.gitattributes, and deliberately duplicated rather
|
||||
# than shared: rig/ must carry its own so it survives being handed over on its
|
||||
# own, and spr had none at all despite shipping ctrl/*.sh and generating
|
||||
# gen/<room>/ctrl/*.sh.
|
||||
#
|
||||
# Line endings are normalised to LF in the repository and on checkout, on every
|
||||
# platform. Without this, a checkout on Windows/WSL rewrites files to CRLF and
|
||||
# every one of them shows up as modified without anyone having touched it.
|
||||
#
|
||||
# For the scripts it is not cosmetic: a shell script with CRLF fails on Linux
|
||||
# with `bad interpreter: /usr/bin/env bash^M`, which reads as a broken installer
|
||||
# rather than a line-ending problem — the worst possible first impression on a
|
||||
# machine where nothing has been proven yet.
|
||||
* text=auto eol=lf
|
||||
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.env text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
|
||||
# Never touch binaries.
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.zip binary
|
||||
*.tar binary
|
||||
*.gz binary
|
||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -34,3 +34,11 @@ cfg/amar/
|
||||
cfg/dlt/
|
||||
# Add new rooms here as they are created
|
||||
# cfg/<room>/
|
||||
|
||||
# Client rigs. A rig is a copy of rig/ renamed after the environment it models,
|
||||
# so its k8s files spell out a real architecture — exactly the thing that must
|
||||
# not land here. They are versioned in their own repo.
|
||||
#
|
||||
# Anchored at the ROOT on purpose: a copy is a SIBLING of rig/, so a rule inside
|
||||
# rig/.gitignore cannot see it.
|
||||
*-rig/
|
||||
|
||||
@@ -129,7 +129,7 @@ Every script stays runnable on its own — the standalone rule holds:
|
||||
```bash
|
||||
python build.py --cfg amar # -> gen/amar/
|
||||
cd gen/standalone && python run.py # bare-metal
|
||||
./ctrl/kind-up.sh # still works directly
|
||||
./ctrl/cluster.sh up # still runs directly; rig builds the cluster
|
||||
cd gen/<room> && ./ctrl/start.sh # each room owns its lifecycle scripts
|
||||
```
|
||||
|
||||
|
||||
8
Makefile
8
Makefile
@@ -13,7 +13,7 @@
|
||||
# make component ARGS="publish soleprint-ui /tmp/out --dist"
|
||||
# make deploy ARGS="--build"
|
||||
#
|
||||
# Every script stays runnable on its own (./ctrl/kind-up.sh still works, and each
|
||||
# Every script stays runnable on its own (./ctrl/cluster.sh up still works, and each
|
||||
# built room keeps its own gen/<room>/ctrl/*.sh) — the standalone rule holds, and
|
||||
# this only saves typing.
|
||||
#
|
||||
@@ -33,6 +33,12 @@ ifneq ($(ARGS),)
|
||||
# then fails with "No rule to make target 'sample'", because make reads every
|
||||
# word on the line as something it has been asked to build.
|
||||
$(eval $(ARGS):;@:)
|
||||
# ...and as PHONY, because some of those words name real directories. `cfg`,
|
||||
# `ctrl`, `docs`, `gen` and `init` all exist at this level, and make considers a
|
||||
# target that is an existing directory already built — so `make build ctrl` ran
|
||||
# the build and then printed "make: 'ctrl' is up to date". The empty rule above
|
||||
# is not enough on its own; only .PHONY stops make consulting the filesystem.
|
||||
.PHONY: $(ARGS)
|
||||
endif
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
29
build.py
29
build.py
@@ -80,6 +80,30 @@ def _rmtree_resilient(path: Path):
|
||||
)
|
||||
|
||||
|
||||
# Never swept into a built room, wherever they appear in a source tree.
|
||||
#
|
||||
# This is a SECURITY boundary, not tidiness. gen/<room>/ is the docker build
|
||||
# context, soleprint/Dockerfile is `COPY . .`, and there is no .dockerignore —
|
||||
# so anything that reaches gen/ reaches an image layer, and registry.mcrn.ar is
|
||||
# public-read. That is how station/tools/tester/.env, gitignored since the last
|
||||
# incident, still ended up baked into soleprint_localtest-soleprint:latest with
|
||||
# its API key intact. .gitignore does not bind shutil.
|
||||
#
|
||||
# Applied to bulk directory copies only. A caller naming a single file is making
|
||||
# an explicit request (cfg/<room>/.env.example is the one that matters) and is
|
||||
# left alone.
|
||||
ALWAYS_IGNORE = {".git", "__pycache__", "node_modules", ".venv", "venv", ".env"}
|
||||
ALWAYS_IGNORE_SUFFIXES = (".pyc", ".pyo")
|
||||
|
||||
|
||||
def is_ignored(name: str) -> bool:
|
||||
return name in ALWAYS_IGNORE or name.endswith(ALWAYS_IGNORE_SUFFIXES)
|
||||
|
||||
|
||||
def _copytree_ignore(directory, files):
|
||||
return {f for f in files if is_ignored(f)}
|
||||
|
||||
|
||||
def copy_path(source: Path, target: Path, quiet: bool = False):
|
||||
"""Copy file or directory, resolving symlinks."""
|
||||
if target.is_symlink():
|
||||
@@ -91,7 +115,7 @@ def copy_path(source: Path, target: Path, quiet: bool = False):
|
||||
target.unlink()
|
||||
|
||||
if source.is_dir():
|
||||
shutil.copytree(source, target, symlinks=False)
|
||||
shutil.copytree(source, target, symlinks=False, ignore=_copytree_ignore)
|
||||
if not quiet:
|
||||
log.info(f" {target.name}/")
|
||||
else:
|
||||
@@ -111,6 +135,8 @@ def merge_into(source: Path, target: Path):
|
||||
for item in source.rglob("*"):
|
||||
if item.is_file():
|
||||
rel = item.relative_to(source)
|
||||
if any(is_ignored(part) for part in rel.parts):
|
||||
continue
|
||||
dest = target / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(item, dest)
|
||||
@@ -687,6 +713,7 @@ def build_soleprint(output_dir: Path, room: str):
|
||||
"index.html",
|
||||
"requirements.txt",
|
||||
"Dockerfile",
|
||||
".dockerignore",
|
||||
]:
|
||||
if (soleprint / name).exists():
|
||||
copy_path(soleprint / name, output_dir / name)
|
||||
|
||||
@@ -6,16 +6,49 @@
|
||||
# ./ctrl/cluster.sh down # delete it (drops every room's namespace)
|
||||
# ./ctrl/cluster.sh status # what's running on it
|
||||
#
|
||||
# One target, one script — the variants live here. The kind-*.sh files stay
|
||||
# exactly as they are and remain runnable on their own; this only dispatches.
|
||||
# spr depends on rig, never the other way round. Building and deleting a cluster
|
||||
# is rig's job, so up and down hand straight to rig/ctrl/cluster.sh, carrying the
|
||||
# four things that make this cluster spr's rather than rig's defaults:
|
||||
#
|
||||
# CLUSTER=spr rooms deploy into the kind-spr context
|
||||
# KIND_CONFIG spr's own shape, which maps the rooms' gateway NodePorts
|
||||
# REGISTRY_MODE=none rooms load images straight into the node
|
||||
# PROFILE=minimal pinned here, so a change to rig's own ctrl/.env can never
|
||||
# quietly add addons to spr's cluster
|
||||
#
|
||||
# status stays here: it answers a question about rooms, not about the cluster.
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RIG_CTRL="$SCRIPT_DIR/../rig/ctrl"
|
||||
|
||||
rig() {
|
||||
CLUSTER=spr \
|
||||
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml" \
|
||||
REGISTRY_MODE=none \
|
||||
PROFILE=minimal \
|
||||
bash "$RIG_CTRL/cluster.sh" "$@"
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
up) exec "$SCRIPT_DIR/kind-up.sh" ;;
|
||||
down) exec "$SCRIPT_DIR/kind-down.sh" ;;
|
||||
status) exec "$SCRIPT_DIR/kind-status.sh" ;;
|
||||
up)
|
||||
rig up
|
||||
echo
|
||||
echo "Per-room deploy:"
|
||||
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
||||
;;
|
||||
down)
|
||||
rig down
|
||||
;;
|
||||
status)
|
||||
if ! kind get clusters 2>/dev/null | grep -qx spr; then
|
||||
echo "No 'spr' kind cluster — run: make cluster up"
|
||||
exit 0
|
||||
fi
|
||||
kubectl --context kind-spr get namespaces -l soleprint-room
|
||||
echo
|
||||
kubectl --context kind-spr get pods -A -l soleprint-room
|
||||
;;
|
||||
*)
|
||||
echo "Unknown subcommand: $1" >&2
|
||||
echo "Usage: cluster.sh [up|down|status]" >&2
|
||||
|
||||
@@ -3,9 +3,15 @@ apiVersion: kind.x-k8s.io/v1alpha4
|
||||
# Single shared cluster for all soleprint rooms.
|
||||
# Each room deploys into its own namespace; gateway Services pick a
|
||||
# NodePort from the 30080-30099 range mapped here.
|
||||
name: spr
|
||||
#
|
||||
# Built by rig, not by spr: ctrl/cluster.sh hands this file to
|
||||
# rig/ctrl/cluster.sh, which substitutes CLUSTER and NODE_IMAGE (named without
|
||||
# braces here so this comment survives the substitution). The shape is spr's —
|
||||
# what its cluster needs is spr's business. Building it is rig's.
|
||||
name: ${CLUSTER}
|
||||
nodes:
|
||||
- role: control-plane
|
||||
image: ${NODE_IMAGE}
|
||||
extraPortMappings:
|
||||
# Room gateway NodePorts (one per active room).
|
||||
- {containerPort: 30080, hostPort: 30080, protocol: TCP}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Delete the shared `spr` kind cluster (drops every room's namespace too).
|
||||
# Use `gen/<room>/ctrl/k8s-down.sh` instead if you only want to remove
|
||||
# a single room's namespace.
|
||||
set -e
|
||||
|
||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
||||
echo "Deleting kind cluster 'spr'..."
|
||||
kind delete cluster --name spr
|
||||
else
|
||||
echo "No kind cluster 'spr' to delete."
|
||||
fi
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Show what's running on the shared `spr` cluster.
|
||||
set -e
|
||||
|
||||
if ! kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
||||
echo "No 'spr' kind cluster — run ctrl/kind-up.sh"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
kubectl --context kind-spr get namespaces -l soleprint-room
|
||||
echo
|
||||
kubectl --context kind-spr get pods -A -l soleprint-room
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Create (or no-op) the single shared `spr` kind cluster used by every
|
||||
# soleprint room. Per-room work happens inside namespaces — see
|
||||
# `gen/<room>/ctrl/k8s-up.sh`.
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml"
|
||||
|
||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
||||
echo "Kind cluster 'spr' already exists."
|
||||
else
|
||||
echo "Creating kind cluster 'spr'..."
|
||||
kind create cluster --config "$KIND_CONFIG"
|
||||
fi
|
||||
|
||||
kubectl config use-context kind-spr >/dev/null
|
||||
|
||||
echo
|
||||
echo "Cluster ready. Per-room deploy:"
|
||||
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
||||
22
rig/.gitattributes
vendored
Normal file
22
rig/.gitattributes
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
# Line endings are normalised to LF in the repository and on checkout, on every
|
||||
# platform. Without this, a checkout on Windows/WSL rewrites files to CRLF and
|
||||
# every one of them shows up as modified without anyone having touched it.
|
||||
#
|
||||
# For the scripts it is not cosmetic: a shell script with CRLF fails on Linux
|
||||
# with `bad interpreter: /usr/bin/env bash^M`, which reads as a broken installer
|
||||
# rather than a line-ending problem — the worst possible first impression on a
|
||||
# machine where nothing has been proven yet.
|
||||
* text=auto eol=lf
|
||||
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.env text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
|
||||
# Never touch binaries.
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.zip binary
|
||||
*.tar binary
|
||||
*.gz binary
|
||||
19
rig/.gitignore
vendored
Normal file
19
rig/.gitignore
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
# def/ — the "default" scratch bucket: always gitignored, never versioned
|
||||
def
|
||||
|
||||
# local env (commit the .env.example, never the .env)
|
||||
.env
|
||||
.env.local
|
||||
ctrl/.env
|
||||
|
||||
# generated: the .dot is a build artifact rendered from arch/*.json, never hand-edited.
|
||||
# The .svg IS committed — onboarding material should render in a repo browser.
|
||||
arch/*.dot
|
||||
ctrl/Tiltfile.gen
|
||||
|
||||
# binaries pulled by `make deps-bundle` for the air-gapped installer image
|
||||
vendor
|
||||
|
||||
# Client rigs are NOT ignored here. A copy is a SIBLING of this directory
|
||||
# (../acme-rig), so a rule in this file cannot see it — the rules live in the
|
||||
# parent repo's .gitignore, anchored at its root, where `*-rig/` matches them.
|
||||
277
rig/BOOTSTRAP.md
Normal file
277
rig/BOOTSTRAP.md
Normal file
@@ -0,0 +1,277 @@
|
||||
# From a machine with nothing on it to a project you can work in
|
||||
|
||||
The README says the prerequisite is Docker and nothing else. This is what that
|
||||
actually looks like end to end: a bare Linux box, and a new project running under
|
||||
Tilt at the end of it.
|
||||
|
||||
A copy of this directory is a sibling of it, named after the environment it
|
||||
models (`acme-rig`). Paths below are relative to the parent checkout.
|
||||
|
||||
It spans three repos because the work does. **rig** prepares the machine — the
|
||||
pinned toolchain, the cluster, the port arithmetic. **all** owns the shape a
|
||||
project takes, in `all/projects/templates/conventions.md` and the `broad`
|
||||
scaffold beside it. **ppl** owns everything after local, and is where this
|
||||
document stops.
|
||||
|
||||
Read it once before running anything. Three of the steps below need root and one
|
||||
needs a logout, so knowing about them in advance is cheaper than meeting them
|
||||
halfway through.
|
||||
|
||||
|
||||
## Docker, and the two sysctls Tilt depends on
|
||||
|
||||
rig installs a toolchain; it does not install Docker. That line is not modesty —
|
||||
Docker is a daemon, a group membership and usually a logout, and a script that
|
||||
did it would have to be trusted with root on a machine it knows nothing about.
|
||||
|
||||
```bash
|
||||
sudo apt-get install -y docker.io && sudo usermod -aG docker "$USER"
|
||||
```
|
||||
|
||||
Then log out and back in, and check `docker info` answers. Until it does, nothing
|
||||
below works and everything below reports the same failure.
|
||||
|
||||
While you have root, raise the inotify limits:
|
||||
|
||||
```bash
|
||||
echo -e 'fs.inotify.max_user_watches=524288\nfs.inotify.max_user_instances=512' \
|
||||
| sudo tee /etc/sysctl.d/99-rig.conf
|
||||
sudo sysctl --system
|
||||
```
|
||||
|
||||
kind and Tilt both watch large trees, and WSL ships 8192 watches and 128
|
||||
instances — far too low. The failure mode is the reason this is here at step
|
||||
zero rather than mentioned later: Tilt does not error, it simply stops noticing
|
||||
that files changed, and you lose an afternoon to a hot reload that silently
|
||||
isn't.
|
||||
|
||||
|
||||
## Read the docs before installing anything
|
||||
|
||||
```bash
|
||||
cd rig
|
||||
make docs
|
||||
```
|
||||
|
||||
`ctrl/docs.sh` runs a throwaway `nginx:alpine` over a read-only bind mount of
|
||||
`docs/` and prints the URL. That is deliberate: the docs are the instructions for
|
||||
building everything else, so they cannot live in the cluster and cannot need
|
||||
`python3 -m http.server` either — a minimal Debian has no python. What it has,
|
||||
by definition, is Docker.
|
||||
|
||||
The port is this environment's `HTTP_PORT + 4`. Nothing is installed and nothing
|
||||
persists; ctrl-c ends it.
|
||||
|
||||
|
||||
## Ask what is wrong with this machine
|
||||
|
||||
```bash
|
||||
make check
|
||||
cp ctrl/.env.example ctrl/.env
|
||||
```
|
||||
|
||||
`check.sh` reports and instructs, and fixes nothing. It runs bare rather than
|
||||
in a container because host detection only ever reads `/proc` and `/etc` — no
|
||||
dependency beyond coreutils.
|
||||
|
||||
Read the whole output, but the `ports` block is the one to read carefully. Every
|
||||
port rig binds derives from this directory's name, so the answer is specific to
|
||||
this copy, and a clash here surfaces as an opaque `failed to bind host port` in
|
||||
the middle of cluster creation if you skip it.
|
||||
|
||||
Copy the `.env` even though the check only warns about it. It is gitignored, it is
|
||||
where a machine-local override goes, and `ports.sh persist` expects it to exist.
|
||||
|
||||
|
||||
## Install the toolchain — through the container
|
||||
|
||||
This is the step where "nothing installed" stops being rhetorical.
|
||||
|
||||
`make deps` runs `ctrl/deps.sh install` directly on the host, and the installer
|
||||
fetches with `curl`. A stock `debian:trixie-slim` has no curl — detection runs
|
||||
fine, then the first download dies with `curl: command not found` and an exit
|
||||
code of 127. That is the bootstrap paradox `ctrl/Dockerfile.deps` exists
|
||||
to kill — the installer carries its own toolchain so the host needs only Docker —
|
||||
but building the image and running it are two different things, and only the
|
||||
build has a Makefile target today. **On a genuinely bare machine, run it by
|
||||
hand:**
|
||||
|
||||
```bash
|
||||
make deps-image # builds rig-deps:deps
|
||||
mkdir -p ~/.local/bin
|
||||
docker run --rm \
|
||||
-v /:/host:ro \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$HOME/.local/bin:/out/bin" \
|
||||
-e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
|
||||
rig-deps:deps install dev
|
||||
```
|
||||
|
||||
The image name follows the directory, like everything else here: in `rig`
|
||||
it is `rig-deps`, in a copy called `acme-rig` it is `acme-rig-deps`. The
|
||||
tag is `deps` (or `full`, below), not `latest`.
|
||||
|
||||
None of the four arguments are guessable, so:
|
||||
|
||||
- **`/:/host:ro`** — the installer reads the *host's* `/etc/os-release` and
|
||||
`/etc/wsl.conf`, not the container's. `HOST_ROOT=/host` is already baked into
|
||||
the image; this is what it points at. Read-only, and it is the only reason
|
||||
detection inside a container tells you anything about the machine.
|
||||
- **the docker socket** — how detection reaches the daemon it is reporting on,
|
||||
and how it counts kind clusters already running.
|
||||
- **`/out/bin`** — the image's `OUT_BIN`. Whatever you mount here is where the
|
||||
four binaries land.
|
||||
- **`HOST_UID` / `HOST_GID`** — the installer runs as root so it can reach that
|
||||
socket, which means everything it writes into a mounted volume is root-owned
|
||||
and useless to you. These drive the `chown` back. Omit them and the install
|
||||
looks like it worked.
|
||||
|
||||
`dev` is kubectl, jq, kind and tilt. `core` is kubectl and jq alone — no cluster
|
||||
tooling — which is the right answer on a managed or corporate-issued machine and
|
||||
is why the split exists.
|
||||
|
||||
Then put them on PATH, which the installer will remind you about because it cannot
|
||||
edit your shell for you:
|
||||
|
||||
```bash
|
||||
export PATH="$HOME/.local/bin:$PATH" # and add the same line to ~/.bashrc
|
||||
```
|
||||
|
||||
If something else on this machine already provides `kubectl`, the installer says so
|
||||
by name rather than shadowing it quietly. `OUT_BIN=$PWD/def/bin` installs
|
||||
somewhere private instead.
|
||||
|
||||
**Two variants worth knowing before you need them.** `make deps-image full` bakes
|
||||
every pinned binary into the image at build time (`DEPS_SOURCE=baked`), so
|
||||
`docker save` gives you the entire installer as one file to carry into an
|
||||
air-gapped network. And `DEPS_SOURCE=artifactory` with `DEPS_ARTIFACTORY_URL`
|
||||
pulls from a generic internal repo, which is usually the only thing a locked-down
|
||||
client allows.
|
||||
|
||||
From here on this machine has curl, so **`make deps` is the short form** for
|
||||
every later run and every later copy of this directory. The container path is
|
||||
the first-time path.
|
||||
|
||||
|
||||
## Prove the machine before blaming the project
|
||||
|
||||
```bash
|
||||
make setup
|
||||
make cluster up
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
`make setup` re-runs every check as a group. It is idempotent and it deliberately
|
||||
does not abort on the first failure — a setup script that dies at step two hides
|
||||
the fact that steps four and five were also going to fail. Run now, it should be
|
||||
`ok` and `done` all the way down, and that is the point: it is the scoreboard,
|
||||
not the installer.
|
||||
|
||||
`make cluster up` builds the default `minimal` profile — one node, no addons,
|
||||
boots fast. You do not need it to develop anything, but you do want to know that
|
||||
kind, the kubeconfig context and the derived port block work *before* a new
|
||||
project has any problems of its own to confuse them with. `make cluster down`
|
||||
when you are done looking.
|
||||
|
||||
Before starting a second cluster, and it will not be long:
|
||||
|
||||
```bash
|
||||
make cluster list
|
||||
```
|
||||
|
||||
Available memory, per-cluster usage and each cluster's port block. On a 16 GiB
|
||||
box four single-node clusters are comfortable and six push into swap, so this is
|
||||
worth reading before rather than after. `make cluster free <names>` stops
|
||||
clusters without deleting them; `docker start` brings them back untouched.
|
||||
|
||||
|
||||
## Scaffold the project
|
||||
|
||||
The canonical layout is [`all/projects/templates/conventions.md`](../all/projects/templates/conventions.md).
|
||||
Read it — it is short, opinionated, and exists precisely so nobody
|
||||
reverse-engineers a layout from whichever repo they happened to open. What
|
||||
follows is only the mechanical part.
|
||||
|
||||
```bash
|
||||
SLUG=<slug> # short, lowercase, no separators
|
||||
cp -r ~/wdir/semester/all/projects/templates/broad ~/wdir/semester/"$SLUG"
|
||||
cd ~/wdir/semester/"$SLUG"
|
||||
grep -rl '<slug>' ctrl | xargs sed -i "s/<slug>/$SLUG/g"
|
||||
cp ctrl/k8s/.env.example ctrl/k8s/.env
|
||||
git init && git add -A && git commit -m "scaffold $SLUG from broad"
|
||||
```
|
||||
|
||||
`<slug>` is the only placeholder and it lives only under `ctrl/` — cluster name,
|
||||
namespace, ConfigMap name, and the `NAME=` in `kind-up.sh` / `kind-down.sh`. One
|
||||
sed does all of it.
|
||||
|
||||
The slug is the folder name, lowercase and short — `mpr`, `unt`, `nvi`. The
|
||||
cluster takes that name and the context becomes `kind-<slug>`, derived by the
|
||||
scaffold's Makefile from the directory, so there is nothing to edit for either.
|
||||
|
||||
**Pick the Tilt port deliberately.** `ctrl/k8s/.env.example` ships a value that
|
||||
is already in use, so copying it unchanged puts two projects on one port:
|
||||
|
||||
```bash
|
||||
grep -h '^TILT_PORT=' ~/wdir/semester/*/ctrl/k8s/.env 2>/dev/null | sort
|
||||
```
|
||||
|
||||
Choose a free one in `10300–10399` — the range ALL reserves in
|
||||
`projects/index.json` under `policy` — avoiding `10350`, which is Tilt's own
|
||||
default. Currently taken: `nvi` 10330, `unt` 10340, `mpr` 10360, `mlv` 10370,
|
||||
`eth` 10380, `lng` 10390. This is the Tilt *web UI* port, not a service port;
|
||||
each project owns its own service ports separately. The scaffold ships it blank
|
||||
on purpose, so there is nothing to collide with until you choose.
|
||||
|
||||
The scaffold's `ctrl/k8s/` is the same shape as every other project here, and it
|
||||
builds as shipped:
|
||||
|
||||
```
|
||||
kind-config.yaml one node; gateway NodePort 30080 -> hostPort 8080
|
||||
base/ namespace, configmap, app (Deployment + Service)
|
||||
overlays/dev/ promotes the app Service to NodePort 30080
|
||||
```
|
||||
|
||||
Check it before `kind` spends minutes on anything — this renders the whole tree
|
||||
without a cluster and catches a broken patch immediately:
|
||||
|
||||
```bash
|
||||
kubectl kustomize ctrl/k8s/overlays/dev
|
||||
```
|
||||
|
||||
The workload is an nginx placeholder so a fresh copy reaches something that
|
||||
answers; replace it. Keep `30080` in step between the overlay patch and
|
||||
`kind-config.yaml`'s `containerPort` — the hostPort is this project's to pick.
|
||||
Reachability is a plain kind port mapping: no ingress controller and no MetalLB.
|
||||
Caddy maps `<slug>.local.ar` onto the host port (`~/wdir/semester/ppl/local/Caddyfile`),
|
||||
with `*.local.ar` resolving to 127.0.0.1 through dnsmasq. That is the whole chain.
|
||||
|
||||
**The one file the scaffold still does not ship is `ctrl/Tiltfile`** — `make
|
||||
tilt-up` runs `cd ctrl && tilt up`, and there is nothing to run until you write
|
||||
one. Copy it from a live project; `unt` and `nvi` are closest to the plain shape.
|
||||
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
make kind-up # idempotent create, then selects the context
|
||||
make tilt-up # context + your assigned port
|
||||
```
|
||||
|
||||
`tilt-up` passes `--context kind-<slug>` every time, which is the point of going
|
||||
through `make` at all: tilt cannot deploy into whichever cluster you last looked
|
||||
at.
|
||||
|
||||
`make tilt-down` and `make kind-down` close the loop, and `make kind-reset` is
|
||||
delete-and-recreate for when a cluster wedges.
|
||||
|
||||
|
||||
## Register it
|
||||
|
||||
The project exists; now it is findable. Add an entry to
|
||||
`~/wdir/semester/all/projects/index.json` and write its `projects/<slug>.md` beside the
|
||||
others. Structured fields in the index, prose in the markdown.
|
||||
|
||||
Putting it on the CI server and deploying it is `ppl`'s half, and it starts at
|
||||
`~/wdir/semester/ppl/ctrl/init-repo.sh` — gitea remote, then Woodpecker. That is a
|
||||
different document.
|
||||
129
rig/Makefile
Normal file
129
rig/Makefile
Normal file
@@ -0,0 +1,129 @@
|
||||
# Thin control Makefile — one target per ctrl/ script, and the subcommand is an
|
||||
# argument rather than a second target: `make cluster down`, not `make cluster-down`.
|
||||
#
|
||||
# The logic lives in the scripts, never here. Each target maps to exactly one
|
||||
# bash file, and that file holds the variants:
|
||||
#
|
||||
# make cluster up -> ctrl/cluster.sh up
|
||||
# make newbox destroy -> ctrl/newbox.sh destroy
|
||||
#
|
||||
# Config layers, weakest first: ctrl/versions.env (pinned toolchain) <
|
||||
# ctrl/env.d/<profile>.env (cluster shape) < ctrl/.env (local, gitignored) <
|
||||
# the environment. So `make cluster up PROFILE=client` beats everything.
|
||||
#
|
||||
# Start with: make setup (then: make cluster up && make docs)
|
||||
|
||||
# Identity follows the FOLDER NAME, so this directory can be copied elsewhere,
|
||||
# renamed, and run as a separate environment with no edits. ctrl/.env overrides
|
||||
# it when you want a name that differs from the directory.
|
||||
SLUG := $(shell echo '$(notdir $(CURDIR))' | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-' | sed 's/^-*//; s/-*$$//')
|
||||
CLUSTER := $(or $(shell sed -n 's/^CLUSTER=//p' ctrl/.env 2>/dev/null),$(SLUG))
|
||||
KCTX := --context kind-$(CLUSTER)
|
||||
TILT_PORT := $(shell sed -n 's/^TILT_PORT=//p' ctrl/.env 2>/dev/null)
|
||||
DEPSIMG := $(SLUG)-deps
|
||||
|
||||
# Words after the target become the script's subcommand. Make would otherwise
|
||||
# treat them as goals of their own, so each gets a no-op rule.
|
||||
ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
|
||||
ifneq ($(ARGS),)
|
||||
$(eval $(ARGS):;@:)
|
||||
# ...and as PHONY, because some of those words name real directories. `cfg`,
|
||||
# `ctrl`, `docs`, `gen` and `init` all exist at this level, and make considers a
|
||||
# target that is an existing directory already built — so `make build ctrl` ran
|
||||
# the build and then printed "make: 'ctrl' is up to date". The empty rule above
|
||||
# is not enough on its own; only .PHONY stops make consulting the filesystem.
|
||||
.PHONY: $(ARGS)
|
||||
endif
|
||||
|
||||
.PHONY: help setup check mem deps deps-image pins cluster registry addons ports \
|
||||
newbox dockerhost docs tilt \
|
||||
kind-up kind-down kind-reset tilt-up tilt-down
|
||||
|
||||
help: ## list targets
|
||||
@grep -hE '^[a-z][a-z-]*:.*?##' $(MAKEFILE_LIST) | sed 's/:.*##/\t/' | expand -t16
|
||||
|
||||
# ── setup ──────────────────────────────────────────────────────────────────
|
||||
|
||||
setup: ## prepare this machine [core] [--share-docker] [--cluster]
|
||||
bash ctrl/setup.sh $(ARGS)
|
||||
|
||||
check: ## is this machine ready? reports, never fixes
|
||||
bash ctrl/check.sh
|
||||
|
||||
mem: ## memory, and any cap holding it [status|backup|restore]
|
||||
bash ctrl/mem.sh $(or $(ARGS),status)
|
||||
|
||||
deps: ## install the toolchain [core|dev] (default dev)
|
||||
bash ctrl/deps.sh install $(or $(ARGS),dev)
|
||||
|
||||
pins: ## standalone/rigdeps.sh still installs what rig pins?
|
||||
bash ctrl/pins.sh
|
||||
|
||||
deps-image: ## build the installer image [full]
|
||||
docker build -f ctrl/Dockerfile.deps \
|
||||
--target $(if $(filter full,$(ARGS)),deps-full,deps) \
|
||||
-t $(DEPSIMG):$(if $(filter full,$(ARGS)),full,deps) .
|
||||
|
||||
# ── cluster ────────────────────────────────────────────────────────────────
|
||||
|
||||
cluster: ## this env + the machine [up|down|reset|list|free]
|
||||
bash ctrl/cluster.sh $(or $(ARGS),up)
|
||||
|
||||
registry: ## registry wiring [up|down|status] (default status)
|
||||
bash ctrl/registry.sh $(or $(ARGS),status)
|
||||
|
||||
addons: ## profile addons [install|list] (default list)
|
||||
bash ctrl/addons.sh $(or $(ARGS),list)
|
||||
|
||||
ports: ## this environment's port block [show|persist]
|
||||
bash ctrl/ports.sh $(or $(ARGS),show)
|
||||
|
||||
# ── host ───────────────────────────────────────────────────────────────────
|
||||
|
||||
newbox: ## throwaway environment [create|status|shell|destroy]
|
||||
bash ctrl/newbox.sh $(or $(ARGS),status)
|
||||
|
||||
dockerhost: ## share Docker between distros [status|share|unshare]
|
||||
$(if $(filter share unshare,$(ARGS)),sudo ,)bash ctrl/dockerhost.sh $(or $(ARGS),status)
|
||||
|
||||
# ── docs + dev loop ────────────────────────────────────────────────────────
|
||||
|
||||
docs: ## documentation [serve|graphs] (default serve)
|
||||
bash ctrl/docs.sh $(or $(ARGS),serve)
|
||||
|
||||
# --port is only passed when TILT_PORT is actually set. It comes from ctrl/.env,
|
||||
# which does NOT carry it by default — ports are derived at runtime in
|
||||
# lib/config.sh unless `make ports persist` has written them. Without the guard
|
||||
# tilt receives a bare `--port` with no value and fails on the flag rather than
|
||||
# on anything real. `make ports show` prints the derived block.
|
||||
tilt: ## dev loop [up|down] (default up)
|
||||
cd ctrl && tilt $(or $(ARGS),up) $(KCTX) $(if $(filter down,$(ARGS)),,$(if $(TILT_PORT),--port $(TILT_PORT)))
|
||||
|
||||
# ── the shape every other project uses ─────────────────────────────────────
|
||||
# Aliases, not a second implementation: each one calls the same script the
|
||||
# canonical target does.
|
||||
#
|
||||
# The header above argues for `make cluster down` over `make cluster-down`, and
|
||||
# that still holds *within* this file. But rig is one repo among several on the
|
||||
# same machine, and every other one answers to kind-up / tilt-up. Muscle memory
|
||||
# spanning six projects beats internal tidiness in one, so both spellings work.
|
||||
#
|
||||
# `cluster list` and `cluster free` have no hyphenated twin on purpose — they
|
||||
# are rig's own, with nothing to be consistent with.
|
||||
|
||||
kind-up: ## alias for `cluster up`
|
||||
bash ctrl/cluster.sh up
|
||||
|
||||
kind-down: ## alias for `cluster down`
|
||||
bash ctrl/cluster.sh down
|
||||
|
||||
kind-reset: ## alias for `cluster reset`
|
||||
bash ctrl/cluster.sh reset
|
||||
|
||||
# These two match the other projects' spelling, but rig has no Tiltfile — there
|
||||
# is nothing to run yet, and they fail the same way `make tilt` does.
|
||||
tilt-up: ## alias for `tilt up` (rig has no Tiltfile yet)
|
||||
cd ctrl && tilt up $(KCTX) $(if $(TILT_PORT),--port $(TILT_PORT))
|
||||
|
||||
tilt-down: ## alias for `tilt down` (rig has no Tiltfile yet)
|
||||
cd ctrl && tilt down $(KCTX)
|
||||
184
rig/README.md
Normal file
184
rig/README.md
Normal file
@@ -0,0 +1,184 @@
|
||||
# rig
|
||||
|
||||
A runnable local model of a large, regulated estate — legacy and new side by
|
||||
side. Its job is onboarding and exploration, not a production replica: most
|
||||
services are deliberately mocked, because what has to be faithful is the
|
||||
topology, not the workloads.
|
||||
|
||||
## Prerequisite
|
||||
|
||||
**Docker.** Nothing else — no curl, no jq, no python, no apt repositories.
|
||||
|
||||
### Starting from plain Windows
|
||||
|
||||
Everything here is bash and runs *inside* a Linux shell, so on a Windows machine
|
||||
that means WSL. Nothing in rig installs WSL, and nothing will: `wsl --install`
|
||||
enables Windows features and requires a reboot, which is not something a script
|
||||
should do to a machine on your behalf — and there is no tested undo for it.
|
||||
|
||||
From an elevated PowerShell or Command Prompt, once:
|
||||
|
||||
```powershell
|
||||
wsl --install
|
||||
```
|
||||
|
||||
Then reboot and open the Linux shell it installed.
|
||||
|
||||
**If you cloned this on the Windows side, copy it into WSL before carrying on.**
|
||||
WSL can reach the Windows drives at `/mnt/c`, and working from there mostly
|
||||
functions — slowly — but file watching does not: that filesystem raises no
|
||||
inotify events, so anything watching for edits silently stops seeing them.
|
||||
|
||||
```bash
|
||||
cp -r /mnt/c/Users/<you>/rig ~/rig
|
||||
cd ~/rig
|
||||
```
|
||||
|
||||
`make deps` reports it if you are running from `/mnt/...`. Then carry on below.
|
||||
|
||||
If it fails, the usual causes give unhelpful messages:
|
||||
|
||||
| symptom | cause |
|
||||
| --- | --- |
|
||||
| "the virtual machine could not be started" | virtualization disabled in BIOS/UEFI |
|
||||
| the command is not recognised | Windows build too old — needs 2004 or later |
|
||||
| the install starts, then nothing works | a reboot is still pending |
|
||||
|
||||
Running the scripts from **Git Bash, MSYS or Cygwin does not work** — those look
|
||||
close enough to a Linux shell to get started and then fail without `/proc` or a
|
||||
docker socket. `ctrl/deps.sh` detects that and says so rather than letting you
|
||||
find out the slow way.
|
||||
|
||||
## Read the docs first
|
||||
|
||||
```bash
|
||||
make docs # serves on localhost, prints the URL
|
||||
```
|
||||
|
||||
They run before anything is installed, which matters because they are the
|
||||
instructions for everything else. No cluster and no toolchain required.
|
||||
|
||||
## Then
|
||||
|
||||
```bash
|
||||
make check # report host and config problems; changes nothing
|
||||
make deps # install the toolchain (add `core` on a managed machine)
|
||||
make cluster up # build the cluster for the active profile
|
||||
```
|
||||
|
||||
`make cluster up` also starts this environment's local registry and wires it
|
||||
into the node, so an image built locally is pullable by the cluster without
|
||||
going near docker.io:
|
||||
|
||||
```bash
|
||||
make registry status # prints: endpoint localhost:<port>
|
||||
docker build -t localhost:<port>/app:1 .
|
||||
docker push localhost:<port>/app:1
|
||||
kubectl --context kind-$(basename $PWD) run app --image=localhost:<port>/app:1
|
||||
```
|
||||
|
||||
The port block is derived from the directory name, so two copies of rig never
|
||||
collide:
|
||||
|
||||
```bash
|
||||
make ports show # HTTP / HTTPS / TILT / REGISTRY
|
||||
make cluster list # every cluster on this machine, with memory
|
||||
make cluster free # stop the others if memory is tight
|
||||
make cluster down # remove this cluster and its registry
|
||||
```
|
||||
|
||||
**`make tilt` has nothing to run yet.** The target and its `tilt-up` / `tilt-down`
|
||||
aliases exist so rig answers to the same spelling as every other project here,
|
||||
but rig ships no `Tiltfile` — it builds the estate, it is not itself a service
|
||||
with a dev loop. Add a `ctrl/Tiltfile` and the target works; until then it fails
|
||||
on the missing file, not on anything rig did.
|
||||
|
||||
`make help` lists every target.
|
||||
|
||||
On a machine where Docker really is the only thing installed, `make deps` has
|
||||
nothing to download with — see [BOOTSTRAP.md](BOOTSTRAP.md), which runs the
|
||||
toolchain through the installer container and carries on to scaffolding and running
|
||||
a new project.
|
||||
|
||||
## One directory is one environment
|
||||
|
||||
Copy this directory, rename it, run it. Cluster name, kubectl context, image
|
||||
tags and the host port block all derive from the directory name, so copies never
|
||||
collide and neither one's teardown can touch the other.
|
||||
|
||||
A copy of this directory is a **sibling** of it, named after the environment it
|
||||
models (`acme-rig`). That is why the ignore rules for copies sit in the *parent*
|
||||
repo's `.gitignore` rather than here: a rule in this directory cannot see a
|
||||
directory beside it.
|
||||
|
||||
## Profiles
|
||||
|
||||
A profile is the shape of the cluster: how many nodes, which addons, whether the
|
||||
apiserver audits. They live in `ctrl/env.d/`, and the active one is `PROFILE`.
|
||||
|
||||
| Profile | For |
|
||||
| --- | --- |
|
||||
| `minimal` | the default. One node, no addons, boots fast. |
|
||||
| `client` | the regulated-estate shape — multi-node, audit on, registry mirror. |
|
||||
| `offline` | air-gapped: everything from a preloaded local registry. |
|
||||
| `data` | the cabinets an environment asks for. |
|
||||
|
||||
```bash
|
||||
PROFILE=data make cluster up
|
||||
PROFILE=data make addons install
|
||||
make addons # what the active profile wants, and what exists
|
||||
```
|
||||
|
||||
A profile names a **cluster shape** — a file in `ctrl/k8s/` — rather than
|
||||
restating node count and audit as variables:
|
||||
|
||||
| shape | nodes | audit | used by |
|
||||
| --- | --- | --- | --- |
|
||||
| `kind-config.yaml.tpl` | 1 | off | `minimal`, `data` |
|
||||
| `kind-config.audit.yaml.tpl` | 1 | on | `offline` |
|
||||
| `kind-config.client.yaml.tpl` | 3 | on | `client` |
|
||||
|
||||
Both numbers are read back out of the chosen file, so the YAML is the only place
|
||||
that decides and there is nothing to drift. The layout under `ctrl/k8s/` is the
|
||||
same as every other project here — a kind config, a kustomize `base/`, an
|
||||
`overlays/dev/` — see [`ctrl/k8s/README.md`](ctrl/k8s/README.md).
|
||||
|
||||
## Addons
|
||||
|
||||
Each addon is its own idempotent script in `ctrl/addons/`, and a profile names
|
||||
the ones it wants in `ADDONS`. Adding one is adding a file — there is no
|
||||
dispatcher to edit.
|
||||
|
||||
**There is no ingress controller, deliberately.** They pin a narrow window of
|
||||
Kubernetes versions, so depending on one would constrain which k8s a rig can be
|
||||
built with — and running a trailing-edge control plane to model a legacy estate
|
||||
is the whole point. Services are reached through MetalLB and
|
||||
`type: LoadBalancer`, which carries no such constraint and is also what a real
|
||||
cluster does.
|
||||
|
||||
| Addon | Does |
|
||||
| --- | --- |
|
||||
| `metallb` | gives `type: LoadBalancer` an address it can actually reach |
|
||||
| `cert-manager` | a local CA, so TLS works offline |
|
||||
| `metrics-server` | makes `kubectl top` work on kind |
|
||||
| `postgres` | database, in the `data` namespace |
|
||||
| `redis` | cache and broker |
|
||||
| `airflow` | scheduled pipelines; needs postgres and redis |
|
||||
|
||||
The last three are **cabinets**: a public service dropped in as-is, the upstream
|
||||
image unmodified, reachable at a known address. A cabinet is declared once and
|
||||
installs on either target — a `service.yml` composes it for a laptop, and these
|
||||
install the same one here. The names match on purpose: each cabinet carries a
|
||||
`rig_addon` field pointing at `ctrl/addons/<name>.sh`.
|
||||
|
||||
Plain manifests rather than helm charts, like every other addon: a chart repo is
|
||||
a network dependency, and the `offline` profile exists precisely so there is a
|
||||
path with none. Images are pinned in `ctrl/versions.env` and can be preloaded.
|
||||
|
||||
Passwords are generated on first install and kept across re-runs, so re-running
|
||||
an addon never rotates a credential out from under something already connected:
|
||||
|
||||
```bash
|
||||
kubectl -n data get secret postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d
|
||||
kubectl -n data port-forward svc/airflow 8080:8080
|
||||
```
|
||||
50
rig/ctrl/.env.example
Normal file
50
rig/ctrl/.env.example
Normal file
@@ -0,0 +1,50 @@
|
||||
# Machine-local config. Copy to ctrl/.env (gitignored) and edit.
|
||||
# Cluster SHAPE lives in ctrl/env.d/<profile>.env — not here.
|
||||
# The architecture MODEL lives in arch/<name>.json — not here either.
|
||||
|
||||
# Which profile in ctrl/env.d/ to build. minimal | client | offline
|
||||
PROFILE=minimal
|
||||
|
||||
# Cluster name; the kubectl context becomes kind-<CLUSTER>.
|
||||
# LEAVE THIS UNSET unless you need a name that differs from the directory —
|
||||
# it defaults to this folder's name, which is what makes the folder copyable:
|
||||
# copy it, rename it, and you get a separate environment with no edits.
|
||||
# CLUSTER=
|
||||
|
||||
# Host ports. LEAVE UNSET — they derive from the directory name so several
|
||||
# environments coexist without negotiating (see ctrl/ports.sh). `make ports`
|
||||
# shows this environment's block; `make ports persist` writes it here so it stops
|
||||
# being derived and becomes fixed. Set a value only to override.
|
||||
# HTTP_PORT=
|
||||
# HTTPS_PORT=
|
||||
# TILT_PORT=
|
||||
# REGISTRY_PORT=
|
||||
|
||||
# Where the application manifests live. The real ones are expected to be
|
||||
# versioned separately from this installer — they change on a different cadence,
|
||||
# by different people. Repoint this at their repo and rig stops owning them:
|
||||
# MANIFESTS_DIR=../platform-manifests/overlays/dev
|
||||
MANIFESTS_DIR=ctrl/k8s/overlays/dev
|
||||
|
||||
# Where the installer fetches the pinned binaries from.
|
||||
# upstream GitHub releases / dl.k8s.io (needs internet)
|
||||
# artifactory a generic repo — what a locked-down client usually allows
|
||||
# baked already inside the installer image; no network at all
|
||||
DEPS_SOURCE=upstream
|
||||
DEPS_ARTIFACTORY_URL=
|
||||
|
||||
# --- Registry -------------------------------------------------------------
|
||||
# Mode comes from the profile (REGISTRY_MODE). These are the secrets it needs.
|
||||
# Required for mirror/remote:
|
||||
REGISTRY_REMOTE_URL=
|
||||
REGISTRY_USER=
|
||||
REGISTRY_PASSWORD=
|
||||
|
||||
# Corporate root CA, if Artifactory is fronted by an internal CA (it usually is).
|
||||
# Trust has to reach THREE places and nothing does it for you: the host docker
|
||||
# daemon, every kind node's containerd, and any in-cluster client. registry.sh
|
||||
# handles the first two; check.sh reports when it's configured but not trusted.
|
||||
# Symptom when missing: x509: certificate signed by unknown authority
|
||||
REGISTRY_CA_FILE=
|
||||
|
||||
# (The local registry's host port is part of the derived block above.)
|
||||
46
rig/ctrl/Dockerfile.deps
Normal file
46
rig/ctrl/Dockerfile.deps
Normal file
@@ -0,0 +1,46 @@
|
||||
# The toolchain installer image. It does NOT run the cluster — it installs a toolchain
|
||||
# onto the host and gets out of the way.
|
||||
#
|
||||
# This exists to kill a bootstrap paradox: a plain bash installer needs curl, jq
|
||||
# and sha256sum to already be present, and a minimal Debian has none of them.
|
||||
# It carries its own toolchain, so the only host prerequisite is Docker.
|
||||
#
|
||||
# Two variants from one file:
|
||||
# docker build -f ctrl/Dockerfile.deps --target deps -t <slug>-deps .
|
||||
# docker build -f ctrl/Dockerfile.deps --target deps-full -t <slug>-deps:full .
|
||||
#
|
||||
# deps-full bakes every pinned binary in at build time. `docker save` it and
|
||||
# you have the whole installer as one file to carry into an air-gapped network.
|
||||
|
||||
FROM debian:trixie-slim AS deps
|
||||
|
||||
# ca-certificates + curl: fetch and verify. graphviz + python3: render diagrams
|
||||
# and validate the arch model, so the host never needs an apt package.
|
||||
#
|
||||
# docker-cli, NOT docker.io: we only ever talk to the host's daemon through the
|
||||
# mounted socket, and under --no-install-recommends the docker.io package ships
|
||||
# docker-init without the actual `docker` binary.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl jq graphviz python3 docker-cli \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /work
|
||||
COPY ctrl/versions.env /work/ctrl/versions.env
|
||||
COPY ctrl/deps.sh /work/ctrl/deps.sh
|
||||
RUN chmod +x /work/ctrl/deps.sh
|
||||
|
||||
# Defaults; every one is overridable with -e at run time.
|
||||
ENV DEPS_SOURCE=upstream \
|
||||
OUT_BIN=/out/bin \
|
||||
HOST_ROOT=/host
|
||||
|
||||
ENTRYPOINT ["/work/ctrl/deps.sh"]
|
||||
CMD ["install"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# deps-full — same image, binaries baked in, works with no network at all.
|
||||
FROM deps AS deps-full
|
||||
RUN /work/ctrl/deps.sh fetch --to /opt/rig/bin
|
||||
ENV DEPS_SOURCE=baked \
|
||||
BAKED_BIN=/opt/rig/bin
|
||||
39
rig/ctrl/addons.sh
Executable file
39
rig/ctrl/addons.sh
Executable file
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the addons the active profile asked for, in the order listed.
|
||||
# Each addon is its own idempotent script in ctrl/addons/ — adding one is adding
|
||||
# a file, not editing a dispatcher.
|
||||
#
|
||||
# Usage: addons.sh install | list
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
install() {
|
||||
if [ -z "${ADDONS// /}" ]; then
|
||||
echo "no addons in profile '$PROFILE_NAME'"
|
||||
return
|
||||
fi
|
||||
local a
|
||||
for a in $ADDONS; do
|
||||
if [ ! -f "addons/${a}.sh" ]; then
|
||||
echo "no such addon: addons/${a}.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "addon: $a"
|
||||
bash "addons/${a}.sh"
|
||||
done
|
||||
}
|
||||
|
||||
list() {
|
||||
echo "profile '$PROFILE_NAME' wants: ${ADDONS:-none}"
|
||||
echo "available:"
|
||||
ls addons/*.sh 2>/dev/null | xargs -n1 basename | sed 's/\.sh$//' | sed 's/^/ /'
|
||||
}
|
||||
|
||||
case "${1:-list}" in
|
||||
install) install ;;
|
||||
list) list ;;
|
||||
*) echo "usage: $0 [install|list]" >&2; exit 1 ;;
|
||||
esac
|
||||
115
rig/ctrl/addons/airflow.sh
Executable file
115
rig/ctrl/addons/airflow.sh
Executable file
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# Apache Airflow — the cluster half of the airflow cabinet.
|
||||
#
|
||||
# Airflow needs a metadata database before it will start at all, so this refuses
|
||||
# rather than rolls a pod that will CrashLoopBackOff while the real problem
|
||||
# (postgres missing from ADDONS) stays invisible in the logs.
|
||||
#
|
||||
# One pod on `standalone`, matching the compose cabinet: migration, admin user,
|
||||
# scheduler and webserver in a single container. The official chart's five
|
||||
# deployments model an installation; switching this on means wanting pipelines.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
|
||||
echo " ! airflow needs the postgres addon, and it is not installed" >&2
|
||||
echo " add it before airflow in the profile's ADDONS:" >&2
|
||||
echo " ADDONS=\"... postgres airflow\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Reuse the credential postgres generated rather than storing a second copy.
|
||||
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
|
||||
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
|
||||
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
|
||||
|
||||
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
|
||||
echo " secret exists, keeping the current admin password and fernet key"
|
||||
else
|
||||
admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||
# Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
|
||||
# restart invalidates every stored connection.
|
||||
fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
|
||||
$K create secret generic airflow -n "$NS" \
|
||||
--from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
|
||||
--from-literal=ADMIN_PASSWORD="$admin_password" \
|
||||
--from-literal=FERNET_KEY="$fernet_key" \
|
||||
--from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/${db_name}" \
|
||||
>/dev/null
|
||||
echo " generated an admin password (read it back with the command below)"
|
||||
fi
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
selector:
|
||||
app: airflow
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: airflow
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: airflow
|
||||
spec:
|
||||
containers:
|
||||
- name: airflow
|
||||
image: ${AIRFLOW_IMAGE}
|
||||
args: ["standalone"]
|
||||
env:
|
||||
- name: AIRFLOW__CORE__EXECUTOR
|
||||
value: LocalExecutor
|
||||
- name: AIRFLOW__CORE__LOAD_EXAMPLES
|
||||
value: "false"
|
||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: SQL_ALCHEMY_CONN}
|
||||
- name: AIRFLOW__CORE__FERNET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: FERNET_KEY}
|
||||
- name: _AIRFLOW_WWW_USER_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_USER}
|
||||
- name: _AIRFLOW_WWW_USER_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
# First boot runs the whole migration before it serves anything.
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 15
|
||||
failureThreshold: 20
|
||||
YAML
|
||||
|
||||
echo " waiting for airflow (the first boot migrates the database, so this is slow)..."
|
||||
$K rollout status deployment/airflow -n "$NS" --timeout=600s
|
||||
|
||||
echo " in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
|
||||
echo " reach it: kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
|
||||
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"
|
||||
65
rig/ctrl/addons/cert-manager.sh
Executable file
65
rig/ctrl/addons/cert-manager.sh
Executable file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# cert-manager plus a self-signed cluster issuer.
|
||||
#
|
||||
# In a regulated estate almost everything is TLS, so the interesting question
|
||||
# during onboarding is "does this service present a cert my client trusts" — not
|
||||
# "can I reach a public ACME server". A local CA answers that offline, which is
|
||||
# also what makes the air-gapped profile usable.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
|
||||
if $K get deployment -n cert-manager cert-manager >/dev/null 2>&1; then
|
||||
echo " already installed"
|
||||
else
|
||||
$K apply -f "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"
|
||||
fi
|
||||
|
||||
echo " waiting for cert-manager..."
|
||||
$K wait --namespace cert-manager \
|
||||
--for=condition=ready pod --selector=app.kubernetes.io/instance=cert-manager \
|
||||
--timeout=240s
|
||||
|
||||
# A self-signed root, then a CA issuer chained off it. Workloads reference
|
||||
# ClusterIssuer/local-ca and get a cert from a CA you can actually distribute.
|
||||
echo " creating local CA issuer"
|
||||
$K apply -f - <<'YAML' >/dev/null
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
name: selfsigned-root
|
||||
spec:
|
||||
selfSigned: {}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: local-ca
|
||||
namespace: cert-manager
|
||||
spec:
|
||||
isCA: true
|
||||
commonName: rig-local-ca
|
||||
secretName: local-ca-key-pair
|
||||
duration: 87600h
|
||||
privateKey:
|
||||
algorithm: ECDSA
|
||||
size: 256
|
||||
issuerRef:
|
||||
name: selfsigned-root
|
||||
kind: ClusterIssuer
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
name: local-ca
|
||||
spec:
|
||||
ca:
|
||||
secretName: local-ca-key-pair
|
||||
YAML
|
||||
|
||||
echo " export the CA for your browser/client with:"
|
||||
echo " kubectl --context ${KUBECONTEXT} -n cert-manager get secret local-ca-key-pair -o jsonpath='{.data.tls\\.crt}' | base64 -d"
|
||||
103
rig/ctrl/addons/metallb.sh
Executable file
103
rig/ctrl/addons/metallb.sh
Executable file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
# MetalLB — makes `Service type: LoadBalancer` actually get an address.
|
||||
#
|
||||
# Why it matters here: real manifests use LoadBalancer, because a real cluster
|
||||
# has one. On a bare kind cluster those Services sit at EXTERNAL-IP <pending>
|
||||
# forever with no error anywhere — the deployment looks fine and simply is not
|
||||
# reachable. Without this, every such Service has to be edited to NodePort,
|
||||
# which means the local manifests stop matching the ones being modelled.
|
||||
#
|
||||
# The address pool is derived from the kind Docker network at install time, not
|
||||
# hardcoded: Docker picks that subnet, it differs between machines, and a pool
|
||||
# outside it is silently unroutable.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
|
||||
# ── work out an address range ──────────────────────────────────────────────
|
||||
# kind hands node addresses out from the bottom of the subnet, so the top is
|
||||
# free. Taking a slice there avoids collisions with current and future nodes.
|
||||
subnet=$(docker network inspect kind \
|
||||
-f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' 2>/dev/null \
|
||||
| tr ' ' '\n' | grep -E '^[0-9]+\.' | head -1)
|
||||
|
||||
if [ -z "$subnet" ]; then
|
||||
echo " ! could not read the kind Docker network subnet" >&2
|
||||
echo " (is the cluster up? MetalLB needs the network to exist first)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
base="${subnet%/*}"; prefix="${subnet#*/}"
|
||||
o1=$(echo "$base" | cut -d. -f1); o2=$(echo "$base" | cut -d. -f2)
|
||||
o3=$(echo "$base" | cut -d. -f3)
|
||||
|
||||
case "$prefix" in
|
||||
16) pool_start="${o1}.${o2}.255.200"; pool_end="${o1}.${o2}.255.250" ;;
|
||||
24) pool_start="${o1}.${o2}.${o3}.200"; pool_end="${o1}.${o2}.${o3}.250" ;;
|
||||
*)
|
||||
# Guessing a range inside an unexpected prefix risks handing out
|
||||
# addresses that belong to something else. Say so instead.
|
||||
echo " ! kind network is $subnet — only /16 and /24 are handled" >&2
|
||||
echo " set the pool by hand in ctrl/addons/metallb.sh" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo " kind network $subnet → pool ${pool_start}-${pool_end}"
|
||||
|
||||
# ── install ────────────────────────────────────────────────────────────────
|
||||
|
||||
if $K get deployment -n metallb-system controller >/dev/null 2>&1; then
|
||||
echo " already installed"
|
||||
else
|
||||
$K apply -f "https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/config/manifests/metallb-native.yaml"
|
||||
fi
|
||||
|
||||
# `kubectl wait` on a selector errors out immediately when nothing matches yet,
|
||||
# and right after apply the ReplicaSet has not created the pod — so it loses a
|
||||
# race it looks like it should win. `rollout status` waits for the Deployment
|
||||
# itself and handles the not-yet-created case.
|
||||
echo " waiting for the controller..."
|
||||
$K rollout status deployment/controller -n metallb-system --timeout=240s
|
||||
$K rollout status daemonset/speaker -n metallb-system --timeout=240s
|
||||
|
||||
# The webhook rejects IPAddressPools until it is actually serving, and it comes
|
||||
# up a moment after the pod is Ready — so retry rather than fail the whole run
|
||||
# on a race that resolves itself in seconds.
|
||||
echo " configuring the address pool"
|
||||
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
||||
if $K apply -f - >/dev/null 2>&1 <<YAML
|
||||
apiVersion: metallb.io/v1beta1
|
||||
kind: IPAddressPool
|
||||
metadata:
|
||||
name: default
|
||||
namespace: metallb-system
|
||||
spec:
|
||||
addresses:
|
||||
- ${pool_start}-${pool_end}
|
||||
---
|
||||
# Layer 2 mode: one node answers ARP for each address. No BGP peer needed, which
|
||||
# is what makes this work on a laptop.
|
||||
apiVersion: metallb.io/v1beta1
|
||||
kind: L2Advertisement
|
||||
metadata:
|
||||
name: default
|
||||
namespace: metallb-system
|
||||
spec:
|
||||
ipAddressPools:
|
||||
- default
|
||||
YAML
|
||||
then
|
||||
echo " pool ready: ${pool_start}-${pool_end}"
|
||||
exit 0
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo " ! the pool was rejected after 10 attempts — is the webhook up?" >&2
|
||||
$K get pods -n metallb-system >&2
|
||||
exit 1
|
||||
25
rig/ctrl/addons/metrics-server.sh
Executable file
25
rig/ctrl/addons/metrics-server.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# metrics-server — makes `kubectl top` work.
|
||||
#
|
||||
# kind nodes serve kubelet metrics over a self-signed cert, so the standard
|
||||
# manifest never becomes ready without --kubelet-insecure-tls. That is fine here
|
||||
# (it is a local cluster) and is the single most common reason metrics-server
|
||||
# sits at 0/1 on kind.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
|
||||
if ! $K get deployment -n kube-system metrics-server >/dev/null 2>&1; then
|
||||
$K apply -f "https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml"
|
||||
fi
|
||||
|
||||
$K patch deployment metrics-server -n kube-system --type=json \
|
||||
-p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' \
|
||||
>/dev/null 2>&1 || true
|
||||
|
||||
echo " waiting for metrics-server..."
|
||||
$K rollout status deployment/metrics-server -n kube-system --timeout=180s
|
||||
119
rig/ctrl/addons/postgres.sh
Executable file
119
rig/ctrl/addons/postgres.sh
Executable file
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# PostgreSQL — the cluster half of the postgres cabinet.
|
||||
#
|
||||
# A cabinet is a public service dropped into the environment as-is — the
|
||||
# upstream image, unmodified, reachable at a known address. This is the cluster
|
||||
# half of it; the compose half is a `service.yml` beside a `cabinet.json`. The
|
||||
# declaration is made once and both paths read it, so nothing is remembered
|
||||
# twice.
|
||||
#
|
||||
# Plain manifests rather than a helm chart, matching the other addons: a chart
|
||||
# repo is a network dependency, and the offline profile exists precisely so
|
||||
# there is a path with none. The image is pinned in ctrl/versions.env and can be
|
||||
# preloaded into a local registry like every other image here.
|
||||
#
|
||||
# One replica on a PVC. This models a dependency for local work, not a
|
||||
# highly-available database, and pretending otherwise on a kind node would be a
|
||||
# more elaborate lie rather than a more useful one.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
$K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
|
||||
|
||||
# The password is generated once and then left alone, so re-running this does
|
||||
# not rotate the credential out from under whatever is already connected.
|
||||
if $K get secret -n "$NS" postgres >/dev/null 2>&1; then
|
||||
echo " secret exists, keeping the current password"
|
||||
else
|
||||
password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||
$K create secret generic postgres -n "$NS" \
|
||||
--from-literal=POSTGRES_DB="${POSTGRES_DB:-postgres}" \
|
||||
--from-literal=POSTGRES_USER="${POSTGRES_USER:-postgres}" \
|
||||
--from-literal=POSTGRES_PASSWORD="$password" >/dev/null
|
||||
echo " generated a password (read it back with the command printed below)"
|
||||
fi
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: postgres-data
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests:
|
||||
storage: ${POSTGRES_STORAGE:-2Gi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: postgres
|
||||
spec:
|
||||
selector:
|
||||
app: postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
targetPort: 5432
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: postgres
|
||||
spec:
|
||||
replicas: 1
|
||||
# One volume, one writer. Rolling would start a second pod against the same
|
||||
# PVC before the first exits, and Postgres refuses to share a data directory.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: ${POSTGRES_IMAGE}
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: postgres
|
||||
env:
|
||||
# The image initialises into the volume root otherwise, and a
|
||||
# lost+found from the PVC makes it refuse to initdb.
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: postgres-data
|
||||
YAML
|
||||
|
||||
echo " waiting for postgres..."
|
||||
$K rollout status deployment/postgres -n "$NS" --timeout=240s
|
||||
|
||||
echo " in-cluster: postgres.${NS}.svc.cluster.local:5432"
|
||||
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d"
|
||||
60
rig/ctrl/addons/redis.sh
Executable file
60
rig/ctrl/addons/redis.sh
Executable file
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Redis — the cluster half of the redis cabinet.
|
||||
#
|
||||
# Cache, and the broker anything queue-shaped runs on. No persistence: a broker
|
||||
# that loses its queue on restart is the honest local model, and a PVC here buys
|
||||
# nothing but a volume to clean up.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
$K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
selector:
|
||||
app: redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: ${REDIS_IMAGE}
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 5
|
||||
YAML
|
||||
|
||||
echo " waiting for redis..."
|
||||
$K rollout status deployment/redis -n "$NS" --timeout=180s
|
||||
|
||||
echo " in-cluster: redis://redis.${NS}.svc.cluster.local:6379/0"
|
||||
221
rig/ctrl/check.sh
Executable file
221
rig/ctrl/check.sh
Executable file
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env bash
|
||||
# Readiness check: is this machine ready to run rig?
|
||||
#
|
||||
# Reports and instructs; never silently fixes anything. Everything it finds is
|
||||
# either already fine, or something a human has to decide on.
|
||||
#
|
||||
# Runs ctrl/deps.sh host detection in a container when Docker is the only thing
|
||||
# installed, or directly when the toolchain is already present. Then adds the
|
||||
# checks that need this repo's config: profile sanity, CA trust, port clashes.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
DEPS_IMAGE="${DEPS_IMAGE:-$(basename "$(cd .. && pwd)")-deps}"
|
||||
|
||||
# Host detection. Prefer running it bare — it needs no dependencies beyond
|
||||
# coreutils — and fall back to the container only if this shell can't.
|
||||
bash ./deps.sh detect
|
||||
|
||||
# ── repo-level checks ──────────────────────────────────────────────────────
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
echo
|
||||
echo "config"
|
||||
echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})"
|
||||
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
|
||||
echo " registry ${REGISTRY_MODE}"
|
||||
echo " ingress ${INGRESS_MODE}"
|
||||
|
||||
if [ ! -f ./.env ]; then
|
||||
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
||||
fi
|
||||
|
||||
# ── memory ─────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# A profile on a box that is already full is the most common first failure, and
|
||||
# it presents as pods stuck Pending rather than anything that says "memory".
|
||||
# Warns; never blocks. Whether to try anyway is the user's call.
|
||||
|
||||
# A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist
|
||||
# only so the tight and does-not-fit branches can be exercised against another
|
||||
# machine's real numbers; in normal use they are the kernel's own files.
|
||||
mb_of() {
|
||||
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||
}
|
||||
|
||||
# What one node costs, measured rather than guessed. On 2026-09-11 a minimal
|
||||
# control-plane node ran at 620 MiB idle and ~728 MiB with a small mock, plus
|
||||
# 16 MiB for the local registry — ~745 MiB of working set. 800 rounds that up,
|
||||
# and agrees with the 800 MB observed independently on a larger rig. Worker
|
||||
# nodes carry no etcd or apiserver and are lighter, so for a multi-node shape
|
||||
# this errs high. It is the cluster alone: whatever you deploy comes on top.
|
||||
NODE_MB=800
|
||||
|
||||
# Every running container's working set in MB, tagged with the kind cluster it
|
||||
# belongs to ('-' when it is not kind). docker stats reports usage minus page
|
||||
# cache, which is what actually competes — cache is handed back under pressure.
|
||||
# Counting only kind would hide the usual culprit on a managed workspace, where
|
||||
# the memory is held by other containers entirely.
|
||||
container_mb() {
|
||||
docker info >/dev/null 2>&1 || return 0
|
||||
awk -F'\t' '
|
||||
FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next }
|
||||
{
|
||||
grp = ($1 in cl ? cl[$1] : "-")
|
||||
# A kind cluster'"'"'s local registry is a plain container with no kind
|
||||
# label, named <cluster>-registry, so on its own it would read as a
|
||||
# stranger. It belongs to its cluster — but only if that cluster exists:
|
||||
# a registry whose cluster is gone is a genuine stray, and says so.
|
||||
if (grp == "-" && $1 ~ /-registry$/) {
|
||||
base = $1; sub(/-registry$/, "", base)
|
||||
if (base in isc) grp = base
|
||||
}
|
||||
split($2, u, " "); v = u[1]; mb = 0
|
||||
if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 }
|
||||
else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v }
|
||||
else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 }
|
||||
else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 }
|
||||
printf "%d\t%s\t%s\n", mb, grp, $1
|
||||
}
|
||||
' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \
|
||||
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
||||
}
|
||||
|
||||
total_mb=$(mb_of MemTotal)
|
||||
avail_mb=$(mb_of MemAvailable)
|
||||
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
||||
overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||
need_mb=$(( NODES * NODE_MB ))
|
||||
|
||||
rows=$(container_mb)
|
||||
# Once this environment's own cluster is running, its real footprint is already
|
||||
# out of MemAvailable and the per-node estimate stops being relevant. Subtracting
|
||||
# the measurement from the estimate would count the same memory twice, and a
|
||||
# running cluster that happens to sit under 800 MB would still "need" the gap.
|
||||
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
||||
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
||||
|
||||
echo
|
||||
echo "memory"
|
||||
printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \
|
||||
"$need_mb" "$NODES" "$NODE_MB"
|
||||
if [ "$ours_mb" -gt 0 ]; then
|
||||
printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER"
|
||||
fi
|
||||
printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb"
|
||||
|
||||
# The biggest things holding memory right now, other than this cluster: kind
|
||||
# clusters summed per cluster, everything else by container name.
|
||||
others=$(awk -F'\t' -v c="$CLUSTER" '
|
||||
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
||||
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
||||
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
||||
if [ -n "$others" ]; then
|
||||
echo " held elsewhere:"
|
||||
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
||||
n_others=$(wc -l <<< "$others")
|
||||
if [ "$n_others" -gt 6 ]; then
|
||||
echo " ... and $((n_others - 6)) more"
|
||||
fi
|
||||
fi
|
||||
|
||||
headroom=$(( avail_mb - still_mb ))
|
||||
if [ "$still_mb" -eq 0 ]; then
|
||||
if [ "$headroom" -ge 512 ]; then
|
||||
printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom"
|
||||
else
|
||||
printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||
fi
|
||||
elif [ "$headroom" -ge 512 ]; then
|
||||
printf " fits — %d MB headroom for what you deploy\n" "$headroom"
|
||||
elif [ "$headroom" -ge 0 ]; then
|
||||
printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||
else
|
||||
printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
||||
# Two failures with opposite fixes, and telling them apart is the point.
|
||||
if [ "$still_mb" -le "$total_mb" ]; then
|
||||
echo " The machine is big enough; something else is holding memory (above)."
|
||||
echo " Stopping that is what helps — a bigger VM would not."
|
||||
if grep -q 'kind cluster' <<< "$others"; then
|
||||
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
||||
fi
|
||||
else
|
||||
echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$swap_used_mb" -gt 0 ]; then
|
||||
printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb"
|
||||
echo " cluster here to be slow well before it fails"
|
||||
fi
|
||||
if [ "$overcommit" = "1" ]; then
|
||||
echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling."
|
||||
echo " A cluster that starts cleanly can still lose processes to the OOM killer."
|
||||
fi
|
||||
|
||||
# The CA reaches three places and only one of them is ours. Report the other two.
|
||||
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
||||
echo
|
||||
echo "registry CA"
|
||||
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
||||
echo " ! REGISTRY_CA_FILE not readable: $REGISTRY_CA_FILE"
|
||||
else
|
||||
echo " file $REGISTRY_CA_FILE"
|
||||
host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
||||
if [ -n "$host" ] && [ ! -f "/etc/docker/certs.d/${host}/ca.crt" ]; then
|
||||
echo " ! the HOST docker daemon does not trust it yet:"
|
||||
echo " sudo mkdir -p /etc/docker/certs.d/${host}"
|
||||
echo " sudo cp ${REGISTRY_CA_FILE} /etc/docker/certs.d/${host}/ca.crt"
|
||||
echo " (kind nodes are handled by registry.sh; in-cluster clients are the workload's job)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Host ports this environment will try to bind. Checked before cluster creation
|
||||
# because docker reports a clash halfway through, as an opaque
|
||||
# "failed to bind host port ...: address already in use".
|
||||
echo
|
||||
echo "ports (block derived from the directory name — see 'make ports')"
|
||||
|
||||
port_busy() {
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn "sport = :$1" 2>/dev/null | grep -q LISTEN && return 0 || return 1
|
||||
fi
|
||||
# iproute2 is absent from a minimal Debian, so fall back to procfs rather
|
||||
# than silently reporting everything as free.
|
||||
local hex; hex=$(printf ':%04X' "$1")
|
||||
grep -qi "^ *[0-9]*: [0-9A-F]*$hex " /proc/net/tcp /proc/net/tcp6 2>/dev/null
|
||||
}
|
||||
|
||||
# A port held by THIS environment's own cluster is not a clash — it is the thing
|
||||
# working. Reporting it as a problem every time the cluster is up would train
|
||||
# people to ignore this section, which is the opposite of the point.
|
||||
# Extract with a second grep rather than `tr -d ':->'`: in tr, ':->' is the
|
||||
# character RANGE ':' to '>', which does not contain '-', so the trailing dash
|
||||
# survives and nothing ever matches.
|
||||
ours=$(docker ps --filter "label=io.x-k8s.kind.cluster=${CLUSTER}" \
|
||||
--format '{{.Ports}}' 2>/dev/null | tr ',' '\n' \
|
||||
| grep -oE ':[0-9]+->' | grep -oE '[0-9]+' || true)
|
||||
|
||||
clash=0
|
||||
for entry in "HTTP:${HTTP_PORT}" "HTTPS:${HTTPS_PORT}" \
|
||||
"TILT:${TILT_PORT}" "REGISTRY:${REGISTRY_PORT}"; do
|
||||
name="${entry%%:*}"; p="${entry#*:}"
|
||||
[ -n "$p" ] || continue
|
||||
if ! port_busy "$p"; then
|
||||
printf " %-9s %-6s free\n" "$name" "$p"
|
||||
elif echo "$ours" | grep -qx "$p"; then
|
||||
printf " %-9s %-6s in use by this environment's cluster\n" "$name" "$p"
|
||||
else
|
||||
printf " ! %-9s %-6s IN USE by something else\n" "$name" "$p"
|
||||
clash=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$clash" -eq 1 ]; then
|
||||
echo " override the clashing one in ctrl/.env, e.g. HTTP_PORT=21080"
|
||||
echo " (or rename this directory — the whole block follows the name)"
|
||||
fi
|
||||
149
rig/ctrl/cluster.sh
Executable file
149
rig/ctrl/cluster.sh
Executable file
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env bash
|
||||
# Cluster lifecycle, plus what else is running on this machine.
|
||||
#
|
||||
# `list` and `free` live here rather than in a separate script because a
|
||||
# near-identical second name (cluster / clusters) is a trap — you reach for one
|
||||
# and get the other. One target, one file, unambiguous subcommands.
|
||||
#
|
||||
# "Idempotent" here means CONVERGENT, not "exits early if the cluster exists".
|
||||
# That distinction matters: an interrupted first run can leave a cluster created
|
||||
# but not finished, and returning early on the re-run would strand it there.
|
||||
# The create step is conditional; every step after it always runs, and each one
|
||||
# is individually idempotent.
|
||||
#
|
||||
# Usage: cluster.sh up | down | reset | list | free
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
up() {
|
||||
if kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then
|
||||
echo "cluster '$CLUSTER' exists — converging"
|
||||
else
|
||||
# Say what this profile locks in BEFORE spending minutes building it:
|
||||
# the audit policy is an apiserver flag and cannot be changed later.
|
||||
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
|
||||
echo " shape ${KIND_CONFIG_SHOWN}"
|
||||
echo " nodes $NODES"
|
||||
echo " image $NODE_IMAGE"
|
||||
echo " audit $AUDIT"
|
||||
echo " ingress $INGRESS_MODE"
|
||||
echo " registry $REGISTRY_MODE"
|
||||
echo " (audit is fixed at creation — 'make cluster reset' to change it)"
|
||||
echo
|
||||
|
||||
render_kind_config | kind create cluster --config -
|
||||
fi
|
||||
|
||||
# The cluster can exist while its context does not — a reset or a switched
|
||||
# KUBECONFIG loses it, and then nothing works despite a healthy cluster.
|
||||
if ! kubectl config get-contexts -o name 2>/dev/null | grep -qx "$KUBECONTEXT"; then
|
||||
echo "context '$KUBECONTEXT' missing from kubeconfig — re-exporting"
|
||||
kind export kubeconfig --name "$CLUSTER"
|
||||
fi
|
||||
kubectl config use-context "$KUBECONTEXT" >/dev/null
|
||||
|
||||
bash registry.sh up
|
||||
|
||||
if [ -n "${ADDONS// /}" ]; then
|
||||
bash addons.sh install
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "cluster '$CLUSTER' ready (context $KUBECONTEXT)"
|
||||
}
|
||||
|
||||
down() {
|
||||
# The registry is a standalone container outside the cluster; take it down
|
||||
# first so a reset doesn't leave it orphaned and holding a port.
|
||||
bash registry.sh down || true
|
||||
|
||||
if kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then
|
||||
echo "deleting cluster '$CLUSTER'..."
|
||||
kind delete cluster --name "$CLUSTER"
|
||||
else
|
||||
echo "no cluster '$CLUSTER' to delete"
|
||||
fi
|
||||
}
|
||||
|
||||
# The escape hatch for a wedged cluster, and the only way to change a
|
||||
# creation-time setting such as the audit policy.
|
||||
reset() {
|
||||
down
|
||||
echo
|
||||
up
|
||||
}
|
||||
|
||||
# ── the whole machine ──────────────────────────────────────────────────────
|
||||
# Every cluster is a running container tree whether or not you are using it, and
|
||||
# an idle one is the usual reason a new one will not fit.
|
||||
|
||||
list() {
|
||||
local total avail
|
||||
total=$(awk '/^MemTotal:/{printf "%.1f", $2/1024/1024}' /proc/meminfo)
|
||||
avail=$(awk '/^MemAvailable:/{printf "%.1f", $2/1024/1024}' /proc/meminfo)
|
||||
echo "memory: ${avail} GB available of ${total} GB"
|
||||
echo
|
||||
|
||||
local names; names=$(kind get clusters 2>/dev/null || true)
|
||||
if [ -z "$names" ]; then
|
||||
echo "no clusters"
|
||||
return
|
||||
fi
|
||||
|
||||
printf "%-16s %-10s %8s %6s %-13s %s\n" CLUSTER STATE MEM NODES PORTS ""
|
||||
local c nodes state mem base
|
||||
for c in $names; do
|
||||
nodes=$(docker ps -a --filter "label=io.x-k8s.kind.cluster=$c" --format '{{.Names}}' | wc -l)
|
||||
state=$(docker inspect -f '{{.State.Status}}' "${c}-control-plane" 2>/dev/null || echo unknown)
|
||||
if [ "$state" = "running" ]; then
|
||||
mem=$(docker stats --no-stream --format '{{.MemUsage}}' \
|
||||
$(docker ps --filter "label=io.x-k8s.kind.cluster=$c" -q) 2>/dev/null \
|
||||
| awk '{gsub(/GiB/,"");gsub(/MiB/,"e-3");s+=$1} END {printf "%.1fG", s}')
|
||||
else
|
||||
mem="-"
|
||||
fi
|
||||
# A cluster's name is its directory slug, so its port block is derivable
|
||||
# here without reading that directory's config.
|
||||
base=$(derive_port_base "$c")
|
||||
printf "%-16s %-10s %8s %6s %-13s %s\n" "$c" "$state" "$mem" "$nodes" \
|
||||
"${base}-$((base + 3))" \
|
||||
"$([ "$c" = "$CLUSTER" ] && echo "<- this one")"
|
||||
done
|
||||
}
|
||||
|
||||
# Stop the OTHER clusters to free memory. Stops, never deletes — a stopped
|
||||
# cluster restarts with `docker start`, so nothing is lost.
|
||||
free() {
|
||||
local targets=("$@")
|
||||
if [ ${#targets[@]} -eq 0 ]; then
|
||||
mapfile -t targets < <(kind get clusters 2>/dev/null | grep -vx "$CLUSTER" || true)
|
||||
fi
|
||||
if [ ${#targets[@]} -eq 0 ]; then
|
||||
echo "nothing to stop"
|
||||
return
|
||||
fi
|
||||
|
||||
local c ids
|
||||
for c in "${targets[@]}"; do
|
||||
ids=$(docker ps --filter "label=io.x-k8s.kind.cluster=$c" -q)
|
||||
if [ -z "$ids" ]; then
|
||||
echo "cluster '$c' is not running"
|
||||
continue
|
||||
fi
|
||||
echo "stopping '$c' (restart with: docker start \$(docker ps -aq -f label=io.x-k8s.kind.cluster=$c))"
|
||||
# shellcheck disable=SC2086
|
||||
docker stop $ids >/dev/null
|
||||
done
|
||||
}
|
||||
|
||||
case "${1:-up}" in
|
||||
up) up ;;
|
||||
down) down ;;
|
||||
reset) reset ;;
|
||||
list) list ;;
|
||||
free) shift; free "$@" ;;
|
||||
*) echo "usage: $0 [up|down|reset|list|free]" >&2; exit 1 ;;
|
||||
esac
|
||||
565
rig/ctrl/deps.sh
Executable file
565
rig/ctrl/deps.sh
Executable file
@@ -0,0 +1,565 @@
|
||||
#!/usr/bin/env bash
|
||||
# Toolchain installer: detect the host, install a pinned toolchain onto it, then
|
||||
# report what it could not do.
|
||||
#
|
||||
# It never runs the cluster, never uses sudo or apt, and writes only into
|
||||
# $OUT_BIN (default ~/.local/bin). Everything that would touch the host proper —
|
||||
# systemd, inotify limits, .wslconfig, docker group — is REPORTED for a human to
|
||||
# decide on, never performed. That is what makes it safe to run on a machine that
|
||||
# already has a working setup.
|
||||
#
|
||||
# Usage (normally via `make deps`, or directly):
|
||||
# deps.sh detect # report host facts only, change nothing
|
||||
# deps.sh fetch [core|dev] [--to DIR] # download + verify into DIR
|
||||
# deps.sh install [core|dev] # detect, fetch, install, report
|
||||
#
|
||||
# Tiers: 'core' is kubectl + jq (talk to a cluster); 'dev' adds kind and tilt
|
||||
# Default is dev.
|
||||
#
|
||||
# Runs both inside the installer container and bare on a host. Inside the
|
||||
# container, host files are read through $HOST_ROOT (mount / as :ro); bare, it
|
||||
# falls back to /.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Keep the caller's cwd so a relative --to resolves where the user expects,
|
||||
# not against ctrl/ once we've moved.
|
||||
INVOKED_FROM="$PWD"
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./versions.env
|
||||
|
||||
# Resolve a possibly-relative path against the caller's original directory.
|
||||
abspath() {
|
||||
case "$1" in
|
||||
/*) echo "$1" ;;
|
||||
*) echo "$INVOKED_FROM/$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
|
||||
HOST_ROOT="${HOST_ROOT:-/}"
|
||||
DEPS_SOURCE="${DEPS_SOURCE:-upstream}"
|
||||
DEPS_ARTIFACTORY_URL="${DEPS_ARTIFACTORY_URL:-}"
|
||||
BAKED_BIN="${BAKED_BIN:-/opt/rig/bin}"
|
||||
|
||||
# Collected by detect(), printed by report_manual() at the very end.
|
||||
MANUAL=()
|
||||
|
||||
# Host FILES (/etc/..., /mnt/c/...) must be read through the mount. Kernel-level
|
||||
# facts (kernel version, meminfo, inotify) are shared with the container, so the
|
||||
# container's own view is already the host's.
|
||||
# A /proc/meminfo field in MB, 0 if the field is absent. MEMINFO exists so the
|
||||
# tight and does-not-fit branches can be exercised against a real machine's
|
||||
# numbers from somewhere else; in normal use it is always /proc/meminfo.
|
||||
mb_of() {
|
||||
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||
}
|
||||
|
||||
host_file() {
|
||||
local p="${1#/}"
|
||||
if [ "$HOST_ROOT" != "/" ] && [ -e "$HOST_ROOT/$p" ]; then
|
||||
echo "$HOST_ROOT/$p"
|
||||
else
|
||||
echo "/$p"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── detect ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||
# the tooling. Detectable, so name it instead.
|
||||
require_linux() {
|
||||
case "$(uname -s)" in
|
||||
MINGW*|MSYS*|CYGWIN*)
|
||||
cat >&2 <<'EOF'
|
||||
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||
|
||||
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||
|
||||
wsl --install
|
||||
|
||||
That enables Windows features and needs a reboot, so it is not something this
|
||||
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||
this from there.
|
||||
|
||||
See "Starting from plain Windows" in README.md.
|
||||
EOF
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||
|
||||
detect() {
|
||||
echo "host"
|
||||
echo " kernel $(uname -r)"
|
||||
|
||||
local osr; osr=$(host_file /etc/os-release)
|
||||
[ -r "$osr" ] && echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' "$osr")"
|
||||
|
||||
# In MB. Whole gigabytes lose nearly half a GB on exactly the machines where
|
||||
# it matters: 1874 MB available used to print as "1 GB". Facts only — whether
|
||||
# that is enough depends on the profile, which check.sh knows and this does not.
|
||||
local total_mb avail_mb swap_total_mb swap_used_mb om
|
||||
total_mb=$(mb_of MemTotal)
|
||||
avail_mb=$(mb_of MemAvailable)
|
||||
swap_total_mb=$(mb_of SwapTotal)
|
||||
swap_used_mb=$(( swap_total_mb - $(mb_of SwapFree) ))
|
||||
printf " memory %d MB total, %d MB available\n" "$total_mb" "$avail_mb"
|
||||
if [ "$swap_total_mb" -gt 0 ]; then
|
||||
printf " swap %d MB used of %d MB\n" "$swap_used_mb" "$swap_total_mb"
|
||||
fi
|
||||
|
||||
# How the kernel answers an allocation it cannot really satisfy. With 1 it
|
||||
# always says yes and settles up later with the OOM killer, so a cluster that
|
||||
# starts cleanly can still lose processes afterwards.
|
||||
om=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||
case "$om" in
|
||||
0) echo " overcommit 0 heuristic — allocations are granted on a guess" ;;
|
||||
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit" ;;
|
||||
2) echo " overcommit 2 strict — an allocation fails honestly instead of killing later" ;;
|
||||
esac
|
||||
|
||||
detect_wsl
|
||||
detect_filesystem
|
||||
detect_docker
|
||||
detect_inotify
|
||||
detect_toolchain
|
||||
}
|
||||
|
||||
detect_wsl() {
|
||||
if ! is_wsl; then
|
||||
echo " platform native linux"
|
||||
return
|
||||
fi
|
||||
|
||||
echo " platform WSL"
|
||||
|
||||
# systemd is off by default in WSL, and the ingress/DNS paths that use a
|
||||
# host service need it. Enabling it requires a Windows-side restart, which
|
||||
# cannot be issued from inside the distro.
|
||||
local wc; wc=$(host_file /etc/wsl.conf)
|
||||
if [ -r "$wc" ] && grep -qE '^\s*systemd\s*=\s*true' "$wc"; then
|
||||
echo " systemd enabled in wsl.conf"
|
||||
else
|
||||
echo " ! systemd not enabled in /etc/wsl.conf"
|
||||
MANUAL+=("Enable systemd — add to /etc/wsl.conf:
|
||||
[boot]
|
||||
systemd=true
|
||||
then from a WINDOWS terminal (not this shell): wsl --shutdown")
|
||||
fi
|
||||
|
||||
# WSL regenerates /etc/resolv.conf on every boot, which silently reverts any
|
||||
# local DNS setup.
|
||||
if [ -r "$wc" ] && grep -qE '^\s*generateResolvConf\s*=\s*false' "$wc"; then
|
||||
echo " resolv.conf pinned (generateResolvConf=false)"
|
||||
else
|
||||
echo " - resolv.conf is WSL-generated; DNS_MODE=dnsmasq would be reverted on reboot"
|
||||
fi
|
||||
|
||||
local wcfg
|
||||
wcfg=$(ls "$HOST_ROOT"/mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||
if [ -n "$wcfg" ] && grep -qE '^\s*memory\s*=' "$wcfg"; then
|
||||
echo " wslconfig memory set: $(grep -E '^\s*memory\s*=' "$wcfg" | tr -d ' ')"
|
||||
else
|
||||
MANUAL+=("Cap/raise the WSL VM memory — see what is set versus what booted:
|
||||
make mem status
|
||||
It prints the edit to make and the command to apply it.")
|
||||
fi
|
||||
}
|
||||
|
||||
# Not a path check: /mnt is an ordinary mount point and an ext4 disk mounted
|
||||
# there is perfectly fine. What matters is the filesystem. The Windows drives
|
||||
# arrive as 9p (WSL2) or drvfs (WSL1); network and fuse mounts behave the same
|
||||
# way. None of them deliver inotify events, so anything watching files goes
|
||||
# quiet without saying why.
|
||||
watch_hostile_fs() {
|
||||
local dir="$1" fstype
|
||||
fstype=$(findmnt -no FSTYPE --target "$dir" 2>/dev/null || true)
|
||||
[ -n "$fstype" ] || fstype=$(stat -f -c %T "$dir" 2>/dev/null || true)
|
||||
case "$fstype" in
|
||||
9p|v9fs|drvfs|cifs|smb3|nfs|nfs4|fuse.sshfs|fuseblk) echo "$fstype" ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
detect_filesystem() {
|
||||
local root fstype
|
||||
root=$(cd .. && pwd -P)
|
||||
fstype=$(watch_hostile_fs "$root")
|
||||
if [ -n "$fstype" ]; then
|
||||
echo " ! this directory is on $fstype — file watching will not work"
|
||||
MANUAL+=("Move this onto the local disk. Nothing watching files sees changes
|
||||
on a $fstype mount, and everything else is slower:
|
||||
cp -r \"$root\" ~/ && cd ~/$(basename "$root")")
|
||||
else
|
||||
echo " filesystem $root ($(findmnt -no FSTYPE --target "$root" 2>/dev/null || echo local))"
|
||||
fi
|
||||
}
|
||||
|
||||
detect_docker() {
|
||||
# Reachability of the daemon is the real question, and the CLI is only how
|
||||
# we ask it. Note that when this runs inside the installer container, Docker
|
||||
# necessarily exists on the host — otherwise nothing would be executing —
|
||||
# so a missing CLI in here is an installer packaging bug, not a host problem.
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
echo " docker socket present (no cli in this context)"
|
||||
else
|
||||
echo " ! docker not found and no socket at /var/run/docker.sock"
|
||||
MANUAL+=("Install Docker — the one true prerequisite:
|
||||
sudo apt-get install -y docker.io && sudo usermod -aG docker \"\$USER\"
|
||||
then log out and back in.")
|
||||
fi
|
||||
return
|
||||
fi
|
||||
if docker info >/dev/null 2>&1; then
|
||||
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null)"
|
||||
local n
|
||||
n=$(docker ps --filter "label=io.x-k8s.kind.cluster" --format '{{.Names}}' 2>/dev/null | wc -l)
|
||||
# Must be an `if`, not `[ ] && echo`: as the last statement in this
|
||||
# function the latter returns 1 when the count is zero, and `set -e`
|
||||
# then kills the caller. That is the fresh-machine case — no clusters
|
||||
# yet — so the bug only ever shows up where it does most harm.
|
||||
if [ "$n" -gt 0 ]; then
|
||||
echo " - $n kind node container(s) already running; see 'make cluster list'"
|
||||
fi
|
||||
else
|
||||
echo " ! docker cli present but the daemon is unreachable"
|
||||
MANUAL+=("Start Docker, or add yourself to the docker group:
|
||||
sudo usermod -aG docker \"\$USER\" # then log out and back in")
|
||||
fi
|
||||
}
|
||||
|
||||
# kind and Tilt both watch large trees. WSL ships defaults (8192/128) far too low,
|
||||
# and the failure mode is silent: Tilt simply stops noticing file changes.
|
||||
detect_inotify() {
|
||||
local w i
|
||||
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||
echo " inotify watches=$w instances=$i"
|
||||
|
||||
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||
echo " ! inotify limits are low — Tilt will silently stop noticing file changes"
|
||||
MANUAL+=("Raise inotify limits (needs root on the host):
|
||||
echo -e 'fs.inotify.max_user_watches=524288\\nfs.inotify.max_user_instances=512' \\
|
||||
| sudo tee /etc/sysctl.d/99-rig.conf
|
||||
sudo sysctl --system")
|
||||
fi
|
||||
}
|
||||
|
||||
# ── fetch ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# Resolve where a given artifact comes from, honouring DEPS_SOURCE.
|
||||
resolve_url() {
|
||||
local upstream="$1"
|
||||
case "$DEPS_SOURCE" in
|
||||
upstream) echo "$upstream" ;;
|
||||
artifactory)
|
||||
if [ -z "$DEPS_ARTIFACTORY_URL" ]; then
|
||||
echo "DEPS_SOURCE=artifactory but DEPS_ARTIFACTORY_URL is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${DEPS_ARTIFACTORY_URL%/}/$(basename "$upstream")"
|
||||
;;
|
||||
*) echo "unsupported DEPS_SOURCE '$DEPS_SOURCE' for a download" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
verify() {
|
||||
local file="$1" want="$2" name="$3" got
|
||||
got=$(sha256sum "$file" | awk '{print $1}')
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "checksum mismatch for $name" >&2
|
||||
echo " expected $want" >&2
|
||||
echo " got $got" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# fetch_bin <name> <url> <sha256> <dest-dir> — a bare binary
|
||||
fetch_bin() {
|
||||
local name="$1" url="$2" sha="$3" dest="$4"
|
||||
local tmp="$dest/.$name.tmp"
|
||||
echo " fetching $name"
|
||||
curl -fsSL --retry 3 -o "$tmp" "$(resolve_url "$url")"
|
||||
verify "$tmp" "$sha" "$name"
|
||||
mv "$tmp" "$dest/$name"
|
||||
chmod +x "$dest/$name"
|
||||
}
|
||||
|
||||
# fetch_tgz <name> <url> <sha256> <dest-dir> <path-inside-archive> <strip>
|
||||
# Archive layouts differ — tilt's is flat (the binary at the root, strip=0),
|
||||
# others nest it a directory down — so the caller says which.
|
||||
fetch_tgz() {
|
||||
local name="$1" url="$2" sha="$3" dest="$4" inner="$5" strip="$6"
|
||||
local tmp="$dest/.$name.tgz"
|
||||
echo " fetching $name"
|
||||
curl -fsSL --retry 3 -o "$tmp" "$(resolve_url "$url")"
|
||||
verify "$tmp" "$sha" "$name"
|
||||
# --no-same-owner: extracting as root would otherwise restore the uid/gid
|
||||
# baked into the archive (some ship as uid 1001), leaving a binary the host
|
||||
# user does not own.
|
||||
tar -xzf "$tmp" -C "$dest" --strip-components="$strip" --no-same-owner "$inner"
|
||||
rm -f "$tmp"
|
||||
chmod +x "$dest/$name"
|
||||
}
|
||||
|
||||
# The installer runs as root so it can reach the docker socket, which means
|
||||
# everything it writes into a mounted volume lands root-owned and unusable from
|
||||
# the host. Hand it back to whoever owns the mount point (the host user created
|
||||
# that directory before mounting it).
|
||||
fix_ownership() {
|
||||
local dir="$1"
|
||||
[ -d "$dir" ] || return 0
|
||||
local owner="${HOST_UID:-}:${HOST_GID:-}"
|
||||
if [ "$owner" = ":" ]; then
|
||||
owner=$(stat -c '%u:%g' "$dir")
|
||||
fi
|
||||
[ "$owner" = "0:0" ] && return 0
|
||||
chown -R "$owner" "$dir" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Two tiers, because not every machine should get cluster tooling.
|
||||
#
|
||||
# core kubectl, jq — talk to a cluster someone else runs. Nothing that
|
||||
# creates one. Appropriate on a managed or corporate-issued machine
|
||||
# where development tools are not wanted by default.
|
||||
# dev core plus kind and tilt — build clusters and hot-reload into them.
|
||||
#
|
||||
# The split exists because "install the toolchain" is not one decision: on a
|
||||
# managed workspace the right answer is kubectl and nothing else.
|
||||
CORE_TOOLS="kubectl jq"
|
||||
# No helm: every addon installs with `kubectl apply -f <url>`, so nothing here
|
||||
# has ever invoked it. Add it back the day something actually needs a chart.
|
||||
#
|
||||
# ctlptl is 'dev' rather than 'core' for the same reason kind is: core is "talk
|
||||
# to a cluster someone else runs", and ctlptl builds them. It earns its place
|
||||
# because it is what wires a cluster to a local registry — without one, an
|
||||
# unqualified image name resolves to docker.io/library/<name> and there is
|
||||
# nothing structural stopping a push there.
|
||||
DEV_TOOLS="kind tilt ctlptl"
|
||||
|
||||
# ── what is already on this machine ───────────────────────────────────────
|
||||
#
|
||||
# A tool already on PATH at its pinned version is left where it is. Without
|
||||
# this, install downloads a second copy into OUT_BIN and then reports the first
|
||||
# one as shadowed — noise, and wrong, when both are the same version. That is
|
||||
# the normal state of any machine someone set up by hand: the AWS Workspace
|
||||
# keeps its toolchain in ~/wdir/bin, all five at exactly these pins.
|
||||
|
||||
pin_of() {
|
||||
case "$1" in
|
||||
kubectl) echo "$KUBECTL_VERSION" ;;
|
||||
jq) echo "$JQ_VERSION" ;;
|
||||
kind) echo "$KIND_VERSION" ;;
|
||||
tilt) echo "$TILT_VERSION" ;;
|
||||
ctlptl) echo "$CTLPTL_VERSION" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# The version string a binary reports. Each tool spells the question
|
||||
# differently, and kubectl has to be told --client or it goes looking for a
|
||||
# server to ask.
|
||||
reported_version() {
|
||||
local tool="$1" path="$2"
|
||||
case "$tool" in
|
||||
kubectl) "$path" version --client 2>/dev/null ;;
|
||||
jq) "$path" --version 2>/dev/null ;;
|
||||
*) "$path" version 2>/dev/null ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Does the binary at PATH report PIN? Matched as a whole version token, so
|
||||
# 0.37.6 never matches 10.37.60, with the leading v optional either side: kind
|
||||
# says v0.32.0, jq says jq-1.8.2, and tilt says v0.37.6 against a pin of 0.37.6.
|
||||
#
|
||||
# Bash's own regex rather than grep, deliberately. grep is not the same program
|
||||
# on every machine — some builds reject patterns that others accept — and a
|
||||
# failed grep inside a count reads exactly like a zero.
|
||||
version_matches() {
|
||||
local tool="$1" path="$2" pin="$3" out v re
|
||||
out=$(reported_version "$tool" "$path") || return 1
|
||||
v="${pin#v}"
|
||||
v="${v//./\\.}"
|
||||
re="(^|[^0-9.])v?${v}([^0-9.]|\$)"
|
||||
[[ $out =~ $re ]]
|
||||
}
|
||||
|
||||
# DEPS_ONLY narrows a fetch to the tools it names. Unset means the whole tier,
|
||||
# which is what an explicit `deps.sh fetch` always gets: "download these into
|
||||
# DIR" must not quietly skip something because this machine happens to have it.
|
||||
# Only install() sets it, to what detect_toolchain found missing or mismatched.
|
||||
want() { [ -z "${DEPS_ONLY:-}" ] || [[ " $DEPS_ONLY " == *" $1 "* ]]; }
|
||||
|
||||
# Every tool in the tier with its state, probed once and reported once. What
|
||||
# still needs fetching is left in TOOLCHAIN_NEED for install() to act on.
|
||||
TOOLCHAIN_NEED=""
|
||||
detect_toolchain() {
|
||||
local tier="${TIER:-dev}" b pin path found
|
||||
TOOLCHAIN_NEED=""
|
||||
echo
|
||||
echo "toolchain (pinned, tier '$tier')"
|
||||
for b in $(tier_tools "$tier"); do
|
||||
pin=$(pin_of "$b")
|
||||
path=$(command -v "$b" 2>/dev/null || true)
|
||||
if [ -z "$path" ]; then
|
||||
printf " - %-8s %-9s not found\n" "$b" "$pin"
|
||||
TOOLCHAIN_NEED+="$b "
|
||||
elif version_matches "$b" "$path" "$pin"; then
|
||||
printf " %-8s %-9s %s\n" "$b" "$pin" "$path"
|
||||
else
|
||||
found=$(reported_version "$b" "$path" 2>/dev/null | head -1 || true)
|
||||
printf " ! %-8s wants %s, %s reports '%s'\n" "$b" "$pin" "$path" "$found"
|
||||
TOOLCHAIN_NEED+="$b "
|
||||
fi
|
||||
done
|
||||
if [ -z "$TOOLCHAIN_NEED" ]; then
|
||||
echo " every pinned tool is already on PATH — nothing to fetch"
|
||||
else
|
||||
echo " 'make deps' fetches only: ${TOOLCHAIN_NEED% }"
|
||||
fi
|
||||
}
|
||||
|
||||
fetch() {
|
||||
local dest="$OUT_BIN" tier="${TIER:-dev}"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--to) dest="$2"; shift 2 ;;
|
||||
core|dev) tier="$1"; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
dest="$(abspath "$dest")"
|
||||
mkdir -p "$dest"
|
||||
TIER="$tier"
|
||||
|
||||
if [ "$DEPS_SOURCE" = "baked" ]; then
|
||||
echo "installing baked binaries from $BAKED_BIN"
|
||||
cp -a "$BAKED_BIN"/. "$dest"/
|
||||
fix_ownership "$dest"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "${DEPS_ONLY:-}" ]; then
|
||||
echo "fetching ${DEPS_ONLY% } (source: $DEPS_SOURCE)"
|
||||
else
|
||||
echo "fetching '$tier' toolchain (source: $DEPS_SOURCE)"
|
||||
fi
|
||||
if want kubectl; then fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"; fi
|
||||
if want jq; then fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"; fi
|
||||
if [ "$tier" = "dev" ]; then
|
||||
if want kind; then fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"; fi
|
||||
if want tilt; then fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0; fi
|
||||
if want ctlptl; then fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0; fi
|
||||
fi
|
||||
|
||||
fix_ownership "$dest"
|
||||
# kind writes the kubeconfig as root too; hand that back as well when it's
|
||||
# a mounted host directory rather than container-local state.
|
||||
fix_ownership "${KUBE_DIR:-/out/kube}"
|
||||
}
|
||||
|
||||
# ── install ────────────────────────────────────────────────────────────────
|
||||
|
||||
report_manual() {
|
||||
echo
|
||||
if [ ${#MANUAL[@]} -eq 0 ]; then
|
||||
echo "nothing left to do by hand."
|
||||
return
|
||||
fi
|
||||
echo "host actions this cannot perform (${#MANUAL[@]}):"
|
||||
echo
|
||||
local n=1
|
||||
for m in "${MANUAL[@]}"; do
|
||||
echo " $n. $m"
|
||||
echo
|
||||
n=$((n + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# Installing into a directory that sits early in PATH silently replaces whatever
|
||||
# the machine was already using — which on a shared or client machine can break
|
||||
# unrelated work (kubectl more than one minor away from a cluster is the common
|
||||
# one). Say so; never decide it for them.
|
||||
tier_tools() { [ "$1" = "core" ] && echo "$CORE_TOOLS" || echo "$CORE_TOOLS $DEV_TOOLS"; }
|
||||
|
||||
warn_shadowing() {
|
||||
local b existing shadowed="" tier="${1:-dev}"
|
||||
for b in $(tier_tools "$tier"); do
|
||||
[ -x "$OUT_BIN/$b" ] || continue
|
||||
# Where would this resolve if OUT_BIN weren't in the way?
|
||||
existing=$(PATH=$(echo "$PATH" | tr ':' '\n' | grep -vx "$OUT_BIN" | paste -sd:) \
|
||||
command -v "$b" 2>/dev/null || true)
|
||||
[ -n "$existing" ] || continue
|
||||
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||
# The same version in both places is not a conflict: nothing changes for
|
||||
# any other project whichever copy PATH happens to find first.
|
||||
if version_matches "$b" "$existing" "$(pin_of "$b")"; then continue; fi
|
||||
shadowed+=" $b $existing"$'\n'
|
||||
done
|
||||
|
||||
[ -n "$shadowed" ] || return 0
|
||||
|
||||
case ":${PATH}:" in
|
||||
*":$OUT_BIN:"*) ;;
|
||||
*) return 0 ;; # not on PATH yet, so nothing is being shadowed
|
||||
esac
|
||||
|
||||
echo
|
||||
echo " ! these were already installed elsewhere and are now shadowed by $OUT_BIN:"
|
||||
printf '%s' "$shadowed"
|
||||
echo " Other projects on this machine will pick up the new versions."
|
||||
MANUAL+=("Decide which toolchain wins. To keep the previous one, remove what
|
||||
was just installed:
|
||||
rm -f $(for b in $(tier_tools "$tier"); do printf '%s ' "$OUT_BIN/$b"; done)
|
||||
Or install somewhere private instead:
|
||||
OUT_BIN=\$PWD/def/bin make deps # then put that dir first in PATH")
|
||||
}
|
||||
|
||||
install() {
|
||||
local tier="${1:-dev}" b
|
||||
TIER="$tier"
|
||||
detect
|
||||
|
||||
# detect_toolchain has already probed PATH. Fetch only what it found missing
|
||||
# or at the wrong version; a tool already present at its pin stays where it is.
|
||||
if [ -n "$TOOLCHAIN_NEED" ]; then
|
||||
echo
|
||||
DEPS_ONLY="$TOOLCHAIN_NEED" fetch "$tier"
|
||||
echo
|
||||
echo "installed to $OUT_BIN ($tier):"
|
||||
for b in $TOOLCHAIN_NEED; do
|
||||
if [ -x "$OUT_BIN/$b" ]; then echo " $b"; fi
|
||||
done
|
||||
if [ "$tier" = "core" ]; then
|
||||
echo " (no kind/tilt — 'make deps dev' adds them)"
|
||||
fi
|
||||
|
||||
# Only worth saying when something actually landed in OUT_BIN. When every
|
||||
# tool was satisfied elsewhere, OUT_BIN may reasonably be off PATH, and
|
||||
# telling the user to add it would be advice to fix nothing.
|
||||
case ":${PATH}:" in
|
||||
*":$OUT_BIN:"*) ;;
|
||||
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||
export PATH=\"${OUT_BIN}:\$PATH\"") ;;
|
||||
esac
|
||||
fi
|
||||
warn_shadowing "$tier"
|
||||
|
||||
report_manual
|
||||
}
|
||||
|
||||
# ── main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
require_linux
|
||||
|
||||
case "${1:-install}" in
|
||||
detect) detect; report_manual ;;
|
||||
fetch) shift; fetch "$@" ;;
|
||||
install) shift; install "${1:-dev}" ;;
|
||||
*) echo "usage: $0 [detect|fetch|install]" >&2; exit 1 ;;
|
||||
esac
|
||||
272
rig/ctrl/dockerhost.sh
Executable file
272
rig/ctrl/dockerhost.sh
Executable file
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env bash
|
||||
# Share ONE Docker daemon across WSL distros, instead of running one per distro.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# WSL2 distros share a kernel and a network stack. Two dockerd instances then
|
||||
# contend over docker0 and iptables, which can disturb the daemon you actually
|
||||
# depend on. Docker Desktop avoids this by running a single daemon in a
|
||||
# dedicated distro and sharing its socket — this is the same idea, without
|
||||
# Docker Desktop.
|
||||
#
|
||||
# So a throwaway rig box does NOT install Docker. It borrows the daemon from
|
||||
# whichever distro is the designated host. That also makes the test more honest:
|
||||
# rig never installs Docker anyway — Docker is its documented prerequisite.
|
||||
#
|
||||
# How
|
||||
# ---
|
||||
# /mnt/wsl is a tmpfs with `shared` mount propagation, visible to every distro
|
||||
# in the WSL VM. The owning distro exposes its socket there; guests point
|
||||
# DOCKER_HOST at it. Two ways, with different costs:
|
||||
#
|
||||
# share bind-mount the existing socket onto the shared tmpfs.
|
||||
# Instant, and dockerd is NEVER restarted. Lasts until the
|
||||
# next WSL shutdown.
|
||||
# share --persist additionally install a systemd drop-in so dockerd listens
|
||||
# there itself. Survives restarts, but requires one Docker
|
||||
# restart now — which stops every container that has no
|
||||
# restart policy, since live-restore is off by default.
|
||||
#
|
||||
# The bind mount is the default precisely because the persistent version's cost
|
||||
# is paid on a machine that is already working.
|
||||
#
|
||||
# Reversibility is the whole design
|
||||
# ---------------------------------
|
||||
# `unshare` removes the bind mount (no restart) and, if present, the drop-in.
|
||||
# The original systemd unit is never edited — only an additive drop-in file is
|
||||
# ever created — so undoing is deletion, not repair. `status` always states
|
||||
# which of the three roles a distro is in, in those words.
|
||||
#
|
||||
# Nothing here runs automatically. It does nothing until invoked.
|
||||
#
|
||||
# Usage:
|
||||
# dockerhost.sh status # which distro owns Docker; what this one uses
|
||||
# dockerhost.sh share # share it (bind mount, no daemon restart)
|
||||
# dockerhost.sh share --persist # ...and survive WSL restarts (restarts Docker)
|
||||
# dockerhost.sh unshare # undo it; this distro owns its Docker again
|
||||
# dockerhost.sh use [--persist] # point THIS distro at the shared socket
|
||||
set -euo pipefail
|
||||
|
||||
SHARED_DIR=/mnt/wsl/shared-docker
|
||||
SHARED_SOCK="$SHARED_DIR/docker.sock"
|
||||
OWNER_FILE="$SHARED_DIR/OWNER"
|
||||
DROPIN=/etc/systemd/system/docker.service.d/10-rig-shared-socket.conf
|
||||
PROFILE_D=/etc/profile.d/rig-docker-host.sh
|
||||
|
||||
distro_name() { echo "${WSL_DISTRO_NAME:-$(hostname)}"; }
|
||||
|
||||
require_wsl() {
|
||||
grep -qi microsoft /proc/version 2>/dev/null && return 0
|
||||
echo "dockerhost is WSL-only: it relies on /mnt/wsl being shared between distros." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── status ─────────────────────────────────────────────────────────────────
|
||||
|
||||
status() {
|
||||
require_wsl
|
||||
echo "distro $(distro_name)"
|
||||
|
||||
if [ -f "$DROPIN" ] || mountpoint -q "$SHARED_SOCK" 2>/dev/null; then
|
||||
echo "role SHARING — this distro's Docker is offered to other distros"
|
||||
elif [ -n "${DOCKER_HOST:-}" ] && [ "${DOCKER_HOST}" = "unix://$SHARED_SOCK" ]; then
|
||||
echo "role BORROWING — using another distro's Docker"
|
||||
else
|
||||
echo "role standalone — this WSL installation has the main host Docker"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ -S "$SHARED_SOCK" ]; then
|
||||
echo "shared sock $SHARED_SOCK (present)"
|
||||
[ -f "$OWNER_FILE" ] && sed 's/^/ /' "$OWNER_FILE"
|
||||
else
|
||||
echo "shared sock none — no distro is sharing right now"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "DOCKER_HOST ${DOCKER_HOST:-(unset — using /var/run/docker.sock)}"
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
echo "docker $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unreachable)"
|
||||
else
|
||||
echo "docker cli not installed"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── share / unshare (run on the host distro) ───────────────────────────────
|
||||
|
||||
# Default: expose the EXISTING socket by bind-mounting it onto the shared tmpfs.
|
||||
# /mnt/wsl has `shared` propagation, so the mount is visible in other distros.
|
||||
#
|
||||
# The point of doing it this way is that dockerd is never restarted. Restarting
|
||||
# it stops every container that has no restart policy (live-restore is off by
|
||||
# default), which on a working machine means quietly killing whatever you had
|
||||
# running. Not a trade worth making just to expose a socket.
|
||||
#
|
||||
# Cost: a bind mount does not survive a WSL VM shutdown. `--persist` adds the
|
||||
# systemd drop-in as well, which does survive but needs that one restart.
|
||||
share_bind() {
|
||||
mkdir -p "$SHARED_DIR"
|
||||
chmod 0755 "$SHARED_DIR"
|
||||
|
||||
if mountpoint -q "$SHARED_SOCK" 2>/dev/null; then
|
||||
echo "already bind-mounted at $SHARED_SOCK"
|
||||
else
|
||||
[ -S /var/run/docker.sock ] || { echo "no /var/run/docker.sock here" >&2; exit 1; }
|
||||
# The target must exist as a file for a bind mount onto it.
|
||||
[ -e "$SHARED_SOCK" ] || : > "$SHARED_SOCK"
|
||||
mount --bind /var/run/docker.sock "$SHARED_SOCK"
|
||||
echo "bind-mounted /var/run/docker.sock -> $SHARED_SOCK (no daemon restart)"
|
||||
fi
|
||||
|
||||
cat > "$OWNER_FILE" <<EOF
|
||||
owner distro: $(distro_name)
|
||||
docker gid: $(getent group docker | cut -d: -f3)
|
||||
socket: $SHARED_SOCK
|
||||
method: bind-mount (until the next WSL shutdown)
|
||||
EOF
|
||||
}
|
||||
|
||||
share() {
|
||||
require_wsl
|
||||
[ "$(id -u)" -eq 0 ] || { echo "run with sudo: sudo bash ctrl/dockerhost.sh share" >&2; exit 1; }
|
||||
|
||||
share_bind
|
||||
|
||||
if [ "${1:-}" != "--persist" ]; then
|
||||
echo
|
||||
echo "This lasts until the next WSL shutdown. To make it survive, re-run with"
|
||||
echo "--persist — but note that adds a systemd drop-in and RESTARTS Docker,"
|
||||
echo "which stops any container that has no restart policy."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -f "$DROPIN" ]; then
|
||||
echo "drop-in already present — sharing persists across restarts."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "--persist: installing a systemd drop-in and restarting Docker."
|
||||
echo "Containers without a restart policy will stop and will NOT come back."
|
||||
docker ps --format ' {{.Names}} restart={{.HostConfig.RestartPolicy.Name}}' 2>/dev/null \
|
||||
|| docker ps --format ' {{.Names}}' 2>/dev/null || true
|
||||
echo
|
||||
|
||||
local exec_line
|
||||
exec_line=$(systemctl cat docker.service | grep -m1 '^ExecStart=')
|
||||
if [ -z "$exec_line" ]; then
|
||||
echo "could not read docker.service ExecStart — refusing to guess" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$DROPIN")" "$SHARED_DIR"
|
||||
# Additive only: blank the inherited ExecStart, then restate it verbatim
|
||||
# with one extra -H. Nothing about the original unit is edited.
|
||||
cat > "$DROPIN" <<EOF
|
||||
# Added by rig (ctrl/dockerhost.sh share).
|
||||
#
|
||||
# Adds a SECOND listening socket on the WSL-shared tmpfs so other distros can
|
||||
# use this daemon instead of running their own. The original socket is
|
||||
# untouched, so this distro behaves exactly as before.
|
||||
#
|
||||
# To undo: sudo bash ctrl/dockerhost.sh unshare
|
||||
[Service]
|
||||
ExecStartPre=-/bin/mkdir -p $SHARED_DIR
|
||||
ExecStartPre=-/bin/chmod 0755 $SHARED_DIR
|
||||
ExecStart=
|
||||
${exec_line} -H unix://$SHARED_SOCK
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl restart docker
|
||||
|
||||
# Guests need a group with a MATCHING GID to use the socket; GIDs are not
|
||||
# consistent across distros, so record ours rather than assume.
|
||||
cat > "$OWNER_FILE" <<EOF
|
||||
owner distro: $(distro_name)
|
||||
docker gid: $(getent group docker | cut -d: -f3)
|
||||
socket: $SHARED_SOCK
|
||||
EOF
|
||||
|
||||
echo "sharing from '$(distro_name)'"
|
||||
echo " guests: export DOCKER_HOST=unix://$SHARED_SOCK"
|
||||
echo " undo: sudo bash ctrl/dockerhost.sh unshare"
|
||||
echo
|
||||
echo "NOTE: /mnt/wsl is tmpfs and is cleared when the WSL VM shuts down."
|
||||
echo " The drop-in recreates the directory on the next Docker start."
|
||||
}
|
||||
|
||||
unshare_() {
|
||||
require_wsl
|
||||
[ "$(id -u)" -eq 0 ] || { echo "run with sudo: sudo bash ctrl/dockerhost.sh unshare" >&2; exit 1; }
|
||||
|
||||
local did=0
|
||||
|
||||
# The bind mount first: undoing it needs no restart, so a plain `share`
|
||||
# is fully reversible without disturbing anything.
|
||||
if mountpoint -q "$SHARED_SOCK" 2>/dev/null; then
|
||||
umount "$SHARED_SOCK"
|
||||
rm -f "$SHARED_SOCK"
|
||||
echo " removed the bind mount (no restart needed)"
|
||||
did=1
|
||||
fi
|
||||
rm -f "$OWNER_FILE"
|
||||
rmdir "$SHARED_DIR" 2>/dev/null || true
|
||||
|
||||
if [ -f "$DROPIN" ]; then
|
||||
rm -f "$DROPIN"
|
||||
rmdir "$(dirname "$DROPIN")" 2>/dev/null || true
|
||||
systemctl daemon-reload
|
||||
systemctl restart docker
|
||||
echo " removed the systemd drop-in and restarted Docker"
|
||||
did=1
|
||||
fi
|
||||
|
||||
if [ "$did" -eq 0 ]; then
|
||||
echo "not sharing — this WSL installation already has the main host Docker."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "restored: this WSL installation has the main host Docker again."
|
||||
echo " (nothing else was changed; the original unit was never edited)"
|
||||
}
|
||||
|
||||
# ── use (run on a guest distro) ────────────────────────────────────────────
|
||||
|
||||
use() {
|
||||
require_wsl
|
||||
if [ ! -S "$SHARED_SOCK" ]; then
|
||||
echo "no shared socket at $SHARED_SOCK" >&2
|
||||
echo "Run 'sudo bash ctrl/dockerhost.sh share' in the distro that owns Docker." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Align the local docker group GID with the owner's, or the socket is
|
||||
# unreadable here even though it is visible.
|
||||
if [ -f "$OWNER_FILE" ] && [ "$(id -u)" -eq 0 ]; then
|
||||
local gid; gid=$(awk '/docker gid:/ {print $3}' "$OWNER_FILE")
|
||||
if [ -n "$gid" ]; then
|
||||
if getent group docker >/dev/null; then
|
||||
[ "$(getent group docker | cut -d: -f3)" = "$gid" ] || groupmod -g "$gid" docker
|
||||
else
|
||||
groupadd -g "$gid" docker
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${1:-}" = "--persist" ]; then
|
||||
[ "$(id -u)" -eq 0 ] || { echo "--persist needs root" >&2; exit 1; }
|
||||
echo "export DOCKER_HOST=unix://$SHARED_SOCK" > "$PROFILE_D"
|
||||
echo "persisted in $PROFILE_D"
|
||||
fi
|
||||
|
||||
echo "export DOCKER_HOST=unix://$SHARED_SOCK"
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
status) status ;;
|
||||
share) shift; share "${1:-}" ;;
|
||||
unshare) unshare_ ;;
|
||||
use) shift; use "${1:-}" ;;
|
||||
*) echo "usage: $0 [status|share|unshare|use [--persist]]" >&2; exit 1 ;;
|
||||
esac
|
||||
58
rig/ctrl/docs.sh
Executable file
58
rig/ctrl/docs.sh
Executable file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# Documentation: render the diagrams, and serve the pages.
|
||||
#
|
||||
# The docs are the instructions for building the cluster, so they must work
|
||||
# BEFORE anything else exists. That rules out serving them from the cluster, and
|
||||
# it rules out python -m http.server too — a minimal Debian has no python3. What
|
||||
# it does have, by definition, is Docker: the single prerequisite rig already
|
||||
# demands. So a throwaway nginx container serves a read-only bind mount.
|
||||
#
|
||||
# Rendered SVGs are committed alongside their .dot sources for the same reason:
|
||||
# the pages have to read on a machine with no Graphviz installed.
|
||||
#
|
||||
# Usage: docs.sh serve | graphs
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
REPO="$(cd .. && pwd)"
|
||||
DOCS_PORT="${DOCS_PORT:-$((HTTP_PORT + 4))}" # +4 sits inside this env's block
|
||||
|
||||
serve() {
|
||||
if [ ! -f "$REPO/docs/index.html" ]; then
|
||||
echo "no docs/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "docs for '$CLUSTER' on http://localhost:${DOCS_PORT}"
|
||||
echo " (ctrl-c to stop; nothing is installed and nothing persists)"
|
||||
docker run --rm \
|
||||
--name "${CLUSTER}-docs" \
|
||||
-p "${DOCS_PORT}:80" \
|
||||
-v "$REPO/docs:/usr/share/nginx/html:ro" \
|
||||
nginx:alpine
|
||||
}
|
||||
|
||||
graphs() {
|
||||
if ! command -v dot >/dev/null 2>&1; then
|
||||
echo "graphviz not found — install with: sudo apt install graphviz" >&2
|
||||
echo "(only needed to re-render; the committed .svg files already work)" >&2
|
||||
exit 1
|
||||
fi
|
||||
shopt -s nullglob
|
||||
local found=0 f out
|
||||
for f in "$REPO"/docs/graphs/*.dot; do
|
||||
out="${f%.dot}.svg"
|
||||
echo " graphviz $(basename "$f") → $(basename "$out")"
|
||||
dot -Tsvg "$f" -o "$out"
|
||||
found=1
|
||||
done
|
||||
[ "$found" -eq 1 ] || echo " no .dot files in docs/graphs/"
|
||||
}
|
||||
|
||||
case "${1:-serve}" in
|
||||
serve) serve ;;
|
||||
graphs) graphs ;;
|
||||
*) echo "usage: $0 [serve|graphs]" >&2; exit 1 ;;
|
||||
esac
|
||||
30
rig/ctrl/env.d/client.env
Normal file
30
rig/ctrl/env.d/client.env
Normal file
@@ -0,0 +1,30 @@
|
||||
# client — the regulated-estate shape. Multi-node so taints, affinity and
|
||||
# topology are real; apiserver audit on; images through a pull-through cache of
|
||||
# the corporate registry.
|
||||
#
|
||||
# Costs roughly 4-6 GB. Check `make cluster list` before starting this alongside
|
||||
# other work — see the memory note in the README.
|
||||
|
||||
PROFILE_NAME=client
|
||||
K8S_VERSION=v1_36
|
||||
KIND_CONFIG=kind-config.client.yaml.tpl
|
||||
ADDONS="metallb cert-manager metrics-server"
|
||||
REGISTRY_MODE=mirror
|
||||
INGRESS_MODE=hostport
|
||||
DNS_MODE=hosts
|
||||
|
||||
# Ports derive from the directory name by default (see ctrl/ports.sh), so
|
||||
# several environments run side by side.
|
||||
#
|
||||
# Opt in to the real ports below only when this is the ONLY environment and
|
||||
# nothing else owns :80. They fail to bind otherwise, and docker reports it as an
|
||||
# opaque "failed to bind host port 0.0.0.0:80/tcp: address already in use"
|
||||
# halfway through cluster creation. `make check` checks before you spend the
|
||||
# time. Uncommenting also means only one environment can exist at a time.
|
||||
# HTTP_PORT=80
|
||||
# HTTPS_PORT=443
|
||||
|
||||
# Set these in ctrl/.env (gitignored), not here:
|
||||
# REGISTRY_REMOTE_URL=https://artifactory.corp.example/artifactory/api/docker/docker-virtual
|
||||
# REGISTRY_USER / REGISTRY_PASSWORD
|
||||
# REGISTRY_CA_FILE=/path/to/corp-root-ca.crt
|
||||
41
rig/ctrl/env.d/data.env
Normal file
41
rig/ctrl/env.d/data.env
Normal file
@@ -0,0 +1,41 @@
|
||||
# data — a cluster with the cabinets an environment asks for.
|
||||
#
|
||||
# A cabinet is a public service dropped in as-is — the upstream image,
|
||||
# unmodified, reachable at a known address. It is declared once and installs on
|
||||
# either target: a `service.yml` composes it for a laptop, and the addons below
|
||||
# install the same one here. The names match deliberately — each cabinet.json
|
||||
# carries a `rig_addon` field pointing at ctrl/addons/<name>.sh.
|
||||
#
|
||||
# Everything lands in the `data` namespace (DATA_NAMESPACE to move it), so
|
||||
# `make cluster reset` on the app namespace leaves the databases alone.
|
||||
#
|
||||
# Costs roughly 2-3 GB with airflow, under 1 without. Airflow's first boot runs
|
||||
# the whole metadata migration, so expect a few minutes before it is ready.
|
||||
|
||||
PROFILE_NAME=data
|
||||
K8S_VERSION=v1_36
|
||||
KIND_CONFIG=kind-config.yaml.tpl
|
||||
# Order matters: addons.sh installs in the order listed, and airflow refuses to
|
||||
# start without a metadata database, so postgres comes first.
|
||||
ADDONS="metallb postgres redis airflow"
|
||||
# local, not none — see minimal.env: `none` has no outward-push guard.
|
||||
REGISTRY_MODE=local
|
||||
INGRESS_MODE=hostport
|
||||
DNS_MODE=hosts
|
||||
|
||||
# Namespace for the dependency containers.
|
||||
DATA_NAMESPACE=data
|
||||
|
||||
# Postgres identity. The password is not here: postgres.sh generates one on
|
||||
# first install and keeps it across re-runs, so re-running the addon never
|
||||
# rotates the credential out from under whatever is already connected.
|
||||
POSTGRES_DB=app
|
||||
POSTGRES_USER=app
|
||||
POSTGRES_STORAGE=2Gi
|
||||
|
||||
AIRFLOW_ADMIN_USER=admin
|
||||
|
||||
# Ports derive from the directory name by default — see ctrl/ports.sh. Reach
|
||||
# the databases with port-forward rather than binding more host ports:
|
||||
# kubectl -n data port-forward svc/postgres 5432:5432
|
||||
# kubectl -n data port-forward svc/airflow 8080:8080
|
||||
21
rig/ctrl/env.d/minimal.env
Normal file
21
rig/ctrl/env.d/minimal.env
Normal file
@@ -0,0 +1,21 @@
|
||||
# minimal — the default. One node, no addons, no registry.
|
||||
# Assumes nothing and boots fast. Start here; move to client.env when you need
|
||||
# the regulated behaviours.
|
||||
#
|
||||
|
||||
PROFILE_NAME=minimal
|
||||
K8S_VERSION=v1_36
|
||||
KIND_CONFIG=kind-config.yaml.tpl
|
||||
ADDONS=""
|
||||
# local, not none: `none` leaves the cluster with no registry to push to, and an
|
||||
# unqualified image name then means docker.io/library/<name>. In a regulated
|
||||
# estate that is a disclosure risk, not a convenience trade — so the default
|
||||
# carries the guard even though it costs one container.
|
||||
REGISTRY_MODE=local
|
||||
INGRESS_MODE=hostport
|
||||
DNS_MODE=hosts
|
||||
|
||||
# Ports are deliberately NOT set here. They derive from the directory name so
|
||||
# several environments coexist — see ctrl/ports.sh, and `make ports` to see the
|
||||
# block this one gets. A fixed default here would collide with whatever else the
|
||||
# machine happens to be running; 8080 in particular is rarely free.
|
||||
18
rig/ctrl/env.d/offline.env
Normal file
18
rig/ctrl/env.d/offline.env
Normal file
@@ -0,0 +1,18 @@
|
||||
# offline — air-gapped. Everything comes from a local registry that was loaded
|
||||
# ahead of time; nothing reaches the internet. Pair with the deps-full image
|
||||
# (DEPS_SOURCE=baked) so the toolchain install is offline too.
|
||||
#
|
||||
# The heavier addons are left out to keep first boot viable.
|
||||
|
||||
PROFILE_NAME=offline
|
||||
K8S_VERSION=v1_36
|
||||
KIND_CONFIG=kind-config.audit.yaml.tpl
|
||||
ADDONS="metallb"
|
||||
REGISTRY_MODE=local
|
||||
INGRESS_MODE=hostport
|
||||
DNS_MODE=hosts
|
||||
|
||||
# Derived from the directory name by default — see ctrl/ports.sh.
|
||||
# Uncomment for the real ports, but only if this is the only environment.
|
||||
# HTTP_PORT=80
|
||||
# HTTPS_PORT=443
|
||||
15
rig/ctrl/hosts.tmpl
Normal file
15
rig/ctrl/hosts.tmpl
Normal file
@@ -0,0 +1,15 @@
|
||||
# /etc/hosts block for this environment. Rendered by newbox.sh; ${CLUSTER} and
|
||||
# ${HTTP_PORT} are substituted.
|
||||
#
|
||||
# Hostnames are a convenience, not a requirement — every service is reachable at
|
||||
# localhost:<port> without any of this, which is why DNS is not touched by
|
||||
# default. Add entries here as the model grows.
|
||||
#
|
||||
# On Windows the same block has to go in
|
||||
# C:\Windows\System32\drivers\etc\hosts for a browser to resolve these. That
|
||||
# file does NOT support wildcards, so every name must be listed explicitly.
|
||||
# newbox.sh prints the block for you to paste rather than editing it.
|
||||
|
||||
127.0.0.1 ${CLUSTER}.local
|
||||
127.0.0.1 api.${CLUSTER}.local
|
||||
127.0.0.1 docs.${CLUSTER}.local
|
||||
70
rig/ctrl/k8s/README.md
Normal file
70
rig/ctrl/k8s/README.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# `ctrl/k8s` — cluster shape, and what runs on it
|
||||
|
||||
Same layout as every other project here: a kind config, a kustomize `base/`,
|
||||
and an `overlays/dev/` that patches it.
|
||||
|
||||
```
|
||||
kind-config*.yaml.tpl the cluster itself — nodes, ports, audit
|
||||
base/ the components, as plain manifests
|
||||
overlays/dev/ how this rig differs from the base
|
||||
audit-policy.yaml mounted into the apiserver by the audit shapes
|
||||
```
|
||||
|
||||
## Why the cluster config is a template
|
||||
|
||||
Every other project checks in a literal `kind-config.yaml`, because there is
|
||||
exactly one `unt` and one `nvi`. A rig is copied and renamed to make a second
|
||||
environment, and both the cluster name and the host port block follow the
|
||||
directory name — so a literal would make every copy collide on both.
|
||||
|
||||
`ctrl/cluster.sh` renders it with `sed`, substituting `${CLUSTER}`,
|
||||
`${NODE_IMAGE}`, `${HTTP_PORT}` and `${HOST_WORKDIR}`. Not `envsubst`: that is
|
||||
`gettext-base`, which a minimal Debian does not have, and Docker being the only
|
||||
prerequisite is the one promise rig makes.
|
||||
|
||||
**The chosen file is the source of truth for node count and audit.**
|
||||
`lib/config.sh` reads both back out of it, so a profile names a shape and does
|
||||
not restate what the YAML already says.
|
||||
|
||||
| file | nodes | audit | profiles |
|
||||
| --- | --- | --- | --- |
|
||||
| `kind-config.yaml.tpl` | 1 | off | `minimal`, `data` |
|
||||
| `kind-config.audit.yaml.tpl` | 1 | on | `offline` |
|
||||
| `kind-config.client.yaml.tpl` | 3 | on | `client` |
|
||||
|
||||
A profile picks one with `KIND_CONFIG` in `ctrl/env.d/<profile>.env`. Adding a
|
||||
shape is adding a file — there is no dispatcher to edit.
|
||||
|
||||
Audit is an apiserver flag and therefore fixed at creation: changing it is
|
||||
`make cluster reset`, not a re-apply.
|
||||
|
||||
## `base/` — replace these
|
||||
|
||||
**The two components in `base/` are examples, not the system.** They exist so
|
||||
the real manifests have a shape to be written against.
|
||||
|
||||
The real ones are expected to be versioned **separately from the installer** —
|
||||
they change on a different cadence, by different people, under different review.
|
||||
Point `MANIFESTS_DIR` in `ctrl/.env` at their overlay and rig stops owning them:
|
||||
|
||||
```
|
||||
MANIFESTS_DIR=../platform-manifests/overlays/dev
|
||||
```
|
||||
|
||||
Until then it defaults to `ctrl/k8s/overlays/dev`.
|
||||
|
||||
### The three states a component can be in
|
||||
|
||||
Switching between them should be a one-line change, never a rewrite. The DNS
|
||||
name stays the same in every case, so callers never know the difference:
|
||||
|
||||
| state | what exists | when |
|
||||
| --- | --- | --- |
|
||||
| **real** | an image built from source, hot-reloaded | the one thing you are working on |
|
||||
| **mock** | a stub returning canned responses (`example-mock.yaml`) | everything else — most of the estate |
|
||||
| **remote** | no pod at all, just a Service (`example-remote.yaml`) | when the real system is reachable and you want it |
|
||||
|
||||
Most components should be **mock**. What has to be faithful is the topology —
|
||||
names, ports, dependency order, who can reach whom, how it fails. The workloads
|
||||
are noise, and mocking them is what makes several copies of a large estate fit
|
||||
on one laptop.
|
||||
44
rig/ctrl/k8s/audit-policy.yaml
Normal file
44
rig/ctrl/k8s/audit-policy.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
# Apiserver audit policy. Mounted into the control plane at creation when a
|
||||
# profile sets AUDIT=on — an apiserver flag, so it cannot be added to a running
|
||||
# cluster without recreating it.
|
||||
#
|
||||
# Deliberately modest: enough to make "who changed what, and when" answerable
|
||||
# during onboarding without filling the disk. Read the log with:
|
||||
# docker exec <cluster>-control-plane cat /var/log/kubernetes/audit.log
|
||||
apiVersion: audit.k8s.io/v1
|
||||
kind: Policy
|
||||
|
||||
# Never log the request body for these — they contain credentials.
|
||||
omitStages:
|
||||
- RequestReceived
|
||||
|
||||
rules:
|
||||
# Secrets/configmaps: record that access happened, never the contents.
|
||||
- level: Metadata
|
||||
resources:
|
||||
- group: ""
|
||||
resources: ["secrets", "configmaps"]
|
||||
|
||||
# Authn/authz decisions — the part an auditor actually asks about.
|
||||
- level: Metadata
|
||||
nonResourceURLs:
|
||||
- /apis*
|
||||
- /api*
|
||||
|
||||
# Mutations to workloads and policy: full request, so a diff is reconstructable.
|
||||
- level: Request
|
||||
verbs: ["create", "update", "patch", "delete"]
|
||||
resources:
|
||||
- group: ""
|
||||
resources: ["pods", "services", "serviceaccounts", "namespaces"]
|
||||
- group: "apps"
|
||||
- group: "networking.k8s.io"
|
||||
- group: "rbac.authorization.k8s.io"
|
||||
|
||||
# Everything else that changes state: metadata only.
|
||||
- level: Metadata
|
||||
verbs: ["create", "update", "patch", "delete"]
|
||||
|
||||
# Reads are dropped entirely — otherwise controller polling drowns the log.
|
||||
- level: None
|
||||
verbs: ["get", "list", "watch"]
|
||||
104
rig/ctrl/k8s/base/example-mock.yaml
Normal file
104
rig/ctrl/k8s/base/example-mock.yaml
Normal file
@@ -0,0 +1,104 @@
|
||||
# EXAMPLE — a mocked component. Copy, rename, replace.
|
||||
#
|
||||
# A stub that answers on the right name and port with canned responses. No image
|
||||
# to build: the script is mounted from the ConfigMap, so changing the behaviour
|
||||
# is a kubectl apply, not a rebuild.
|
||||
#
|
||||
# Deliberately boring and readable. This is onboarding material — someone should
|
||||
# be able to read the generated object and recognise what it is.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: example-service-stub
|
||||
data:
|
||||
# Canned responses by path. Add entries as the contract becomes clear;
|
||||
# anything unmatched returns 404 so a missing route is visible, not silent.
|
||||
routes.json: |
|
||||
{
|
||||
"/health": {"status": 200, "body": {"status": "ok"}},
|
||||
"/v1/example": {"status": 200, "body": {"items": [], "mocked": true}}
|
||||
}
|
||||
serve.py: |
|
||||
import json, os
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
ROUTES = json.load(open("/etc/stub/routes.json"))
|
||||
NAME = os.environ.get("STUB_NAME", "stub")
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
r = ROUTES.get(self.path)
|
||||
if r is None:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
# Say which stub rejected it — with everything mocked, "404"
|
||||
# alone tells you nothing about where the call actually landed.
|
||||
self.wfile.write(json.dumps(
|
||||
{"error": "no canned route", "stub": NAME, "path": self.path}
|
||||
).encode())
|
||||
return
|
||||
body = json.dumps(r["body"]).encode()
|
||||
self.send_response(r["status"])
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("X-Mocked-By", NAME)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
print("%s %s" % (NAME, fmt % args), flush=True)
|
||||
|
||||
HTTPServer(("0.0.0.0", 8080), H).serve_forever()
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: example-service
|
||||
labels:
|
||||
app: example-service
|
||||
rig.component/impl: mock # so `kubectl get deploy -L rig.component/impl`
|
||||
# shows at a glance what is real and what is not
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: example-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: example-service
|
||||
spec:
|
||||
containers:
|
||||
- name: stub
|
||||
image: python:3.12-slim
|
||||
command: ["python3", "/etc/stub/serve.py"]
|
||||
env:
|
||||
- name: STUB_NAME
|
||||
value: example-service
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: stub
|
||||
mountPath: /etc/stub
|
||||
readinessProbe:
|
||||
httpGet: { path: /health, port: 8080 }
|
||||
initialDelaySeconds: 2
|
||||
# Small enough that a whole estate of these fits alongside the real
|
||||
# thing you are working on.
|
||||
resources:
|
||||
requests: { memory: 32Mi, cpu: 10m }
|
||||
limits: { memory: 64Mi }
|
||||
volumes:
|
||||
- name: stub
|
||||
configMap:
|
||||
name: example-service-stub
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: example-service
|
||||
spec:
|
||||
selector:
|
||||
app: example-service
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
46
rig/ctrl/k8s/base/example-remote.yaml
Normal file
46
rig/ctrl/k8s/base/example-remote.yaml
Normal file
@@ -0,0 +1,46 @@
|
||||
# EXAMPLE — a component that is NOT simulated, pointed at the real system.
|
||||
#
|
||||
# This is the payoff of keeping the topology honest: there is no pod here at
|
||||
# all, yet `example-remote.<namespace>.svc.cluster.local` resolves exactly as it
|
||||
# does when the same component is mocked. Callers are identical in both cases,
|
||||
# so moving a dependency from mocked to real is a one-line change and nothing
|
||||
# downstream is touched.
|
||||
#
|
||||
# Use this when the real system is reachable and you want it in the loop.
|
||||
# Note that reachability depends on where you are running: systems restricted to
|
||||
# a managed workspace will not resolve from a laptop at all, which is the whole
|
||||
# reason most components should stay mocked.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: example-remote
|
||||
labels:
|
||||
rig.component/impl: remote
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: real-system.internal.example.com
|
||||
---
|
||||
# If the real system has no DNS name — only an IP, which is common for legacy
|
||||
# hosts — ExternalName cannot express it. Use a bare Service plus manual
|
||||
# Endpoints instead, and delete the block above.
|
||||
#
|
||||
# apiVersion: v1
|
||||
# kind: Service
|
||||
# metadata:
|
||||
# name: example-remote
|
||||
# labels:
|
||||
# rig.component/impl: remote
|
||||
# spec:
|
||||
# ports:
|
||||
# - port: 80
|
||||
# targetPort: 8080
|
||||
# ---
|
||||
# apiVersion: v1
|
||||
# kind: Endpoints
|
||||
# metadata:
|
||||
# name: example-remote # must match the Service name exactly
|
||||
# subsets:
|
||||
# - addresses:
|
||||
# - ip: 10.0.0.42
|
||||
# ports:
|
||||
# - port: 8080
|
||||
11
rig/ctrl/k8s/base/kustomization.yaml
Normal file
11
rig/ctrl/k8s/base/kustomization.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
# The namespace every component lands in. The overlay overrides it, so a rig
|
||||
# modelling two estates can apply the same base twice under different names.
|
||||
namespace: rig
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- example-mock.yaml
|
||||
- example-remote.yaml
|
||||
4
rig/ctrl/k8s/base/namespace.yaml
Normal file
4
rig/ctrl/k8s/base/namespace.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: rig
|
||||
57
rig/ctrl/k8s/kind-config.audit.yaml.tpl
Normal file
57
rig/ctrl/k8s/kind-config.audit.yaml.tpl
Normal file
@@ -0,0 +1,57 @@
|
||||
# Cluster shape: one node, apiserver audit ON. Used by the `offline` profile.
|
||||
#
|
||||
# Audit is an apiserver flag, so it is fixed when the cluster is created —
|
||||
# changing it means `make cluster reset`, not a re-apply. That is why it is a
|
||||
# property of the cluster file rather than something switched at runtime.
|
||||
#
|
||||
# k8s >= 1.31 uses kubeadm v1beta4, where extraArgs is a LIST of name/value
|
||||
# pairs. The older map form is silently ignored — it does not error, audit
|
||||
# simply never turns on.
|
||||
#
|
||||
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
|
||||
# (named without the ${...} braces so this line survives the substitution)
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: ${CLUSTER}
|
||||
|
||||
containerdConfigPatches:
|
||||
- |-
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
kind: ClusterConfiguration
|
||||
apiServer:
|
||||
extraArgs:
|
||||
- name: audit-policy-file
|
||||
value: /etc/kubernetes/audit/policy.yaml
|
||||
- name: audit-log-path
|
||||
value: /var/log/kubernetes/audit.log
|
||||
- name: audit-log-maxage
|
||||
value: "7"
|
||||
extraVolumes:
|
||||
- name: audit-policy
|
||||
hostPath: /etc/kubernetes/audit
|
||||
mountPath: /etc/kubernetes/audit
|
||||
readOnly: true
|
||||
- name: audit-log
|
||||
hostPath: /var/log/kubernetes
|
||||
mountPath: /var/log/kubernetes
|
||||
readOnly: false
|
||||
|
||||
nodes:
|
||||
- role: control-plane
|
||||
image: ${NODE_IMAGE}
|
||||
# hostPath is resolved by the HOST dockerd, so this must be a host path even
|
||||
# when cluster.sh runs inside the installer container. HOST_WORKDIR says where
|
||||
# this rig lives on the host; bare on a host it is just the repo root.
|
||||
extraMounts:
|
||||
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
|
||||
containerPath: /etc/kubernetes/audit/policy.yaml
|
||||
readOnly: true
|
||||
extraPortMappings:
|
||||
- containerPort: 30080
|
||||
hostPort: ${HTTP_PORT}
|
||||
listenAddress: "0.0.0.0"
|
||||
protocol: TCP
|
||||
55
rig/ctrl/k8s/kind-config.client.yaml.tpl
Normal file
55
rig/ctrl/k8s/kind-config.client.yaml.tpl
Normal file
@@ -0,0 +1,55 @@
|
||||
# Cluster shape: three nodes, apiserver audit ON. Used by the `client` profile —
|
||||
# the regulated-estate shape.
|
||||
#
|
||||
# Multi-node so taints, affinity and topology spread are real rather than
|
||||
# vacuously satisfied by a single node. It costs roughly 4-6 GB; run
|
||||
# `make cluster list` before starting this alongside other work.
|
||||
#
|
||||
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
|
||||
# (named without the ${...} braces so this line survives the substitution)
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: ${CLUSTER}
|
||||
|
||||
containerdConfigPatches:
|
||||
- |-
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
|
||||
kubeadmConfigPatches:
|
||||
- |
|
||||
kind: ClusterConfiguration
|
||||
apiServer:
|
||||
extraArgs:
|
||||
- name: audit-policy-file
|
||||
value: /etc/kubernetes/audit/policy.yaml
|
||||
- name: audit-log-path
|
||||
value: /var/log/kubernetes/audit.log
|
||||
- name: audit-log-maxage
|
||||
value: "7"
|
||||
extraVolumes:
|
||||
- name: audit-policy
|
||||
hostPath: /etc/kubernetes/audit
|
||||
mountPath: /etc/kubernetes/audit
|
||||
readOnly: true
|
||||
- name: audit-log
|
||||
hostPath: /var/log/kubernetes
|
||||
mountPath: /var/log/kubernetes
|
||||
readOnly: false
|
||||
|
||||
nodes:
|
||||
- role: control-plane
|
||||
image: ${NODE_IMAGE}
|
||||
extraMounts:
|
||||
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
|
||||
containerPath: /etc/kubernetes/audit/policy.yaml
|
||||
readOnly: true
|
||||
extraPortMappings:
|
||||
- containerPort: 30080
|
||||
hostPort: ${HTTP_PORT}
|
||||
listenAddress: "0.0.0.0"
|
||||
protocol: TCP
|
||||
- role: worker
|
||||
image: ${NODE_IMAGE}
|
||||
- role: worker
|
||||
image: ${NODE_IMAGE}
|
||||
36
rig/ctrl/k8s/kind-config.yaml.tpl
Normal file
36
rig/ctrl/k8s/kind-config.yaml.tpl
Normal file
@@ -0,0 +1,36 @@
|
||||
# Cluster shape: one node, no audit. Used by the `minimal` and `data` profiles.
|
||||
#
|
||||
# A TEMPLATE rather than a plain kind-config.yaml because a rig is copied and
|
||||
# renamed to make a second environment, and both the cluster name and the host
|
||||
# port follow the directory. A checked-in literal would make every copy collide
|
||||
# on both. ctrl/cluster.sh renders it with sed — not envsubst, which is
|
||||
# gettext-base and absent from a minimal Debian, and rig's whole premise is that
|
||||
# Docker is the only prerequisite.
|
||||
#
|
||||
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
|
||||
# (named without the ${...} braces so this line survives the substitution)
|
||||
# Node count and audit are READ BACK from this file by lib/config.sh, so this
|
||||
# YAML is the source of truth for both — there is no second place to update.
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: ${CLUSTER}
|
||||
|
||||
# Point containerd at a certs.d directory. registry.sh drops per-host hosts.toml
|
||||
# files in there afterwards, so switching registry mode never requires
|
||||
# recreating the cluster.
|
||||
containerdConfigPatches:
|
||||
- |-
|
||||
[plugins."io.containerd.grpc.v1.cri".registry]
|
||||
config_path = "/etc/containerd/certs.d"
|
||||
|
||||
nodes:
|
||||
- role: control-plane
|
||||
image: ${NODE_IMAGE}
|
||||
# One NodePort bridged to the host; an in-cluster gateway owns it. There is
|
||||
# deliberately no ingress controller — they pin a narrow window of k8s
|
||||
# versions, and running a trailing-edge control plane is the point.
|
||||
extraPortMappings:
|
||||
- containerPort: 30080
|
||||
hostPort: ${HTTP_PORT}
|
||||
listenAddress: "0.0.0.0"
|
||||
protocol: TCP
|
||||
22
rig/ctrl/k8s/overlays/dev/kustomization.yaml
Normal file
22
rig/ctrl/k8s/overlays/dev/kustomization.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- ../../base
|
||||
|
||||
# The dev overlay is where a rig says how its estate differs from the base —
|
||||
# which components are real, which are mocked, which point at a live system.
|
||||
# Kept empty on purpose: the base already boots, and an overlay full of examples
|
||||
# is harder to read than one that starts blank.
|
||||
#
|
||||
# The shape a patch takes, for when the first one is needed:
|
||||
#
|
||||
# patches:
|
||||
# - target: {kind: Service, name: example-service}
|
||||
# patch: |
|
||||
# - op: replace
|
||||
# path: /spec/type
|
||||
# value: NodePort
|
||||
# - op: add
|
||||
# path: /spec/ports/0/nodePort
|
||||
# value: 30080
|
||||
158
rig/ctrl/lib/config.sh
Normal file
158
rig/ctrl/lib/config.sh
Normal file
@@ -0,0 +1,158 @@
|
||||
# Shared config loading. Sourced, never executed.
|
||||
#
|
||||
# The ecosystem convention is that scripts are standalone with no shared log
|
||||
# library — that still holds. This file is not a logging lib; it is the single
|
||||
# definition of how the config layers compose, which every script has to agree
|
||||
# on exactly. Precedence, weakest first:
|
||||
#
|
||||
# ctrl/versions.env pinned toolchain + image digests (committed)
|
||||
# ctrl/env.d/<profile> cluster shape (committed)
|
||||
# ctrl/.env machine-local values and secrets (gitignored)
|
||||
# the caller's env `make cluster up PROFILE=client` (always wins)
|
||||
#
|
||||
# That last rule is why this is more than a few `source` lines: .env sets
|
||||
# PROFILE, so without snapshotting it would silently override the PROFILE the
|
||||
# user just typed on the command line.
|
||||
#
|
||||
# Run from ctrl/.
|
||||
|
||||
# Values a user can reasonably override per-invocation. Anything set in the
|
||||
# environment when load_config runs is restored after the files are read.
|
||||
# NODES and AUDIT are deliberately NOT here: they are properties of the chosen
|
||||
# ctrl/k8s/kind-config*.yaml.tpl and are read back out of it below, so there is
|
||||
# one place that decides the shape of the cluster rather than two that can drift.
|
||||
CONFIG_OVERRIDABLE="PROFILE CLUSTER K8S_VERSION KIND_CONFIG ADDONS
|
||||
REGISTRY_MODE INGRESS_MODE DNS_MODE TILT_PORT
|
||||
SOURCE ARCH DEPS_SOURCE HTTP_PORT HTTPS_PORT"
|
||||
|
||||
# The containing folder's name, reduced to something kind accepts as a cluster
|
||||
# name (a DNS label: lowercase alphanumerics and dashes). Run from ctrl/, so the
|
||||
# repo root is the parent.
|
||||
default_cluster_name() {
|
||||
local n
|
||||
n=$(basename "$(cd .. && pwd)")
|
||||
n=$(echo "$n" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-')
|
||||
n=$(echo "$n" | sed 's/^-*//; s/-*$//')
|
||||
echo "${n:-rig}"
|
||||
}
|
||||
|
||||
# Base of this environment's 10-port block. cksum is used rather than $RANDOM or
|
||||
# bash hashing because it is POSIX and returns the same value on every machine,
|
||||
# which is what makes the block reproducible instead of merely unique.
|
||||
derive_port_base() {
|
||||
local h; h=$(printf '%s' "$1" | cksum | awk '{print $1}')
|
||||
echo $((20000 + (h % 200) * 10))
|
||||
}
|
||||
|
||||
load_config() {
|
||||
local k saved=""
|
||||
for k in $CONFIG_OVERRIDABLE; do
|
||||
# ${!k+x} distinguishes "set but empty" from "unset" — an explicit
|
||||
# FOO= on the command line is a real choice and must survive.
|
||||
if [ -n "${!k+x}" ]; then
|
||||
saved+="$k=$(printf '%q' "${!k}")"$'\n'
|
||||
fi
|
||||
done
|
||||
|
||||
set -a
|
||||
source ./versions.env
|
||||
[ -f ./.env ] && source ./.env
|
||||
set +a
|
||||
|
||||
# Re-apply overrides now so PROFILE is the caller's before we pick the file.
|
||||
_config_restore "$saved"
|
||||
|
||||
local profile="${PROFILE:-minimal}"
|
||||
if [ ! -f "./env.d/${profile}.env" ]; then
|
||||
echo "no such profile: env.d/${profile}.env" >&2
|
||||
echo "available: $(ls env.d/*.env 2>/dev/null | xargs -n1 basename | sed 's/\.env$//' | tr '\n' ' ')" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -a
|
||||
source "./env.d/${profile}.env"
|
||||
[ -f ./.env ] && source ./.env
|
||||
set +a
|
||||
|
||||
_config_restore "$saved"
|
||||
|
||||
# Identity follows the FOLDER, so copying this directory somewhere else and
|
||||
# renaming it yields a distinct environment with no further edits. Without
|
||||
# this, two copies would share one cluster and `make cluster down` in either
|
||||
# would destroy the other's.
|
||||
CLUSTER="${CLUSTER:-$(default_cluster_name)}"
|
||||
KUBECONTEXT="kind-${CLUSTER}"
|
||||
|
||||
# Host ports are a single shared namespace, so unlike the cluster name they
|
||||
# cannot just follow the directory — they have to be spread out. Anything
|
||||
# already set (ctrl/.env, a profile, the command line) wins; only the gaps
|
||||
# are filled. See ports.sh for the reasoning.
|
||||
local base; base=$(derive_port_base "$CLUSTER")
|
||||
HTTP_PORT="${HTTP_PORT:-$base}"
|
||||
HTTPS_PORT="${HTTPS_PORT:-$((base + 1))}"
|
||||
TILT_PORT="${TILT_PORT:-$((base + 2))}"
|
||||
REGISTRY_PORT="${REGISTRY_PORT:-$((base + 3))}"
|
||||
|
||||
# Profiles name a k8s minor (v1_36); versions.env holds the pinned digest.
|
||||
local var="NODE_IMAGE_${K8S_VERSION}"
|
||||
NODE_IMAGE="${!var:-}"
|
||||
if [ -z "$NODE_IMAGE" ]; then
|
||||
echo "K8S_VERSION='${K8S_VERSION}' has no NODE_IMAGE_${K8S_VERSION} in versions.env" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
|
||||
# shape is adding a file; there is no dispatcher to edit.
|
||||
#
|
||||
# A host that needs its own shape — extra port mappings, more nodes — passes
|
||||
# an absolute path instead, and rig renders it exactly like one of its own:
|
||||
# ${CLUSTER} and ${NODE_IMAGE} are substituted either way. The shape stays in
|
||||
# the host's tree, because what a host's cluster needs is the host's business;
|
||||
# rig only knows how to build whatever it is handed.
|
||||
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
|
||||
case "$KIND_CONFIG" in
|
||||
/*) KIND_CONFIG_PATH="$KIND_CONFIG"; KIND_CONFIG_SHOWN="$KIND_CONFIG" ;;
|
||||
*) KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"; KIND_CONFIG_SHOWN="ctrl/k8s/${KIND_CONFIG}" ;;
|
||||
esac
|
||||
if [ ! -f "$KIND_CONFIG_PATH" ]; then
|
||||
echo "no such cluster shape: ${KIND_CONFIG_SHOWN}" >&2
|
||||
echo "rig's own: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
|
||||
echo "or pass an absolute path to a shape of your own" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Read the shape back out of the YAML rather than trusting a profile to
|
||||
# restate it. check.sh sizes the memory warning on NODES, and cluster.sh
|
||||
# prints AUDIT before spending minutes building something that cannot be
|
||||
# changed afterwards — both would mislead if the numbers drifted.
|
||||
NODES=$(grep -c '^ - role:' "$KIND_CONFIG_PATH")
|
||||
if grep -q 'audit-policy-file' "$KIND_CONFIG_PATH"; then AUDIT=on; else AUDIT=off; fi
|
||||
}
|
||||
|
||||
# Render a cluster shape to stdout. sed rather than envsubst: envsubst is
|
||||
# gettext-base, absent from a minimal Debian, and Docker is meant to be the only
|
||||
# prerequisite. The variable list is explicit so a template cannot quietly start
|
||||
# depending on something the caller does not set.
|
||||
#
|
||||
# hostPath entries are resolved by the HOST dockerd, so HOST_WORKDIR must stay a
|
||||
# host path even when this runs inside the installer container.
|
||||
render_kind_config() {
|
||||
local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}"
|
||||
sed -e "s|\${CLUSTER}|${CLUSTER}|g" \
|
||||
-e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" \
|
||||
-e "s|\${HTTP_PORT}|${HTTP_PORT}|g" \
|
||||
-e "s|\${HOST_WORKDIR}|${host_workdir}|g" \
|
||||
"$KIND_CONFIG_PATH"
|
||||
}
|
||||
|
||||
_config_restore() {
|
||||
local line
|
||||
while IFS= read -r line; do
|
||||
if [ -n "$line" ]; then
|
||||
eval "export $line"
|
||||
fi
|
||||
done <<< "$1"
|
||||
# A while loop returns its last body command's status; the trailing empty
|
||||
# line would otherwise make this return 1 and trip `set -e` in the caller.
|
||||
return 0
|
||||
}
|
||||
233
rig/ctrl/mem.sh
Executable file
233
rig/ctrl/mem.sh
Executable file
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env bash
|
||||
# What memory this machine has, what is left, and — where there is one — what
|
||||
# cap is holding it there.
|
||||
#
|
||||
# Runs on native Linux and under WSL, because rig is developed on one and used
|
||||
# on the other. The difference is not cosmetic: on WSL the memory you see is a
|
||||
# VM allocation that can be raised, and the commonest failure is raising it
|
||||
# without restarting, so the number on disk and the number in /proc disagree.
|
||||
# On native Linux there is no such cap and pretending otherwise sends you to a
|
||||
# file that does not exist.
|
||||
#
|
||||
# This reports and instructs. It never writes a .wslconfig — applying one costs
|
||||
# a full VM restart that takes every shell, mount and container with it, and
|
||||
# choosing that moment is yours.
|
||||
#
|
||||
# `backup` exists so `restore` has something to read: back up, hand-edit
|
||||
# following the printed instruction, restore if it goes wrong. Both are
|
||||
# WSL-only, because .wslconfig is the only thing here worth backing up.
|
||||
#
|
||||
# Usage: mem.sh status | backup | restore
|
||||
set -euo pipefail
|
||||
|
||||
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||
# the tooling. Detectable, so name it instead.
|
||||
require_linux() {
|
||||
case "$(uname -s)" in
|
||||
MINGW*|MSYS*|CYGWIN*)
|
||||
cat >&2 <<'EOF'
|
||||
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||
|
||||
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||
|
||||
wsl --install
|
||||
|
||||
That enables Windows features and needs a reboot, so it is not something this
|
||||
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||
this from there.
|
||||
|
||||
See "Starting from plain Windows" in README.md.
|
||||
EOF
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||
|
||||
require_wsl() {
|
||||
if ! is_wsl; then
|
||||
echo "$1 acts on .wslconfig, which only exists under WSL." >&2
|
||||
echo "This is native Linux — there is no VM allocation to save or roll back." >&2
|
||||
echo "Use 'mem.sh status' to see what the machine actually has." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
||||
|
||||
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
||||
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
||||
# Windows, then fall back to whichever profile actually owns a config.
|
||||
wslconfig_path() {
|
||||
local profile winpath found
|
||||
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
||||
case "$profile" in
|
||||
""|*%*) ;;
|
||||
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
||||
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
||||
echo "$winpath/.wslconfig"; return
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||
if [ -n "$found" ]; then echo "$found"; return; fi
|
||||
|
||||
echo "cannot tell which Windows profile owns .wslconfig. Candidates:" >&2
|
||||
ls -d /mnt/c/Users/*/ 2>/dev/null \
|
||||
| grep -viE "/(All Users|Default|Default User|Public)/$" | sed "s/^/ /" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
configured_memory() {
|
||||
[ -r "$1" ] || { echo ""; return; }
|
||||
sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$1" | tail -1 | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
# "9GB" / "8192MB" / "9G" -> MB, so it can be compared with /proc/meminfo.
|
||||
to_mb() {
|
||||
local v="${1^^}" n
|
||||
n=$(echo "$v" | tr -dc '0-9')
|
||||
[ -n "$n" ] || { echo ""; return; }
|
||||
case "$v" in
|
||||
*GB|*G) echo $(( n * 1024 )) ;;
|
||||
*MB|*M) echo "$n" ;;
|
||||
*) echo $(( n / 1024 / 1024 )) ;;
|
||||
esac
|
||||
}
|
||||
|
||||
hogs() {
|
||||
echo "holding the most:"
|
||||
ps -eo rss,comm --sort=-rss 2>/dev/null | awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
||||
}
|
||||
|
||||
status() {
|
||||
local total avail swap_total swap_free
|
||||
total=$(mb MemTotal); avail=$(mb MemAvailable)
|
||||
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
||||
|
||||
if is_wsl; then
|
||||
local cfg conf conf_mb
|
||||
cfg=$(wslconfig_path)
|
||||
conf=$(configured_memory "$cfg")
|
||||
echo "platform WSL"
|
||||
echo "config $cfg"
|
||||
if [ -n "$conf" ]; then
|
||||
conf_mb=$(to_mb "$conf")
|
||||
echo "configured $conf (${conf_mb} MB)"
|
||||
else
|
||||
conf_mb=""
|
||||
echo "configured (no memory= set — WSL defaults to 50% of host RAM, or 8GB, whichever is less)"
|
||||
fi
|
||||
echo "booted ${total} MB"
|
||||
echo "available ${avail} MB"
|
||||
echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||
|
||||
if [ -n "$conf_mb" ]; then
|
||||
# The VM reports a little less than allocated; 15% covers the kernel
|
||||
# without calling every healthy machine a mismatch.
|
||||
if [ "$total" -lt $(( conf_mb * 85 / 100 )) ]; then
|
||||
echo
|
||||
echo "! configured ${conf_mb} MB but booted ${total} MB."
|
||||
echo " The change has not been applied. From a WINDOWS terminal:"
|
||||
echo
|
||||
echo " wsl --shutdown"
|
||||
echo
|
||||
echo " then start the distro again."
|
||||
fi
|
||||
else
|
||||
echo
|
||||
echo "To raise it, add to $cfg on the Windows side:"
|
||||
echo
|
||||
echo " [wsl2]"
|
||||
echo " memory=8GB"
|
||||
echo
|
||||
echo "then, from a WINDOWS terminal: wsl --shutdown"
|
||||
fi
|
||||
|
||||
local n
|
||||
n=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
||||
[ "$n" -gt 0 ] && echo "backups $n (newest: $(ls -t "$cfg".*.bak 2>/dev/null | head -1))"
|
||||
else
|
||||
echo "platform native linux"
|
||||
echo "total ${total} MB"
|
||||
echo "available ${avail} MB"
|
||||
echo "swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||
echo
|
||||
echo "No VM allocation to raise here — this is the machine's own memory."
|
||||
echo "If it is tight the levers are freeing something or adding swap."
|
||||
fi
|
||||
|
||||
# Under a fifth left is worth naming wherever you are running.
|
||||
if [ "$avail" -lt $(( total / 5 )) ]; then
|
||||
echo
|
||||
hogs
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
backup() {
|
||||
require_wsl backup
|
||||
local cfg dest
|
||||
cfg=$(wslconfig_path)
|
||||
[ -r "$cfg" ] || { echo "nothing to back up: $cfg does not exist" >&2; exit 1; }
|
||||
# Timestamped and never overwritten: a backup that can destroy itself on a
|
||||
# second run is not a backup.
|
||||
dest="${cfg}.$(date +%Y%m%d-%H%M%S).bak"
|
||||
cp "$cfg" "$dest"
|
||||
echo "backed up $dest"
|
||||
echo
|
||||
echo "Edit $cfg by hand, then from a WINDOWS terminal: wsl --shutdown"
|
||||
}
|
||||
|
||||
restore() {
|
||||
require_wsl restore
|
||||
local cfg newest count
|
||||
cfg=$(wslconfig_path)
|
||||
newest=$(ls -t "$cfg".*.bak 2>/dev/null | head -1 || true)
|
||||
[ -n "$newest" ] || { echo "no backups found beside $cfg" >&2; exit 1; }
|
||||
|
||||
echo "restoring $newest"
|
||||
echo " -> $cfg"
|
||||
echo
|
||||
|
||||
# Newest is the right default — undo the last edit — but if you backed up
|
||||
# *after* editing, the state you want is older. Show the rest so a no-op
|
||||
# restore is obviously a no-op rather than a mystery.
|
||||
count=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
||||
if [ "$count" -gt 1 ]; then
|
||||
echo "$count backups exist, newest first:"
|
||||
ls -t "$cfg".*.bak | sed 's/^/ /'
|
||||
echo " (restoring the newest; copy another by hand to pick an older one)"
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ -r "$cfg" ]; then
|
||||
echo "what changes:"
|
||||
if diff "$cfg" "$newest" > /tmp/mem.diff 2>&1 && [ ! -s /tmp/mem.diff ]; then
|
||||
echo " nothing — that backup is identical to the current config"
|
||||
else
|
||||
sed 's/^/ /' /tmp/mem.diff
|
||||
fi
|
||||
rm -f /tmp/mem.diff
|
||||
echo
|
||||
fi
|
||||
|
||||
printf "proceed? [y/N] "
|
||||
read -r reply
|
||||
case "$reply" in
|
||||
y|Y|yes|Yes) ;;
|
||||
*) echo "left alone"; return 0 ;;
|
||||
esac
|
||||
cp "$newest" "$cfg"
|
||||
echo "restored. From a WINDOWS terminal: wsl --shutdown"
|
||||
}
|
||||
|
||||
require_linux
|
||||
|
||||
case "${1:-status}" in
|
||||
status) status ;;
|
||||
backup) backup ;;
|
||||
restore) restore ;;
|
||||
*) echo "usage: $0 [status|backup|restore]" >&2; exit 1 ;;
|
||||
esac
|
||||
316
rig/ctrl/newbox.sh
Executable file
316
rig/ctrl/newbox.sh
Executable file
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create a disposable Linux environment to validate the installer from a
|
||||
# genuinely clean slate — one that can be thrown away without touching the
|
||||
# environment you actually work in.
|
||||
#
|
||||
# This is the ONLY host-aware file in the tree. Everything else needs just a
|
||||
# Linux with Docker, which is what keeps other host types a later addition
|
||||
# rather than a rewrite.
|
||||
#
|
||||
# On WSL it creates a second distro. There is no .bat and no PowerShell script:
|
||||
# wsl.exe is callable from inside WSL, and wslpath converts the paths it wants.
|
||||
# A machine with no WSL at all needs `wsl --install` run once by hand first —
|
||||
# scripting a reboot-requiring Windows feature install is not worth it.
|
||||
#
|
||||
# Docker: borrowed by default, never installed twice
|
||||
# --------------------------------------------------
|
||||
# WSL2 distros share one kernel and one network stack, so two dockerd instances
|
||||
# contend over docker0 and iptables and can disturb the daemon you depend on.
|
||||
# (That is why Docker Desktop runs one daemon in a dedicated distro and shares
|
||||
# its socket rather than installing one per distro.)
|
||||
#
|
||||
# REUSE_DOCKER=1 (default) borrow the host distro's daemon over /mnt/wsl.
|
||||
# Nothing is installed; nothing can conflict.
|
||||
# Requires `ctrl/dockerhost.sh share` once on the
|
||||
# distro that owns Docker.
|
||||
# REUSE_DOCKER=0 install a second daemon in the new distro. Only
|
||||
# if you specifically want to test a from-scratch
|
||||
# Docker install, and not on a machine you need.
|
||||
#
|
||||
# Borrowing is also the more honest test: rig never installs Docker anyway — it
|
||||
# is the documented prerequisite — so a clean box does not need its own to
|
||||
# exercise everything rig actually does.
|
||||
#
|
||||
# Usage: newbox.sh create | destroy [--purge] | status | shell
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
REPO="$(cd .. && pwd)"
|
||||
|
||||
# The distro is named after this environment, and that derived name is the ONLY
|
||||
# thing this script will ever destroy. See guard_name().
|
||||
BOX="${BOX:-${CLUSTER}box}"
|
||||
BOX_USER="${BOX_USER:-dev}"
|
||||
|
||||
# Borrow the host distro's Docker rather than installing a second daemon.
|
||||
REUSE_DOCKER="${REUSE_DOCKER:-1}"
|
||||
SHARED_SOCK=/mnt/wsl/shared-docker/docker.sock
|
||||
|
||||
WSL_EXE=/mnt/c/Windows/System32/wsl.exe
|
||||
|
||||
# ── host detection ─────────────────────────────────────────────────────────
|
||||
|
||||
require_wsl() {
|
||||
if ! grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
cat >&2 <<'EOF'
|
||||
newbox is WSL-only for now.
|
||||
|
||||
If WSL is not installed, run `wsl --install` from an elevated Windows prompt
|
||||
first — see "Starting from plain Windows" in README.md.
|
||||
|
||||
On native Linux you do not need it: rig already isolates environments by
|
||||
directory (own cluster, context, images and port block), so a second copy in a
|
||||
second directory is the clean slate. To validate the installer itself against a
|
||||
bare system, run ctrl/deps.sh against a stock Debian container instead.
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$WSL_EXE" ]; then
|
||||
echo "wsl.exe not found at $WSL_EXE" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
wsl_list() { "$WSL_EXE" -l -q 2>/dev/null | tr -d '\0\r'; }
|
||||
box_exists() { wsl_list | grep -qx "$BOX"; }
|
||||
|
||||
# `wsl --unregister` permanently deletes a distro's filesystem. The whole safety
|
||||
# story is this function: only the name derived from this directory can ever be
|
||||
# a target, so a typo or a stray argument cannot destroy the distro you work in.
|
||||
guard_name() {
|
||||
local derived="${CLUSTER}box"
|
||||
if [ "$BOX" != "$derived" ]; then
|
||||
echo "refusing: BOX='$BOX' is not the name derived from this directory ('$derived')." >&2
|
||||
echo "That guard exists because --unregister is irreversible." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$CLUSTER" ] || [ "$BOX" = "box" ]; then
|
||||
echo "refusing: empty environment name" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── create ─────────────────────────────────────────────────────────────────
|
||||
|
||||
rootfs_path() {
|
||||
local win_home; win_home=$(wslpath "$("$WSL_EXE" -d "$(wsl_list | head -1)" -e printf '%s' "$USERPROFILE" 2>/dev/null || true)" 2>/dev/null || true)
|
||||
# Simpler and reliable: use the current user's Windows home via /mnt/c.
|
||||
ls -d /mnt/c/Users/*/ 2>/dev/null | grep -viE '/(All Users|Default|Default User|Public)/$' | head -1
|
||||
}
|
||||
|
||||
build_rootfs() {
|
||||
local tar="$1"
|
||||
if [ -f "$tar" ]; then
|
||||
echo " rootfs cached: $(basename "$tar")"
|
||||
return
|
||||
fi
|
||||
echo " exporting a stock Debian rootfs (cached for next time)"
|
||||
local cid; cid=$(docker create debian:trixie-slim)
|
||||
docker export "$cid" > "$tar"
|
||||
docker rm -f "$cid" >/dev/null
|
||||
}
|
||||
|
||||
provision() {
|
||||
echo " provisioning (root)"
|
||||
local hosts_block
|
||||
hosts_block=$(CLUSTER="$CLUSTER" HTTP_PORT="$HTTP_PORT" \
|
||||
envsubst < ./hosts.tmpl 2>/dev/null || sed "s/\${CLUSTER}/$CLUSTER/g" ./hosts.tmpl)
|
||||
|
||||
# Piped as stdin rather than a second script file, the same shape as any
|
||||
# remote provisioning heredoc. Everything here is idempotent so a failed run
|
||||
# can simply be repeated.
|
||||
"$WSL_EXE" -d "$BOX" -u root -- bash -s <<PROVISION
|
||||
set -euo pipefail
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq ca-certificates curl gnupg sudo >/dev/null
|
||||
|
||||
if [ "$REUSE_DOCKER" = "1" ]; then
|
||||
# Borrow the host distro's daemon: CLI only, no dockerd, nothing to
|
||||
# conflict with. The GID must match the owner's or the shared socket is
|
||||
# unreadable here even though it is visible.
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
if [ ! -f /etc/apt/keyrings/docker.asc ]; then
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
fi
|
||||
echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \$(. /etc/os-release && echo \$VERSION_CODENAME) stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq docker-ce-cli >/dev/null
|
||||
|
||||
echo "export DOCKER_HOST=unix://$SHARED_SOCK" > /etc/profile.d/rig-docker-host.sh
|
||||
|
||||
if [ -f /mnt/wsl/shared-docker/OWNER ]; then
|
||||
gid=\$(awk '/docker gid:/ {print \$3}' /mnt/wsl/shared-docker/OWNER)
|
||||
if [ -n "\$gid" ]; then
|
||||
getent group docker >/dev/null && groupmod -g "\$gid" docker || groupadd -g "\$gid" docker
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# A second daemon. Only when deliberately testing a from-scratch install.
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
if [ ! -f /etc/apt/keyrings/docker.asc ]; then
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
fi
|
||||
echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \$(. /etc/os-release && echo \$VERSION_CODENAME) stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
|
||||
fi
|
||||
|
||||
id -u "$BOX_USER" >/dev/null 2>&1 || useradd -m -s /bin/bash "$BOX_USER"
|
||||
usermod -aG sudo,docker "$BOX_USER"
|
||||
echo "$BOX_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-$BOX_USER
|
||||
chmod 0440 /etc/sudoers.d/90-$BOX_USER
|
||||
|
||||
# systemd is off by default in WSL, and Docker needs it. Takes effect on the
|
||||
# next start of this distro, which is why create() terminates it below.
|
||||
cat > /etc/wsl.conf <<WSLCONF
|
||||
[boot]
|
||||
systemd=true
|
||||
|
||||
[user]
|
||||
default=$BOX_USER
|
||||
WSLCONF
|
||||
|
||||
# The default inotify limits are low enough that file watching silently stops
|
||||
# working — no error, changes just stop being noticed. Fix it before it bites.
|
||||
cat > /etc/sysctl.d/99-rig.conf <<SYSCTL
|
||||
fs.inotify.max_user_watches=524288
|
||||
fs.inotify.max_user_instances=512
|
||||
SYSCTL
|
||||
|
||||
if ! grep -q 'rig environment' /etc/hosts 2>/dev/null; then
|
||||
{ echo ""; echo "# rig environment"; cat <<'HOSTS'
|
||||
$hosts_block
|
||||
HOSTS
|
||||
} >> /etc/hosts
|
||||
fi
|
||||
|
||||
touch /etc/rig-provisioned
|
||||
PROVISION
|
||||
}
|
||||
|
||||
create() {
|
||||
require_wsl
|
||||
guard_name
|
||||
|
||||
local winhome; winhome=$(rootfs_path)
|
||||
[ -n "$winhome" ] || { echo "could not locate the Windows user directory" >&2; exit 1; }
|
||||
local tar="${winhome}rig-rootfs.tar"
|
||||
local installdir="${winhome}WSL/${BOX}"
|
||||
|
||||
echo "creating '$BOX'"
|
||||
if [ "$REUSE_DOCKER" = "1" ]; then
|
||||
echo " docker: borrowing the host distro's daemon (nothing installed)"
|
||||
if [ ! -S "$SHARED_SOCK" ]; then
|
||||
echo
|
||||
echo " No shared socket yet. In the distro that owns Docker, run once:"
|
||||
echo " sudo bash ctrl/dockerhost.sh share"
|
||||
echo " That adds one systemd drop-in and nothing else; undo with 'unshare'."
|
||||
echo " Continuing — the box will be created, but Docker won't work in it"
|
||||
echo " until you do that."
|
||||
fi
|
||||
else
|
||||
echo
|
||||
echo " REUSE_DOCKER=0: installing a SECOND Docker daemon."
|
||||
echo " WSL distros share a network stack, so this can disturb Docker in"
|
||||
echo " the distro you work in. Ctrl-C now if that is a bad trade today."
|
||||
echo
|
||||
sleep 4
|
||||
fi
|
||||
echo
|
||||
|
||||
if box_exists; then
|
||||
echo " distro already registered"
|
||||
else
|
||||
build_rootfs "$tar"
|
||||
mkdir -p "$installdir"
|
||||
"$WSL_EXE" --import "$BOX" "$(wslpath -w "$installdir")" "$(wslpath -w "$tar")" --version 2
|
||||
fi
|
||||
|
||||
# Resumable: a partially-created box is finished rather than restarted.
|
||||
if "$WSL_EXE" -d "$BOX" -u root -- test -f /etc/rig-provisioned 2>/dev/null; then
|
||||
echo " already provisioned"
|
||||
else
|
||||
provision
|
||||
echo " restarting the distro so systemd and group membership apply"
|
||||
"$WSL_EXE" --terminate "$BOX" # ONLY this distro; never --shutdown
|
||||
fi
|
||||
|
||||
echo " copying rig in"
|
||||
tar c -C "$REPO" --exclude=def --exclude=.git --exclude=ctrl/.env . \
|
||||
| "$WSL_EXE" -d "$BOX" -u "$BOX_USER" -- bash -lc "mkdir -p ~/rig && tar x -C ~/rig"
|
||||
|
||||
echo
|
||||
echo " docker: $("$WSL_EXE" -d "$BOX" -u "$BOX_USER" -- bash -lc 'systemctl is-active docker 2>/dev/null || echo inactive')"
|
||||
echo
|
||||
echo "next:"
|
||||
echo " make newbox shell # a shell inside it"
|
||||
echo " then: cd ~/rig && make check && make deps && make cluster up"
|
||||
echo
|
||||
echo "For a browser on Windows to resolve the hostnames, paste this into"
|
||||
echo "C:\\Windows\\System32\\drivers\\etc\\hosts (it has no wildcard support):"
|
||||
CLUSTER="$CLUSTER" envsubst < ./hosts.tmpl 2>/dev/null | grep -v '^#' | grep -v '^$' | sed 's/^/ /'
|
||||
}
|
||||
|
||||
# ── the rest ───────────────────────────────────────────────────────────────
|
||||
|
||||
destroy() {
|
||||
require_wsl
|
||||
guard_name
|
||||
|
||||
if ! box_exists; then
|
||||
echo "no distro '$BOX' to remove"
|
||||
else
|
||||
echo "about to PERMANENTLY delete the distro '$BOX' and its filesystem."
|
||||
"$WSL_EXE" --terminate "$BOX" 2>/dev/null || true
|
||||
"$WSL_EXE" --unregister "$BOX"
|
||||
echo " unregistered"
|
||||
fi
|
||||
|
||||
local winhome; winhome=$(rootfs_path)
|
||||
rm -rf "${winhome}WSL/${BOX}" 2>/dev/null || true
|
||||
|
||||
if [ "${1:-}" = "--purge" ]; then
|
||||
rm -f "${winhome}rig-rootfs.tar"
|
||||
echo " cached rootfs removed"
|
||||
fi
|
||||
}
|
||||
|
||||
status() {
|
||||
require_wsl
|
||||
echo "environment $CLUSTER"
|
||||
echo "distro $BOX"
|
||||
if box_exists; then
|
||||
echo "registered yes"
|
||||
echo "provisioned $("$WSL_EXE" -d "$BOX" -u root -- test -f /etc/rig-provisioned 2>/dev/null && echo yes || echo no)"
|
||||
echo "docker $("$WSL_EXE" -d "$BOX" -u root -- bash -lc 'systemctl is-active docker 2>/dev/null' || echo unknown)"
|
||||
echo "rig copied $("$WSL_EXE" -d "$BOX" -u "$BOX_USER" -- bash -lc 'test -f ~/rig/Makefile && echo yes || echo no' 2>/dev/null)"
|
||||
else
|
||||
echo "registered no"
|
||||
fi
|
||||
echo
|
||||
echo "all distros (this one is never touched unless it is '$BOX'):"
|
||||
wsl_list | sed 's/^/ /'
|
||||
}
|
||||
|
||||
shell() {
|
||||
require_wsl
|
||||
box_exists || { echo "no distro '$BOX' — run 'make newbox' first" >&2; exit 1; }
|
||||
"$WSL_EXE" -d "$BOX" -u "$BOX_USER" --cd '~'
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
create) create ;;
|
||||
destroy) shift; destroy "${1:-}" ;;
|
||||
status) status ;;
|
||||
shell) shell ;;
|
||||
*) echo "usage: $0 [create|destroy [--purge]|status|shell]" >&2; exit 1 ;;
|
||||
esac
|
||||
57
rig/ctrl/pins.sh
Executable file
57
rig/ctrl/pins.sh
Executable file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# Do the standalone scripts still install what rig pins?
|
||||
#
|
||||
# standalone/rigdeps.sh carries its toolchain pins inline, because it exists for
|
||||
# a machine that will never have ctrl/versions.env. That makes two copies of the
|
||||
# same versions and checksums, and two copies drift the day one is edited and
|
||||
# the other forgotten. This is the check that notices.
|
||||
#
|
||||
# ctrl/versions.env is the source of truth. Only the keys rigdeps.sh itself
|
||||
# defines are compared: versions.env also pins addon images (cert-manager,
|
||||
# metallb, metrics-server) that rigdeps.sh never installs, and demanding those
|
||||
# would make this fail forever for no reason.
|
||||
#
|
||||
# Exits non-zero on any mismatch — unlike the host checks, this one is a test.
|
||||
#
|
||||
# Usage: pins.sh
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
SOURCE=./versions.env
|
||||
COPY=../standalone/rigdeps.sh
|
||||
|
||||
[ -r "$COPY" ] || { echo "no $COPY to compare" >&2; exit 1; }
|
||||
|
||||
# KEY=value for the pin keys a file defines, quotes stripped. awk rather than a
|
||||
# grep regex, which is not the same program everywhere.
|
||||
pins() {
|
||||
awk -F= '/^[A-Z_]+_(VERSION|SHA256)=/ {
|
||||
v = substr($0, index($0, "=") + 1); gsub(/^["\x27]|["\x27]$/, "", v)
|
||||
print $1 "=" v }' "$1"
|
||||
}
|
||||
|
||||
echo "pins: standalone/rigdeps.sh against ctrl/versions.env"
|
||||
bad=0
|
||||
while IFS='=' read -r key copy_val; do
|
||||
[ -n "$key" ] || continue
|
||||
src_val=$(pins "$SOURCE" | sed -n "s/^${key}=//p" | head -1)
|
||||
if [ -z "$src_val" ]; then
|
||||
printf " ! %-16s in rigdeps.sh but not in versions.env\n" "$key"
|
||||
bad=1
|
||||
elif [ "$src_val" = "$copy_val" ]; then
|
||||
printf " %-16s %s\n" "$key" "$( [ ${#src_val} -gt 20 ] && echo "${src_val:0:12}…" || echo "$src_val" )"
|
||||
else
|
||||
printf " ! %-16s versions.env %s\n" "$key" "$src_val"
|
||||
printf " %-16s rigdeps.sh %s\n" "" "$copy_val"
|
||||
bad=1
|
||||
fi
|
||||
done < <(pins "$COPY")
|
||||
|
||||
echo
|
||||
if [ "$bad" -eq 0 ]; then
|
||||
echo "in step — rigdeps.sh installs exactly what rig pins."
|
||||
else
|
||||
echo "DRIFT. versions.env is the source of truth: copy the differing lines from it"
|
||||
echo "into standalone/rigdeps.sh, taking checksums from the publisher's release list."
|
||||
exit 1
|
||||
fi
|
||||
103
rig/ctrl/ports.sh
Executable file
103
rig/ctrl/ports.sh
Executable file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
# Give each environment its own block of host ports.
|
||||
#
|
||||
# New versions of a system mean new clusters on ONE machine, not new machines.
|
||||
# Cluster name, kubectl context, registry container and image tag already derive
|
||||
# from the directory name, so two copies never collide there — but host ports are
|
||||
# a single shared namespace and would.
|
||||
#
|
||||
# The block is derived from the directory name: stateless, stable, and requiring
|
||||
# no coordination between copies that know nothing about each other.
|
||||
#
|
||||
# base = 20000 + (hash(slug) % 200) * 10
|
||||
# +0 HTTP +1 HTTPS +2 TILT +3 REGISTRY (+4..9 reserved)
|
||||
#
|
||||
# 20000+ deliberately avoids the ports something is already likely to hold: 80,
|
||||
# 443, 3000, 5432, 8000, 8080.
|
||||
#
|
||||
# Derivation is a default, not a decision. On first use the resolved block is
|
||||
# written into ctrl/.env, so it becomes pinned, visible and editable rather than
|
||||
# a number that appears from nowhere. Anything already in ctrl/.env wins.
|
||||
#
|
||||
# Usage: ports.sh show | derive | persist
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
|
||||
# derive_port_base lives in lib/config.sh so every script resolves the same block
|
||||
# without going through this one.
|
||||
derive_base() { derive_port_base "$1"; }
|
||||
|
||||
derive() {
|
||||
load_config
|
||||
local base; base=$(derive_base "$CLUSTER")
|
||||
DERIVED_HTTP=$base
|
||||
DERIVED_HTTPS=$((base + 1))
|
||||
DERIVED_TILT=$((base + 2))
|
||||
DERIVED_REGISTRY=$((base + 3))
|
||||
}
|
||||
|
||||
show() {
|
||||
derive
|
||||
echo "environment $CLUSTER"
|
||||
echo "derived base $(derive_base "$CLUSTER")"
|
||||
echo
|
||||
printf " %-14s %-8s %-8s %s\n" KEY DERIVED ACTIVE SOURCE
|
||||
_row HTTP_PORT "$DERIVED_HTTP"
|
||||
_row HTTPS_PORT "$DERIVED_HTTPS"
|
||||
_row TILT_PORT "$DERIVED_TILT"
|
||||
_row REGISTRY_PORT "$DERIVED_REGISTRY"
|
||||
}
|
||||
|
||||
_row() {
|
||||
local key="$1" derived="$2" active="${!1:-}" src="derived"
|
||||
if [ -n "$active" ] && [ "$active" != "$derived" ]; then
|
||||
src="override"
|
||||
elif [ -z "$active" ]; then
|
||||
active="$derived"
|
||||
fi
|
||||
printf " %-14s %-8s %-8s %s\n" "$key" "$derived" "$active" "$src"
|
||||
}
|
||||
|
||||
# Write the derived block into ctrl/.env, once. Existing keys are never
|
||||
# rewritten — an override stays an override.
|
||||
persist() {
|
||||
derive
|
||||
[ -f ./.env ] || cp ./.env.example ./.env
|
||||
|
||||
local wrote=0 key val
|
||||
for key in HTTP_PORT:$DERIVED_HTTP \
|
||||
HTTPS_PORT:$DERIVED_HTTPS \
|
||||
TILT_PORT:$DERIVED_TILT \
|
||||
REGISTRY_PORT:$DERIVED_REGISTRY; do
|
||||
val="${key#*:}"; key="${key%%:*}"
|
||||
if grep -qE "^${key}=[0-9]" ./.env 2>/dev/null; then
|
||||
continue
|
||||
fi
|
||||
if [ "$wrote" -eq 0 ]; then
|
||||
{
|
||||
echo ""
|
||||
echo "# Port block for this environment, derived from the directory name"
|
||||
echo "# so copies never collide. Pinned here on first use — edit freely."
|
||||
} >> ./.env
|
||||
wrote=1
|
||||
fi
|
||||
# Replace a commented/empty placeholder if present, else append.
|
||||
if grep -qE "^#?\s*${key}=" ./.env 2>/dev/null; then
|
||||
sed -i "s|^#\?\s*${key}=.*|${key}=${val}|" ./.env
|
||||
else
|
||||
echo "${key}=${val}" >> ./.env
|
||||
fi
|
||||
done
|
||||
|
||||
[ "$wrote" -eq 1 ] && echo "pinned port block into ctrl/.env" || echo "ports already set in ctrl/.env"
|
||||
return 0
|
||||
}
|
||||
|
||||
case "${1:-show}" in
|
||||
show) show ;;
|
||||
derive) derive; echo "$DERIVED_HTTP $DERIVED_HTTPS $DERIVED_TILT $DERIVED_REGISTRY" ;;
|
||||
persist) persist ;;
|
||||
*) echo "usage: $0 [show|derive|persist]" >&2; exit 1 ;;
|
||||
esac
|
||||
216
rig/ctrl/registry.sh
Executable file
216
rig/ctrl/registry.sh
Executable file
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env bash
|
||||
# Registry plumbing. THIS is the seam — not a tool.
|
||||
#
|
||||
# Four modes, selected by REGISTRY_MODE in the active profile:
|
||||
#
|
||||
# none Tilt builds straight into the node. No registry at all — and so no
|
||||
# guard against an outward push: an unqualified image name means
|
||||
# docker.io/library/<name>, and only Tilt's kind detection stands
|
||||
# between that and a real push. Throwaway use only; every profile
|
||||
# here now defaults to `local` instead.
|
||||
# local a registry:2 container wired into the cluster.
|
||||
# mirror the same container, but configured as a pull-through CACHE of the
|
||||
# corporate registry. What a locked-down client actually looks like:
|
||||
# images originate from corp, you don't hammer it, and you keep
|
||||
# working when the VPN drops.
|
||||
# remote no local container; pull straight from the corporate registry using
|
||||
# an imagePullSecret.
|
||||
#
|
||||
# Deliberately a script rather than a tool. ctlptl collapses the `local` wiring
|
||||
# into one line, but its Registry spec only accepts name/port/image/listenAddress
|
||||
# — there is no way to set REGISTRY_PROXY_REMOTEURL, so it cannot express
|
||||
# `mirror` at all. Keeping the seam here is what keeps the corporate registry
|
||||
# swappable.
|
||||
#
|
||||
# Usage: registry.sh up | down | status
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
REG_NAME="${CLUSTER}-registry"
|
||||
REG_PORT="${REGISTRY_PORT:-5005}"
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
|
||||
# ── CA trust ───────────────────────────────────────────────────────────────
|
||||
# A corporate registry is almost always fronted by an internal CA, and trust has
|
||||
# to reach three separate places. Nothing does this for you, and the symptom when
|
||||
# it's missing is an opaque:
|
||||
# x509: certificate signed by unknown authority
|
||||
#
|
||||
# 1. the host docker daemon — /etc/docker/certs.d/<host>/ca.crt (needs root)
|
||||
# 2. every kind node's containerd — nodes do NOT inherit host trust
|
||||
# 3. anything doing HTTPS from inside the cluster, in its own trust store
|
||||
#
|
||||
# We handle (2) here because it's ours to handle. (1) is reported by check.sh
|
||||
# since it needs root. (3) belongs to the workload.
|
||||
install_ca_into_nodes() {
|
||||
[ -n "${REGISTRY_CA_FILE:-}" ] || return 0
|
||||
|
||||
if [ ! -r "$REGISTRY_CA_FILE" ]; then
|
||||
echo "REGISTRY_CA_FILE is set but not readable: $REGISTRY_CA_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " distributing CA to kind nodes"
|
||||
local node
|
||||
for node in $(kind get nodes --name "$CLUSTER"); do
|
||||
docker cp "$REGISTRY_CA_FILE" "$node:/usr/local/share/ca-certificates/corp-registry.crt"
|
||||
docker exec "$node" update-ca-certificates >/dev/null 2>&1
|
||||
docker exec "$node" systemctl restart containerd
|
||||
done
|
||||
}
|
||||
|
||||
# Point containerd at a registry host. The cluster config already set
|
||||
# config_path=/etc/containerd/certs.d, so this is a per-node drop-in and needs no
|
||||
# cluster recreate — which is what lets registry mode change on a live cluster.
|
||||
write_hosts_toml() {
|
||||
local host="$1" upstream="$2" skip_verify="${3:-false}"
|
||||
local node
|
||||
for node in $(kind get nodes --name "$CLUSTER"); do
|
||||
docker exec "$node" mkdir -p "/etc/containerd/certs.d/${host}"
|
||||
docker exec -i "$node" cp /dev/stdin "/etc/containerd/certs.d/${host}/hosts.toml" <<TOML
|
||||
server = "${upstream}"
|
||||
|
||||
[host."${upstream}"]
|
||||
capabilities = ["pull", "resolve"]
|
||||
skip_verify = ${skip_verify}
|
||||
TOML
|
||||
done
|
||||
}
|
||||
|
||||
# ── the local container (local + mirror) ───────────────────────────────────
|
||||
|
||||
start_registry_container() {
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$REG_NAME" 2>/dev/null || true)" = "true" ]; then
|
||||
echo " registry container '$REG_NAME' already running"
|
||||
return
|
||||
fi
|
||||
docker rm -f "$REG_NAME" >/dev/null 2>&1 || true
|
||||
|
||||
local args=(-d --restart=always --name "$REG_NAME"
|
||||
-p "127.0.0.1:${REG_PORT}:5000")
|
||||
|
||||
if [ "$REGISTRY_MODE" = "mirror" ]; then
|
||||
if [ -z "${REGISTRY_REMOTE_URL:-}" ]; then
|
||||
echo "REGISTRY_MODE=mirror needs REGISTRY_REMOTE_URL in ctrl/.env" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " starting pull-through cache of ${REGISTRY_REMOTE_URL}"
|
||||
args+=(-e "REGISTRY_PROXY_REMOTEURL=${REGISTRY_REMOTE_URL}")
|
||||
[ -n "${REGISTRY_USER:-}" ] && args+=(-e "REGISTRY_PROXY_USERNAME=${REGISTRY_USER}")
|
||||
[ -n "${REGISTRY_PASSWORD:-}" ] && args+=(-e "REGISTRY_PROXY_PASSWORD=${REGISTRY_PASSWORD}")
|
||||
if [ -n "${REGISTRY_CA_FILE:-}" ]; then
|
||||
args+=(-v "$(readlink -f "$REGISTRY_CA_FILE"):/etc/ssl/certs/corp-ca.crt:ro")
|
||||
fi
|
||||
else
|
||||
echo " starting local registry"
|
||||
fi
|
||||
|
||||
docker run "${args[@]}" "$REGISTRY_IMAGE" >/dev/null
|
||||
}
|
||||
|
||||
# The registry must share a network with the nodes so they can resolve it by
|
||||
# container name; localhost inside a node is the node, not the host.
|
||||
join_kind_network() {
|
||||
if docker inspect -f '{{json .NetworkSettings.Networks}}' "$REG_NAME" | grep -q '"kind"'; then
|
||||
return
|
||||
fi
|
||||
docker network connect kind "$REG_NAME" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# The documented contract that tells tooling (Tilt, skaffold) where the local
|
||||
# registry is, so they don't have to be configured separately.
|
||||
apply_hosting_configmap() {
|
||||
$K apply -f - <<YAML >/dev/null
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: local-registry-hosting
|
||||
namespace: kube-public
|
||||
data:
|
||||
localRegistryHosting.v1: |
|
||||
host: "localhost:${REG_PORT}"
|
||||
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
|
||||
YAML
|
||||
}
|
||||
|
||||
# ── modes ──────────────────────────────────────────────────────────────────
|
||||
|
||||
up() {
|
||||
echo "registry: ${REGISTRY_MODE}"
|
||||
case "$REGISTRY_MODE" in
|
||||
none)
|
||||
echo " no registry — images are built straight into the node"
|
||||
;;
|
||||
|
||||
local|mirror)
|
||||
start_registry_container
|
||||
join_kind_network
|
||||
install_ca_into_nodes
|
||||
# Nodes reach the registry by container name on the shared network;
|
||||
# the host reaches it on localhost:PORT. Both names must resolve.
|
||||
write_hosts_toml "localhost:${REG_PORT}" "http://${REG_NAME}:5000"
|
||||
if [ "$REGISTRY_MODE" = "mirror" ]; then
|
||||
# Anything asking for docker.io transparently goes to the cache.
|
||||
write_hosts_toml "docker.io" "http://${REG_NAME}:5000"
|
||||
fi
|
||||
apply_hosting_configmap
|
||||
echo " ready at localhost:${REG_PORT}"
|
||||
;;
|
||||
|
||||
remote)
|
||||
if [ -z "${REGISTRY_REMOTE_URL:-}" ]; then
|
||||
echo "REGISTRY_MODE=remote needs REGISTRY_REMOTE_URL in ctrl/.env" >&2
|
||||
exit 1
|
||||
fi
|
||||
install_ca_into_nodes
|
||||
local host="${REGISTRY_REMOTE_URL#*://}"; host="${host%%/*}"
|
||||
if [ -n "${REGISTRY_USER:-}" ]; then
|
||||
echo " creating imagePullSecret for ${host}"
|
||||
$K create secret docker-registry regcred \
|
||||
--docker-server="$host" \
|
||||
--docker-username="$REGISTRY_USER" \
|
||||
--docker-password="$REGISTRY_PASSWORD" \
|
||||
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
||||
# Attach to the default ServiceAccount so plain pods inherit it.
|
||||
$K patch serviceaccount default \
|
||||
-p '{"imagePullSecrets":[{"name":"regcred"}]}' >/dev/null
|
||||
fi
|
||||
echo " pulling directly from ${host}"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "unknown REGISTRY_MODE '$REGISTRY_MODE' (expected none|local|mirror|remote)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
down() {
|
||||
if docker inspect "$REG_NAME" >/dev/null 2>&1; then
|
||||
echo "removing registry container '$REG_NAME'"
|
||||
docker rm -f "$REG_NAME" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
status() {
|
||||
echo "mode ${REGISTRY_MODE}"
|
||||
if docker inspect "$REG_NAME" >/dev/null 2>&1; then
|
||||
echo "container ${REG_NAME} $(docker inspect -f '{{.State.Status}}' "$REG_NAME")"
|
||||
echo "endpoint localhost:${REG_PORT}"
|
||||
else
|
||||
echo "container none"
|
||||
fi
|
||||
[ -n "${REGISTRY_REMOTE_URL:-}" ] && echo "upstream ${REGISTRY_REMOTE_URL}"
|
||||
[ -n "${REGISTRY_CA_FILE:-}" ] && echo "ca ${REGISTRY_CA_FILE}"
|
||||
return 0
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
up) up ;;
|
||||
down) down ;;
|
||||
status) status ;;
|
||||
*) echo "usage: $0 [up|down|status]" >&2; exit 1 ;;
|
||||
esac
|
||||
249
rig/ctrl/setup.sh
Executable file
249
rig/ctrl/setup.sh
Executable file
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prepare a machine to run rig, and say plainly what worked, what was already
|
||||
# done, and what is left for a human.
|
||||
#
|
||||
# This is the grouped entry point: `make setup`. Every step is idempotent and
|
||||
# independently checked, so running it twice is safe and running it on a
|
||||
# half-configured machine finishes the job rather than starting over.
|
||||
#
|
||||
# It deliberately does NOT abort on the first failure. A setup script that dies
|
||||
# at step 2 hides the fact that steps 4 and 5 were also going to fail — and on
|
||||
# an unfamiliar machine, the full picture is the whole point. Failures are
|
||||
# collected and reported together, and the exit code reflects the worst outcome.
|
||||
#
|
||||
# The same script runs inside a fresh throwaway distro (newbox), so the
|
||||
# provisioning path and the everyday path cannot drift apart.
|
||||
#
|
||||
# Usage:
|
||||
# setup.sh # host checks + the dev toolchain
|
||||
# setup.sh core # kubectl and jq only — no cluster tooling
|
||||
# setup.sh --share-docker # ...and offer this distro's Docker to others
|
||||
# setup.sh --cluster # ...and bring the cluster up
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
WITH_SHARE=0
|
||||
WITH_CLUSTER=0
|
||||
# Cluster tooling is not wanted everywhere: a managed or corporate-issued
|
||||
# machine may legitimately want kubectl and nothing that builds clusters.
|
||||
TIER=dev
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
core|dev) TIER="$a" ;;
|
||||
--share-docker) WITH_SHARE=1 ;;
|
||||
--cluster) WITH_CLUSTER=1 ;;
|
||||
*) echo "unknown option: $a" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$TIER" = "core" ] && [ "$WITH_CLUSTER" -eq 1 ]; then
|
||||
echo "core tier installs no cluster tooling, so --cluster cannot work" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── step framework ─────────────────────────────────────────────────────────
|
||||
# Statuses are deliberately distinct: "already" and "done" both mean success but
|
||||
# tell you very different things about the machine you are on.
|
||||
STEP_NAMES=()
|
||||
STEP_STATUS=()
|
||||
STEP_NOTE=()
|
||||
WORST=0
|
||||
|
||||
record() {
|
||||
STEP_NAMES+=("$1"); STEP_STATUS+=("$2"); STEP_NOTE+=("${3:-}")
|
||||
# Only a genuine failure is a non-zero exit. "manual" means the machine is
|
||||
# fine and you have something to do — reporting that as an error makes the
|
||||
# whole run look broken and trains people to ignore the output.
|
||||
[ "$2" = "fail" ] && WORST=1 || true
|
||||
local mark
|
||||
case "$2" in
|
||||
already) mark=" ok " ;;
|
||||
done) mark=" done " ;;
|
||||
skip) mark=" skip " ;;
|
||||
manual) mark="MANUAL" ;;
|
||||
fail) mark=" FAIL " ;;
|
||||
esac
|
||||
printf "[%s] %-22s %s\n" "$mark" "$1" "${3:-}"
|
||||
}
|
||||
|
||||
# ── steps ──────────────────────────────────────────────────────────────────
|
||||
|
||||
step_host() {
|
||||
local out
|
||||
if ! out=$(bash ./deps.sh detect 2>&1); then
|
||||
record host fail "detection failed"
|
||||
return
|
||||
fi
|
||||
# Anything flagged with '!' needs a human; surface the count here
|
||||
# and the detail below rather than burying it.
|
||||
local warns; warns=$(echo "$out" | grep -c '^\s*!' || true)
|
||||
HOST_DETAIL="$out"
|
||||
if [ "$warns" -gt 0 ]; then
|
||||
record host manual "$warns item(s) need attention — see below"
|
||||
else
|
||||
record host already "no problems detected"
|
||||
fi
|
||||
}
|
||||
|
||||
step_toolchain() {
|
||||
local want="kubectl jq"
|
||||
[ "$TIER" = "dev" ] && want="$want kind tilt"
|
||||
|
||||
local missing=""
|
||||
for b in $want; do
|
||||
command -v "$b" >/dev/null 2>&1 || missing="$missing $b"
|
||||
done
|
||||
|
||||
if [ -z "$missing" ]; then
|
||||
record toolchain already "$TIER: $want"
|
||||
return
|
||||
fi
|
||||
|
||||
if bash ./deps.sh install "$TIER" >/tmp/rig-deps.$$ 2>&1; then
|
||||
local still=""
|
||||
for b in $want; do
|
||||
[ -x "${OUT_BIN:-$HOME/.local/bin}/$b" ] || still="$still $b"
|
||||
done
|
||||
if [ -n "$still" ]; then
|
||||
record toolchain fail "still missing:$still (see /tmp/rig-deps.$$)"
|
||||
else
|
||||
record toolchain done "$TIER, installed:$missing"
|
||||
rm -f "/tmp/rig-deps.$$"
|
||||
fi
|
||||
else
|
||||
record toolchain fail "install failed — see /tmp/rig-deps.$$"
|
||||
fi
|
||||
}
|
||||
|
||||
step_path() {
|
||||
local bin="${OUT_BIN:-$HOME/.local/bin}"
|
||||
case ":$PATH:" in
|
||||
*":$bin:"*) ;;
|
||||
*) record path manual "add to ~/.bashrc: export PATH=\"$bin:\$PATH\""; return ;;
|
||||
esac
|
||||
if grep -qs "$bin" "$HOME/.bashrc" "$HOME/.profile" 2>/dev/null; then
|
||||
record path already "$bin on PATH and persisted"
|
||||
else
|
||||
record path manual "on PATH now, but not persisted in ~/.bashrc"
|
||||
fi
|
||||
}
|
||||
|
||||
step_docker() {
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
record docker fail "no docker cli — this is the one prerequisite rig cannot install"
|
||||
return
|
||||
fi
|
||||
if docker info >/dev/null 2>&1; then
|
||||
record docker already "$(docker version --format '{{.Server.Version}}' 2>/dev/null)"
|
||||
else
|
||||
record docker fail "daemon unreachable (in the docker group? logged out and back in?)"
|
||||
fi
|
||||
}
|
||||
|
||||
step_share_docker() {
|
||||
if [ "$WITH_SHARE" -ne 1 ]; then
|
||||
record docker-share skip "not requested (--share-docker)"
|
||||
return
|
||||
fi
|
||||
if ! grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
record docker-share skip "not WSL — sharing only applies between WSL distros"
|
||||
return
|
||||
fi
|
||||
if [ -f /etc/systemd/system/docker.service.d/10-rig-shared-socket.conf ]; then
|
||||
record docker-share already "this distro is offering its Docker to others"
|
||||
return
|
||||
fi
|
||||
# Needs root, and asking mid-script is worse than telling the user the
|
||||
# single command to run.
|
||||
if [ "$(id -u)" -ne 0 ] && ! sudo -n true 2>/dev/null; then
|
||||
record docker-share manual "run: sudo bash ctrl/dockerhost.sh share"
|
||||
return
|
||||
fi
|
||||
if sudo bash ./dockerhost.sh share >/tmp/rig-share.$$ 2>&1; then
|
||||
record docker-share done "this distro now owns the shared Docker"
|
||||
rm -f "/tmp/rig-share.$$"
|
||||
else
|
||||
record docker-share fail "see /tmp/rig-share.$$"
|
||||
fi
|
||||
}
|
||||
|
||||
step_ports() {
|
||||
local busy=""
|
||||
for entry in "HTTP:$HTTP_PORT" "HTTPS:$HTTPS_PORT" "TILT:$TILT_PORT" "REGISTRY:$REGISTRY_PORT"; do
|
||||
local p="${entry#*:}"
|
||||
if command -v ss >/dev/null 2>&1 && ss -ltn "sport = :$p" 2>/dev/null | grep -q LISTEN; then
|
||||
busy="$busy ${entry%%:*}($p)"
|
||||
fi
|
||||
done
|
||||
if [ -n "$busy" ]; then
|
||||
record ports fail "in use:$busy — override in ctrl/.env or rename the directory"
|
||||
else
|
||||
record ports already "$HTTP_PORT-$REGISTRY_PORT free"
|
||||
fi
|
||||
}
|
||||
|
||||
step_cluster() {
|
||||
if [ "$TIER" = "core" ]; then
|
||||
record cluster skip "core tier — no cluster tooling on this machine"
|
||||
return
|
||||
fi
|
||||
if [ "$WITH_CLUSTER" -ne 1 ]; then
|
||||
record cluster skip "not requested (--cluster)"
|
||||
return
|
||||
fi
|
||||
if kind get clusters 2>/dev/null | grep -qx "$CLUSTER"; then
|
||||
record cluster already "'$CLUSTER' exists"
|
||||
return
|
||||
fi
|
||||
if bash ./cluster.sh up >/tmp/rig-cluster.$$ 2>&1; then
|
||||
record cluster done "'$CLUSTER' created"
|
||||
rm -f "/tmp/rig-cluster.$$"
|
||||
else
|
||||
record cluster fail "see /tmp/rig-cluster.$$"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── run ────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "setting up '$CLUSTER'"
|
||||
echo
|
||||
HOST_DETAIL=""
|
||||
step_host
|
||||
step_toolchain
|
||||
step_path
|
||||
step_docker
|
||||
step_share_docker
|
||||
step_ports
|
||||
step_cluster
|
||||
|
||||
echo
|
||||
if [ -n "$HOST_DETAIL" ]; then
|
||||
echo "host detail"
|
||||
echo "$HOST_DETAIL" | sed 's/^/ /'
|
||||
echo
|
||||
fi
|
||||
|
||||
# Repeat only what still needs action, so the tail of the output is a to-do list
|
||||
# rather than a transcript.
|
||||
outstanding=0
|
||||
for i in "${!STEP_NAMES[@]}"; do
|
||||
case "${STEP_STATUS[$i]}" in
|
||||
fail|manual)
|
||||
[ "$outstanding" -eq 0 ] && echo "outstanding:"
|
||||
outstanding=1
|
||||
printf " %-8s %-16s %s\n" "${STEP_STATUS[$i]}" "${STEP_NAMES[$i]}" "${STEP_NOTE[$i]}"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$outstanding" -eq 0 ]; then
|
||||
echo "ready. next: make cluster up && make docs"
|
||||
else
|
||||
echo
|
||||
echo "(nothing was aborted — every step ran so the list above is complete)"
|
||||
fi
|
||||
|
||||
exit "$WORST"
|
||||
73
rig/ctrl/versions.env
Normal file
73
rig/ctrl/versions.env
Normal file
@@ -0,0 +1,73 @@
|
||||
# Pinned toolchain — the single manifest ctrl/deps.sh installs from.
|
||||
# Every entry is a single binary; none of them needs an apt repo.
|
||||
# kubectl fully static
|
||||
# kind libc only
|
||||
# tilt libc + libstdc++ + libgcc (present in base Debian)
|
||||
# jq upstream static build (Debian's is linked against libjq/libonig)
|
||||
#
|
||||
# Checksums are the upstream-published SHA256 of the linux/amd64 artifact.
|
||||
#
|
||||
# To bump: change the version, then take the checksum from the release's own
|
||||
# published list — never hand-edit or hand-copy one from a download you did.
|
||||
# For anything hosted on GitHub releases that is:
|
||||
#
|
||||
# curl -sSL https://github.com/<org>/<repo>/releases/download/<tag>/checksums.txt \
|
||||
# | grep linux.x86_64
|
||||
#
|
||||
# (kubectl publishes its own instead: <KUBECTL_URL>.sha256.)
|
||||
#
|
||||
# There was a `ctrl/versions-refresh.sh` named here that has never existed. If
|
||||
# bumping stops being rare enough to do by hand, write it — but a comment
|
||||
# pointing at a missing script is worse than no comment.
|
||||
|
||||
KIND_VERSION=v0.32.0
|
||||
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||
KIND_URL=https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64
|
||||
|
||||
KUBECTL_VERSION=v1.36.3
|
||||
KUBECTL_SHA256=ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336
|
||||
KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
|
||||
|
||||
TILT_VERSION=0.37.6
|
||||
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
||||
TILT_URL=https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz
|
||||
|
||||
# ctlptl — creates a kind cluster WITH a local registry wired in, which is what
|
||||
# keeps images off docker.io (an unqualified name means docker.io/library/<name>).
|
||||
# Same publisher and same archive shape as tilt: binary at the archive root, so
|
||||
# fetch_tgz handles it with strip=0 and no special case.
|
||||
CTLPTL_VERSION=0.9.4
|
||||
CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e
|
||||
CTLPTL_URL=https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz
|
||||
|
||||
JQ_VERSION=1.8.2
|
||||
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
||||
JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64
|
||||
|
||||
# Node images shipped with KIND_VERSION above, pinned by digest so a kind upgrade
|
||||
# can never silently move the k8s version. Profiles select one via K8S_VERSION.
|
||||
# Older entries are kept deliberately: running a trailing-edge control plane is
|
||||
# part of simulating a legacy estate.
|
||||
NODE_IMAGE_v1_36=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
|
||||
NODE_IMAGE_v1_35=kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95
|
||||
NODE_IMAGE_v1_34=kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256
|
||||
NODE_IMAGE_v1_33=kindest/node:v1.33.12@sha256:3f5c8443c620245e4d355cfe09e96a91ead32ceaa569d3f1ca9edf0cb2fe2ff4
|
||||
|
||||
# Images pulled at runtime (registry, mocks). Pinned by tag; the registry mode
|
||||
# decides where they are pulled FROM.
|
||||
REGISTRY_IMAGE=registry:2
|
||||
STUB_IMAGE=python:3.12-slim
|
||||
|
||||
# Addons, installed by ctrl/addons/<name>.sh when listed in a profile's ADDONS.
|
||||
CERT_MANAGER_VERSION=v1.21.1
|
||||
METRICS_SERVER_VERSION=v0.9.0
|
||||
METALLB_VERSION=v0.16.0
|
||||
|
||||
# Cabinets — public services dropped in as-is, the upstream image unmodified.
|
||||
# The same declaration installs on compose or in the cluster, so a dependency is
|
||||
# named once and works either way. Pinned by tag rather than
|
||||
# digest because they are ordinary upstream images with no supply chain claim
|
||||
# attached — bump freely, and preload them for the offline profile.
|
||||
POSTGRES_IMAGE=postgres:16-alpine
|
||||
REDIS_IMAGE=redis:7-alpine
|
||||
AIRFLOW_IMAGE=apache/airflow:2.10.4
|
||||
47
rig/docs/graphs/01-install.dot
Normal file
47
rig/docs/graphs/01-install.dot
Normal file
@@ -0,0 +1,47 @@
|
||||
digraph rig_install {
|
||||
rankdir=LR
|
||||
bgcolor="#0a0e17"
|
||||
fontname="Helvetica"
|
||||
node [fontname="Helvetica" fontsize=11 style=filled color="#1e2a4a" fontcolor="#e8eaf0" shape=box]
|
||||
edge [fontname="Helvetica" fontsize=9 fontcolor="#8892a8" color="#4a5568"]
|
||||
|
||||
label="Installation — the only host prerequisite is Docker"
|
||||
labelloc=t
|
||||
fontsize=16
|
||||
fontcolor="#0066ff"
|
||||
|
||||
subgraph cluster_host {
|
||||
label="Your machine"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
docker [label="Docker\n(the one prerequisite)" fillcolor="#1a1a3a" fontcolor="#0066ff" shape=octagon]
|
||||
bin [label="~/.local/bin\nkind · kubectl · tilt\njq" fillcolor="#121829" shape=cylinder]
|
||||
}
|
||||
|
||||
subgraph cluster_installer {
|
||||
label="Installer container (transient)"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
installer [label="deps installer\ncurl · jq · python · graphviz" fillcolor="#121829"]
|
||||
detect [label="detect host\nWSL · memory · inotify · docker" fillcolor="#121829"]
|
||||
fetch [label="fetch + verify\nSHA256, pinned versions" fillcolor="#121829"]
|
||||
}
|
||||
|
||||
upstream [label="upstream\nreleases / corporate mirror" fillcolor="#1a3a1a" fontcolor="#00c853" shape=octagon]
|
||||
report [label="report what it\nCANNOT do" fillcolor="#3a1a1a" fontcolor="#ffc107"]
|
||||
|
||||
docker -> installer [label="docker run"]
|
||||
installer -> detect
|
||||
detect -> fetch
|
||||
fetch -> upstream [label="pinned + checksummed" color="#00c853"]
|
||||
fetch -> bin [label="install"]
|
||||
detect -> report [style=dashed label="sudo / Windows-side steps" color="#ffc107"]
|
||||
|
||||
// The container is gone after this; nothing depends on it at run time.
|
||||
installer -> gone [style=dotted label="exits"]
|
||||
gone [label="(container discarded)" fillcolor="#0a0e17" fontcolor="#4a5568" color="#1e2a4a" style="filled,dashed"]
|
||||
}
|
||||
128
rig/docs/graphs/01-install.svg
Normal file
128
rig/docs/graphs/01-install.svg
Normal file
@@ -0,0 +1,128 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 14.1.2 (0)
|
||||
-->
|
||||
<!-- Title: rig_install Pages: 1 -->
|
||||
<svg width="1145pt" height="287pt"
|
||||
viewBox="0.00 0.00 1145.00 287.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 283.29)">
|
||||
<title>rig_install</title>
|
||||
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-283.29 1141.06,-283.29 1141.06,4 -4,4"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="568.53" y="-260.09" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#0066ff">Installation — the only host prerequisite is Docker</text>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_host</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="898.49,-8 898.49,-190 1123.82,-190 1123.82,-8 898.49,-8"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-170.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Your machine</text>
|
||||
</g>
|
||||
<g id="clust2" class="cluster">
|
||||
<title>cluster_installer</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-95 8,-175 745.5,-175 745.5,-95 8,-95"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="376.75" y="-155.8" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Installer container (transient)</text>
|
||||
</g>
|
||||
<!-- docker -->
|
||||
<g id="node1" class="node">
|
||||
<title>docker</title>
|
||||
<polygon fill="#1a1a3a" stroke="#1e2a4a" points="1115.82,-31.9 1115.82,-54.1 1054.51,-69.79 967.8,-69.79 906.49,-54.1 906.49,-31.9 967.8,-16.21 1054.51,-16.21 1115.82,-31.9"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-46.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">Docker</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-32.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">(the one prerequisite)</text>
|
||||
</g>
|
||||
<!-- installer -->
|
||||
<g id="node3" class="node">
|
||||
<title>installer</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="181.25,-139 16,-139 16,-103 181.25,-103 181.25,-139"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">deps installer</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="98.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">curl · jq · python · graphviz</text>
|
||||
</g>
|
||||
<!-- docker->installer -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>docker->installer</title>
|
||||
<path fill="none" stroke="#4a5568" d="M906.12,-45.7C757.47,-50.51 475.98,-63.12 238.25,-94 223.38,-95.93 207.7,-98.48 192.45,-101.24"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="192.13,-97.74 182.93,-103.01 193.4,-104.63 192.13,-97.74"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="506.62" y="-74.39" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">docker run</text>
|
||||
</g>
|
||||
<!-- bin -->
|
||||
<g id="node2" class="node">
|
||||
<title>bin</title>
|
||||
<path fill="#121829" stroke="#1e2a4a" d="M1069.03,-148.28C1069.03,-151.63 1043.09,-154.34 1011.15,-154.34 979.22,-154.34 953.28,-151.63 953.28,-148.28 953.28,-148.28 953.28,-93.72 953.28,-93.72 953.28,-90.37 979.22,-87.66 1011.15,-87.66 1043.09,-87.66 1069.03,-90.37 1069.03,-93.72 1069.03,-93.72 1069.03,-148.28 1069.03,-148.28"/>
|
||||
<path fill="none" stroke="#1e2a4a" d="M1069.03,-148.28C1069.03,-144.94 1043.09,-142.22 1011.15,-142.22 979.22,-142.22 953.28,-144.94 953.28,-148.28"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-130.8" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">~/.local/bin</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-117.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">kind · kubectl · tilt</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-103.8" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">jq</text>
|
||||
</g>
|
||||
<!-- detect -->
|
||||
<g id="node4" class="node">
|
||||
<title>detect</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="429,-139 238.25,-139 238.25,-103 429,-103 429,-139"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">detect host</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">WSL · memory · inotify · docker</text>
|
||||
</g>
|
||||
<!-- installer->detect -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>installer->detect</title>
|
||||
<path fill="none" stroke="#4a5568" d="M181.44,-121C195.97,-121 211.28,-121 226.37,-121"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="226.33,-124.5 236.33,-121 226.33,-117.5 226.33,-124.5"/>
|
||||
</g>
|
||||
<!-- gone -->
|
||||
<g id="node8" class="node">
|
||||
<title>gone</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="400.5,-47 266.75,-47 266.75,-11 400.5,-11 400.5,-47"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="333.62" y="-25.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#4a5568">(container discarded)</text>
|
||||
</g>
|
||||
<!-- installer->gone -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>installer->gone</title>
|
||||
<path fill="none" stroke="#4a5568" stroke-dasharray="1,5" d="M119.29,-102.62C138.26,-86 168.54,-62.29 199.25,-49.75 216.76,-42.6 236.49,-37.9 255.29,-34.81"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="255.67,-38.29 265.04,-33.35 254.64,-31.37 255.67,-38.29"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="209.75" y="-52.45" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">exits</text>
|
||||
</g>
|
||||
<!-- fetch -->
|
||||
<g id="node5" class="node">
|
||||
<title>fetch</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="737.5,-139 584.25,-139 584.25,-103 737.5,-103 737.5,-139"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="660.88" y="-124.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">fetch + verify</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="660.88" y="-110.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">SHA256, pinned versions</text>
|
||||
</g>
|
||||
<!-- detect->fetch -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>detect->fetch</title>
|
||||
<path fill="none" stroke="#4a5568" d="M429.14,-121C474.43,-121 528.32,-121 572.63,-121"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="572.46,-124.5 582.46,-121 572.46,-117.5 572.46,-124.5"/>
|
||||
</g>
|
||||
<!-- report -->
|
||||
<g id="node7" class="node">
|
||||
<title>report</title>
|
||||
<polygon fill="#3a1a1a" stroke="#1e2a4a" points="706.75,-219 615,-219 615,-183 706.75,-183 706.75,-219"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="660.88" y="-204.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">report what it</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="660.88" y="-190.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">CANNOT do</text>
|
||||
</g>
|
||||
<!-- detect->report -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>detect->report</title>
|
||||
<path fill="none" stroke="#ffc107" stroke-dasharray="5,2" d="M409.65,-139.45C468.85,-154.02 550.13,-174.01 603.78,-187.2"/>
|
||||
<polygon fill="#ffc107" stroke="#ffc107" points="602.77,-190.56 613.32,-189.55 604.45,-183.76 602.77,-190.56"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="506.62" y="-180.06" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">sudo / Windows-side steps</text>
|
||||
</g>
|
||||
<!-- fetch->bin -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>fetch->bin</title>
|
||||
<path fill="none" stroke="#4a5568" d="M737.88,-121C798.58,-121 882.9,-121 941.56,-121"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="941.43,-124.5 951.43,-121 941.43,-117.5 941.43,-124.5"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="811.38" y="-123.7" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">install</text>
|
||||
</g>
|
||||
<!-- upstream -->
|
||||
<g id="node6" class="node">
|
||||
<title>upstream</title>
|
||||
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="1137.06,-213.9 1137.06,-236.1 1063.3,-251.79 959,-251.79 885.25,-236.1 885.25,-213.9 959,-198.21 1063.3,-198.21 1137.06,-213.9"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-228.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">upstream</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="1011.15" y="-214.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">releases / corporate mirror</text>
|
||||
</g>
|
||||
<!-- fetch->upstream -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>fetch->upstream</title>
|
||||
<path fill="none" stroke="#00c853" d="M714.8,-139.5C759.85,-154.95 826.43,-177.11 885.25,-194 894.79,-196.74 904.78,-199.46 914.78,-202.09"/>
|
||||
<polygon fill="#00c853" stroke="#00c853" points="913.59,-205.4 924.15,-204.52 915.35,-198.62 913.59,-205.4"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="811.38" y="-191.36" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">pinned + checksummed</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 9.0 KiB |
54
rig/docs/graphs/02-environment.dot
Normal file
54
rig/docs/graphs/02-environment.dot
Normal file
@@ -0,0 +1,54 @@
|
||||
digraph rig_environment {
|
||||
rankdir=TB
|
||||
bgcolor="#0a0e17"
|
||||
fontname="Helvetica"
|
||||
node [fontname="Helvetica" fontsize=11 style=filled color="#1e2a4a" fontcolor="#e8eaf0" shape=box]
|
||||
edge [fontname="Helvetica" fontsize=9 fontcolor="#8892a8" color="#4a5568"]
|
||||
|
||||
label="One environment per directory — copies never collide"
|
||||
labelloc=t
|
||||
fontsize=16
|
||||
fontcolor="#0066ff"
|
||||
|
||||
dirname [label="directory name\ne.g. acmebank/" fillcolor="#1f6feb" fontcolor="#ffffff" shape=octagon]
|
||||
|
||||
subgraph cluster_derived {
|
||||
label="Everything below is derived from it"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
cname [label="cluster name\nacmebank" fillcolor="#121829"]
|
||||
ctx [label="kubectl context\nkind-acmebank" fillcolor="#121829"]
|
||||
img [label="image tag\nacmebank-deps" fillcolor="#121829"]
|
||||
ports [label="port block\n21300–21309" fillcolor="#121829"]
|
||||
reg [label="registry container\nacmebank-registry" fillcolor="#121829"]
|
||||
}
|
||||
|
||||
subgraph cluster_config {
|
||||
label="Configuration — weakest first, later wins"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
versions [label="versions.env\npinned toolchain" fillcolor="#121829"]
|
||||
profile [label="env.d/<profile>.env\nnodes · CNI · audit · addons" fillcolor="#121829"]
|
||||
localenv [label="ctrl/.env\nsecrets, overrides" fillcolor="#121829"]
|
||||
shell [label="the environment\nPROFILE=client make …" fillcolor="#1a3a1a" fontcolor="#00c853"]
|
||||
}
|
||||
|
||||
dirname -> cname
|
||||
dirname -> ctx
|
||||
dirname -> img
|
||||
dirname -> ports
|
||||
dirname -> reg
|
||||
|
||||
versions -> profile [label="overridden by"]
|
||||
profile -> localenv [label="overridden by"]
|
||||
localenv -> shell [label="overridden by" color="#00c853"]
|
||||
|
||||
cluster [label="kind cluster" fillcolor="#1a1a3a" fontcolor="#0066ff" shape=octagon]
|
||||
cname -> cluster
|
||||
ports -> cluster
|
||||
shell -> cluster [style=dashed]
|
||||
}
|
||||
169
rig/docs/graphs/02-environment.svg
Normal file
169
rig/docs/graphs/02-environment.svg
Normal file
@@ -0,0 +1,169 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 14.1.2 (0)
|
||||
-->
|
||||
<!-- Title: rig_environment Pages: 1 -->
|
||||
<svg width="962pt" height="481pt"
|
||||
viewBox="0.00 0.00 962.00 481.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 476.83)">
|
||||
<title>rig_environment</title>
|
||||
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-476.83 958,-476.83 958,4 -4,4"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="477" y="-453.63" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#0066ff">One environment per directory — copies never collide</text>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_derived</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-65 8,-144.5 596,-144.5 596,-65 8,-65"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="302" y="-125.3" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Everything below is derived from it</text>
|
||||
</g>
|
||||
<g id="clust2" class="cluster">
|
||||
<title>cluster_config</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="604,-65 604,-437.33 946,-437.33 946,-65 604,-65"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="775" y="-418.13" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Configuration — weakest first, later wins</text>
|
||||
</g>
|
||||
<!-- dirname -->
|
||||
<g id="node1" class="node">
|
||||
<title>dirname</title>
|
||||
<polygon fill="#1f6feb" stroke="#1e2a4a" points="370.11,-197.44 370.11,-219.63 324.94,-235.33 261.06,-235.33 215.89,-219.63 215.89,-197.44 261.06,-181.75 324.94,-181.75 370.11,-197.44"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="293" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">directory name</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="293" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffffff">e.g. acmebank/</text>
|
||||
</g>
|
||||
<!-- cname -->
|
||||
<g id="node2" class="node">
|
||||
<title>cname</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="104,-109 16,-109 16,-73 104,-73 104,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="60" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">cluster name</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="60" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank</text>
|
||||
</g>
|
||||
<!-- dirname->cname -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>dirname->cname</title>
|
||||
<path fill="none" stroke="#4a5568" d="M228.68,-192.51C192.88,-182.36 148.52,-166.7 113,-144.5 101.4,-137.25 90.34,-127.04 81.36,-117.55"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="84.07,-115.32 74.76,-110.26 78.88,-120.02 84.07,-115.32"/>
|
||||
</g>
|
||||
<!-- ctx -->
|
||||
<g id="node3" class="node">
|
||||
<title>ctx</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="223.75,-109 122.25,-109 122.25,-73 223.75,-73 223.75,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="173" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">kubectl context</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="173" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">kind-acmebank</text>
|
||||
</g>
|
||||
<!-- dirname->ctx -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>dirname->ctx</title>
|
||||
<path fill="none" stroke="#4a5568" d="M265.77,-181.32C245.77,-162.07 218.77,-136.06 199.05,-117.08"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="201.55,-114.63 191.91,-110.21 196.69,-119.67 201.55,-114.63"/>
|
||||
</g>
|
||||
<!-- img -->
|
||||
<g id="node4" class="node">
|
||||
<title>img</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="344.12,-109 241.88,-109 241.88,-73 344.12,-73 344.12,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="293" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">image tag</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="293" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-deps</text>
|
||||
</g>
|
||||
<!-- dirname->img -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>dirname->img</title>
|
||||
<path fill="none" stroke="#4a5568" d="M293,-181.32C293,-163.19 293,-139.07 293,-120.47"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="296.5,-120.67 293,-110.67 289.5,-120.67 296.5,-120.67"/>
|
||||
</g>
|
||||
<!-- ports -->
|
||||
<g id="node5" class="node">
|
||||
<title>ports</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="451.38,-109 362.62,-109 362.62,-73 451.38,-73 451.38,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">port block</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">21300–21309</text>
|
||||
</g>
|
||||
<!-- dirname->ports -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>dirname->ports</title>
|
||||
<path fill="none" stroke="#4a5568" d="M318.87,-181.32C337.78,-162.15 363.29,-136.3 382,-117.34"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="384.47,-119.81 389,-110.24 379.49,-114.9 384.47,-119.81"/>
|
||||
</g>
|
||||
<!-- reg -->
|
||||
<g id="node6" class="node">
|
||||
<title>reg</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="588.38,-109 469.62,-109 469.62,-73 588.38,-73 588.38,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="529" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">registry container</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="529" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">acmebank-registry</text>
|
||||
</g>
|
||||
<!-- dirname->reg -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>dirname->reg</title>
|
||||
<path fill="none" stroke="#4a5568" d="M350.86,-190.3C383.87,-179.35 425.43,-163.64 460,-144.5 474.27,-136.6 488.83,-125.98 500.87,-116.36"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="502.85,-119.26 508.36,-110.21 498.41,-113.84 502.85,-119.26"/>
|
||||
</g>
|
||||
<!-- cluster -->
|
||||
<g id="node11" class="node">
|
||||
<title>cluster</title>
|
||||
<polygon fill="#1a1a3a" stroke="#1e2a4a" points="460.81,-10.54 460.81,-25.46 429.29,-36 384.71,-36 353.19,-25.46 353.19,-10.54 384.71,0 429.29,0 460.81,-10.54"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="407" y="-14.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#0066ff">kind cluster</text>
|
||||
</g>
|
||||
<!-- cname->cluster -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>cname->cluster</title>
|
||||
<path fill="none" stroke="#4a5568" d="M93.35,-72.51C99.75,-69.67 106.48,-67 113,-65 189.55,-41.49 281.19,-29.58 341.58,-23.85"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="341.71,-27.35 351.35,-22.96 341.07,-20.38 341.71,-27.35"/>
|
||||
</g>
|
||||
<!-- ports->cluster -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>ports->cluster</title>
|
||||
<path fill="none" stroke="#4a5568" d="M407,-72.81C407,-65.23 407,-56.1 407,-47.54"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="410.5,-47.54 407,-37.54 403.5,-47.54 410.5,-47.54"/>
|
||||
</g>
|
||||
<!-- versions -->
|
||||
<g id="node7" class="node">
|
||||
<title>versions</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="750.38,-401.83 643.62,-401.83 643.62,-365.83 750.38,-365.83 750.38,-401.83"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="697" y="-386.88" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">versions.env</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="697" y="-373.38" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">pinned toolchain</text>
|
||||
</g>
|
||||
<!-- profile -->
|
||||
<g id="node8" class="node">
|
||||
<title>profile</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="781.5,-318.58 612.5,-318.58 612.5,-282.58 781.5,-282.58 781.5,-318.58"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="697" y="-303.63" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">env.d/<profile>.env</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="697" y="-290.13" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">nodes · CNI · audit · addons</text>
|
||||
</g>
|
||||
<!-- versions->profile -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>versions->profile</title>
|
||||
<path fill="none" stroke="#4a5568" d="M697,-365.59C697,-355.32 697,-342.03 697,-330.21"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="700.5,-330.58 697,-320.58 693.5,-330.58 700.5,-330.58"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="728.5" y="-339.28" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||
</g>
|
||||
<!-- localenv -->
|
||||
<g id="node9" class="node">
|
||||
<title>localenv</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="752.88,-226.54 637.12,-226.54 637.12,-190.54 752.88,-190.54 752.88,-226.54"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="695" y="-211.59" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">ctrl/.env</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="695" y="-198.09" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">secrets, overrides</text>
|
||||
</g>
|
||||
<!-- profile->localenv -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>profile->localenv</title>
|
||||
<path fill="none" stroke="#4a5568" d="M696.61,-282.22C696.34,-269.76 695.96,-252.69 695.64,-238.23"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="699.14,-238.28 695.42,-228.36 692.14,-238.43 699.14,-238.28"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="727.68" y="-256.03" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||
</g>
|
||||
<!-- shell -->
|
||||
<g id="node10" class="node">
|
||||
<title>shell</title>
|
||||
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="763.38,-109 614.62,-109 614.62,-73 763.38,-73 763.38,-109"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="689" y="-94.05" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">the environment</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="689" y="-80.55" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">PROFILE=client make …</text>
|
||||
</g>
|
||||
<!-- localenv->shell -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>localenv->shell</title>
|
||||
<path fill="none" stroke="#00c853" d="M694.11,-190.49C693.16,-172.16 691.63,-142.72 690.49,-120.79"/>
|
||||
<polygon fill="#00c853" stroke="#00c853" points="694,-120.81 689.99,-111.01 687.01,-121.18 694,-120.81"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="724.21" y="-155.2" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">overridden by</text>
|
||||
</g>
|
||||
<!-- shell->cluster -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>shell->cluster</title>
|
||||
<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M628.29,-72.6C618.83,-69.99 609.17,-67.38 600,-65 553.72,-52.99 500.89,-40.48 462.22,-31.54"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="463.2,-28.18 452.67,-29.35 461.63,-35 463.2,-28.18"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 11 KiB |
56
rig/docs/graphs/03-architecture.dot
Normal file
56
rig/docs/graphs/03-architecture.dot
Normal file
@@ -0,0 +1,56 @@
|
||||
// TODO: PLACEHOLDER — replace with the real estate topology.
|
||||
//
|
||||
// This is where the extracted platform diagrams land. The shape below is
|
||||
// illustrative only: it shows how a mocked dependency, a real service and a
|
||||
// remote system are meant to sit together, not what the system actually is.
|
||||
//
|
||||
// The intended end state is that this file stops being hand-written and is
|
||||
// generated from the running cluster, so the diagram becomes a report of what
|
||||
// exists rather than a drawing of what was once intended.
|
||||
digraph estate {
|
||||
rankdir=LR
|
||||
bgcolor="#0a0e17"
|
||||
fontname="Helvetica"
|
||||
node [fontname="Helvetica" fontsize=11 style=filled color="#1e2a4a" fontcolor="#e8eaf0" shape=box]
|
||||
edge [fontname="Helvetica" fontsize=9 fontcolor="#8892a8" color="#4a5568"]
|
||||
|
||||
label="Estate topology — PLACEHOLDER"
|
||||
labelloc=t
|
||||
fontsize=16
|
||||
fontcolor="#ffc107"
|
||||
|
||||
subgraph cluster_new {
|
||||
label="New"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
api [label="service under work\n(real: built and hot-reloaded)" fillcolor="#1a3a1a" fontcolor="#00c853"]
|
||||
}
|
||||
|
||||
subgraph cluster_core {
|
||||
label="Core (mocked)"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
svc_a [label="upstream service\n(mock: canned responses)" fillcolor="#121829"]
|
||||
db [label="datastore\n(mock)" fillcolor="#121829" shape=cylinder]
|
||||
}
|
||||
|
||||
subgraph cluster_legacy {
|
||||
label="Legacy estate (mocked)"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
batch [label="batch drop\n(mock: writes files on a timer)" fillcolor="#121829"]
|
||||
}
|
||||
|
||||
remote [label="external system\n(remote: ExternalName,\nreachable only from a VDI)" fillcolor="#3a1a1a" fontcolor="#ffc107" shape=octagon]
|
||||
|
||||
api -> svc_a [label="HTTP"]
|
||||
api -> db [label="query"]
|
||||
api -> batch [label="file handoff" style=dashed]
|
||||
api -> remote [label="only when reachable" style=dashed color="#ffc107"]
|
||||
}
|
||||
94
rig/docs/graphs/03-architecture.svg
Normal file
94
rig/docs/graphs/03-architecture.svg
Normal file
@@ -0,0 +1,94 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 14.1.2 (0)
|
||||
-->
|
||||
<!-- Title: estate Pages: 1 -->
|
||||
<svg width="579pt" height="362pt"
|
||||
viewBox="0.00 0.00 579.00 362.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 357.62)">
|
||||
<title>estate</title>
|
||||
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-357.62 575.41,-357.62 575.41,4 -4,4"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="285.7" y="-334.42" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#ffc107">Estate topology — PLACEHOLDER</text>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_new</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-130.12 8,-210.12 199,-210.12 199,-130.12 8,-130.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-190.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">New</text>
|
||||
</g>
|
||||
<g id="clust2" class="cluster">
|
||||
<title>cluster_core</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="357.33,-172.12 357.33,-318.12 534.83,-318.12 534.83,-172.12 357.33,-172.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-298.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Core (mocked)</text>
|
||||
</g>
|
||||
<g id="clust3" class="cluster">
|
||||
<title>cluster_legacy</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="341.83,-84.12 341.83,-164.12 551.33,-164.12 551.33,-84.12 341.83,-84.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.58" y="-144.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Legacy estate (mocked)</text>
|
||||
</g>
|
||||
<!-- api -->
|
||||
<g id="node1" class="node">
|
||||
<title>api</title>
|
||||
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="191,-174.12 16,-174.12 16,-138.12 191,-138.12 191,-174.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-159.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">service under work</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-145.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">(real: built and hot-reloaded)</text>
|
||||
</g>
|
||||
<!-- svc_a -->
|
||||
<g id="node2" class="node">
|
||||
<title>svc_a</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="526.83,-282.12 365.33,-282.12 365.33,-246.12 526.83,-246.12 526.83,-282.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-267.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">upstream service</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-253.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: canned responses)</text>
|
||||
</g>
|
||||
<!-- api->svc_a -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>api->svc_a</title>
|
||||
<path fill="none" stroke="#4a5568" d="M149.07,-174.56C167.54,-182.07 189.23,-190.7 209,-198.12 258.25,-216.6 270.12,-222.85 320.75,-237.12 331.43,-240.13 342.71,-243 353.94,-245.67"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="353.12,-249.07 363.66,-247.92 354.71,-242.25 353.12,-249.07"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-234.24" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">HTTP</text>
|
||||
</g>
|
||||
<!-- db -->
|
||||
<g id="node3" class="node">
|
||||
<title>db</title>
|
||||
<path fill="#121829" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-226.22 465.18,-228.18 446.08,-228.18 426.97,-228.18 411.45,-226.22 411.45,-223.81 411.45,-223.81 411.45,-184.43 411.45,-184.43 411.45,-182.02 426.97,-180.06 446.08,-180.06 465.18,-180.06 480.7,-182.02 480.7,-184.43 480.7,-184.43 480.7,-223.81 480.7,-223.81"/>
|
||||
<path fill="none" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-221.39 465.18,-219.43 446.08,-219.43 426.97,-219.43 411.45,-221.39 411.45,-223.81"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-207.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">datastore</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-193.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock)</text>
|
||||
</g>
|
||||
<!-- api->db -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>api->db</title>
|
||||
<path fill="none" stroke="#4a5568" d="M191.32,-168.36C257.83,-177.73 346.91,-190.28 399.91,-197.75"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="399.4,-201.22 409.79,-199.15 400.38,-194.29 399.4,-201.22"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-185.69" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">query</text>
|
||||
</g>
|
||||
<!-- batch -->
|
||||
<g id="node4" class="node">
|
||||
<title>batch</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="537.33,-128.12 354.83,-128.12 354.83,-92.12 537.33,-92.12 537.33,-128.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-113.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">batch drop</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-99.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: writes files on a timer)</text>
|
||||
</g>
|
||||
<!-- api->batch -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>api->batch</title>
|
||||
<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M191.32,-144.39C237.67,-138.13 294.98,-130.39 343.4,-123.85"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="343.61,-127.36 353.05,-122.55 342.67,-120.42 343.61,-127.36"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-143.94" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">file handoff</text>
|
||||
</g>
|
||||
<!-- remote -->
|
||||
<g id="node5" class="node">
|
||||
<title>remote</title>
|
||||
<polygon fill="#3a1a1a" stroke="#1e2a4a" points="571.41,-21.74 571.41,-52.5 497.99,-74.24 394.17,-74.24 320.75,-52.5 320.75,-21.74 394.17,0 497.99,0 571.41,-21.74"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-46.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">external system</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-33.42" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">(remote: ExternalName,</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-19.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">reachable only from a VDI)</text>
|
||||
</g>
|
||||
<!-- api->remote -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>api->remote</title>
|
||||
<path fill="none" stroke="#ffc107" stroke-dasharray="5,2" d="M149.55,-137.68C167.9,-130.34 189.37,-121.97 209,-114.87 254.45,-98.43 305.32,-81.49 348.17,-67.64"/>
|
||||
<polygon fill="#ffc107" stroke="#ffc107" points="349,-71.05 357.44,-64.65 346.85,-64.39 349,-71.05"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-117.57" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">only when reachable</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.9 KiB |
588
rig/docs/index.html
Normal file
588
rig/docs/index.html
Normal file
@@ -0,0 +1,588 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>rig — local environment installer</title>
|
||||
<style>
|
||||
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap');
|
||||
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
background: #0a0e17;
|
||||
color: #e8eaf0;
|
||||
font-family: 'Inter', sans-serif;
|
||||
line-height: 1.6;
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 16px 24px;
|
||||
border-bottom: 1px solid #1e2a4a;
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 16px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 22px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 3px;
|
||||
color: #0066ff;
|
||||
}
|
||||
|
||||
header .subtitle {
|
||||
font-size: 13px;
|
||||
color: #4a5568;
|
||||
letter-spacing: 1px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.layout { display: flex; flex: 1; min-height: 0; }
|
||||
|
||||
nav {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 200px;
|
||||
flex-shrink: 0;
|
||||
background: #121829;
|
||||
border-right: 1px solid #1e2a4a;
|
||||
padding: 8px 0;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
nav a {
|
||||
padding: 10px 20px;
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 12px;
|
||||
color: #8892a8;
|
||||
text-decoration: none;
|
||||
border-left: 2px solid transparent;
|
||||
transition: all 0.15s;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
nav a:hover { color: #e8eaf0; background: #1a2340; }
|
||||
nav a.active { color: #0066ff; border-left-color: #0066ff; background: #0d1a33; }
|
||||
|
||||
main { flex: 1; overflow: auto; padding: 32px 48px; }
|
||||
|
||||
.section { display: none; animation: fadeIn 0.2s ease; }
|
||||
.section.active { display: block; }
|
||||
@keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
|
||||
|
||||
.section h2 {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 15px;
|
||||
font-weight: 500;
|
||||
color: #8892a8;
|
||||
margin-bottom: 8px;
|
||||
letter-spacing: 1px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.section > p.lede {
|
||||
font-size: 13px;
|
||||
color: #4a5568;
|
||||
margin-bottom: 24px;
|
||||
max-width: 800px;
|
||||
}
|
||||
|
||||
.prose { max-width: 820px; }
|
||||
.prose p { font-size: 14px; color: #b4bccf; line-height: 1.7; margin-bottom: 14px; }
|
||||
.prose p b { color: #e8eaf0; }
|
||||
.prose ul { margin: 0 0 16px 20px; }
|
||||
.prose li { font-size: 14px; color: #b4bccf; margin-bottom: 6px; }
|
||||
|
||||
.prose h3 {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 13px;
|
||||
text-transform: uppercase;
|
||||
color: #e8eaf0;
|
||||
margin: 32px 0 10px;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
|
||||
.prose code, pre code {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 12px;
|
||||
color: #7ab0ff;
|
||||
background: #121829;
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
pre {
|
||||
background: #121829;
|
||||
border: 1px solid #1e2a4a;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
pre code { background: none; padding: 0; }
|
||||
pre .c { color: #4a5568; }
|
||||
pre .k { color: #0066ff; }
|
||||
|
||||
.graph-container { margin: 16px 0; }
|
||||
.graph-container img {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
background: #0a0e17;
|
||||
border: 1px solid #1e2a4a;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
dl {
|
||||
display: grid;
|
||||
grid-template-columns: max-content 1fr;
|
||||
gap: 10px 24px;
|
||||
margin: 16px 0;
|
||||
max-width: 820px;
|
||||
}
|
||||
|
||||
dt {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
color: #0066ff;
|
||||
font-size: 13px;
|
||||
padding-top: 2px;
|
||||
}
|
||||
|
||||
dd { font-size: 14px; color: #b4bccf; line-height: 1.6; }
|
||||
|
||||
table { border-collapse: collapse; margin: 16px 0; max-width: 820px; }
|
||||
th, td {
|
||||
text-align: left;
|
||||
padding: 7px 16px 7px 0;
|
||||
font-size: 13px;
|
||||
border-bottom: 1px solid #1e2a4a;
|
||||
color: #b4bccf;
|
||||
}
|
||||
th {
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
color: #8892a8;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
td code { white-space: nowrap; }
|
||||
|
||||
.note {
|
||||
border-left: 2px solid #ffc107;
|
||||
background: #17130a;
|
||||
padding: 12px 16px;
|
||||
margin: 16px 0;
|
||||
max-width: 820px;
|
||||
}
|
||||
.note p { margin: 0; font-size: 13px; color: #b4bccf; }
|
||||
.note b { color: #ffc107; }
|
||||
|
||||
.menu-toggle {
|
||||
display: none;
|
||||
background: transparent;
|
||||
border: 1px solid #1e2a4a;
|
||||
color: #8892a8;
|
||||
padding: 6px 10px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
line-height: 1;
|
||||
margin-left: auto;
|
||||
}
|
||||
.menu-toggle:hover { background: #1a2340; }
|
||||
|
||||
.nav-backdrop {
|
||||
display: none;
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
z-index: 10;
|
||||
}
|
||||
.layout.nav-open .nav-backdrop { display: block; }
|
||||
|
||||
@media (max-width: 720px) {
|
||||
header { padding: 10px 12px; gap: 8px; }
|
||||
header h1 { font-size: 16px; letter-spacing: 1px; }
|
||||
header .subtitle { display: none; }
|
||||
.menu-toggle { display: inline-block; }
|
||||
.layout { position: relative; }
|
||||
nav {
|
||||
position: absolute; left: 0; top: 0; bottom: 0;
|
||||
width: 200px; z-index: 20;
|
||||
transform: translateX(-100%);
|
||||
transition: transform 0.2s ease;
|
||||
box-shadow: 2px 0 8px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
.layout.nav-open nav { transform: translateX(0); }
|
||||
main { padding: 16px; }
|
||||
.section h2 { font-size: 13px; }
|
||||
.prose p, .prose li { font-size: 13px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<h1>RIG</h1>
|
||||
<span class="subtitle">local environment installer</span>
|
||||
<button class="menu-toggle">☰</button>
|
||||
</header>
|
||||
|
||||
<div class="layout">
|
||||
<div class="nav-backdrop"></div>
|
||||
<nav>
|
||||
<a href="#start">Start here</a>
|
||||
<a href="#steps">The steps</a>
|
||||
<a href="#install">Installation</a>
|
||||
<a href="#environments">Environments</a>
|
||||
<a href="#profiles">Profiles</a>
|
||||
<a href="#registry">Registry</a>
|
||||
<a href="#architecture">Architecture</a>
|
||||
<a href="#troubleshooting">Troubleshooting</a>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
|
||||
<section class="section" id="start">
|
||||
<h2>Start here</h2>
|
||||
<p class="lede">A runnable local model of a large, regulated estate — legacy and new side by side.</p>
|
||||
<div class="prose">
|
||||
<p>rig builds a disposable Kubernetes environment on your machine so you can
|
||||
explore how a system fits together without needing access to any of it. Its
|
||||
job is <b>onboarding and exploration</b>, not a production replica.</p>
|
||||
|
||||
<p>Most services in it are deliberately <b>not real</b>. What has to be faithful
|
||||
is the topology — the names, the ports, the dependency order, who can reach whom,
|
||||
and how it fails. The workloads themselves are noise. This is what makes the
|
||||
whole estate fit on a laptop: a real 20-service platform will not fit even once
|
||||
on 14 GB, but mocks are about 30 MB each, so three faithful copies do.</p>
|
||||
|
||||
<h3>The only prerequisite</h3>
|
||||
<p><b>Docker.</b> No curl, no jq, no python, no apt repositories to configure.</p>
|
||||
<pre><code><span class="c"># then, in the environment directory:</span>
|
||||
make check <span class="c"># is this machine ready? reports, never fixes</span>
|
||||
make deps <span class="c"># install the pinned toolchain</span>
|
||||
make cluster up <span class="c"># build the cluster for the active profile</span>
|
||||
</code></pre>
|
||||
<p>Read <code>make check</code> before <code>make deps</code>. It never changes
|
||||
anything — it prints what it found and, at the end, the steps it cannot perform
|
||||
for you.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="steps">
|
||||
<h2>The steps</h2>
|
||||
<p class="lede">Start to finish, in order, with what each one actually does.</p>
|
||||
<div class="prose">
|
||||
|
||||
<h3>1 · make check</h3>
|
||||
<p>Asks whether this machine is ready. It <b>changes nothing</b> — it
|
||||
reports what it found and, at the end, the things only a human can do
|
||||
(anything needing <code>sudo</code>, or a Windows-side restart). Read it
|
||||
before installing anything; it is faster than discovering the same problems
|
||||
one failure at a time.</p>
|
||||
<pre><code>make check</code></pre>
|
||||
|
||||
<h3>2 · make setup</h3>
|
||||
<p>Does the preparation that can be automated: installs the pinned
|
||||
toolchain if it is missing, checks PATH, Docker, and this environment's
|
||||
ports. Every step is independently checked, so running it twice is safe and
|
||||
running it half-configured finishes the job.</p>
|
||||
<p>It <b>does not stop at the first failure</b>. A setup script that dies at
|
||||
step two hides that steps four and five would also have failed, and on an
|
||||
unfamiliar machine the complete list is the point. The tail of the output is
|
||||
a to-do list of only what is outstanding.</p>
|
||||
<pre><code>make setup <span class="c"># host + toolchain</span>
|
||||
make setup --share-docker <span class="c"># ...and offer this machine's Docker to other distros</span>
|
||||
</code></pre>
|
||||
|
||||
<h3>3 · make cluster up</h3>
|
||||
<p>Builds the cluster for the active profile. It prints what the profile
|
||||
locks in <i>before</i> spending the time, because the CNI and the audit
|
||||
policy are fixed at creation and cannot be changed afterwards.</p>
|
||||
<p>Re-running is safe and, more importantly, <b>convergent</b>: if a first
|
||||
attempt was interrupted before the CNI was installed, running it again
|
||||
finishes the job rather than reporting "already exists" and leaving every
|
||||
node permanently NotReady.</p>
|
||||
<pre><code>make cluster up <span class="c"># default profile</span>
|
||||
make cluster up PROFILE=client <span class="c"># three nodes, audit on, cached registry</span>
|
||||
make cluster reset <span class="c"># destroy and rebuild — the only way to change CNI or audit</span>
|
||||
</code></pre>
|
||||
|
||||
<h3>4 · make docs</h3>
|
||||
<p>Serves this page from a throwaway container. Works with no cluster and
|
||||
no toolchain, which is deliberate: these pages are the instructions for
|
||||
building everything else, so they cannot depend on it.</p>
|
||||
<pre><code>make docs</code></pre>
|
||||
|
||||
<h3>Checking on things</h3>
|
||||
<dl>
|
||||
<dt>make cluster list</dt><dd>Every cluster on the machine, its memory cost and its port block. The usual reason a new one will not start is an old one you forgot about; <code>make cluster free</code> frees them without deleting.</dd>
|
||||
<dt>make ports</dt><dd>This environment's port block, and whether each is derived or overridden.</dd>
|
||||
<dt>make registry</dt><dd>Which of the four registry modes is active, and where it points.</dd>
|
||||
<dt>make dockerhost</dt><dd>Which WSL distro owns Docker and what this one is using.</dd>
|
||||
</dl>
|
||||
|
||||
<h3>Running more than one</h3>
|
||||
<p>Copy the directory, rename it, and repeat from step 2. Cluster name,
|
||||
context, image tags and the port block all follow the directory name, so
|
||||
the second environment collides with nothing and neither one's teardown can
|
||||
reach the other.</p>
|
||||
<pre><code>cp -r rig ../platform-v2 && cd ../platform-v2
|
||||
make setup && make cluster up
|
||||
</code></pre>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="install">
|
||||
<h2>Installation</h2>
|
||||
<p class="lede">A container installs onto the host and then gets out of the way.</p>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/01-install.svg"><img src="graphs/01-install.svg" alt="Installation flow"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>The installer is a container, not a shell script, for a specific reason: a
|
||||
stock slim Debian has no <code>curl</code>, no <code>wget</code>, no
|
||||
<code>jq</code>, no <code>python3</code> and <b>no CA bundle</b>. A shell
|
||||
installer could not make a verified HTTPS request, let alone check one. The
|
||||
container carries its own toolchain, so the host needs nothing but Docker.</p>
|
||||
|
||||
<p>The cluster never runs inside that container. Everything it installs —
|
||||
kind, kubectl, tilt, jq — runs natively afterwards, so nothing pays a
|
||||
container tax during daily work.</p>
|
||||
|
||||
<h3>Pinned and verified</h3>
|
||||
<p>Every tool is a single binary fetched at a pinned version and checked
|
||||
against a published SHA256. Node images are pinned <b>by digest</b>, so
|
||||
upgrading kind cannot silently move your Kubernetes version.</p>
|
||||
|
||||
<h3>Not every machine should get cluster tooling</h3>
|
||||
<p>A managed or corporate-issued machine — the kind that holds the access
|
||||
you cannot get anywhere else — is not somewhere to install development
|
||||
tools by default. So the toolchain comes in two tiers:</p>
|
||||
<table>
|
||||
<tr><th>tier</th><th>installs</th><th>for</th></tr>
|
||||
<tr><td><code>core</code></td><td>kubectl, jq</td><td>talk to a cluster someone else runs</td></tr>
|
||||
<tr><td><code>dev</code></td><td>+ kind, tilt</td><td>build clusters and hot-reload into them</td></tr>
|
||||
</table>
|
||||
<pre><code>make deps core <span class="c"># kubectl and jq only — nothing that creates a cluster</span>
|
||||
make deps <span class="c"># dev, the default</span>
|
||||
make setup core <span class="c"># same distinction, via setup</span>
|
||||
</code></pre>
|
||||
<p>Testing <i>in situ</i> on a managed machine is still possible — install
|
||||
the <code>dev</code> tier deliberately when you need it. The point is that
|
||||
it should be a decision rather than a side effect of running setup.</p>
|
||||
<p>The documentation itself needs neither tier: <code>make docs</code>
|
||||
wants only Docker.</p>
|
||||
|
||||
<h3>Air-gapped</h3>
|
||||
<pre><code>make deps-image full <span class="c"># bakes every binary into the image</span>
|
||||
docker save …-deps:full | gzip > rig.tgz
|
||||
<span class="c"># carry that one file in, then:</span>
|
||||
docker load < rig.tgz && make cluster up PROFILE=offline
|
||||
</code></pre>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="environments">
|
||||
<h2>Environments</h2>
|
||||
<p class="lede">One directory is one environment. Copy it, rename it, run it.</p>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/02-environment.svg"><img src="graphs/02-environment.svg" alt="Environment derivation"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>Running several versions of a system at once means several clusters on one
|
||||
machine, not several machines. Everything that could collide is derived from
|
||||
the directory name:</p>
|
||||
<dl>
|
||||
<dt>cluster + context</dt><dd><code>acmebank/</code> builds <code>acmebank</code> on <code>kind-acmebank</code>.</dd>
|
||||
<dt>port block</dt><dd>Ten ports from a hash of the name, in the 20000+ range — clear of 80, 443, 3000, 5432, 8000 and 8080.</dd>
|
||||
<dt>registry + images</dt><dd>Named after the environment, so two copies never share one.</dd>
|
||||
</dl>
|
||||
<p>Two copies therefore never collide, and neither one's
|
||||
<code>make cluster down</code> can touch the other. <code>make ports</code>
|
||||
shows the block; <code>make ports persist</code> freezes it into
|
||||
<code>ctrl/.env</code> if you want it fixed rather than derived.</p>
|
||||
|
||||
<h3>Configuration layers</h3>
|
||||
<p>Weakest first, later wins: pinned versions → the profile →
|
||||
<code>ctrl/.env</code> → the environment. So
|
||||
<code>make cluster up PROFILE=client</code> always beats every file.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="profiles">
|
||||
<h2>Profiles</h2>
|
||||
<p class="lede">Cluster shape is declared, not baked in.</p>
|
||||
<div class="prose">
|
||||
<table>
|
||||
<tr><th>profile</th><th>nodes</th><th>audit</th><th>registry</th><th>for</th></tr>
|
||||
<tr><td><code>minimal</code></td><td>1</td><td>off</td><td>none</td><td>first boot; assumes nothing</td></tr>
|
||||
<tr><td><code>client</code></td><td>3</td><td>on</td><td>mirror</td><td>the regulated shape</td></tr>
|
||||
<tr><td><code>offline</code></td><td>1</td><td>on</td><td>local</td><td>air-gapped</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="note"><p><b>The audit policy cannot be changed later.</b> It is an
|
||||
apiserver flag, fixed when the cluster is created. <code>cluster up</code>
|
||||
prints what a profile locks in before spending the time, and
|
||||
<code>make cluster reset</code> is the way out.</p></div>
|
||||
|
||||
<h3>LoadBalancer services</h3>
|
||||
<p>Real manifests use <code>type: LoadBalancer</code>, because a real
|
||||
cluster has one. On a bare local cluster those Services sit at
|
||||
<code>EXTERNAL-IP <pending></code> forever, with no error anywhere —
|
||||
the deployment looks healthy and simply is not reachable.</p>
|
||||
<p>The <code>metallb</code> addon fixes that, so the same manifests work
|
||||
here as upstream and nothing has to be rewritten to NodePort. Its address
|
||||
pool is derived from the cluster's Docker network at install time rather
|
||||
than hardcoded, because Docker picks that subnet and it differs between
|
||||
machines.</p>
|
||||
<div class="note"><p><b>Where those addresses are reachable from.</b> The
|
||||
pool lives on the Docker bridge, so LoadBalancer IPs work from the Linux
|
||||
side — including from inside WSL. A browser on Windows has no route to
|
||||
them. Use the ingress host ports for anything you need to open in a
|
||||
browser.</p></div>
|
||||
|
||||
<h3>Networking</h3>
|
||||
<p>The cluster uses kind's built-in networking, which <b>does</b> enforce
|
||||
standard NetworkPolicy — verified against a no-policy control, not assumed.
|
||||
The widely repeated claim that it accepts policies and silently ignores
|
||||
them is out of date.</p>
|
||||
<p>A pluggable CNI was tried and removed: it only added
|
||||
GlobalNetworkPolicy, policy tiers and egress-CIDR rules, none of which are
|
||||
needed yet, in exchange for a slower boot and one more thing that has to be
|
||||
right at creation time. Worth revisiting only when a policy the built-in
|
||||
cannot express actually comes up.</p>
|
||||
|
||||
<h3>Memory</h3>
|
||||
<p>Every cluster is a running container tree whether you are using it or not.
|
||||
<code>make cluster list</code> shows what exists and what it costs;
|
||||
<code>make cluster free</code> stops the others without deleting them.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="registry">
|
||||
<h2>Registry</h2>
|
||||
<p class="lede">Local, cached, or straight to the corporate registry.</p>
|
||||
<div class="prose">
|
||||
<table>
|
||||
<tr><th>mode</th><th>what it does</th></tr>
|
||||
<tr><td><code>none</code></td><td>images are built straight into the node</td></tr>
|
||||
<tr><td><code>local</code></td><td>a registry container wired into the cluster</td></tr>
|
||||
<tr><td><code>mirror</code></td><td>that container as a <b>pull-through cache</b> of the corporate registry</td></tr>
|
||||
<tr><td><code>remote</code></td><td>no local container; pull direct with an imagePullSecret</td></tr>
|
||||
</table>
|
||||
<p><code>mirror</code> is what a locked-down network actually looks like:
|
||||
images originate from the corporate registry, you do not hammer it, and you
|
||||
keep working when the connection drops.</p>
|
||||
|
||||
<div class="note"><p><b>The corporate CA will bite you.</b> A corporate
|
||||
registry is usually behind an internal CA, and trust has to reach
|
||||
<b>three</b> places: the host Docker daemon, every cluster node's containerd
|
||||
(nodes do <i>not</i> inherit host trust), and any in-cluster client. Set
|
||||
<code>REGISTRY_CA_FILE</code> and <code>make check</code> reports which is
|
||||
still missing. The symptom otherwise is an opaque
|
||||
<code>x509: certificate signed by unknown authority</code>.</p></div>
|
||||
|
||||
<p>Reachability also depends on where you are: if the registry is only
|
||||
routable from a managed workspace, <code>mirror</code> and <code>remote</code>
|
||||
will not resolve from a laptop at all. That is what <code>local</code> and
|
||||
<code>offline</code> are for.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="architecture">
|
||||
<h2>Architecture</h2>
|
||||
<p class="lede">The estate being modelled.</p>
|
||||
<div class="note"><p><b>TODO — placeholder.</b> The diagram below is
|
||||
illustrative only: it shows how a mocked dependency, a service under active
|
||||
work, and an unreachable remote system sit together. It is not the real
|
||||
topology. Replace <code>docs/graphs/03-architecture.dot</code> with the
|
||||
extracted platform diagrams, then run <code>make docs graphs</code>.</p></div>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/03-architecture.svg"><img src="graphs/03-architecture.svg" alt="Estate topology (placeholder)"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>Each component is one of three things, and switching between them should be
|
||||
a one-line change rather than a rewrite:</p>
|
||||
<dl>
|
||||
<dt>real</dt><dd>Built from source and hot-reloaded. The thing you are actually working on — usually exactly one.</dd>
|
||||
<dt>mock</dt><dd>A generic stub with canned responses. Everything you do not care about today.</dd>
|
||||
<dt>remote</dt><dd>No pod at all: a Service of type ExternalName pointing at the real system. In-cluster DNS resolves identically, so callers never change.</dd>
|
||||
</dl>
|
||||
<p>The intended end state is that this diagram is <b>generated from the
|
||||
running cluster</b> rather than drawn by hand — so it becomes a report of
|
||||
what exists instead of a picture of what was once intended.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="troubleshooting">
|
||||
<h2>Troubleshooting</h2>
|
||||
<p class="lede">The failures that are hard to diagnose from their symptoms.</p>
|
||||
<div class="prose">
|
||||
<h3>Tilt stops noticing file changes</h3>
|
||||
<p>Almost always <code>inotify</code> limits, and it fails <i>silently</i> —
|
||||
nothing errors, changes just stop being picked up. Defaults on WSL are far too
|
||||
low. <code>make check</code> reports it and prints the fix.</p>
|
||||
|
||||
<h3>Cluster creation dies halfway with a port error</h3>
|
||||
<p>Docker reports <code>failed to bind host port … address already in use</code>
|
||||
partway through creating the cluster. Run <code>make check</code> first — it
|
||||
checks every port in this environment's block before anything is built.</p>
|
||||
|
||||
<h3>Every node stays NotReady</h3>
|
||||
<p>Usually a cluster created with the default CNI disabled but the real CNI
|
||||
never installed — typically an interrupted first run. Just run
|
||||
<code>make cluster up</code> again: it converges rather than exiting early, and
|
||||
will finish the missing steps.</p>
|
||||
|
||||
<h3>x509: certificate signed by unknown authority</h3>
|
||||
<p>Corporate CA trust has not reached one of the three places it needs to be.
|
||||
See <a href="#registry">Registry</a>.</p>
|
||||
|
||||
<h3>kubectl says the context does not exist</h3>
|
||||
<p>The cluster can exist while its context does not — a reset or a switched
|
||||
<code>KUBECONFIG</code> loses it. <code>make cluster up</code> detects this and
|
||||
re-exports the context.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
var layout = document.querySelector('.layout');
|
||||
var main = document.querySelector('main');
|
||||
|
||||
function syncActive() {
|
||||
var hash = location.hash.slice(1) || 'start';
|
||||
document.querySelectorAll('.section').forEach(function (s) { s.classList.remove('active'); });
|
||||
document.querySelectorAll('nav a').forEach(function (a) { a.classList.remove('active'); });
|
||||
var section = document.getElementById(hash);
|
||||
if (section) section.classList.add('active');
|
||||
var link = document.querySelector('nav a[href="#' + hash + '"]');
|
||||
if (link) link.classList.add('active');
|
||||
if (main) main.scrollTop = 0;
|
||||
layout.classList.remove('nav-open');
|
||||
}
|
||||
|
||||
window.addEventListener('hashchange', syncActive);
|
||||
window.addEventListener('DOMContentLoaded', syncActive);
|
||||
syncActive();
|
||||
|
||||
document.addEventListener('click', function (e) {
|
||||
if (e.target.closest('.menu-toggle') || e.target.closest('.nav-backdrop')) {
|
||||
layout.classList.toggle('nav-open');
|
||||
}
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
101
rig/docs/viewer.html
Normal file
101
rig/docs/viewer.html
Normal file
@@ -0,0 +1,101 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Graph Viewer</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; }
|
||||
body {
|
||||
background: #0a0e17;
|
||||
overflow: hidden;
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
}
|
||||
#container {
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
cursor: grab;
|
||||
}
|
||||
#container.dragging { cursor: grabbing; }
|
||||
img {
|
||||
transform-origin: 0 0;
|
||||
user-select: none;
|
||||
-webkit-user-drag: none;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="container">
|
||||
<img id="img" />
|
||||
</div>
|
||||
<script>
|
||||
var src = new URLSearchParams(location.search).get('src');
|
||||
var img = document.getElementById('img');
|
||||
var container = document.getElementById('container');
|
||||
|
||||
img.src = src;
|
||||
|
||||
var scale = 1;
|
||||
var x = 0, y = 0;
|
||||
var dragging = false;
|
||||
var startX, startY, startPanX, startPanY;
|
||||
|
||||
function apply() {
|
||||
img.style.transform = 'translate(' + x + 'px,' + y + 'px) scale(' + scale + ')';
|
||||
}
|
||||
|
||||
// Fit to screen on load
|
||||
img.onload = function() {
|
||||
var sw = window.innerWidth / img.naturalWidth;
|
||||
var sh = window.innerHeight / img.naturalHeight;
|
||||
scale = Math.min(sw, sh) * 0.95;
|
||||
x = (window.innerWidth - img.naturalWidth * scale) / 2;
|
||||
y = (window.innerHeight - img.naturalHeight * scale) / 2;
|
||||
apply();
|
||||
};
|
||||
|
||||
// Wheel zoom toward cursor
|
||||
container.addEventListener('wheel', function(e) {
|
||||
e.preventDefault();
|
||||
var factor = e.deltaY < 0 ? 1.12 : 0.89;
|
||||
var rect = container.getBoundingClientRect();
|
||||
var mx = e.clientX - rect.left;
|
||||
var my = e.clientY - rect.top;
|
||||
x = mx - (mx - x) * factor;
|
||||
y = my - (my - y) * factor;
|
||||
scale *= factor;
|
||||
apply();
|
||||
}, { passive: false });
|
||||
|
||||
// Pan
|
||||
container.addEventListener('mousedown', function(e) {
|
||||
if (e.button !== 0) return;
|
||||
dragging = true;
|
||||
startX = e.clientX;
|
||||
startY = e.clientY;
|
||||
startPanX = x;
|
||||
startPanY = y;
|
||||
container.classList.add('dragging');
|
||||
e.preventDefault();
|
||||
});
|
||||
|
||||
window.addEventListener('mousemove', function(e) {
|
||||
if (!dragging) return;
|
||||
x = startPanX + (e.clientX - startX);
|
||||
y = startPanY + (e.clientY - startY);
|
||||
apply();
|
||||
});
|
||||
|
||||
window.addEventListener('mouseup', function() {
|
||||
dragging = false;
|
||||
container.classList.remove('dragging');
|
||||
});
|
||||
|
||||
// Double-click to reset
|
||||
container.addEventListener('dblclick', function() {
|
||||
img.onload();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
37
rig/standalone/README.md
Normal file
37
rig/standalone/README.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# standalone — single files for a machine the full rig is not going to
|
||||
|
||||
Each script here does one of rig's jobs without the rest of the tree. Copy one
|
||||
file onto a machine, run it, read the output. Nothing to clone, nothing to
|
||||
install first.
|
||||
|
||||
| file | does | full-rig equivalent |
|
||||
| --- | --- | --- |
|
||||
| `rigdeps.sh` | installs kind, kubectl, tilt, ctlptl and jq at rig's pins, checksum-verified, no sudo | `make deps` (`ctrl/deps.sh`) |
|
||||
| `rigmini.sh` | reports how much memory the machine *advertises* and what caps it; `push` measures what it will actually *survive* | `make mem`, and the memory section of `make check` |
|
||||
|
||||
**These are transitional.** Where the full rig is installed, use its own
|
||||
targets instead; they read `ctrl/versions.env` and the profile, which these
|
||||
cannot.
|
||||
|
||||
## Why single files
|
||||
|
||||
`rigdeps.sh` carries its pins inline, because `ctrl/versions.env` is not on the
|
||||
machine it is for. That makes two copies of the same versions and checksums.
|
||||
`make pins` compares them and fails on any difference — `ctrl/versions.env` is
|
||||
the source of truth.
|
||||
|
||||
`rigmini.sh` exists because on a container or managed workspace `/proc/meminfo`
|
||||
reports the *host's* memory while a cgroup cap kills processes at a fraction of
|
||||
it. `status` reads the caps; `push` allocates until something stops it.
|
||||
|
||||
## Use
|
||||
|
||||
```bash
|
||||
bash rigdeps.sh detect # report, change nothing
|
||||
bash rigdeps.sh install dev # install into ~/.local/bin
|
||||
bash rigmini.sh status # advertised memory and caps; safe
|
||||
bash rigmini.sh push # allocates until it stops — not on a machine you need
|
||||
```
|
||||
|
||||
`rigmini.sh push` deliberately consumes memory. Run `status` first, and only run
|
||||
`push` somewhere it is acceptable for other processes to be squeezed.
|
||||
574
rig/standalone/rigdeps.sh
Executable file
574
rig/standalone/rigdeps.sh
Executable file
@@ -0,0 +1,574 @@
|
||||
#!/usr/bin/env bash
|
||||
# Put kind, tilt and kubectl on a machine that has none of them.
|
||||
#
|
||||
# The single file companion to rigmini.sh, for the same reason: rig installs its
|
||||
# toolchain from ctrl/deps.sh reading ctrl/versions.env, and neither of those is
|
||||
# going to a fresh AWS WorkSpace. The pins live inline here instead.
|
||||
#
|
||||
# What it will not do, deliberately:
|
||||
#
|
||||
# * no sudo, no apt, no yum. It writes ONLY into $OUT_BIN (default
|
||||
# ~/.local/bin). Everything needing root — installing Docker, joining the
|
||||
# docker group, raising inotify limits — is REPORTED for you to decide on.
|
||||
# That is what makes it safe to run on a machine that already works.
|
||||
# * no unverified download. Every artifact is checked against a SHA256 taken
|
||||
# from the publisher's own release list. A mismatch aborts.
|
||||
# * no guessing at another architecture. See ARCHITECTURE below.
|
||||
#
|
||||
# Two tiers, because "install the toolchain" is not one decision:
|
||||
#
|
||||
# core kubectl, jq — talk to a cluster someone else runs. Nothing that
|
||||
# creates one. The right answer on a managed or corporate machine.
|
||||
# dev core plus kind, tilt and ctlptl — build clusters and hot-reload
|
||||
# into them. The default, and what you want on a workspace of your own.
|
||||
#
|
||||
# Usage:
|
||||
# rigdeps.sh detect report the host, change nothing
|
||||
# rigdeps.sh list the pinned versions and where they come from
|
||||
# rigdeps.sh install [core|dev] detect, download, verify, install, report
|
||||
# rigdeps.sh fetch [core|dev] [--to DIR] download + verify only
|
||||
# rigdeps.sh verify run what is installed and see if it works
|
||||
set -euo pipefail
|
||||
|
||||
OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
|
||||
|
||||
# ── the pinned toolchain ───────────────────────────────────────────────────
|
||||
#
|
||||
# ARCHITECTURE. These checksums are the upstream-published SHA256 of the
|
||||
# **linux/amd64** artifact and of nothing else. An arm64 WorkSpace bundle needs
|
||||
# a different binary with a different checksum, and this script refuses rather
|
||||
# than reusing these — a checksum that is merely plausible is worse than none,
|
||||
# because it turns a verified download into a ceremony.
|
||||
#
|
||||
# To bump a version, or to add arm64: take the checksum from the release's own
|
||||
# published list, never from a download you did.
|
||||
#
|
||||
# curl -sSL https://github.com/<org>/<repo>/releases/download/<tag>/checksums.txt
|
||||
#
|
||||
# kubectl publishes its own instead, at <KUBECTL_URL>.sha256.
|
||||
|
||||
KIND_VERSION=v0.32.0
|
||||
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
|
||||
|
||||
KUBECTL_VERSION=v1.36.3
|
||||
KUBECTL_SHA256=ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336
|
||||
KUBECTL_URL="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
|
||||
|
||||
TILT_VERSION=0.37.6
|
||||
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
||||
TILT_URL="https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz"
|
||||
|
||||
# ctlptl creates a kind cluster WITH a local registry wired in, which is what
|
||||
# keeps images off docker.io — an unqualified image name resolves to
|
||||
# docker.io/library/<name>, and there is nothing structural stopping a push there.
|
||||
CTLPTL_VERSION=0.9.4
|
||||
CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e
|
||||
CTLPTL_URL="https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz"
|
||||
|
||||
# Upstream's static build. Debian's jq is linked against libjq/libonig, which is
|
||||
# fine on Debian and not portable anywhere else.
|
||||
JQ_VERSION=1.8.2
|
||||
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
||||
JQ_URL="https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
|
||||
|
||||
CORE_TOOLS="kubectl jq"
|
||||
DEV_TOOLS="kind tilt ctlptl"
|
||||
|
||||
# No helm: every rig addon installs with `kubectl apply -f`, so nothing has ever
|
||||
# invoked it. Add it the day something actually needs a chart.
|
||||
|
||||
# Collected as we go, printed by report_manual() at the very end. Anything that
|
||||
# needs root or a decision lands here instead of being done.
|
||||
MANUAL=()
|
||||
|
||||
# ── platform ───────────────────────────────────────────────────────────────
|
||||
|
||||
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||
# the tooling. Detectable, so name it instead.
|
||||
require_linux() {
|
||||
case "$(uname -s)" in
|
||||
MINGW*|MSYS*|CYGWIN*)
|
||||
cat >&2 <<'EOF'
|
||||
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||
|
||||
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||
|
||||
wsl --install
|
||||
|
||||
That enables Windows features and needs a reboot, so it is not something this
|
||||
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||
this from there.
|
||||
EOF
|
||||
exit 1 ;;
|
||||
Linux) ;;
|
||||
*) echo "$(uname -s) is not Linux. These are linux binaries; nothing here" >&2
|
||||
echo "would run even if it downloaded." >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
arch() {
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64) echo amd64 ;;
|
||||
aarch64|arm64) echo arm64 ;;
|
||||
*) uname -m ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# The pins above are amd64. Rather than download something that cannot execute
|
||||
# and let it fail as "cannot execute binary file: Exec format error", say so
|
||||
# here and hand over the commands that produce the right checksums.
|
||||
require_amd64() {
|
||||
local a; a=$(arch)
|
||||
[ "$a" = "amd64" ] && return 0
|
||||
cat >&2 <<EOF
|
||||
This machine is ${a} ($(uname -m)); every pin in this script is linux/amd64.
|
||||
|
||||
Nothing here would run, so it does not download. To make an ${a} version, the
|
||||
URLs need the ${a} artifact and the checksums need to come from each project's
|
||||
own published list — not from these values, and not from a download you did:
|
||||
|
||||
curl -sSL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/checksums.txt
|
||||
curl -sSL https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${a}/kubectl.sha256
|
||||
curl -sSL https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/checksums.txt
|
||||
curl -sSL https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/checksums.txt
|
||||
curl -sSL https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt
|
||||
|
||||
Edit the pinned block at the top of this file with what those print.
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||
|
||||
# ── the tools this script itself needs ─────────────────────────────────────
|
||||
|
||||
# A fresh minimal image may genuinely have neither curl nor wget. Find out once,
|
||||
# up front, rather than half way through the first download.
|
||||
DL=""
|
||||
pick_downloader() {
|
||||
if command -v curl >/dev/null 2>&1; then DL=curl
|
||||
elif command -v wget >/dev/null 2>&1; then DL=wget
|
||||
else
|
||||
echo "neither curl nor wget is installed, so nothing can be downloaded." >&2
|
||||
echo "Install one first: $(pkg_install_cmd curl)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
download() {
|
||||
local url="$1" out="$2"
|
||||
case "$DL" in
|
||||
curl) curl -fsSL --retry 3 -o "$out" "$url" ;;
|
||||
wget) wget -q --tries=3 -O "$out" "$url" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# sha256sum is coreutils; shasum is the perl one that turns up on stripped
|
||||
# images. Verification is not optional, so if neither exists that is fatal.
|
||||
SHA=""
|
||||
pick_sha() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then SHA=sha256sum
|
||||
elif command -v shasum >/dev/null 2>&1; then SHA="shasum -a 256"
|
||||
else
|
||||
echo "no sha256sum and no shasum — downloads could not be verified." >&2
|
||||
echo "Refusing to install unverified binaries." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── package manager, for the instructions only ─────────────────────────────
|
||||
# This never runs a package manager. It names one so the reported action is
|
||||
# something you can paste, on the distro you are actually on — an apt line on
|
||||
# Amazon Linux 2 is a wrong answer dressed up as help.
|
||||
|
||||
pkg_install_cmd() {
|
||||
local pkg="$1"
|
||||
if command -v apt-get >/dev/null 2>&1; then echo "sudo apt-get update && sudo apt-get install -y $pkg"
|
||||
elif command -v dnf >/dev/null 2>&1; then echo "sudo dnf install -y $pkg"
|
||||
elif command -v yum >/dev/null 2>&1; then echo "sudo yum install -y $pkg"
|
||||
elif command -v zypper >/dev/null 2>&1; then echo "sudo zypper install -y $pkg"
|
||||
elif command -v apk >/dev/null 2>&1; then echo "sudo apk add $pkg"
|
||||
else echo "install '$pkg' with this system's package manager"
|
||||
fi
|
||||
}
|
||||
|
||||
docker_pkg() {
|
||||
# Debian and Ubuntu call it docker.io; the RPM distros call it docker.
|
||||
if command -v apt-get >/dev/null 2>&1; then echo docker.io; else echo docker; fi
|
||||
}
|
||||
|
||||
# ── detect ─────────────────────────────────────────────────────────────────
|
||||
|
||||
detect() {
|
||||
echo "host"
|
||||
echo " kernel $(uname -r)"
|
||||
echo " arch $(arch) ($(uname -m))"
|
||||
[ -r /etc/os-release ] && \
|
||||
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||
if is_wsl; then echo " platform WSL"; else echo " platform native linux"; fi
|
||||
|
||||
local total_kb avail_kb
|
||||
total_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo)
|
||||
avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
|
||||
printf " memory %d GB total, %d GB available\n" \
|
||||
$((total_kb / 1024 / 1024)) $((avail_kb / 1024 / 1024))
|
||||
if [ $((avail_kb / 1024 / 1024)) -lt 4 ]; then
|
||||
echo " ! under 4 GB available — a cluster will struggle here."
|
||||
echo " rigmini.sh says how much this box will actually give you."
|
||||
fi
|
||||
|
||||
echo " install to $OUT_BIN"
|
||||
detect_libc
|
||||
detect_prereqs
|
||||
detect_docker
|
||||
detect_inotify
|
||||
return 0
|
||||
}
|
||||
|
||||
# tilt is the one binary here that needs a recent glibc. MEASURED, not guessed:
|
||||
# tilt 0.37.6 on Amazon Linux 2 (glibc 2.26) fails with
|
||||
#
|
||||
# /lib64/libc.so.6: version `GLIBC_2.34' not found (required by .../tilt)
|
||||
#
|
||||
# which names a symbol rather than the problem. Amazon Linux 2 is a stock
|
||||
# WorkSpaces bundle, so this is the likely case, not an exotic one. Report the
|
||||
# version now; `verify` catches the actual failure after installing.
|
||||
detect_libc() {
|
||||
local v=""
|
||||
if command -v ldd >/dev/null 2>&1; then
|
||||
v=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$' || true)
|
||||
fi
|
||||
if [ -z "$v" ]; then
|
||||
echo " libc unknown (no ldd) — 'verify' is the real test"
|
||||
return 0
|
||||
fi
|
||||
echo " libc glibc $v"
|
||||
if [ "$(printf '%s\n2.34\n' "$v" | sort -V | head -1)" != "2.34" ]; then
|
||||
echo " ! older than glibc 2.34, which tilt needs. kubectl, kind, jq and"
|
||||
echo " ctlptl are static or libc-only and work here; tilt will not start."
|
||||
echo " Install the core tier, or run tilt from a container."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# What this script needs to do its own job. Reported here so `detect` answers
|
||||
# "will install work?" instead of leaving you to find out one download in.
|
||||
# Amazon Linux 2 ships without tar, which is exactly the surprise this catches.
|
||||
detect_prereqs() {
|
||||
local missing=""
|
||||
if command -v curl >/dev/null 2>&1; then echo " download curl"
|
||||
elif command -v wget >/dev/null 2>&1; then echo " download wget"
|
||||
else echo " ! no curl and no wget — nothing can be downloaded"; missing+=" curl"
|
||||
fi
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1; then
|
||||
echo " checksums ok"
|
||||
else
|
||||
echo " ! no sha256sum or shasum — downloads could not be verified"
|
||||
missing+=" coreutils"
|
||||
fi
|
||||
|
||||
if command -v tar >/dev/null 2>&1 && command -v gzip >/dev/null 2>&1; then
|
||||
echo " archives tar + gzip"
|
||||
else
|
||||
echo " ! no tar/gzip — tilt and ctlptl ship as tarballs, so the dev tier"
|
||||
echo " cannot be unpacked. The core tier is two bare binaries and is fine."
|
||||
missing+=" tar gzip"
|
||||
fi
|
||||
|
||||
if [ -n "$missing" ]; then
|
||||
MANUAL+=("Install what this script needs to run at all:
|
||||
$(pkg_install_cmd "${missing# }")")
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
detect_docker() {
|
||||
# kind builds a cluster out of containers. Without a reachable daemon,
|
||||
# everything here installs perfectly and then does nothing.
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
echo " docker socket present, no cli"
|
||||
return 0
|
||||
fi
|
||||
echo " ! docker not installed — kind has nothing to build a cluster in"
|
||||
MANUAL+=("Install Docker. It is the one real prerequisite, and the only
|
||||
thing here that needs root:
|
||||
$(pkg_install_cmd "$(docker_pkg)")
|
||||
sudo systemctl enable --now docker
|
||||
sudo usermod -aG docker \"\$USER\"
|
||||
then log out and back in, so the new group applies to your shell.")
|
||||
return 0
|
||||
fi
|
||||
if docker info >/dev/null 2>&1; then
|
||||
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null)"
|
||||
local n
|
||||
n=$(docker ps --filter "label=io.x-k8s.kind.cluster" --format '{{.Names}}' 2>/dev/null | wc -l)
|
||||
# Must be an `if`, not `[ ] && echo`: as the last statement here the
|
||||
# latter returns 1 when the count is zero, and `set -e` kills the
|
||||
# caller. That is the fresh-machine case, where it does most harm.
|
||||
if [ "$n" -gt 0 ]; then
|
||||
echo " - $n kind node container(s) already running"
|
||||
fi
|
||||
else
|
||||
echo " ! docker cli present but the daemon is unreachable"
|
||||
MANUAL+=("Start Docker, or add yourself to the docker group:
|
||||
sudo systemctl enable --now docker
|
||||
sudo usermod -aG docker \"\$USER\" # then log out and back in")
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# kind and tilt both watch large trees. Distro defaults are far too low and the
|
||||
# failure mode is silent: tilt simply stops noticing that files changed.
|
||||
detect_inotify() {
|
||||
local w i
|
||||
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||
echo " inotify watches=$w instances=$i"
|
||||
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||
echo " ! low — tilt will silently stop seeing file changes"
|
||||
MANUAL+=("Raise the inotify limits (needs root):
|
||||
echo -e 'fs.inotify.max_user_watches=524288\\nfs.inotify.max_user_instances=512' \\
|
||||
| sudo tee /etc/sysctl.d/99-rig.conf
|
||||
sudo sysctl --system")
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── fetch ──────────────────────────────────────────────────────────────────
|
||||
|
||||
verify_sha() {
|
||||
local file="$1" want="$2" name="$3" got
|
||||
got=$($SHA "$file" | awk '{print $1}')
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo >&2
|
||||
echo "CHECKSUM MISMATCH for $name — not installing it." >&2
|
||||
echo " expected $want" >&2
|
||||
echo " got $got" >&2
|
||||
echo >&2
|
||||
echo "Either the pin in this script is stale, or what arrived is not what" >&2
|
||||
echo "the publisher released. Neither is worth guessing about." >&2
|
||||
rm -f "$file"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# fetch_bin <name> <url> <sha256> <dest-dir> — a bare binary
|
||||
fetch_bin() {
|
||||
local name="$1" url="$2" sha="$3" dest="$4"
|
||||
local tmp="$dest/.$name.tmp"
|
||||
printf ' %-8s ' "$name"
|
||||
download "$url" "$tmp"
|
||||
verify_sha "$tmp" "$sha" "$name"
|
||||
mv "$tmp" "$dest/$name"
|
||||
chmod +x "$dest/$name"
|
||||
echo "ok"
|
||||
}
|
||||
|
||||
# fetch_tgz <name> <url> <sha256> <dest-dir> <path-inside> <strip>
|
||||
# Archive layouts differ, so the caller says which. tilt and ctlptl both ship
|
||||
# the binary at the archive root, hence strip=0.
|
||||
fetch_tgz() {
|
||||
local name="$1" url="$2" sha="$3" dest="$4" inner="$5" strip="$6"
|
||||
local tmp="$dest/.$name.tgz"
|
||||
printf ' %-8s ' "$name"
|
||||
download "$url" "$tmp"
|
||||
verify_sha "$tmp" "$sha" "$name"
|
||||
# --no-same-owner: some archives ship as uid 1001, and extracting as root
|
||||
# would otherwise restore an owner that is not you.
|
||||
tar -xzf "$tmp" -C "$dest" --strip-components="$strip" --no-same-owner "$inner"
|
||||
rm -f "$tmp"
|
||||
chmod +x "$dest/$name"
|
||||
echo "ok"
|
||||
}
|
||||
|
||||
fetch() {
|
||||
local dest="$OUT_BIN" tier="dev"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--to) dest="${2:?--to needs a directory}"; shift 2 ;;
|
||||
core|dev) tier="$1"; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
mkdir -p "$dest"
|
||||
|
||||
if ! command -v tar >/dev/null 2>&1 && [ "$tier" = "dev" ]; then
|
||||
echo "tar is missing, and tilt and ctlptl ship as tarballs." >&2
|
||||
echo " $(pkg_install_cmd tar)" >&2
|
||||
echo "Or install the core tier, which is two bare binaries: $0 install core" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "fetching '$tier' into $dest (verifying every checksum)"
|
||||
fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"
|
||||
fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"
|
||||
if [ "$tier" = "dev" ]; then
|
||||
fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"
|
||||
fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0
|
||||
fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── verify ─────────────────────────────────────────────────────────────────
|
||||
|
||||
tier_tools() { [ "$1" = "core" ] && echo "$CORE_TOOLS" || echo "$CORE_TOOLS $DEV_TOOLS"; }
|
||||
|
||||
# Downloading a verified binary proves it is the right file, not that this
|
||||
# machine can run it. On an old distro tilt fails here, with a linker error
|
||||
# about a missing symbol, and finding that out now beats finding out during a
|
||||
# first cluster build.
|
||||
verify_tools() {
|
||||
local tier="${1:-dev}" b bin out rc broke=0
|
||||
echo "checking that each one actually runs"
|
||||
for b in $(tier_tools "$tier"); do
|
||||
bin="$OUT_BIN/$b"
|
||||
if [ ! -x "$bin" ]; then
|
||||
printf ' %-8s not installed\n' "$b"
|
||||
continue
|
||||
fi
|
||||
rc=0
|
||||
case "$b" in
|
||||
kubectl) out=$("$bin" version --client 2>&1 | head -1) || rc=$? ;;
|
||||
jq) out=$("$bin" --version 2>&1 | head -1) || rc=$? ;;
|
||||
*) out=$("$bin" version 2>&1 | head -1) || rc=$? ;;
|
||||
esac
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
printf ' %-8s %s\n' "$b" "$out"
|
||||
else
|
||||
printf ' ! %-6s does not run here: %s\n' "$b" "$out"
|
||||
broke=1
|
||||
fi
|
||||
done
|
||||
if [ "$broke" -eq 1 ]; then
|
||||
echo
|
||||
echo " A binary that downloads and verifies but will not start is almost"
|
||||
echo " always this distro's libc being older than the release needs."
|
||||
echo " 'detect' prints the glibc version. The core tier (kubectl + jq)"
|
||||
echo " has no such dependency and will work regardless."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── install ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Installing into a directory early in PATH silently replaces whatever the
|
||||
# machine was already using, which on a shared or corporate machine can break
|
||||
# unrelated work — kubectl more than one minor away from its cluster is the
|
||||
# common one. Say so; never decide it.
|
||||
warn_shadowing() {
|
||||
local b existing shadowed="" tier="${1:-dev}"
|
||||
case ":${PATH}:" in
|
||||
*":$OUT_BIN:"*) ;;
|
||||
*) return 0 ;; # not on PATH, so nothing is being shadowed yet
|
||||
esac
|
||||
for b in $(tier_tools "$tier"); do
|
||||
[ -x "$OUT_BIN/$b" ] || continue
|
||||
existing=$(PATH=$(echo "$PATH" | tr ':' '\n' | grep -vx "$OUT_BIN" | paste -sd:) \
|
||||
command -v "$b" 2>/dev/null || true)
|
||||
[ -n "$existing" ] || continue
|
||||
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||
shadowed+=" $b $existing"$'\n'
|
||||
done
|
||||
[ -n "$shadowed" ] || return 0
|
||||
|
||||
echo
|
||||
echo " ! these were already installed elsewhere and are now shadowed:"
|
||||
printf '%s' "$shadowed"
|
||||
MANUAL+=("Decide which toolchain wins. To keep the previous one:
|
||||
rm -f $(for b in $(tier_tools "$tier"); do printf '%s ' "$OUT_BIN/$b"; done)
|
||||
Or install somewhere private instead:
|
||||
OUT_BIN=\$PWD/bin $0 install")
|
||||
return 0
|
||||
}
|
||||
|
||||
report_manual() {
|
||||
echo
|
||||
if [ ${#MANUAL[@]} -eq 0 ]; then
|
||||
echo "nothing left to do by hand."
|
||||
return 0
|
||||
fi
|
||||
echo "host actions this cannot perform (${#MANUAL[@]}):"
|
||||
echo
|
||||
local n=1 m
|
||||
for m in "${MANUAL[@]}"; do
|
||||
echo " $n. $m"
|
||||
echo
|
||||
n=$((n + 1))
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
install() {
|
||||
local tier="${1:-dev}"
|
||||
detect
|
||||
echo
|
||||
fetch "$tier"
|
||||
echo
|
||||
verify_tools "$tier"
|
||||
warn_shadowing "$tier"
|
||||
|
||||
if [ "$tier" = "core" ]; then
|
||||
echo
|
||||
echo " core tier: no kind, tilt or ctlptl. '$0 install dev' adds them."
|
||||
fi
|
||||
|
||||
case ":${PATH}:" in
|
||||
*":$OUT_BIN:"*) ;;
|
||||
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||
export PATH=\"${OUT_BIN}:\$PATH\"
|
||||
then: source ~/.bashrc") ;;
|
||||
esac
|
||||
|
||||
report_manual
|
||||
|
||||
if [ "$tier" = "dev" ]; then
|
||||
echo "Once Docker is reachable and this is on PATH:"
|
||||
echo
|
||||
echo " kind create cluster --name scratch"
|
||||
echo " kubectl cluster-info --context kind-scratch"
|
||||
echo " kind delete cluster --name scratch"
|
||||
echo
|
||||
echo "That round trip is the real test that this machine can host a rig."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
list() {
|
||||
echo "pinned, linux/amd64 only:"
|
||||
printf ' %-8s %s\n' kubectl "$KUBECTL_VERSION"
|
||||
printf ' %-8s %s\n' jq "$JQ_VERSION"
|
||||
printf ' %-8s %s\n' kind "$KIND_VERSION"
|
||||
printf ' %-8s %s\n' tilt "$TILT_VERSION"
|
||||
printf ' %-8s %s\n' ctlptl "$CTLPTL_VERSION"
|
||||
echo
|
||||
echo " core = $CORE_TOOLS"
|
||||
echo " dev = $CORE_TOOLS $DEV_TOOLS"
|
||||
echo
|
||||
echo "Checksums are pinned in the block at the top of this file. To bump one,"
|
||||
echo "take the new checksum from the publisher's own release list — the header"
|
||||
echo "comment has the exact commands."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
require_linux
|
||||
|
||||
case "${1:-install}" in
|
||||
detect) detect; report_manual ;;
|
||||
list) list ;;
|
||||
verify) verify_tools "${2:-dev}" ;;
|
||||
fetch) shift; require_amd64; pick_downloader; pick_sha; fetch "$@" ;;
|
||||
install) shift; require_amd64; pick_downloader; pick_sha; install "${1:-dev}" ;;
|
||||
*) echo "usage: $0 [detect|list|install|fetch|verify]" >&2
|
||||
echo " install [core|dev] (default dev)" >&2
|
||||
echo " fetch [core|dev] [--to DIR]" >&2
|
||||
echo " OUT_BIN=<dir> overrides the install directory" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
635
rig/standalone/rigmini.sh
Executable file
635
rig/standalone/rigmini.sh
Executable file
@@ -0,0 +1,635 @@
|
||||
#!/usr/bin/env bash
|
||||
# How much memory this box will actually give you before something dies.
|
||||
#
|
||||
# rig answers this for a machine it is installed on. This is the single file
|
||||
# version, for a machine rig is not going to: paste it onto a fresh AWS
|
||||
# WorkSpace, an EC2 box or a container, run it, and get the same numbers in the
|
||||
# same order so two machines can be read side by side.
|
||||
#
|
||||
# There are two numbers and they are rarely the same. `status` reports what the
|
||||
# machine ADVERTISES and what is quietly capping it. `push` finds what it will
|
||||
# SURVIVE, by allocating until it stops.
|
||||
#
|
||||
# The gap between them is the whole reason this exists. Under WSL the cap lives
|
||||
# in .wslconfig; in a container or a managed workspace it is a cgroup limit, and
|
||||
# there /proc/meminfo reports the HOST's memory while the kernel kills you at a
|
||||
# fraction of it. A script that only read MemTotal would confidently report 32 GB
|
||||
# on a box that OOMs at 2.
|
||||
#
|
||||
# Reports and instructs. It never raises a limit, frees anything, writes a
|
||||
# config or installs a package — on a machine you are still evaluating, a probe
|
||||
# that changes what it is measuring is worse than no probe.
|
||||
#
|
||||
# Usage:
|
||||
# rigmini.sh status what it has, what caps it
|
||||
# rigmini.sh push [--to GB] [--to-oom] climb until it stops
|
||||
# rigmini.sh all [--budget GB] both, then the verdict
|
||||
set -euo pipefail
|
||||
|
||||
# ── defaults ───────────────────────────────────────────────────────────────
|
||||
|
||||
STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push().
|
||||
STEP_EXPLICIT=no # whether --step was given, which turns the scaling off.
|
||||
TO_MB="" # --to: stop here regardless. Empty means no hard cap.
|
||||
TO_OOM=no # --to-oom: opt in to running until the kernel intervenes.
|
||||
BUDGET_GB=6 # what the rig data profile is assumed to want; see all().
|
||||
BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below.
|
||||
|
||||
# ── platform ───────────────────────────────────────────────────────────────
|
||||
|
||||
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||
# the tooling. Detectable, so name it instead.
|
||||
require_linux() {
|
||||
case "$(uname -s)" in
|
||||
MINGW*|MSYS*|CYGWIN*)
|
||||
cat >&2 <<'EOF'
|
||||
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||
|
||||
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||
|
||||
wsl --install
|
||||
|
||||
That enables Windows features and needs a reboot, so it is not something this
|
||||
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||
this from there.
|
||||
EOF
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
# Everything below reads /proc. Without it there is nothing to measure, and
|
||||
# failing here beats printing a page of empty fields.
|
||||
if [ ! -r /proc/meminfo ]; then
|
||||
echo "no readable /proc/meminfo — this needs a Linux kernel." >&2
|
||||
echo "On macOS or a BSD none of the numbers below exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||
|
||||
is_container() {
|
||||
[ -f /.dockerenv ] && return 0
|
||||
grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null
|
||||
}
|
||||
|
||||
platform() {
|
||||
if is_wsl; then echo WSL
|
||||
elif is_container; then echo container
|
||||
else echo "native linux"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── reading memory ─────────────────────────────────────────────────────────
|
||||
|
||||
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
||||
|
||||
# MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on
|
||||
# corporate images — need the estimate it replaced, which is worse but not wrong.
|
||||
avail_meminfo_mb() {
|
||||
if grep -q '^MemAvailable:' /proc/meminfo; then
|
||||
mb MemAvailable
|
||||
else
|
||||
awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo
|
||||
fi
|
||||
}
|
||||
|
||||
# Where a cgroup records this cgroup's own limit and usage. Set once by
|
||||
# find_cgroup, because every later reading needs both and hunting for the files
|
||||
# on each call would be the slow part of the poll loop.
|
||||
CG_MAX_FILE=""
|
||||
CG_CUR_FILE=""
|
||||
CG_VERSION=""
|
||||
|
||||
find_cgroup() {
|
||||
local rel
|
||||
|
||||
# Inside a container the cgroup namespace makes the top of the tree BE the
|
||||
# container's own cgroup, so the unqualified path is already the right one.
|
||||
# On a host it is the root cgroup, which is never limited — hence the second
|
||||
# attempt via /proc/self/cgroup, which names the slice this shell is in.
|
||||
if [ -r /sys/fs/cgroup/memory.max ]; then
|
||||
CG_VERSION=v2
|
||||
CG_MAX_FILE=/sys/fs/cgroup/memory.max
|
||||
CG_CUR_FILE=/sys/fs/cgroup/memory.current
|
||||
elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
|
||||
CG_VERSION=v1
|
||||
CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes
|
||||
CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes
|
||||
fi
|
||||
|
||||
rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||
if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then
|
||||
CG_VERSION=v2
|
||||
CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max"
|
||||
CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current"
|
||||
return 0
|
||||
fi
|
||||
|
||||
rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||
if [ -n "$rel" ] && [ "$rel" != "/" ] \
|
||||
&& [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then
|
||||
CG_VERSION=v1
|
||||
CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes"
|
||||
CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# The cap in MB, or "" when there is none worth reporting. v2 spells unlimited
|
||||
# "max"; v1 spells it as a number near 2^63, which is why this compares against
|
||||
# MemTotal rather than testing for a magic value — a "limit" above the machine's
|
||||
# own memory is not a limit, however it is written.
|
||||
cgroup_cap_mb() {
|
||||
local raw cap
|
||||
[ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; }
|
||||
raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max)
|
||||
[ "$raw" = "max" ] && { echo ""; return 0; }
|
||||
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||
cap=$((raw / 1024 / 1024))
|
||||
[ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; }
|
||||
echo "$cap"
|
||||
}
|
||||
|
||||
cgroup_used_mb() {
|
||||
local raw
|
||||
[ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; }
|
||||
raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "")
|
||||
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||
echo $((raw / 1024 / 1024))
|
||||
}
|
||||
|
||||
# ulimit -v is a per-process address-space cap. It stops YOU long before the box
|
||||
# does, and because it is inherited from a login shell it is easy to hit without
|
||||
# knowing it is set.
|
||||
ulimit_v_mb() {
|
||||
local v; v=$(ulimit -v 2>/dev/null || echo unlimited)
|
||||
[ "$v" = "unlimited" ] && { echo ""; return 0; }
|
||||
case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||
echo $((v / 1024))
|
||||
}
|
||||
|
||||
# The number everything else is about: the lowest of the things that can stop
|
||||
# you. Printed at the end of `status` and used as the sanity bound in `push`.
|
||||
effective_ceiling_mb() {
|
||||
local c; c=$(mb MemTotal)
|
||||
local cap; cap=$(cgroup_cap_mb)
|
||||
local ul; ul=$(ulimit_v_mb)
|
||||
[ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap"
|
||||
[ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul"
|
||||
echo "$c"
|
||||
}
|
||||
|
||||
# How much room is left RIGHT NOW, from whichever accounting actually governs.
|
||||
# In a capped container /proc/meminfo describes the host and is worse than
|
||||
# useless for this — it would report tens of gigabytes free on a box that is one
|
||||
# allocation from being killed.
|
||||
headroom_mb() {
|
||||
local cap used
|
||||
cap=$(cgroup_cap_mb)
|
||||
used=$(cgroup_used_mb)
|
||||
if [ -n "$cap" ] && [ -n "$used" ]; then
|
||||
echo $(( cap - used ))
|
||||
else
|
||||
avail_meminfo_mb
|
||||
fi
|
||||
}
|
||||
|
||||
# ── status ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
||||
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
||||
# Windows, then fall back to whichever profile actually owns a config.
|
||||
wslconfig_path() {
|
||||
local profile winpath found
|
||||
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
||||
case "$profile" in
|
||||
""|*%*) ;;
|
||||
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
||||
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
||||
echo "$winpath/.wslconfig"; return 0
|
||||
fi ;;
|
||||
esac
|
||||
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||
[ -n "$found" ] && echo "$found"
|
||||
return 0
|
||||
}
|
||||
|
||||
hogs() {
|
||||
echo " holding the most:"
|
||||
ps -eo rss,comm --sort=-rss 2>/dev/null \
|
||||
| awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
||||
return 0
|
||||
}
|
||||
|
||||
status() {
|
||||
local total avail swap_total swap_free cap ul cur
|
||||
|
||||
echo "host"
|
||||
echo " platform $(platform)"
|
||||
echo " kernel $(uname -r)"
|
||||
[ -r /etc/os-release ] && \
|
||||
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||
echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)"
|
||||
|
||||
# ── the caps first, because they decide what the totals below are worth ──
|
||||
echo
|
||||
echo "caps"
|
||||
cap=$(cgroup_cap_mb)
|
||||
if [ -n "$cap" ]; then
|
||||
cur=$(cgroup_used_mb)
|
||||
echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)"
|
||||
echo " ! /proc/meminfo below describes the HOST, not this cgroup."
|
||||
echo " $(mb MemTotal) MB total is not yours; ${cap} MB is."
|
||||
elif [ -n "$CG_VERSION" ]; then
|
||||
echo " cgroup none (${CG_VERSION} present, no memory limit set)"
|
||||
else
|
||||
echo " cgroup no memory controller found"
|
||||
fi
|
||||
|
||||
ul=$(ulimit_v_mb)
|
||||
if [ -n "$ul" ]; then
|
||||
echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell"
|
||||
echo " it stops this process long before the machine runs out"
|
||||
else
|
||||
echo " ulimit -v unlimited"
|
||||
fi
|
||||
|
||||
# overcommit_memory=0 is the default heuristic: a large allocation is
|
||||
# granted on a guess, and the reckoning arrives later as an OOM kill rather
|
||||
# than as a failed malloc. It is why `push` touches every page it asks for.
|
||||
local om or_
|
||||
om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?')
|
||||
or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?')
|
||||
case "$om" in
|
||||
0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;;
|
||||
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;;
|
||||
2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;;
|
||||
*) echo " overcommit ${om}" ;;
|
||||
esac
|
||||
[ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for"
|
||||
|
||||
# ── what it says it has ──
|
||||
total=$(mb MemTotal); avail=$(avail_meminfo_mb)
|
||||
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
||||
echo
|
||||
echo "memory"
|
||||
echo " total ${total} MB"
|
||||
echo " available ${avail} MB"
|
||||
echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||
if [ "$swap_total" -eq 0 ]; then
|
||||
echo " - no swap: this box has no cushion. It goes from fine to OOM-killed"
|
||||
echo " with nothing in between, which is the abrupt failure you get in a VM."
|
||||
fi
|
||||
|
||||
# postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB,
|
||||
# and the resulting failure names neither shm nor the size.
|
||||
if [ -d /dev/shm ]; then
|
||||
local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}')
|
||||
if [ -n "$shm" ]; then
|
||||
if [ "$shm" -le 64 ]; then
|
||||
echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB"
|
||||
echo " is docker's default. Raise it with --shm-size when the cabinet fails."
|
||||
else
|
||||
echo " /dev/shm ${shm} MB"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "disk"
|
||||
local d
|
||||
for d in / /tmp /var/lib/docker; do
|
||||
[ -d "$d" ] || continue
|
||||
df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}'
|
||||
done
|
||||
|
||||
# kind and Tilt both watch large trees, and the failure mode is silent:
|
||||
# they simply stop noticing file changes. Cheap to report while we are here.
|
||||
local w i
|
||||
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||
echo
|
||||
echo "tooling"
|
||||
echo " inotify watches=$w instances=$i"
|
||||
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||
echo " ! low — anything watching files will silently stop seeing changes"
|
||||
fi
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
if [ -S /var/run/docker.sock ]; then
|
||||
echo " docker socket present, no cli"
|
||||
else
|
||||
echo " docker not installed"
|
||||
fi
|
||||
elif docker info >/dev/null 2>&1; then
|
||||
local n
|
||||
n=$(docker ps -q 2>/dev/null | wc -l)
|
||||
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running"
|
||||
else
|
||||
echo " ! docker cli present but the daemon is unreachable"
|
||||
fi
|
||||
|
||||
# WSL keeps its cap on the Windows side, in a file this shell can read but
|
||||
# not usefully apply — the change costs a full VM restart. Report it, and
|
||||
# report the commonest mistake, which is editing it and not restarting.
|
||||
if is_wsl; then
|
||||
local cfg conf
|
||||
cfg=$(wslconfig_path)
|
||||
echo
|
||||
echo "wsl"
|
||||
if [ -z "$cfg" ]; then
|
||||
echo " ! cannot tell which Windows profile owns .wslconfig"
|
||||
else
|
||||
echo " config $cfg"
|
||||
conf=$(sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$cfg" 2>/dev/null \
|
||||
| tail -1 | tr -d '[:space:]')
|
||||
if [ -n "$conf" ]; then
|
||||
echo " configured $conf (booted ${total} MB)"
|
||||
echo " - if those disagree the edit has not been applied."
|
||||
echo " From a WINDOWS terminal: wsl --shutdown"
|
||||
else
|
||||
echo " configured no memory= set (WSL defaults to half the host RAM, or 8 GB,"
|
||||
echo " whichever is less — which is where your Airflow ceiling comes from)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "effective ceiling $(effective_ceiling_mb) MB"
|
||||
echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box"
|
||||
echo " claims. 'push' measures what it will actually hand over."
|
||||
|
||||
[ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; }
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── push ───────────────────────────────────────────────────────────────────
|
||||
|
||||
STATE=""
|
||||
CHILD=""
|
||||
|
||||
cleanup() {
|
||||
if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then
|
||||
kill -KILL "$CHILD" 2>/dev/null || true
|
||||
wait "$CHILD" 2>/dev/null || true
|
||||
fi
|
||||
[ -n "$STATE" ] && rm -f "$STATE"
|
||||
return 0
|
||||
}
|
||||
|
||||
# The child allocates and stops itself; the parent only watches. That split is
|
||||
# the point: under --to-oom the allocating process is expected to be killed, and
|
||||
# something has to survive to say how far it got.
|
||||
allocator() {
|
||||
# Raise our own OOM score to the maximum so the kernel picks THIS process
|
||||
# first. Raising needs no privilege (only lowering does). Without it, the
|
||||
# kernel is free to choose your shell, your ssh session or dockerd — on a
|
||||
# box you are still using, that is not an acceptable coin toss.
|
||||
echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true
|
||||
|
||||
local arr=() held=0 i=0 rss swapped avail first_swap=0
|
||||
local bytes=$((STEP_MB * 1024 * 1024))
|
||||
local swap_used_start
|
||||
swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) ))
|
||||
|
||||
while :; do
|
||||
# Written STRAIGHT INTO the array element. The obvious spelling —
|
||||
# build one chunk and `arr+=("$chunk")` — costs three copies per step,
|
||||
# not one: the template stays resident, expanding "$chunk" makes a
|
||||
# temporary word, and the append makes the element. A 128 MB step then
|
||||
# needs 384 MB transiently, and on a small box it is killed on the
|
||||
# first append while reporting a third of the true ceiling.
|
||||
#
|
||||
# printf -v into a subscript also means every page is written, so it is
|
||||
# resident rather than merely promised — the only kind of allocation
|
||||
# that measures anything under heuristic overcommit.
|
||||
printf -v "arr[$i]" '%*s' "$bytes" ''
|
||||
i=$((i + 1)); held=$((held + STEP_MB))
|
||||
|
||||
rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0)
|
||||
avail=$(headroom_mb)
|
||||
swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start ))
|
||||
[ "$swapped" -lt 0 ] && swapped=0
|
||||
|
||||
printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \
|
||||
"$held" "$rss" "$avail" "$swapped"
|
||||
printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE"
|
||||
|
||||
# Worth calling out separately from the ceiling: this is where the box
|
||||
# stops being fast and starts being unusable, which for a scheduler is
|
||||
# a different and earlier problem than being killed.
|
||||
if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then
|
||||
first_swap=$held
|
||||
echo " - first swap page at ${held} MB — past here it works but crawls"
|
||||
echo "swapat $held" >> "$STATE"
|
||||
fi
|
||||
|
||||
if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then
|
||||
echo "stop reached-the-cap" >> "$STATE"; return 0
|
||||
fi
|
||||
if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then
|
||||
echo "stop floor" >> "$STATE"; return 0
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
push() {
|
||||
local total ceiling rc=0 last held rss swapat stop
|
||||
total=$(mb MemTotal)
|
||||
ceiling=$(effective_ceiling_mb)
|
||||
|
||||
# A step is worth about a sixty-fourth of the ceiling: enough resolution to
|
||||
# find the edge, few enough lines to read, and small enough that the
|
||||
# transient cost of one allocation never dominates a small box. A fixed
|
||||
# size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB.
|
||||
if [ "$STEP_EXPLICIT" = no ]; then
|
||||
STEP_MB=$(( ceiling / 64 ))
|
||||
[ "$STEP_MB" -lt 4 ] && STEP_MB=4
|
||||
[ "$STEP_MB" -gt 256 ] && STEP_MB=256
|
||||
fi
|
||||
|
||||
# Stop with a cushion rather than riding it to the kill. How big a cushion
|
||||
# depends on what it is protecting. Under a cgroup cap, running out kills
|
||||
# only this container's own processes, so it need cover no more than the
|
||||
# shell that prints the result — and a 512 MB cushion on a 1 GB box would
|
||||
# halve the answer. On a host there is everything else to protect, and the
|
||||
# OOM killer does not promise to pick the process that caused the problem.
|
||||
if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi
|
||||
[ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 ))
|
||||
|
||||
STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX")
|
||||
trap cleanup EXIT
|
||||
# INT kills the child and lets the summary below print anyway, so an
|
||||
# impatient Ctrl-C still tells you how far it got — and, more importantly,
|
||||
# still gives the memory back.
|
||||
trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT
|
||||
|
||||
echo "push"
|
||||
echo " step ${STEP_MB} MB per allocation, every page touched"
|
||||
echo " ceiling ${ceiling} MB claimed"
|
||||
if [ -n "$TO_MB" ]; then
|
||||
echo " stopping at ${TO_MB} MB (--to)"
|
||||
elif [ "$TO_OOM" = yes ]; then
|
||||
echo " ! stopping only when the kernel stops it (--to-oom)"
|
||||
echo " the allocating child is marked as the preferred OOM victim,"
|
||||
echo " but nothing about an OOM kill is entirely polite. Not on a box"
|
||||
echo " running anything you mind losing."
|
||||
else
|
||||
echo " stopping when headroom drops below ${FLOOR_MB} MB"
|
||||
fi
|
||||
echo
|
||||
|
||||
allocator &
|
||||
CHILD=$!
|
||||
wait "$CHILD" || rc=$?
|
||||
CHILD=""
|
||||
trap - INT
|
||||
|
||||
last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true)
|
||||
held=$(echo "$last" | awk '{print $1}')
|
||||
rss=$(echo "$last" | awk '{print $2}')
|
||||
swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||
stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||
|
||||
echo
|
||||
if [ -z "$held" ]; then
|
||||
echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —"
|
||||
echo " try a smaller --step, or check ulimit -v in 'status'."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo " reached ${rss:-$held} MB resident"
|
||||
[ -n "$swapat" ] && echo " swapping from ${swapat} MB"
|
||||
|
||||
case "$stop" in
|
||||
reached-the-cap)
|
||||
echo " outcome stopped at the --to cap, not at a limit."
|
||||
echo " The box held ${TO_MB} MB without complaint; there is more." ;;
|
||||
floor)
|
||||
echo " outcome stopped with a cushion intact, by choice."
|
||||
echo " The real ceiling is higher — --to-oom finds it, at the"
|
||||
echo " cost of an actual OOM kill." ;;
|
||||
interrupted)
|
||||
echo " outcome interrupted at ${rss:-$held} MB — where you stopped it,"
|
||||
echo " not where the box did." ;;
|
||||
*)
|
||||
# No stop line means the child did not decide to stop: it was ended.
|
||||
if [ "$rc" -ge 128 ]; then
|
||||
echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB."
|
||||
elif [ "$rc" -ne 0 ]; then
|
||||
echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})."
|
||||
echo " bash could not get the next chunk — an honest malloc"
|
||||
echo " failure rather than a kill. That is the strict-overcommit"
|
||||
echo " or ulimit path."
|
||||
else
|
||||
echo " outcome ended at ${rss:-$held} MB."
|
||||
fi
|
||||
local ev
|
||||
ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true)
|
||||
if [ -n "$ev" ]; then
|
||||
echo " kernel ${ev#*] }"
|
||||
else
|
||||
echo " - dmesg is unreadable here (dmesg_restrict, or no privilege),"
|
||||
echo " so the kill cannot be confirmed from this side. The number stands."
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
# The gap between the claim and the measurement is the finding — but only
|
||||
# when the BOX chose where to stop. An empty $stop means the child was ended
|
||||
# rather than deciding to end; anything else (--to, the floor) is a stop we
|
||||
# asked for, and flagging those as short of the ceiling would put a warning
|
||||
# on every deliberately small run.
|
||||
local got="${rss:-$held}"
|
||||
echo
|
||||
if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then
|
||||
echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it."
|
||||
echo " Something is taking the difference. 'status' names the candidates:"
|
||||
echo " a cgroup cap, ulimit -v, or memory already resident."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── all ────────────────────────────────────────────────────────────────────
|
||||
|
||||
all() {
|
||||
status
|
||||
echo
|
||||
echo "────────────────────────────────────────────────────────────"
|
||||
echo
|
||||
push
|
||||
|
||||
local got budget_mb ceiling
|
||||
budget_mb=$(( BUDGET_GB * 1024 ))
|
||||
ceiling=$(effective_ceiling_mb)
|
||||
got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true)
|
||||
[ -n "$got" ] || got=0
|
||||
|
||||
echo
|
||||
echo "verdict"
|
||||
echo " budget ${BUDGET_GB} GB for kind + postgres + redis + airflow"
|
||||
# Only worth explaining while it is still a guess. Once --budget is given
|
||||
# the number came from somewhere better than this reasoning, and repeating
|
||||
# the derivation would describe a figure that is no longer in use.
|
||||
if [ "$BUDGET_EXPLICIT" = no ]; then
|
||||
echo " - that is 2 GB per kind node, which is rig's own figure, plus about"
|
||||
echo " 4 GB for the three cabinets. THE 4 GB IS AN ESTIMATE, not something"
|
||||
echo " measured. Re-run with --budget once you have watched the real thing."
|
||||
fi
|
||||
echo " measured ${got} MB handed over"
|
||||
|
||||
if [ "$got" -ge "$budget_mb" ]; then
|
||||
echo " fits, with $(( got - budget_mb )) MB spare."
|
||||
if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then
|
||||
echo " - under 30% spare is thin for a scheduler. Airflow's memory use"
|
||||
echo " is spiky, and the spikes are what get killed."
|
||||
fi
|
||||
else
|
||||
echo " ! short by $(( budget_mb - got )) MB."
|
||||
if [ "$ceiling" -ge "$budget_mb" ]; then
|
||||
echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it."
|
||||
echo " Free something, or read the caps section again."
|
||||
else
|
||||
echo " The box does not have it to give. A bigger bundle, or a smaller"
|
||||
echo " profile: PROFILE=minimal drops the cabinets entirely."
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
parse_flags() {
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;;
|
||||
--to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;;
|
||||
--step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;;
|
||||
--to-oom) TO_OOM=yes; shift ;;
|
||||
--budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then
|
||||
echo "--to and --to-oom contradict each other: one stops early, the other" >&2
|
||||
echo "refuses to stop at all. Pick one." >&2
|
||||
exit 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
require_linux
|
||||
find_cgroup
|
||||
|
||||
cmd="${1:-status}"
|
||||
[ $# -gt 0 ] && shift
|
||||
|
||||
case "$cmd" in
|
||||
status) parse_flags "$@"; status ;;
|
||||
push) parse_flags "$@"; push ;;
|
||||
all) parse_flags "$@"; all ;;
|
||||
*) echo "usage: $0 [status|push|all]" >&2
|
||||
echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2
|
||||
echo " all [--budget GB]" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
24
soleprint/.dockerignore
Normal file
24
soleprint/.dockerignore
Normal file
@@ -0,0 +1,24 @@
|
||||
# The build context is gen/<room>/, and the Dockerfile is `COPY . .` — so this
|
||||
# file is the last thing standing between a stray secret and a public image
|
||||
# layer. build.py already filters these out of the copy into gen/; this repeats
|
||||
# the rule at the docker boundary so a hand-built context, or a future copy path
|
||||
# that forgets, still cannot bake one in.
|
||||
#
|
||||
# Copied into gen/<room>/ by build.py's named-file list alongside the Dockerfile.
|
||||
|
||||
.env
|
||||
.env.*
|
||||
**/.env
|
||||
**/.env.*
|
||||
!.env.example
|
||||
!**/.env.example
|
||||
|
||||
__pycache__/
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
|
||||
.git/
|
||||
.venv/
|
||||
venv/
|
||||
node_modules/
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Render per-room manifests for deploy into the shared `spr` kind cluster.
|
||||
|
||||
The `spr` cluster itself is created via `ctrl/kind-up.sh` at the repo root
|
||||
(one cluster, all rooms). Each room becomes a namespace inside it.
|
||||
The `spr` cluster itself is created by `make cluster up` at the repo root,
|
||||
which hands spr's shape to rig to build (one cluster, all rooms). Each room becomes a namespace inside it.
|
||||
|
||||
Called from build.py when a room opts in to k8s output. Emits:
|
||||
|
||||
|
||||
@@ -395,21 +395,21 @@ resources:
|
||||
|
||||
|
||||
# ─── Lifecycle scripts ──────────────────────────────────────────────
|
||||
# These target the shared `spr` kind cluster (created via repo-root
|
||||
# ctrl/kind-up.sh). Each room owns a namespace inside that cluster.
|
||||
# These target the shared `spr` kind cluster (created by `make cluster up`
|
||||
# at the repo root, which builds it with rig). Each room owns a namespace inside that cluster.
|
||||
|
||||
def k8s_up_sh(*, room: str, cluster: str, nodeport: int) -> str:
|
||||
return f"""\
|
||||
#!/bin/bash
|
||||
# Apply the "{room}" room into the shared `{cluster}` kind cluster.
|
||||
# (Run repo-root ctrl/kind-up.sh first if the cluster doesn't exist.)
|
||||
# (Run 'make cluster up' at the repo root first if the cluster doesn't exist.)
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)"
|
||||
K8S_DIR="$SCRIPT_DIR/k8s"
|
||||
|
||||
if ! kind get clusters 2>/dev/null | grep -q '^{cluster}$'; then
|
||||
echo "Kind cluster '{cluster}' not found — run ctrl/kind-up.sh from the repo root first."
|
||||
echo "Kind cluster '{cluster}' not found — run 'make cluster up' from the repo root first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -432,7 +432,7 @@ def k8s_down_sh(*, room: str, cluster: str) -> str:
|
||||
return f"""\
|
||||
#!/bin/bash
|
||||
# Remove the "{room}" namespace from the shared `{cluster}` cluster.
|
||||
# Leaves the cluster itself running (use repo-root ctrl/kind-down.sh to drop everything).
|
||||
# Leaves the cluster itself running (use 'make cluster down' at the repo root to drop everything).
|
||||
set -e
|
||||
|
||||
CTX="kind-{cluster}"
|
||||
|
||||
@@ -577,7 +577,7 @@ def station_index(request: Request):
|
||||
# not the server. Route registration raises more than ImportError — FastAPI
|
||||
# turns a missing optional dependency into a RuntimeError at decoration time —
|
||||
# and catching only ImportError meant one such tool took the whole app down.
|
||||
for _tool in ("tester", "graphgen", "datagen", "shuntgen"):
|
||||
for _tool in ("tester", "graphgen", "datagen", "shuntgen", "histgen"):
|
||||
try:
|
||||
_module = importlib.import_module(f"station.tools.{_tool}.api")
|
||||
app.include_router(_module.router, prefix="/station")
|
||||
|
||||
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# Local job: which repos this machine distills is not a fact about the tool, and
|
||||
# the list names whatever those repos are. Copy distill-example.json to
|
||||
# distill.json and edit that; it stays here.
|
||||
distill.json
|
||||
26
soleprint/station/tools/distill/distill-example.json
Normal file
26
soleprint/station/tools/distill/distill-example.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that. Everything that shapes the run is at the top; 'repos' is just a list of paths. Preview any change with: ./distill.sh list",
|
||||
|
||||
"_command": "tree = a directory per repo. digest = one .md per repo, flattened into a single readable file. both = each repo as a directory AND a .md. list = write nothing, just report what would be kept.",
|
||||
"command": "list",
|
||||
|
||||
"out": "distilled",
|
||||
|
||||
"_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base (it gets a _PARTIAL.md saying so).",
|
||||
"branch_mode": "full",
|
||||
"diff_base": "main",
|
||||
|
||||
"_filters": "Applied to every repo. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, images, minified, maps). max_bytes skips anything larger and names it in MANIFEST.md.",
|
||||
"exclude": [],
|
||||
"include": [],
|
||||
"all": false,
|
||||
"max_bytes": null,
|
||||
"skip_unchanged": true,
|
||||
"prune": true,
|
||||
|
||||
"_repos": "'path' is absolute, or a name resolved under --root. 'branches' is optional — leave it out for the working tree as it stands, uncommitted changes included. A path may appear more than once.",
|
||||
"repos": [
|
||||
{ "path": "/path/to/some-repo" },
|
||||
{ "path": "/path/to/another-repo", "branches": ["origin/main", "origin/feature/example"] }
|
||||
]
|
||||
}
|
||||
1639
soleprint/station/tools/distill/distill.sh
Executable file
1639
soleprint/station/tools/distill/distill.sh
Executable file
File diff suppressed because it is too large
Load Diff
75
soleprint/station/tools/distill/explode.md
Normal file
75
soleprint/station/tools/distill/explode.md
Normal file
@@ -0,0 +1,75 @@
|
||||
# explode — one file back into the tree it describes
|
||||
|
||||
The other half of `distill.sh`. Together they are a round trip:
|
||||
|
||||
```
|
||||
distill.sh digest -> one file -> paste into a chat -> the reply -> explode.sh
|
||||
```
|
||||
|
||||
`distill` flattens repos into a single readable file so they fit somewhere that
|
||||
only takes text. `explode` takes the answer and writes it back onto disk. Neither
|
||||
is much use without the other.
|
||||
|
||||
```bash
|
||||
./explode.sh --list reply.md # what is in there; writes nothing
|
||||
./explode.sh -o ./restored reply.md # write the tree
|
||||
./explode.sh -o ./restored --force x.md # overwrite what is already there
|
||||
./explode.sh --contract > contract.txt # the format to hand to the model
|
||||
./explode.sh --selftest # check this copy against known input
|
||||
```
|
||||
|
||||
## Ask for `@@`, and attach it
|
||||
|
||||
`--contract` prints the output format to give whatever writes the reply. **Attach
|
||||
that file; do not paste it into the message.** A chat box renders markdown before
|
||||
the model sees it, and `===` alone under a line of text is setext syntax for a
|
||||
heading — so a pasted spec gets rendered as a title and the model is told nothing.
|
||||
`---` is worse, `##` is a heading, backticks open a fence. `@@` means nothing in
|
||||
markdown, which is exactly why it is the marker to ask for.
|
||||
|
||||
Keeping the wording in `--contract` rather than in a note somewhere means what you
|
||||
ask for cannot drift from what the parser accepts.
|
||||
|
||||
## Layouts
|
||||
|
||||
Four shapes are recognised, picked automatically; `--format` overrides the guess.
|
||||
|
||||
| shape | when |
|
||||
| --- | --- |
|
||||
| `@@ FILE: path` … `@@ END` | **ask for this** — explicit, and invisible to markdown |
|
||||
| `=== FILE: path` … `=== END` | the same thing, still read; do not ask for it |
|
||||
| `=== path` marker | a marker line, then the file until the next one |
|
||||
| `## path` + fenced block | `distill.sh`'s own digest |
|
||||
|
||||
Explicit open and close is worth insisting on: a writer emitting plain three-backtick
|
||||
fences silently truncates any file that itself contains a fence — every README with a
|
||||
shell example — because the nested fence looks exactly like the closing one.
|
||||
|
||||
## One reply, several projects
|
||||
|
||||
A thread usually touches more than one repo, and produces one file regardless. The
|
||||
contract asks for paths that begin with the project name, so point `-o` at the
|
||||
directory those projects sit in and each file lands in its own worktree:
|
||||
|
||||
```bash
|
||||
./explode.sh -o ~/wdir ~/Downloads/reply.md
|
||||
```
|
||||
|
||||
No mapping table to maintain, and a new project needs no change here.
|
||||
|
||||
## Refusals
|
||||
|
||||
Paths come out of a text file, so they are untrusted. Anything absolute or reaching
|
||||
upward with `..` is refused and **nothing** is written — the check runs over the whole
|
||||
input before the first file is created. An unclosed block is refused too, rather than
|
||||
writing the file short. Existing files are never overwritten without `--force`.
|
||||
|
||||
A digest whose files `distill` **clipped** to fit a token budget is refused for the
|
||||
same reason: the document holds only their head and tail, and a truncated file that
|
||||
reads complete is the failure the whole format exists to prevent. The tree copy beside
|
||||
the digest has them whole — take them from there.
|
||||
|
||||
Two limits worth knowing: a file whose last line had no trailing newline comes back
|
||||
with one, and in the bare `=== path` layout a line starting with `=== ` inside a file's
|
||||
own content cannot be told from a real marker. The digest and `@@` layouts have no such
|
||||
ambiguity.
|
||||
466
soleprint/station/tools/distill/explode.sh
Executable file
466
soleprint/station/tools/distill/explode.sh
Executable file
@@ -0,0 +1,466 @@
|
||||
#!/usr/bin/env bash
|
||||
# Explode one file back into the tree of files it describes.
|
||||
#
|
||||
# The inverse of distill.sh's digest: something hands you a single text
|
||||
# file with many files inside it, each introduced by its path, and you want the
|
||||
# directory back.
|
||||
#
|
||||
# Three layouts are understood, picked automatically. Prefer the first if you
|
||||
# control what writes the file:
|
||||
#
|
||||
# @@ FILE: pkg/models/domain.py explicit open and close. Nothing has to be
|
||||
# <the file> counted or inferred, and a block that is
|
||||
# @@ END never closed is an error rather than a
|
||||
# file quietly missing its tail.
|
||||
#
|
||||
# === FILE: pkg/models/domain.py the same thing with '===' instead of '@@'.
|
||||
# <the file> Still read, but do not ask for it: see the
|
||||
# === END note on markdown below.
|
||||
#
|
||||
# === ./pkg/models/domain.py a marker line, then the file, until the
|
||||
# <the file> next marker or the end
|
||||
#
|
||||
# ## pkg/models/domain.py distill.sh's own digest: a heading, then
|
||||
# ```python a fenced block. The fence may be longer
|
||||
# <the file> than three backticks, and the closing one
|
||||
# ``` has to match it exactly.
|
||||
#
|
||||
# Usage:
|
||||
# explode.sh [opts] FILE
|
||||
#
|
||||
# Options:
|
||||
# -o DEST where to write the tree (default: the current directory)
|
||||
# --list print what the file contains and write nothing
|
||||
# -n same as --list
|
||||
# --force overwrite files that already exist
|
||||
# --format F fenced | marker | digest | auto (default: auto)
|
||||
# --contract print the output format to hand to whatever generates the file
|
||||
# --selftest check this copy of the script against known input and exit
|
||||
#
|
||||
# Examples:
|
||||
# explode.sh --list bundle.txt
|
||||
# explode.sh -o ./restored bundle.txt
|
||||
# explode.sh -o ./restored --force repo.md
|
||||
# explode.sh --contract > /tmp/contract.txt # attach this, do not paste it
|
||||
# explode.sh -o ~/wdir ~/Downloads/reply.md # one reply, every project
|
||||
#
|
||||
# Why the explicit form is worth asking for: a writer that emits plain three-
|
||||
# backtick fences truncates any file that itself contains a fence — every README
|
||||
# with a shell example — and does it silently, because the nested fence looks
|
||||
# exactly like the closing one. distill.sh avoids that by making its fences
|
||||
# longer than anything inside the file, but nothing else will bother.
|
||||
#
|
||||
# Ask for '@@', not '==='. A chat box renders markdown before the model sees the
|
||||
# message, and '===' alone on a line directly under text is setext syntax for a
|
||||
# level-one heading — so the format spec you paste gets swallowed and rendered as
|
||||
# a title, and the model is told nothing. '---' is worse (heading AND horizontal
|
||||
# rule), '##' is a heading, backticks open a fence. '@@' has no meaning in
|
||||
# markdown at all, which is the whole reason to use it. Both are parsed here, so
|
||||
# nothing already written stops working.
|
||||
#
|
||||
# The other half of that: put the spec in an attached file rather than the
|
||||
# message body. Attachments are not rendered. '--contract' prints the exact text
|
||||
# to attach, so the wording cannot drift from the parser that reads the reply
|
||||
# back.
|
||||
#
|
||||
# One reply, several projects. A thread produces one file however many repos it
|
||||
# touched, so --contract asks for paths that start with the project name. Point
|
||||
# -o at the directory those projects are siblings in and each file lands in its
|
||||
# own worktree — no table to maintain, and a new worktree needs no change here.
|
||||
#
|
||||
# Two limits worth knowing. A file whose last line has no trailing newline comes
|
||||
# back with one: the digest has to put a newline before the closing fence, so the
|
||||
# distinction is not in the input to recover. And in marker layout a line
|
||||
# starting with "=== " inside a file's own content is indistinguishable from a
|
||||
# real marker — there are no fences to say otherwise. The digest layout has no
|
||||
# such ambiguity, which is the reason to prefer it when something else is
|
||||
# generating the file.
|
||||
#
|
||||
# Paths come out of a text file, so they are treated as untrusted: anything
|
||||
# absolute, or reaching upward with .., is refused and nothing is written. A
|
||||
# file that describes /etc/cron.d/x is not a file you want to expand blindly.
|
||||
set -euo pipefail
|
||||
|
||||
SELF="$(basename "$0")"
|
||||
usage() { awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"; }
|
||||
die() { echo "$SELF: $*" >&2; exit 1; }
|
||||
|
||||
DEST="."
|
||||
LIST=""
|
||||
FORCE=""
|
||||
FORMAT="auto"
|
||||
SRC=""
|
||||
SELFTEST=""
|
||||
CONTRACT=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-o) shift; DEST="${1:-}" ;;
|
||||
--list|-n) LIST=1 ;;
|
||||
--force) FORCE=1 ;;
|
||||
--format) shift; FORMAT="${1:-}" ;;
|
||||
--contract) CONTRACT=1 ;;
|
||||
--selftest) SELFTEST=1 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
-*) die "unknown option: $1" ;;
|
||||
*) [ -z "$SRC" ] && SRC="$1" || die "one input file at a time (got '$SRC' and '$1')" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ── the contract ───────────────────────────────────────────────────────────
|
||||
# One copy of the wording, printed rather than remembered, so what you ask for
|
||||
# and what this parses cannot drift apart. Attach it; do not paste it into the
|
||||
# message body, where markdown gets a say first.
|
||||
contract() {
|
||||
cat <<'CONTRACT'
|
||||
OUTPUT FORMAT
|
||||
|
||||
Return every file you changed or created in full, one after another, using
|
||||
exactly this shape and nothing else:
|
||||
|
||||
@@ FILE: <project>/relative/path/to/file.py
|
||||
<the complete contents of the file>
|
||||
@@ END
|
||||
|
||||
Rules:
|
||||
|
||||
- One @@ FILE: line per file, and a matching @@ END line after its last line.
|
||||
- Start every path with the project it belongs to, spelled exactly as the
|
||||
heading of the document it came from, then the path relative to that
|
||||
project's root. One reply covers every project we touched; the prefix is
|
||||
the only thing that says which file goes where, so it is never optional
|
||||
and never abbreviated.
|
||||
- No leading ./ or /.
|
||||
- Between @@ FILE: and @@ END, emit the file verbatim. Do not wrap it in
|
||||
markdown fences, do not add line numbers, do not elide anything as
|
||||
"unchanged" or "...". A partial file is worse than no file.
|
||||
- Anything you want to say to me goes outside the blocks, before the first
|
||||
@@ FILE: or after the last @@ END. Text between blocks is ignored.
|
||||
- Return whole files only. No diffs, no patches, no hunks.
|
||||
- If a file's own content happens to contain a line starting with @@, say so
|
||||
in your prose so I know to check that block by hand.
|
||||
CONTRACT
|
||||
}
|
||||
|
||||
if [ -n "$CONTRACT" ]; then contract; exit 0; fi
|
||||
|
||||
# ── self-test ──────────────────────────────────────────────────────────────
|
||||
# So a copy of this script on another machine can be checked without any real
|
||||
# input, and without asking whether it is the version that knows a given format.
|
||||
# Every case here is one that has actually gone wrong.
|
||||
selftest() {
|
||||
local t rc=0 got want
|
||||
t="$(mktemp -d)"; trap 'rm -rf "$t"' RETURN
|
||||
|
||||
check() { # name, expected, actual
|
||||
if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1"
|
||||
else printf ' FAIL %s\n expected: %s\n got: %s\n' "$1" "$2" "$3"; rc=1
|
||||
fi
|
||||
}
|
||||
|
||||
# A file containing a fence and a stray === line: the two things that break
|
||||
# naive parsers.
|
||||
cat > "$t/a.txt" <<'FIXTURE'
|
||||
Here is the code.
|
||||
|
||||
=== FILE: pkg/core/client.py
|
||||
class Client:
|
||||
pass
|
||||
=== END
|
||||
|
||||
=== FILE: README.md
|
||||
# proj
|
||||
|
||||
```bash
|
||||
pip install proj
|
||||
```
|
||||
|
||||
A line that says === not a marker
|
||||
=== END
|
||||
FIXTURE
|
||||
"$0" -o "$t/a" "$t/a.txt" >/dev/null 2>&1 || true
|
||||
check "fenced: file count" "2" "$(find "$t/a" -type f 2>/dev/null | wc -l)"
|
||||
check "fenced: nested fences" "2" "$(grep -c '```' "$t/a/README.md" 2>/dev/null || echo 0)"
|
||||
check "fenced: === in content" "1" "$(grep -c 'not a marker' "$t/a/README.md" 2>/dev/null || echo 0)"
|
||||
check "fenced: subdirectory" "class Client:" "$(head -1 "$t/a/pkg/core/client.py" 2>/dev/null)"
|
||||
check "fenced: prose skipped" "0" "$(find "$t/a" -name 'Here*' 2>/dev/null | wc -l)"
|
||||
|
||||
# An unclosed block must be refused, not written short.
|
||||
printf '=== FILE: a.py\nx = 1\n=== END\n\n=== FILE: b.py\ny = 2\n' > "$t/b.txt"
|
||||
"$0" -o "$t/b" "$t/b.txt" >/dev/null 2>&1 || true
|
||||
check "unterminated: refused" "1" "$([ -e "$t/b" ] && echo 0 || echo 1)"
|
||||
|
||||
# Paths out of a text file are untrusted.
|
||||
printf '=== FILE: ../escape.py\nx\n=== END\n' > "$t/c.txt"
|
||||
"$0" -o "$t/c" "$t/c.txt" >/dev/null 2>&1 || true
|
||||
check "traversal: refused" "1" "$([ -e "$t/c" ] && echo 0 || echo 1)"
|
||||
|
||||
# The stale-copy failure: explicit format read by the marker parser.
|
||||
"$0" --format marker -o "$t/d" "$t/a.txt" >/dev/null 2>&1 || true
|
||||
check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)"
|
||||
|
||||
# The other two layouts still work.
|
||||
# The @@ markers, which are the ones to ask a chat model for.
|
||||
printf '@@ FILE: pkg/a.py\nx = 1\n@@ END\n@@ FILE: b.md\n# t\n\n```sh\nls\n```\n@@ END\n' > "$t/g.txt"
|
||||
"$0" -o "$t/g" "$t/g.txt" >/dev/null 2>&1 || true
|
||||
check "at-markers: file count" "2" "$(find "$t/g" -type f 2>/dev/null | wc -l)"
|
||||
check "at-markers: fenced body" "2" "$(grep -c '```' "$t/g/b.md" 2>/dev/null || echo 0)"
|
||||
printf '@@ FILE: a.py\nx = 1\n' > "$t/h.txt"
|
||||
"$0" -o "$t/h" "$t/h.txt" >/dev/null 2>&1 || true
|
||||
check "at-markers: unterminated" "1" "$([ -e "$t/h" ] && echo 0 || echo 1)"
|
||||
|
||||
# One reply, several projects: the prefix is just the first directory.
|
||||
printf '@@ FILE: projA/a.py\nx = 1\n@@ END\n@@ FILE: projB/deep/b.py\ny = 2\n@@ END\n' > "$t/i.txt"
|
||||
"$0" -o "$t/i" "$t/i.txt" >/dev/null 2>&1 || true
|
||||
check "multi-project: first" "x = 1" "$(cat "$t/i/projA/a.py" 2>/dev/null)"
|
||||
check "multi-project: nested" "y = 2" "$(cat "$t/i/projB/deep/b.py" 2>/dev/null)"
|
||||
|
||||
printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt"
|
||||
"$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true
|
||||
check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)"
|
||||
|
||||
printf '# d\n\n## x/y.py\n\n```python\nz = 1\n```\n' > "$t/f.txt"
|
||||
"$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true
|
||||
check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)"
|
||||
|
||||
# What distill actually writes: a metadata line between the heading and the
|
||||
# fence, and prose sections whose heading is followed by no fence at all.
|
||||
printf '# d\n\n## Tree\n\nx/\n y.py\n\n## x/y.py\n\n_1 lines · 6 bytes_\n\n```python\nz = 1\n```\n' > "$t/j.txt"
|
||||
"$0" -o "$t/j" "$t/j.txt" >/dev/null 2>&1 || true
|
||||
check "digest: metadata line" "z = 1" "$(cat "$t/j/x/y.py" 2>/dev/null)"
|
||||
check "digest: prose skipped" "1" "$(find "$t/j" -type f 2>/dev/null | wc -l)"
|
||||
|
||||
# A clipped file is head and tail only. Writing it would truncate the real
|
||||
# one, so the whole run is refused.
|
||||
printf '# d\n\n## x/y.py\n\n_900 lines · 60000 bytes · CLIPPED — head and tail only_\n\n```python\nz = 1\n```\n' > "$t/k.txt"
|
||||
"$0" -o "$t/k" "$t/k.txt" >/dev/null 2>&1 || true
|
||||
check "digest: clipped refused" "1" "$([ -e "$t/k" ] && echo 0 || echo 1)"
|
||||
|
||||
echo
|
||||
if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current"
|
||||
else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
if [ -n "$SELFTEST" ]; then selftest; exit $?; fi
|
||||
|
||||
[ -n "$SRC" ] || { usage >&2; exit 1; }
|
||||
[ -f "$SRC" ] || die "no such file: $SRC"
|
||||
case "$FORMAT" in fenced|marker|digest|auto) ;; *) die "--format must be fenced, marker, digest or auto" ;; esac
|
||||
|
||||
# Which layout is it? Count the two shapes and take the commoner one, rather
|
||||
# than trusting the first line that happens to match: a digest of a repo full of
|
||||
# markdown will contain plenty of '=== ' inside its own fenced content, and a
|
||||
# marker file can quote a '## ' heading just as easily.
|
||||
if [ "$FORMAT" = auto ]; then
|
||||
n_fenced=$(grep -cE '^(===|@@) +FILE: +[^ ]' "$SRC" || true)
|
||||
n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true)
|
||||
n_marker=$((n_marker - n_fenced - $(grep -cE '^(===|@@) +END[ \t]*$' "$SRC" || true)))
|
||||
[ "$n_marker" -lt 0 ] && n_marker=0
|
||||
n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true)
|
||||
if [ "$n_fenced" -gt 0 ]; then
|
||||
FORMAT=fenced
|
||||
elif [ "$n_marker" -eq 0 ] && [ "$n_digest" -eq 0 ]; then
|
||||
die "found no '=== FILE:' blocks, no '=== path' markers and no '## path' headings in $SRC"
|
||||
elif [ "$n_marker" -ge "$n_digest" ]; then FORMAT=marker
|
||||
else FORMAT=digest
|
||||
fi
|
||||
echo "format: $FORMAT"
|
||||
fi
|
||||
|
||||
# ── the parser ─────────────────────────────────────────────────────────────
|
||||
# One awk, two modes. In list mode it prints "path<TAB>lines"; otherwise it
|
||||
# writes each file under DEST. Reading the whole thing in awk rather than a
|
||||
# bash read-loop matters once the input is a few megabytes.
|
||||
#
|
||||
# In digest mode a heading only opens a file if a fence follows it. distill.sh
|
||||
# writes '## Tree' and '## Binary files ...' sections that are prose, and
|
||||
# treating those as files would scatter junk through the output.
|
||||
parse() {
|
||||
awk -v dest="$DEST" -v mode="$1" -v fmt="$FORMAT" '
|
||||
function flush() {
|
||||
if (path != "") {
|
||||
if (mode == "list") { printf "%s\t%d\n", path, n }
|
||||
path = ""
|
||||
}
|
||||
n = 0
|
||||
}
|
||||
function clean(p) {
|
||||
sub(/^\.\//, "", p)
|
||||
sub(/[ \t\r]+$/, "", p)
|
||||
return p
|
||||
}
|
||||
function unsafe(p) {
|
||||
return (p == "" || p ~ /^\// || p ~ /^[A-Za-z]:/ || p ~ /(^|\/)\.\.(\/|$)/)
|
||||
}
|
||||
function open_file(p) {
|
||||
path = p
|
||||
n = 0
|
||||
if (mode == "write") {
|
||||
out = dest "/" path
|
||||
d = out; sub(/\/[^\/]*$/, "", d)
|
||||
system("mkdir -p \"" d "\"")
|
||||
printf "" > out
|
||||
}
|
||||
}
|
||||
function emit(line) {
|
||||
n++
|
||||
if (mode == "write") print line >> (dest "/" path)
|
||||
}
|
||||
|
||||
# Explicit open/close. The whole point is that nothing is inferred:
|
||||
# content is content until the END line, whatever it looks like.
|
||||
fmt == "fenced" && path == "" && /^(===|@@) +FILE: +/ {
|
||||
p = substr($0, index($0, "FILE:") + 5)
|
||||
sub(/^[ \t]+/, "", p)
|
||||
p = clean(p)
|
||||
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
||||
open_file(p)
|
||||
next
|
||||
}
|
||||
fmt == "fenced" && path != "" && /^(===|@@) +END[ \t]*$/ { flush(); next }
|
||||
fmt == "fenced" && path == "" { next } # anything between blocks is prose
|
||||
|
||||
fmt == "marker" && /^=== +/ {
|
||||
flush()
|
||||
p = clean(substr($0, index($0, " ") + 1))
|
||||
# "FILE: ./x.py" and "END" are not paths, they are the explicit
|
||||
# format being read by the wrong parser. Left alone this writes a
|
||||
# directory literally called "FILE: ." and a file called "END",
|
||||
# which is what an out-of-date copy of this script did once.
|
||||
if (p ~ /^FILE:/ || p == "END") { print "WRONGFMT\t" p; bad = 1; next }
|
||||
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
||||
open_file(p)
|
||||
next
|
||||
}
|
||||
|
||||
# Only when NOT already inside a file. The content of any markdown file
|
||||
# in the input is full of "## " headings, and the fence is the only
|
||||
# thing that says which ones are structure and which are text. Once a
|
||||
# file is open, the matching close fence is the sole way out.
|
||||
fmt == "digest" && path == "" && /^## +/ {
|
||||
flush()
|
||||
pending = clean(substr($0, 4))
|
||||
expect = 1
|
||||
next
|
||||
}
|
||||
fmt == "digest" && expect == 1 {
|
||||
if ($0 ~ /^[ \t]*$/) next # blank line between the two
|
||||
# distill.sh puts an italic "N lines, B bytes" line under each heading,
|
||||
# so the fence is no longer the next thing after it. Step over that
|
||||
# line rather than reading it as prose — without this, every file in
|
||||
# a current digest is skipped and the whole document looks empty.
|
||||
if ($0 ~ /^_.*_[ \t]*$/) {
|
||||
# Unless it says the file was clipped. A clipped body is head
|
||||
# and tail with a marker in between; writing it out would
|
||||
# replace a real file with a truncated one that reads complete,
|
||||
# which is the exact failure the unterminated check exists for.
|
||||
if ($0 ~ /CLIPPED/) {
|
||||
print "CLIPPED\t" pending
|
||||
bad = 1; expect = 0; pending = ""
|
||||
next
|
||||
}
|
||||
next
|
||||
}
|
||||
if ($0 ~ /^`{3,}/) { # a fence: this is a file
|
||||
match($0, /^`+/)
|
||||
fence = substr($0, 1, RLENGTH)
|
||||
expect = 0
|
||||
if (unsafe(pending)) { print "UNSAFE\t" pending; bad = 1; next }
|
||||
open_file(pending)
|
||||
next
|
||||
}
|
||||
expect = 0 # prose section, not a file
|
||||
pending = ""
|
||||
next
|
||||
}
|
||||
fmt == "digest" && path != "" && $0 == fence { flush(); next }
|
||||
|
||||
|
||||
{ if (path != "") emit($0) }
|
||||
|
||||
END {
|
||||
if (fmt == "fenced" && path != "") {
|
||||
print "UNTERMINATED\t" path
|
||||
bad = 1
|
||||
}
|
||||
flush()
|
||||
exit (bad ? 3 : 0)
|
||||
}
|
||||
' "$SRC"
|
||||
}
|
||||
|
||||
# Validate before writing anything: a refusal after half the tree is on disk is
|
||||
# not a refusal.
|
||||
# awk exits non-zero when it found something wrong; that is the signal, not a
|
||||
# crash, so let it through and report it properly below.
|
||||
scan="$(parse list || true)"
|
||||
|
||||
refused="$(printf '%s\n' "$scan" | grep '^UNSAFE' || true)"
|
||||
if [ -n "$refused" ]; then
|
||||
echo "$SELF: refusing — these paths escape the destination:" >&2
|
||||
printf '%s\n' "$refused" | sed 's/^UNSAFE\t/ /' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A block that never closed means the input is malformed, or a file contained
|
||||
# the END line. Either way the tail is missing, and a truncated source file that
|
||||
# looks complete is the failure this format exists to prevent.
|
||||
wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)"
|
||||
if [ -n "$wrongfmt" ]; then
|
||||
echo "$SELF: this file uses 'FILE: path' / 'END' markers, but it was read as" >&2
|
||||
echo "the plain marker format, which would create a directory called 'FILE: .'" >&2
|
||||
echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)"
|
||||
if [ -n "$unterminated" ]; then
|
||||
echo "$SELF: refusing — this block was never closed with '@@ END' or '=== END':" >&2
|
||||
printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2
|
||||
echo "the file it describes would be silently truncated" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# distill clips the largest files to fit a budget, and says so. The tree copy
|
||||
# beside the digest has them whole, so the fix is to take them from there — not
|
||||
# to write out the head and tail under the real name.
|
||||
clipped="$(printf '%s\n' "$scan" | grep '^CLIPPED' || true)"
|
||||
if [ -n "$clipped" ]; then
|
||||
echo "$SELF: refusing — distill clipped these, so the digest has only their" >&2
|
||||
echo "head and tail:" >&2
|
||||
printf '%s\n' "$clipped" | sed 's/^CLIPPED\t/ /' >&2
|
||||
echo "take them from the tree copy instead; writing these would truncate them" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT|CLIPPED)' || true)"
|
||||
[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)"
|
||||
count=$(printf '%s\n' "$listing" | grep -c . )
|
||||
|
||||
if [ -n "$LIST" ]; then
|
||||
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %-60s %5d lines\n", $1, $2 }'
|
||||
echo "$count files"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Existing files are someone's work until proven otherwise.
|
||||
if [ -z "$FORCE" ]; then
|
||||
clashes=""
|
||||
while IFS=$'\t' read -r p _; do
|
||||
[ -e "$DEST/$p" ] && clashes="$clashes $p"$'\n'
|
||||
done <<< "$listing"
|
||||
if [ -n "$clashes" ]; then
|
||||
echo "$SELF: these already exist under $DEST:" >&2
|
||||
printf '%s' "$clashes" >&2
|
||||
echo "re-run with --force to overwrite" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "$DEST"
|
||||
parse write >/dev/null
|
||||
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %s\n", $1 }'
|
||||
echo "wrote $count files to $DEST"
|
||||
5
soleprint/station/tools/histgen/.gitignore
vendored
Normal file
5
soleprint/station/tools/histgen/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Local settings: whose repo this machine points at is not a fact about the
|
||||
# tool. Copy the folder, run `make init-config`, and the answer stays here.
|
||||
histgen.json
|
||||
def
|
||||
__pycache__/
|
||||
137
soleprint/station/tools/histgen/Makefile
Normal file
137
soleprint/station/tools/histgen/Makefile
Normal file
@@ -0,0 +1,137 @@
|
||||
# histgen — one target per verb.
|
||||
#
|
||||
# The folder is meant to be copied out of soleprint and used on its own, so
|
||||
# everything here is derived from where this file sits rather than written down:
|
||||
# copy the directory anywhere, `cd` into it, and `make` works. Renaming it works
|
||||
# too, since the package name comes from the directory.
|
||||
#
|
||||
# Two directories, and the whole tool hangs off the difference:
|
||||
#
|
||||
# SOURCE the tree to read. Read-only, always. Nothing is written into it.
|
||||
# OUT the plan, the briefs, and OUT/<name>/ — a copy of the source with
|
||||
# the designed history committed into it.
|
||||
#
|
||||
# make check prove it works, on its own fixture
|
||||
# make copy SOURCE=~/work/x OUT=~/out just the files, no repo, no keys
|
||||
# make run SOURCE=~/work/x OUT=~/out scan + plan + brief
|
||||
# make list the commits, to confirm
|
||||
# make commands copy the files, hand back git commands
|
||||
# make export ...or have it commit them for you
|
||||
#
|
||||
# Set them once and the verbs take no arguments:
|
||||
#
|
||||
# make init-config SOURCE=... OUT=...
|
||||
# make config what everything resolves to
|
||||
# make status what is in OUT, and what is left
|
||||
#
|
||||
# The logic lives in the Python, never here. Each target is one invocation.
|
||||
|
||||
HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
|
||||
PKG := $(notdir $(HERE))
|
||||
PARENT := $(patsubst %/,%,$(dir $(HERE)))
|
||||
PY ?= python3
|
||||
PREFIX ?= $(HOME)/.local
|
||||
|
||||
# Run the package from its parent, which is what `python -m` needs and what
|
||||
# lets this work without installing anything.
|
||||
HISTGEN := PYTHONPATH=$(PARENT) $(PY) -m $(PKG)
|
||||
|
||||
# Extra flags for the verb being run: make plan REPO=x ARGS=--max-files=12
|
||||
ARGS ?=
|
||||
|
||||
# Left empty, these say nothing and the config file decides. Passing REPO= or
|
||||
# OUT= on the command line overrides it, which is the precedence the tool
|
||||
# already applies — the Makefile just has to not invent a default of its own.
|
||||
SOURCE ?=
|
||||
OUT ?=
|
||||
CONFIG ?=
|
||||
|
||||
WHERE := $(if $(SOURCE),--source $(SOURCE)) $(if $(OUT),--out $(OUT)) \
|
||||
$(if $(CONFIG),--config $(CONFIG))
|
||||
|
||||
.PHONY: help check run copy scan plan list brief export keep commands dry-run verify status \
|
||||
against-history clean install uninstall doctor config init-config
|
||||
|
||||
help: ## List every target
|
||||
@echo "histgen — seed a clean, logical history into a repo"
|
||||
@echo
|
||||
@grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \
|
||||
| awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-16s\033[0m %s\n", $$1, $$2}'
|
||||
@echo
|
||||
@echo " SOURCE=/path/to/tree what to read (read-only, never written to)"
|
||||
@echo " OUT=/path/to/out what to write (plan, briefs, and OUT/<name>/)"
|
||||
@echo " ARGS=... extra flags, e.g. ARGS=\"--max-files 25\""
|
||||
@echo
|
||||
@echo " Both can live in histgen.json instead: make init-config SOURCE=.. OUT=.."
|
||||
|
||||
check: ## Prove the whole pipeline works, needing no repo and nothing installed
|
||||
@$(PY) $(HERE)/selftest.py
|
||||
|
||||
config: ## Show what repo, out and max-files resolve to
|
||||
@$(HISTGEN) config $(WHERE)
|
||||
|
||||
init-config: ## Write a starter histgen.json beside the tool
|
||||
@$(HISTGEN) config --init $(WHERE)
|
||||
|
||||
doctor: ## Report whether this machine can run it
|
||||
@printf 'python : '; $(PY) --version 2>&1 || echo MISSING
|
||||
@printf 'git : '; git --version 2>&1 || echo MISSING
|
||||
@printf 'package: %s (from %s)\n' '$(PKG)' '$(PARENT)'
|
||||
@$(HISTGEN) --help >/dev/null 2>&1 \
|
||||
&& echo 'import : ok' || echo 'import : FAILED — is the folder intact?'
|
||||
|
||||
run: ## scan + plan + brief, everything before the messages are needed
|
||||
@$(HISTGEN) run $(WHERE) $(ARGS)
|
||||
|
||||
list: ## Print the commits, to confirm before exporting
|
||||
@$(HISTGEN) list $(WHERE) $(ARGS)
|
||||
|
||||
status: ## What is in OUT, what state it is in, and what is left to do
|
||||
@$(HISTGEN) status $(WHERE) $(ARGS)
|
||||
|
||||
copy: ## Copy the files out: no .git, nothing ignored, no keys, no build output
|
||||
@$(HISTGEN) copy $(WHERE) $(ARGS)
|
||||
|
||||
scan: ## Read the source and cache what was read
|
||||
@$(HISTGEN) scan $(WHERE) $(ARGS)
|
||||
|
||||
plan: ## Order the files and cut them into commits
|
||||
@$(HISTGEN) plan $(WHERE) $(ARGS)
|
||||
|
||||
brief: ## Write one brief per commit, for the messages
|
||||
@$(HISTGEN) brief $(WHERE) $(ARGS)
|
||||
|
||||
against-history: ## Report how an existing history compares. Reads only
|
||||
@$(HISTGEN) plan $(WHERE) --against-history $(ARGS)
|
||||
|
||||
dry-run: ## Write the export as a reviewable regen.sh instead of running it
|
||||
@$(HISTGEN) export $(WHERE) --dry-run $(ARGS)
|
||||
|
||||
export: ## Copy the source into OUT and commit the history. Resumes if interrupted
|
||||
@$(HISTGEN) export $(WHERE) $(ARGS)
|
||||
|
||||
keep: ## Export, carrying an existing history over onto its own branch
|
||||
@$(HISTGEN) export $(WHERE) --keep-history $(ARGS)
|
||||
|
||||
commands: ## Copy the files, create no repo, print the git commands to run yourself
|
||||
@$(HISTGEN) export $(WHERE) --commands $(ARGS)
|
||||
|
||||
verify: ## Nothing left untracked, and the exported tree matches the source
|
||||
@$(HISTGEN) verify $(WHERE) $(ARGS)
|
||||
|
||||
clean: ## Delete the whole OUT directory. The source is not touched
|
||||
@d=$$($(HISTGEN) config $(WHERE) | awk '/^out /{print $$2}'); \
|
||||
test -n "$$d" -a "$$d" != "(unset)" || { echo "Error: no OUT set." >&2; exit 1; }; \
|
||||
rm -rf "$$d" && echo "Removed $$d. The source was never written to."
|
||||
|
||||
install: ## Put a `histgen` command on PATH, pointing back at this folder
|
||||
@mkdir -p $(PREFIX)/bin
|
||||
@printf '#!/bin/sh\n# Generated by histgen'"'"'s Makefile; points at the folder it was run from.\nPYTHONPATH="%s" exec "%s" -m %s "$$@"\n' \
|
||||
'$(PARENT)' '$(shell command -v $(PY))' '$(PKG)' > $(PREFIX)/bin/histgen
|
||||
@chmod +x $(PREFIX)/bin/histgen
|
||||
@echo "Installed $(PREFIX)/bin/histgen -> $(HERE)"
|
||||
@case ":$$PATH:" in *":$(PREFIX)/bin:"*) ;; \
|
||||
*) echo "Note: $(PREFIX)/bin is not on PATH." ;; esac
|
||||
|
||||
uninstall: ## Remove that command
|
||||
@rm -f $(PREFIX)/bin/histgen && echo "Removed $(PREFIX)/bin/histgen"
|
||||
545
soleprint/station/tools/histgen/README.md
Normal file
545
soleprint/station/tools/histgen/README.md
Normal file
@@ -0,0 +1,545 @@
|
||||
# histgen
|
||||
|
||||
Seeds a clean, logical history into a repo — reading one directory and writing
|
||||
another, so the tree it reads is never touched.
|
||||
|
||||
```bash
|
||||
histgen copy # just the files: no .git, nothing ignored, no keys
|
||||
histgen run # scan the source, plan the commits, write the briefs
|
||||
histgen list # the commits, to confirm
|
||||
histgen commands # copy the files, hand back the git add/commit commands
|
||||
histgen export # ...or have it do the committing itself
|
||||
```
|
||||
|
||||
Two directories, and the whole tool hangs off the difference:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **source** | the tree to read. Opened read-only, always. Not a commit, not a `.git`, not a state file is written into it — it can be a checkout you do not own or a read-only mount. |
|
||||
| **out** | everything produced. The index, the plan, the briefs, and `out/<name>/` — a copy of the source with the designed history committed into it. |
|
||||
|
||||
That separation is what makes the history safe to argue with. It is an argument
|
||||
you will have more than once, and every attempt is a directory you can delete
|
||||
rather than a repo you have to put back.
|
||||
|
||||
Stdlib only, no network, no API key. Also readable in the browser at
|
||||
`/station/tools/histgen/`, which shows a plan and never writes one.
|
||||
|
||||
## Copying it out
|
||||
|
||||
Copy the folder anywhere, `cd` into it, and use the Makefile. It derives the
|
||||
package name and path from where it sits, so the directory can be renamed and
|
||||
still work, and nothing has to be installed.
|
||||
|
||||
```bash
|
||||
cp -r histgen ~/tools/ && cd ~/tools/histgen
|
||||
|
||||
make check # prove it works, on its own fixture
|
||||
make doctor # what this machine has
|
||||
make init-config SOURCE=~/work/x OUT=~/out # set both once
|
||||
make run # scan + plan + brief
|
||||
make list # the commits, to confirm
|
||||
make export # write them into ~/out/x
|
||||
make help # every target
|
||||
```
|
||||
|
||||
One target per verb, plus `dry-run`, `against-history`, `verify` and `clean`.
|
||||
Extra flags go in `ARGS`:
|
||||
|
||||
```bash
|
||||
make plan REPO=/path/to/repo ARGS="--max-files 25"
|
||||
```
|
||||
|
||||
`make install` drops a `histgen` command in `~/.local/bin` pointing back at the
|
||||
folder, if you would rather not `cd` into it.
|
||||
|
||||
## Just the files, without the repo
|
||||
|
||||
```bash
|
||||
make copy SOURCE=~/code/myproject OUT=~/clean
|
||||
```
|
||||
|
||||
Gives you `~/clean/myproject` holding what the project actually is — no `.git`,
|
||||
nothing gitignored, nothing a build regenerates, and nothing that looks like a
|
||||
key. No history is planned and nothing is committed; this is the plain utility
|
||||
underneath the rest.
|
||||
|
||||
```
|
||||
14 files tracked, 7 to copy, 7 left behind.
|
||||
|
||||
secret — looks like a key or a credential (2):
|
||||
.env
|
||||
certs/server.key
|
||||
|
||||
ignored — tracked, but the ignore rules say they should not be (1):
|
||||
data/raw/dump.sql
|
||||
|
||||
derived — a build regenerates these (3):
|
||||
dist/bundle.js.map
|
||||
dist/bundle.min.js
|
||||
package-lock.json
|
||||
|
||||
oversize — larger than --max-bytes (1):
|
||||
data/big.csv
|
||||
|
||||
Copied to /home/you/clean/myproject
|
||||
no .git — the source has one and it was not copied
|
||||
what was left behind: /home/you/clean/copied.md
|
||||
```
|
||||
|
||||
**Every drop is named**, on screen and in `copied.md`. A file quietly missing
|
||||
from a copy is the same class of failure as a file quietly missing from a
|
||||
history, one directory earlier.
|
||||
|
||||
The same filter runs on the history path. `scan` drops secrets and
|
||||
ignored-but-tracked files before they ever reach a plan, and says so:
|
||||
|
||||
```
|
||||
Scanned 5 files: 5 read, 0 reused from cache.
|
||||
3 left out of the history:
|
||||
.env (looks like a key or a credential)
|
||||
certs/tls.key (looks like a key or a credential)
|
||||
data/dump.sql (tracked, but the ignore rules say they should not be)
|
||||
```
|
||||
|
||||
so `make commands` and `make export` cannot commit a key that `make copy` would
|
||||
have left behind. `make status` keeps saying it afterwards, and
|
||||
`--keep-secrets` turns it off.
|
||||
|
||||
**One deliberate difference between the two.** `copy` also drops what a build
|
||||
regenerates — lockfiles, maps, minified output — because a snapshot is for
|
||||
reading. `scan` keeps them, because a lockfile is content in a repo somebody is
|
||||
going to use. `copy --all` keeps them too.
|
||||
|
||||
### What gets left behind, and why
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **secret** | `.env`, `*.pem`, `*.key`, `id_rsa`, `.netrc`, `credentials.json`, `service-account*.json`, `.ssh/` |
|
||||
| **ignored** | tracked *despite* the repo's own ignore rules — someone ran `git add -f` once |
|
||||
| **derived** | lockfiles, `*.map`, `*.min.js`, `*.pyc`, `*.so`, `__pycache__/`, `node_modules/` |
|
||||
| **oversize** | whatever `--max-bytes` says |
|
||||
|
||||
The derived list is ported from `ppl/ctrl/distill.sh`, including the lesson in
|
||||
its comments: **the line is derived-vs-content, not text-vs-binary.** Images,
|
||||
fonts, spreadsheets and PDFs are content and are kept — none of them can be
|
||||
regenerated from what is left, which is the only thing that makes a file safe to
|
||||
drop. That distinction was wrong in distill once and cost real files.
|
||||
|
||||
**Lookalikes are kept on purpose.** `.env.example` is the documented way to say
|
||||
what the real one needs, and dropping it takes the documentation with the
|
||||
secret. Same for `server.key.pub` — a public key is not a private one.
|
||||
|
||||
The **ignored** case is the one worth reading. `git add -f` is not always a
|
||||
mistake, so these are named rather than assumed either way; but a dump or a
|
||||
credentials file that went in once and was never noticed since looks exactly
|
||||
like this, and the repo is already contradicting itself about them.
|
||||
|
||||
### Knobs
|
||||
|
||||
`--keep-secrets`, `--exclude` and `--include` work on `copy`, `scan` and `run`
|
||||
alike, and can live in `histgen.json`. The rest are `copy`'s own.
|
||||
|
||||
```bash
|
||||
make copy ARGS="--dry-run" # report only, write nothing
|
||||
make copy ARGS="--max-bytes 100000" # leave anything bigger
|
||||
make copy ARGS="--exclude '*.csv' --exclude data/"
|
||||
make copy ARGS="--include package-lock.json" # keep it, whatever the filters say
|
||||
make copy ARGS="--all --keep-secrets" # turn the two filters off
|
||||
```
|
||||
|
||||
`--exclude` follows distill's rule: a pattern with no `/` matches basenames at
|
||||
any depth. `--include` is checked first and wins outright, so one file can be
|
||||
rescued without turning a whole filter off.
|
||||
|
||||
To then plan a history from the cleaned tree, point a fresh run at it:
|
||||
|
||||
```bash
|
||||
make run SOURCE=~/clean/myproject OUT=~/history
|
||||
```
|
||||
|
||||
## Recipe: a copy with a clean history
|
||||
|
||||
The common case. You have a repo, you want the same files somewhere new with a
|
||||
history that reads like the thing was built on purpose, and you do not care what
|
||||
the old history said.
|
||||
|
||||
Nothing is written to the original. The old history is simply not carried over —
|
||||
that is the default, and `--keep-history` is the opt-in for when you do want it.
|
||||
|
||||
**`OUT` is the parent directory, not the repo.** The copy keeps the source's own
|
||||
name underneath it. `OUT` does not have to exist yet; it is created on the first
|
||||
command.
|
||||
|
||||
```bash
|
||||
cd ~/tools/histgen # wherever you copied the folder
|
||||
|
||||
make init-config SOURCE=~/code/myproject OUT=~/clean
|
||||
make config # check both paths before anything runs
|
||||
```
|
||||
|
||||
```
|
||||
source /home/you/code/myproject read-only
|
||||
out /home/you/clean
|
||||
exported to /home/you/clean/myproject <- the copy ends up here
|
||||
```
|
||||
|
||||
### 1. Plan it
|
||||
|
||||
```bash
|
||||
make run # reads the source, groups the files, writes a brief per commit
|
||||
make list # the proposed commits, in order
|
||||
```
|
||||
|
||||
`make list` is the thing to look at. Every commit is marked `*` until it has a
|
||||
message. If a commit holds two unrelated ideas, move a path between groups in
|
||||
`~/clean/plan.json` and run `make plan && make list` again — the grouping is a
|
||||
proposal, and re-planning keeps every message whose group still holds the same
|
||||
files.
|
||||
|
||||
### 2. Write the messages
|
||||
|
||||
`~/clean/briefs/` has one markdown file per commit, holding each file's opening
|
||||
comment. Read them and write a `title` and `body` into each group in
|
||||
`~/clean/plan.json`.
|
||||
|
||||
This is the part worth doing properly: the briefs carry the reasoning already in
|
||||
the code, which is what makes a message worth reading. A message reconstructed
|
||||
from the diff just restates the diff.
|
||||
|
||||
```bash
|
||||
make list # again — the titles you wrote now show instead of the * marks
|
||||
```
|
||||
|
||||
To see the shape end to end before writing any of them, use
|
||||
`ARGS=--allow-untitled` in the next step; the commits get their group name as a
|
||||
subject, which is fine for a throwaway pass and not fine for anything you keep.
|
||||
|
||||
### 3. Get the commands, and run them yourself
|
||||
|
||||
```bash
|
||||
make commands
|
||||
```
|
||||
|
||||
This copies the planned files into `~/clean/myproject` and **creates no repo** —
|
||||
no `git init`, no `.git`. What comes back is the list, also saved to
|
||||
`~/clean/commands.sh`:
|
||||
|
||||
```
|
||||
cd /home/you/clean/myproject
|
||||
git init
|
||||
|
||||
# 01 Repo skeleton: ignore rules and line-endings policy
|
||||
git add -- .gitattributes .gitignore
|
||||
git commit -F /home/you/clean/messages/01-skeleton.txt
|
||||
|
||||
# 02 Pin the toolchain in one manifest
|
||||
git add -- versions.env
|
||||
git commit -F /home/you/clean/messages/02-versions.txt
|
||||
|
||||
...
|
||||
|
||||
# Worth running afterwards. The first says no file was silently
|
||||
# missed; the second says the result is byte-identical to the source.
|
||||
git status --porcelain
|
||||
git rev-parse HEAD^{tree} # expect 3132703a817922f9f83bafa0e86e6bdf002ce8cb
|
||||
```
|
||||
|
||||
`git init` is the first line of the list rather than something already done: a
|
||||
repo that appeared without you asking is exactly what someone reaching for this
|
||||
mode does not want. Read the list, edit it, reorder it, run it a line at a time.
|
||||
|
||||
Messages go in files rather than `-m` because bodies are multi-line, and the
|
||||
body is where the *why* lives. Edit the message files directly if you want to
|
||||
reword something — nothing has been committed yet.
|
||||
|
||||
The last two commands are worth running when you are done. `git status
|
||||
--porcelain` printing nothing means no file was silently missed, which is the
|
||||
failure this whole exercise exists to prevent. The tree hash matching means the
|
||||
result is byte-identical to the source.
|
||||
|
||||
Only the files in the plan are copied: not the source's `.git`, not anything
|
||||
gitignored. If `~/clean/myproject` already contains a repo, this refuses rather
|
||||
than handing you commands that would commit into it.
|
||||
|
||||
### Or let it do the committing
|
||||
|
||||
```bash
|
||||
make dry-run # optional: writes ~/clean/regen.sh, a script that does everything
|
||||
make export # copy and commit, checking both guards itself
|
||||
```
|
||||
|
||||
```
|
||||
18 commits on main in /home/you/clean/myproject. Checking:
|
||||
nothing left untracked: ok
|
||||
tree matches source (3132703a8179): ok
|
||||
```
|
||||
|
||||
Three modes, and the difference is who does what:
|
||||
|
||||
| | copies the files | makes the repo | commits |
|
||||
|---|---|---|---|
|
||||
| `make commands` | yes | no — you run `git init` | you |
|
||||
| `make dry-run` | no — writes a script that would | in the script | in the script |
|
||||
| `make export` | yes | yes | yes, and checks both guards |
|
||||
|
||||
```bash
|
||||
cd ~/clean/myproject
|
||||
git log --oneline
|
||||
```
|
||||
|
||||
### If something goes wrong
|
||||
|
||||
```bash
|
||||
make status # says which of the four states out is in, and what to do next
|
||||
```
|
||||
|
||||
- Interrupted partway? Run `make export` again — it continues from the commit
|
||||
after the last one recorded, rather than starting over or refusing. (This
|
||||
applies to `make export`; with `make commands` the repo is yours, so a
|
||||
half-finished run is yours to continue from the list.)
|
||||
- Changed the plan after exporting? `make status` says so; `make export
|
||||
ARGS=--force` discards the copy and redoes it.
|
||||
- Want to start completely fresh? `make clean` deletes the whole `OUT`
|
||||
directory. The source is not touched, so there is nothing to put back.
|
||||
|
||||
### Handing this to someone else
|
||||
|
||||
Everything above needs the folder, `python3` and `git` — nothing installed, no
|
||||
network, no API key. Copy the directory, then:
|
||||
|
||||
```bash
|
||||
cd histgen && make check # proves the whole pipeline on a fixture it builds
|
||||
make help # every target
|
||||
```
|
||||
|
||||
## What is already in out
|
||||
|
||||
`export` writes, so it starts by working out what it is writing into. Four
|
||||
states, and they are genuinely different — `histgen status` prints which one:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **absent** | nothing there yet. Copy the tree, init, commit. |
|
||||
| **unfinished** | commits this tool made, and a record of where it stopped. Something interrupted the run. **Continue from the group after the last one recorded.** |
|
||||
| **foreign** | commits this tool did *not* make. That history is someone's, so nothing is rewritten, moved or deleted: the designed account goes on its own orphan branch and the existing branch is left exactly where it was. |
|
||||
| **stale** | the plan changed, or the copy moved underneath us. Refuse, and say which of the two it was. `--force` discards and starts over. |
|
||||
|
||||
Telling **unfinished** from **foreign** is the whole reason `progress.json`
|
||||
exists. Without it both read as "there are commits here", and the tool either
|
||||
destroys work it should have kept or refuses to finish work it started — which
|
||||
is exactly what it used to do.
|
||||
|
||||
```
|
||||
$ histgen status
|
||||
source /home/mariano/wdir/rdir/adapter
|
||||
out /home/mariano/histories/adapter
|
||||
census 91 files
|
||||
plan 24 commits, 6 without a message
|
||||
export unfinished — 18 group(s) committed by this tool, 6 to go
|
||||
committed: 1-18
|
||||
remaining: 19-24
|
||||
|
||||
Run `export` again; it continues from where it stopped.
|
||||
```
|
||||
|
||||
The record is written after **each** commit, not at the end — the point is to
|
||||
survive the run not reaching the end. It stores the plan's fingerprint (the
|
||||
groups and their paths, never the messages, so rewording commit 20 does not
|
||||
invalidate the nineteen already made) and each commit's sha, which must still
|
||||
be where the branch tip is or the copy has moved and it says so.
|
||||
|
||||
## Keeping a history that already exists
|
||||
|
||||
```bash
|
||||
histgen export --keep-history
|
||||
```
|
||||
|
||||
Carries the source's `.git` into the copy and commits the designed account to an
|
||||
orphan branch, leaving the original branch pointing exactly where it did. Two
|
||||
tiers, which is what `all/ctrl/handover.sh` has been saying all along:
|
||||
|
||||
```
|
||||
main o-o-o-o "updates 33.1 139" (untouched)
|
||||
designed-history o-o-o-o-o-o-o-o the designed account (no shared parent)
|
||||
```
|
||||
|
||||
Both are present in `out/<name>/`; which one to publish is a decision for later
|
||||
and by hand. Nothing is rewritten and the source is not touched either way.
|
||||
|
||||
## Settings
|
||||
|
||||
```bash
|
||||
make init-config SOURCE=~/work/thing OUT=~/histories/thing
|
||||
make config # what everything resolves to, and where it came from
|
||||
make run # no arguments
|
||||
```
|
||||
|
||||
Writes `histgen.json` beside the tool — the arrangement `ppl/ctrl/distill.sh`
|
||||
already uses, where the JSON next to the script is picked up when nothing else
|
||||
says otherwise.
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "~/work/thing",
|
||||
"out": "~/histories/thing",
|
||||
"max_files": null,
|
||||
"keep_history": false,
|
||||
"branch": null
|
||||
}
|
||||
```
|
||||
|
||||
Precedence, most specific first:
|
||||
|
||||
```
|
||||
the command line -> --config FILE -> histgen.json beside the tool -> defaults
|
||||
```
|
||||
|
||||
so a config sets a starting point and never wins an argument with a flag typed
|
||||
deliberately. A mistyped key is refused rather than ignored, because a setting
|
||||
plainly written in the file and silently not applied is a bad thing to debug.
|
||||
`out` inside `source` is refused too — the source is read-only by design, and
|
||||
`out` would end up in its own census.
|
||||
|
||||
`histgen.json` is gitignored by the folder's own `.gitignore`: which tree this
|
||||
machine points at is not a fact about the tool.
|
||||
|
||||
## The verbs
|
||||
|
||||
Each writes one file under the repo's `.histgen/`, so the step before it is
|
||||
never repeated.
|
||||
|
||||
| | | |
|
||||
|---|---|---|
|
||||
| `scan` | `out/index.json` | what is in the source, cached by content hash |
|
||||
| `plan` | `out/plan.json` | the order, cut into commits |
|
||||
| `brief` | `out/briefs/*.md` | one pack per commit, for the messages |
|
||||
| `copy` | `out/<name>/` | the files alone, no repo — needs no plan |
|
||||
| `list` | — | the commits, printed to confirm |
|
||||
| `export` | `out/<name>/` | the copy and its history, with both guards |
|
||||
| `status` | — | which of the four states `out` is in |
|
||||
| `verify` | — | the guards, on their own |
|
||||
|
||||
`plan.json` is the seam. Everything above it is analysis that can be recomputed
|
||||
from the tree; everything below is git commands. That split is the whole design:
|
||||
the expensive half is a model reading code, and it should run once.
|
||||
|
||||
## Where the messages come from
|
||||
|
||||
`brief` writes a markdown pack per commit holding each file's **opening
|
||||
comment** — not the file. An agent reads the packs and writes `title` and
|
||||
`body` back into `plan.json`.
|
||||
|
||||
That is deliberate. A commit message reconstructed from a diff restates the
|
||||
diff, and the thing worth recording was never in the diff: it was in the comment
|
||||
explaining why the ignore rules exist before the code they exclude, or why a
|
||||
port offset has to mean the same thing in two different projects. Handing over
|
||||
the reasoning that is already written down produces a message worth reading;
|
||||
handing over the diff produces `Update files`.
|
||||
|
||||
Keeping the model outside the tool is also what keeps the tool offline, keeps
|
||||
every message editable before a single commit exists, and keeps the cost of a
|
||||
500-file repo to the comments rather than the code.
|
||||
|
||||
## The order
|
||||
|
||||
Role first, references second.
|
||||
|
||||
```
|
||||
skeleton (.gitignore) -> README -> version pins -> config layer -> profiles
|
||||
-> templates -> the things that source them -> front door (Makefile) LATE
|
||||
-> the bootstrap account LAST
|
||||
```
|
||||
|
||||
The front door is late because it only dispatches; the bootstrap account is last
|
||||
because it narrates everything above it. References refine within that, so a
|
||||
config lands before the script that sources it.
|
||||
|
||||
**References never override roles.** A reference in code is a dependency; a
|
||||
reference in a comment is a footnote. `.gitignore` names `ctrl/wizard.sh` to say
|
||||
the opposite of "I need this", and a README names every file in the repo.
|
||||
Counting those as edges commits the ignore rules after the code they exclude —
|
||||
consistent, and unreadable. So refs are taken from non-comment lines only, and
|
||||
narrative files (`.gitignore`, README, docs, BOOTSTRAP) contribute no outgoing
|
||||
edges at all.
|
||||
|
||||
## The grouping is a proposal
|
||||
|
||||
One coherent idea per commit, not one directory per commit. What holds a group
|
||||
together is that its files name each other; a hub and the directory named after
|
||||
it (`addons.sh` and `addons/`) always travel together, because committing a
|
||||
loader without the things it loads produces a commit that cannot run.
|
||||
|
||||
Where it cannot know — five scripts in one directory that never mention each
|
||||
other are five ideas or one, and nothing in the text says which — it guesses and
|
||||
says so. **Moving a path from one group to another in `plan.json` is the
|
||||
expected way to use this**, and `plan` re-run afterwards keeps every message
|
||||
whose group still holds the same files.
|
||||
|
||||
`--max-files` sets the cap. The default is about a twentieth of the tree with a
|
||||
floor of eight, which lands near how these repos were actually built — rig plans
|
||||
18 against a real 18, spr 77 against a real 78. Raising it gives fewer, larger
|
||||
commits; lowering it gives more.
|
||||
|
||||
## The two guards
|
||||
|
||||
`export` refuses a plan whose paths are missing, duplicated, or do not cover the
|
||||
source — before it writes anything. After the last commit it checks both:
|
||||
|
||||
1. **nothing left untracked**, with nothing exempt. The state lives in `out`
|
||||
and the copy lives inside it, so there is genuinely nothing of ours in the
|
||||
tree being checked. A file silently missed is the failure this whole tool
|
||||
exists to prevent. It is quiet at the time and surfaces much later, when
|
||||
something does not build on a fresh clone and the history offers no clue
|
||||
which commit should have carried it.
|
||||
2. **the exported tree still matches the source**, by tree hash. Every path
|
||||
committed is not the same claim as the same tree: a stale index, a path in
|
||||
two groups, or a file edited mid-plan all pass the first check and fail this
|
||||
one.
|
||||
|
||||
`--dry-run` writes `out/regen.sh` and `out/messages/` instead — ordinary git
|
||||
commands that copy the files and make the commits, both guards included,
|
||||
reviewable before anything runs.
|
||||
|
||||
## Reporting on a history that already exists
|
||||
|
||||
```bash
|
||||
python -m station.tools.histgen plan /path/to/repo --against-history
|
||||
```
|
||||
|
||||
Maps each of the source's existing commits onto the group holding most of the files it touched,
|
||||
then reports what agrees and what does not:
|
||||
|
||||
```
|
||||
= 01 skeleton a127b1d matched one commit
|
||||
~ 03 ctrl split across 2 one idea, committed piecemeal
|
||||
+ 04 ctrl-lib no commit never landed as its own change
|
||||
! 4 commit(s) land earlier in the proposed order than work already done
|
||||
? 31 commit(s) carry no usable account of the change ("updates 33.1 139")
|
||||
```
|
||||
|
||||
It reads and prints. It never rewrites: published history is someone else's
|
||||
clone.
|
||||
|
||||
## Verified against
|
||||
|
||||
`rig`'s 18-commit history, which was built by hand and is what this reproduces.
|
||||
Run over the same 64 files, histgen plans 18 commits; the profiles, the cluster
|
||||
templates, the addons hub, the k8s manifests, the Makefile, `sample-rig` and
|
||||
`BOOTSTRAP.md` all land as their own commits in the same places. Replaying it
|
||||
produces a tree hash identical to the one rig ships.
|
||||
|
||||
Where it differs is where the difference is semantic: rig splits its wizard,
|
||||
host checks, cluster lifecycle and registry into four commits, and nothing in
|
||||
those four files' text says they are four things.
|
||||
|
||||
## The CLI shape
|
||||
|
||||
`cli.py` is a shared scaffold — subcommand registration, one spelling for
|
||||
`--source/-o/-n/--force/--dry-run`, `Error: … -> stderr -> exit 1` as the only
|
||||
exit path, deferred heavy imports so `--help` stays instant, and
|
||||
`refuse_to_clobber`. It exists because every tool here grew its own slightly
|
||||
different copy of the same three things.
|
||||
|
||||
histgen is its first user. Nothing else was rewritten to use it: a scaffold
|
||||
earns adoption by being there when the next tool is written.
|
||||
27
soleprint/station/tools/histgen/__init__.py
Normal file
27
soleprint/station/tools/histgen/__init__.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Histgen — seed a clean, logical history into a repo.
|
||||
|
||||
The general case is a tree with no git at all: read the code, then commit it in
|
||||
parts, ordered so each commit stands on what came before. Done by hand it means
|
||||
uploading everything and asking a model; done twice it stops being worth the
|
||||
time.
|
||||
|
||||
The expensive half (reading the code) and the mechanical half (applying a plan)
|
||||
are split on purpose, and `plan.json` is the seam. Everything before it is
|
||||
analysis and can be cached; everything after it is git commands that fail fast.
|
||||
|
||||
python -m station.tools.histgen scan /path/to/repo
|
||||
python -m station.tools.histgen plan /path/to/repo
|
||||
python -m station.tools.histgen brief /path/to/repo
|
||||
# an agent reads briefs/ and writes title+body back into plan.json
|
||||
python -m station.tools.histgen apply /path/to/repo --dry-run
|
||||
python -m station.tools.histgen verify /path/to/repo
|
||||
|
||||
Stdlib only, no network. The directory can be copied out of soleprint and run
|
||||
on its own — a repo that needs a history seeded is, by definition, not one that
|
||||
already has this framework on its path.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
__all__ = ["census", "order", "brief", "snapshot", "export"]
|
||||
318
soleprint/station/tools/histgen/__main__.py
Normal file
318
soleprint/station/tools/histgen/__main__.py
Normal file
@@ -0,0 +1,318 @@
|
||||
"""
|
||||
Histgen CLI — seed a clean, logical history into a repo.
|
||||
|
||||
python -m station.tools.histgen run --source ~/work/thing --out ~/out
|
||||
python -m station.tools.histgen list # the commits, to confirm
|
||||
python -m station.tools.histgen export # write them into out/thing
|
||||
|
||||
Two directories and one rule: the source is read-only, everything is written
|
||||
under out. Set both once with `config --init` and the verbs take no arguments.
|
||||
|
||||
Run from the soleprint/ directory so `station.tools...` resolves, or copy the
|
||||
folder out and use its Makefile.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .cli import Tool, fail
|
||||
|
||||
VERBS = (
|
||||
("copy", "Copy the files out, with no .git and nothing private."),
|
||||
("scan", "Read the source and cache what was read."),
|
||||
("plan", "Order the files and cut them into commits."),
|
||||
("list", "Print the commits, to confirm before exporting."),
|
||||
("brief", "Write one brief per commit, for the messages."),
|
||||
("export", "Copy the source into out and commit the history."),
|
||||
("verify", "Nothing left untracked, and the tree still matches."),
|
||||
("status", "What is in the out directory, and what is left to do."),
|
||||
("run", "scan + plan + brief."),
|
||||
("config", "Show the resolved settings, or write a starter file."),
|
||||
)
|
||||
|
||||
|
||||
def _settings(args, need_out=True):
|
||||
"""
|
||||
Where to read and where to write, with the command line on top.
|
||||
|
||||
Resolved once per invocation and passed down, rather than each module
|
||||
working it out again — two places deciding where things live is how `scan`
|
||||
and `plan` end up disagreeing about it.
|
||||
"""
|
||||
from . import config
|
||||
s = config.resolve(args, getattr(args, "config", None))
|
||||
|
||||
if not s["source"]:
|
||||
fail("No source given.",
|
||||
f"Pass --source, or set it in {config.default_path()} "
|
||||
"(see `histgen config --init`).")
|
||||
if not s["source"].is_dir():
|
||||
fail(f"Not a directory: {s['source']}")
|
||||
|
||||
if need_out and not s["out"]:
|
||||
fail("No out directory given.",
|
||||
"Pass --out, or set it in the config. It is where the plan and "
|
||||
"the exported repo go; the source is never written to.")
|
||||
if s["out"]:
|
||||
# The source must stay clean, and an out inside it would be scanned as
|
||||
# part of the tree it describes on the very next run.
|
||||
try:
|
||||
if s["out"] == s["source"] or s["out"].is_relative_to(s["source"]):
|
||||
fail("out is inside source.",
|
||||
"The source is read-only by design, and out would end up "
|
||||
"in its own census. Put out somewhere else.")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
def _index(out):
|
||||
"""The census, which must already exist — scanning is its own verb."""
|
||||
from . import census
|
||||
index = census.load_index(out)
|
||||
if not index.get("files"):
|
||||
fail(f"No census at {census.state_path(out)}.", "Run `scan` first.")
|
||||
return index
|
||||
|
||||
|
||||
def cmd_copy(args):
|
||||
"""The plain utility: files out, repo and private things left behind."""
|
||||
from . import snapshot
|
||||
s = _settings(args)
|
||||
snapshot.take(s["source"], s["out"],
|
||||
keep_noise=args.all, keep_secrets=s["keep_secrets"],
|
||||
max_bytes=args.max_bytes, exclude=s["exclude"],
|
||||
include=s["include"], force=args.force, dry_run=args.dry_run)
|
||||
|
||||
|
||||
def cmd_scan(args):
|
||||
from . import census
|
||||
s = _settings(args)
|
||||
census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"],
|
||||
exclude=s["exclude"], include=s["include"])
|
||||
|
||||
|
||||
def cmd_plan(args):
|
||||
from . import order
|
||||
s = _settings(args)
|
||||
plan = order.build_plan(_index(s["out"]), s["out"], max_files=s["max_files"])
|
||||
if args.against_history:
|
||||
from . import history
|
||||
print()
|
||||
history.compare(s["source"], plan)
|
||||
|
||||
|
||||
def cmd_list(args):
|
||||
"""
|
||||
Print the commits as a numbered list, which is the thing to confirm.
|
||||
|
||||
Deliberately the plainest output here: a number, a title, and the files
|
||||
under it. Deciding whether a commit is one idea is done by reading it, and
|
||||
anything else on the line is in the way.
|
||||
"""
|
||||
from . import export as exporter
|
||||
s = _settings(args)
|
||||
plan = exporter.load_plan(s["out"])
|
||||
files = _index(s["out"])["files"] if args.roles else {}
|
||||
for g in plan["groups"]:
|
||||
title = (g.get("title") or "").strip()
|
||||
mark = " " if title else "*"
|
||||
print(f"{mark}{g['n']:3}. {title or g['slug'] + ' (no message yet)'}")
|
||||
for p in g["paths"]:
|
||||
role = f" [{files[p]['role']}]" if args.roles and p in files else ""
|
||||
print(f" {p}{role}")
|
||||
untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||
print(f"\n{len(plan['groups'])} commits, "
|
||||
f"{sum(len(g['paths']) for g in plan['groups'])} files.")
|
||||
if untitled:
|
||||
print(f"* {len(untitled)} still without a message — read "
|
||||
f"{s['out']}/briefs/ and write them into {s['out']}/plan.json.")
|
||||
|
||||
|
||||
def cmd_brief(args):
|
||||
from . import brief, order
|
||||
s = _settings(args)
|
||||
if not order.plan_path(s["out"]).exists():
|
||||
fail(f"No plan at {order.plan_path(s['out'])}.", "Run `plan` first.")
|
||||
brief.write_briefs(s["out"])
|
||||
|
||||
|
||||
def cmd_export(args):
|
||||
from . import export as exporter
|
||||
s = _settings(args)
|
||||
exporter.export(s["source"], s["out"], dry_run=args.dry_run,
|
||||
commands=args.commands, allow_untitled=args.allow_untitled,
|
||||
keep_history=s["keep_history"], branch=s["branch"],
|
||||
force=args.force)
|
||||
|
||||
|
||||
def cmd_verify(args):
|
||||
from . import export as exporter
|
||||
s = _settings(args)
|
||||
plan = exporter.load_plan(s["out"])
|
||||
planned = [p for g in plan["groups"] for p in g["paths"]]
|
||||
copy = exporter.repo_dir(s["source"], s["out"])
|
||||
if not copy.is_dir():
|
||||
fail(f"Nothing exported at {copy}.", "Run `export` first.")
|
||||
print("Checking:", flush=True)
|
||||
if not exporter.verify(copy, exporter.source_tree_hash(s["source"], planned)):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_status(args):
|
||||
"""What is in out, what state it is in, and what is left."""
|
||||
from . import census, export as exporter, order
|
||||
s = _settings(args)
|
||||
|
||||
index = census.load_index(s["out"])
|
||||
plan = None
|
||||
if order.plan_path(s["out"]).exists():
|
||||
plan = exporter.load_plan(s["out"])
|
||||
|
||||
print(f"source {s['source']}")
|
||||
print(f"out {s['out']}")
|
||||
if index.get("files"):
|
||||
left = index.get("left_out", [])
|
||||
print(f"census {len(index['files'])} files"
|
||||
+ (f", {len(left)} left out" if left else ""))
|
||||
for item in left:
|
||||
print(f" left out: {item['path']} ({item['why']})")
|
||||
else:
|
||||
print("census none — run `scan`")
|
||||
if plan:
|
||||
untitled = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||
print(f"plan {len(plan['groups'])} commits"
|
||||
+ (f", {len(untitled)} without a message" if untitled else
|
||||
", all messages written"))
|
||||
else:
|
||||
print("plan none — run `plan`")
|
||||
|
||||
report = exporter.inspect(s["source"], s["out"], plan)
|
||||
print(f"export {report['state']} — {report['detail']}")
|
||||
if report["done"]:
|
||||
print(f" committed: {_ranges(report['done'])}")
|
||||
if report["remaining"]:
|
||||
print(f" remaining: {_ranges(report['remaining'])}")
|
||||
|
||||
advice = {
|
||||
"absent": "Run `export`.",
|
||||
"unfinished": "Run `export` again; it continues from where it stopped.",
|
||||
"foreign": "Run `export`; the designed history goes on its own branch "
|
||||
"and nothing existing is touched.",
|
||||
"stale": "Run `export --force` to discard and redo, or point --out elsewhere.",
|
||||
"complete": "Nothing to do.",
|
||||
}
|
||||
print(f"\n{advice.get(report['state'], '')}")
|
||||
|
||||
|
||||
def _ranges(numbers):
|
||||
"""[1,2,3,7,8] -> '1-3, 7-8'. A list of eighteen numbers is unreadable."""
|
||||
if not numbers:
|
||||
return "none"
|
||||
out, start, previous = [], numbers[0], numbers[0]
|
||||
for n in numbers[1:] + [None]:
|
||||
if n == previous + 1:
|
||||
previous = n
|
||||
continue
|
||||
out.append(str(start) if start == previous else f"{start}-{previous}")
|
||||
start = previous = n
|
||||
return ", ".join(out)
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
from . import brief, census, order
|
||||
s = _settings(args)
|
||||
index = census.scan(s["source"], s["out"], keep_secrets=s["keep_secrets"],
|
||||
exclude=s["exclude"], include=s["include"])
|
||||
order.build_plan(index, s["out"], max_files=s["max_files"])
|
||||
brief.write_briefs(s["out"])
|
||||
|
||||
|
||||
def cmd_config(args):
|
||||
from . import config
|
||||
if args.init:
|
||||
path = config.write_template(
|
||||
Path(args.config).expanduser() if args.config else config.default_path(),
|
||||
source=args.source, out=args.out)
|
||||
print(f"Wrote {path}. Edit \"source\" and \"out\".")
|
||||
return
|
||||
s = config.resolve(args, args.config)
|
||||
print(f"config {s['config_path'] or '(none; using defaults)'}")
|
||||
print(f"source {s['source'] or '(unset)'} read-only")
|
||||
print(f"out {s['out'] or '(unset)'}")
|
||||
if s["source"] and s["out"]:
|
||||
from .export import repo_dir
|
||||
print(f"exported to {repo_dir(s['source'], s['out'])}")
|
||||
print(f"max-files {s['max_files'] or '(scales with the source)'}")
|
||||
print(f"keep-history {s['keep_history']}")
|
||||
print(f"keep-secrets {s['keep_secrets']}"
|
||||
+ ("" if s["keep_secrets"] else " keys and credentials are left out"))
|
||||
if s["exclude"]:
|
||||
print(f"exclude {', '.join(s['exclude'])}")
|
||||
if s["include"]:
|
||||
print(f"include {', '.join(s['include'])}")
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
tool = Tool("histgen", __doc__, package=__package__)
|
||||
handlers = {name: globals()[f"cmd_{name}"] for name, _ in VERBS}
|
||||
|
||||
for verb, help_text in VERBS:
|
||||
tool.command(verb, handlers[verb], help_text)
|
||||
tool.argument(verb, "--source", "-s", default=None, metavar="DIR",
|
||||
help="The tree to read. Never written to.")
|
||||
tool.argument(verb, "--out", "-o", default=None, metavar="DIR",
|
||||
help="Where the plan and the exported repo go.")
|
||||
tool.argument(verb, "--config", "-c", default=None, metavar="FILE",
|
||||
help="Settings file, instead of histgen.json beside the tool.")
|
||||
|
||||
for verb in ("plan", "run"):
|
||||
tool.argument(verb, "--max-files", type=int, default=None, metavar="N",
|
||||
help="Files per commit before a group is cut. Default "
|
||||
"scales with the source (about N/20, floor 8).")
|
||||
tool.argument("plan", "--against-history", action="store_true",
|
||||
help="Also report how the source's existing history compares.")
|
||||
tool.argument("list", "--roles", action="store_true",
|
||||
help="Show each file's detected role.")
|
||||
|
||||
# Whatever reads the source can filter it, so the same three answers hold
|
||||
# for a snapshot and for a history. Keeping them on `copy` alone was how a
|
||||
# tracked key stayed out of one and went straight into the other.
|
||||
for verb in ("copy", "scan", "run"):
|
||||
tool.argument(verb, "--keep-secrets", action="store_true",
|
||||
help="Keep files that look like keys or credentials.")
|
||||
tool.argument(verb, "--exclude", action="append", default=[], metavar="GLOB",
|
||||
help="Leave these out. Repeatable; a pattern with no / "
|
||||
"matches basenames at any depth.")
|
||||
tool.argument(verb, "--include", action="append", default=[], metavar="GLOB",
|
||||
help="Keep these whatever the filters say. Repeatable.")
|
||||
|
||||
tool.common("copy", "dry_run")
|
||||
tool.argument("copy", "--all", action="store_true",
|
||||
help="Keep what a build regenerates too: lockfiles, maps, "
|
||||
"minified and compiled output.")
|
||||
tool.argument("copy", "--max-bytes", type=int, default=None, metavar="N",
|
||||
help="Leave behind anything larger.")
|
||||
tool.argument("copy", "--force", action="store_true",
|
||||
help="Write into a destination that is not empty.")
|
||||
tool.common("export", "dry_run")
|
||||
tool.argument("export", "--commands", action="store_true",
|
||||
help="Copy the files, create no repo, and print the git add "
|
||||
"and git commit commands to run yourself.")
|
||||
tool.argument("export", "--allow-untitled", action="store_true",
|
||||
help="Commit groups whose message was never written.")
|
||||
tool.argument("export", "--keep-history", action="store_true",
|
||||
help="Carry the source's existing history into the copy, and "
|
||||
"put the designed one on its own branch.")
|
||||
tool.argument("export", "--branch", default=None, metavar="NAME",
|
||||
help="Branch for the designed history when one is kept.")
|
||||
tool.argument("export", "--force", action="store_true",
|
||||
help="Discard an out directory that no longer matches the plan.")
|
||||
tool.argument("config", "--init", action="store_true",
|
||||
help="Write a starter config file rather than reading one.")
|
||||
|
||||
tool.run(argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
98
soleprint/station/tools/histgen/api.py
Normal file
98
soleprint/station/tools/histgen/api.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
Histgen's HTTP surface.
|
||||
|
||||
Read-only on purpose. The CLI seeds histories; this shows what a plan looks
|
||||
like before anyone runs it, because the interesting failure — a group that
|
||||
holds two unrelated ideas — is one you see by reading, not by testing.
|
||||
|
||||
Nothing here writes commits. A browser tab is the wrong place to decide that a
|
||||
repository's history is about to be rebuilt.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/tools/histgen", tags=["histgen"])
|
||||
|
||||
SPR_ROOT = Path(__file__).parents[3]
|
||||
HERE = Path(__file__).parent
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pages
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
def index():
|
||||
template = HERE / "templates" / "index.html"
|
||||
if template.exists():
|
||||
return template.read_text()
|
||||
return "<h1>histgen</h1>"
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "tool": "histgen"}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# API
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _resolve(out: str) -> Path:
|
||||
"""
|
||||
An out directory, kept inside the tree this instance was built from.
|
||||
|
||||
The parameter is a filesystem path from a query string, so it is the one
|
||||
input here worth distrusting: without the containment check, `../..` reads
|
||||
any directory the server can.
|
||||
"""
|
||||
target = Path(out).resolve() if Path(out).is_absolute() else (SPR_ROOT.parent / out).resolve()
|
||||
root = SPR_ROOT.parent.resolve()
|
||||
if root not in target.parents and target != root:
|
||||
raise HTTPException(400, f"Outside the tree: {out}")
|
||||
if not target.is_dir():
|
||||
raise HTTPException(404, f"Not a directory: {out}")
|
||||
return target
|
||||
|
||||
|
||||
@router.get("/api/plan")
|
||||
def get_plan(out: str):
|
||||
"""The plan as it stands, with each group's files and message."""
|
||||
import json
|
||||
|
||||
from .order import plan_path
|
||||
|
||||
path = plan_path(_resolve(out))
|
||||
if not path.exists():
|
||||
raise HTTPException(404, "No plan yet. Run `histgen run`.")
|
||||
plan = json.loads(path.read_text())
|
||||
return {
|
||||
"groups": plan["groups"],
|
||||
"commits": len(plan["groups"]),
|
||||
"files": sum(len(g["paths"]) for g in plan["groups"]),
|
||||
"untitled": [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/census")
|
||||
def get_census(out: str):
|
||||
"""What the scan found, without the per-file detail."""
|
||||
from collections import Counter
|
||||
|
||||
from .census import load_index
|
||||
|
||||
index = load_index(_resolve(out))
|
||||
if not index.get("files"):
|
||||
raise HTTPException(404, "No census yet. Run `scan` first.")
|
||||
files = index["files"]
|
||||
return {
|
||||
"files": len(files),
|
||||
"roles": dict(Counter(f["role"] for f in files.values())),
|
||||
"edges": sum(len(f["refs"]) for f in files.values()),
|
||||
}
|
||||
111
soleprint/station/tools/histgen/brief.py
Normal file
111
soleprint/station/tools/histgen/brief.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
One pack per commit, for whoever writes the message.
|
||||
|
||||
This is the interface to the expensive reader, and it is a directory of
|
||||
markdown rather than a network call. The tool does not know how to reach a
|
||||
model and does not want to: an agent already in this repo reads the briefs and
|
||||
writes titles and bodies back into plan.json, which keeps the messages editable
|
||||
before a single commit exists and keeps an API key out of a tool that otherwise
|
||||
runs offline.
|
||||
|
||||
What travels is the reasoning already in the code — each file's opening
|
||||
comment — and not the file. That is both what makes the pack cheap and what
|
||||
makes the message right: a commit message reconstructed from a diff restates
|
||||
the diff, and the thing worth recording was never in the diff. It was in the
|
||||
comment explaining why the port offsets have to match another project's, or why
|
||||
the ignore rules exist before the code they exclude.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from .census import INDEX_FILE, state_dir
|
||||
from .order import plan_path
|
||||
|
||||
BRIEF_DIR = "briefs"
|
||||
|
||||
WHY_CHARS = 700 # an opening comment past this is an essay; the head carries it
|
||||
MAX_LISTED = 40
|
||||
|
||||
|
||||
HEADER = """# {n:02d} — {slug}
|
||||
|
||||
**{count} file(s), commit {n} of {total}.**
|
||||
|
||||
Write a title and a body for this commit, then put them in
|
||||
`{plan}` under group {n} as `"title"` and `"body"`.
|
||||
|
||||
- The title says what this commit establishes, in the repo's own words.
|
||||
- The body carries the **why** — take it from the reasoning already in the
|
||||
comments below. Do not restate the diff; the diff is already in the commit.
|
||||
- If a file below does not belong in this commit, move its path to another
|
||||
group in plan.json. The grouping is a proposal.
|
||||
"""
|
||||
|
||||
|
||||
def _fmt_why(text):
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return "_(no opening comment)_"
|
||||
if len(text) > WHY_CHARS:
|
||||
text = text[:WHY_CHARS].rsplit("\n", 1)[0] + "\n…"
|
||||
return "\n".join("> " + line if line.strip() else ">" for line in text.split("\n"))
|
||||
|
||||
|
||||
def write_briefs(out, quiet=False):
|
||||
state = state_dir(out)
|
||||
plan = json.loads(plan_path(out).read_text())
|
||||
index = json.loads((state / INDEX_FILE).read_text())
|
||||
files = index["files"]
|
||||
groups = plan["groups"]
|
||||
|
||||
# Which commit each path lands in, so a dependency can be named by the
|
||||
# commit that introduced it rather than by a bare path. "stands on 04" is
|
||||
# the sentence the ordering exists to make true.
|
||||
landed = {p: g["n"] for g in groups for p in g["paths"]}
|
||||
|
||||
out_dir = state / BRIEF_DIR
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stale in out_dir.glob("*.md"):
|
||||
stale.unlink()
|
||||
|
||||
for g in groups:
|
||||
lines = [HEADER.format(n=g["n"], slug=g["slug"], count=len(g["paths"]),
|
||||
total=len(groups), plan=plan_path(out))]
|
||||
|
||||
earlier, later = {}, set()
|
||||
for p in g["paths"]:
|
||||
for dep in files.get(p, {}).get("refs", []):
|
||||
n = landed.get(dep)
|
||||
if n is None or dep in g["paths"]:
|
||||
continue
|
||||
(earlier.setdefault(n, set()).add(dep) if n < g["n"] else later.add(dep))
|
||||
|
||||
if earlier:
|
||||
lines.append("\n## Stands on\n")
|
||||
for n in sorted(earlier):
|
||||
names = ", ".join(f"`{d}`" for d in sorted(earlier[n])[:MAX_LISTED])
|
||||
lines.append(f"- commit {n:02d}: {names}")
|
||||
if later:
|
||||
# Worth stating plainly rather than hiding: it is the one thing a
|
||||
# reader of the finished history would notice and the tool cannot
|
||||
# fix, because the fix is a judgement about which comes first.
|
||||
names = ", ".join(f"`{d}`" for d in sorted(later)[:MAX_LISTED])
|
||||
lines.append("\n## Forward references (this commit names things not yet committed)\n")
|
||||
lines.append(f"- {names}")
|
||||
|
||||
lines.append("\n## Files\n")
|
||||
for p in g["paths"]:
|
||||
e = files.get(p, {})
|
||||
meta = f"{e.get('role', '?')}, {e.get('lines', 0)} lines"
|
||||
if e.get("binary"):
|
||||
meta += ", binary"
|
||||
lines.append(f"\n### `{p}`\n\n_{meta}_\n")
|
||||
lines.append(_fmt_why(e.get("why")))
|
||||
|
||||
path = out_dir / f"{g['n']:02d}-{g['slug']}.md"
|
||||
path.write_text("\n".join(lines) + "\n")
|
||||
|
||||
if not quiet:
|
||||
print(f"Wrote {len(groups)} briefs -> {out_dir}")
|
||||
print(f"Read them, then write title and body into {plan_path(out)}.")
|
||||
return out_dir
|
||||
456
soleprint/station/tools/histgen/census.py
Normal file
456
soleprint/station/tools/histgen/census.py
Normal file
@@ -0,0 +1,456 @@
|
||||
"""
|
||||
What is in the tree, and what each file says about itself.
|
||||
|
||||
This is the expensive half. Walking is cheap; reading is not, so what gets read
|
||||
is cached by content hash and a second run costs only the files that changed.
|
||||
The cache lives beside the output, in the repo's own .histgen/, because a
|
||||
destination that carries its own state cannot be confused with another one's.
|
||||
|
||||
Two things are extracted from every file, and both are used twice:
|
||||
|
||||
the opening comment why the file exists, in the author's words. `order`
|
||||
does not read it; `brief` hands it to whoever writes
|
||||
the commit message, because that reasoning is the
|
||||
message. A commit that restates its own diff is noise.
|
||||
|
||||
path references which other files this one names. `order` turns them
|
||||
into edges, so a config lands before the script that
|
||||
sources it. `brief` reports them as the seam between
|
||||
one commit and the last.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
STATE_DIR = ".histgen"
|
||||
INDEX_FILE = "index.json"
|
||||
|
||||
# Read caps. A file's opening comment is at the top by definition, and nothing
|
||||
# below the cap has ever been the reason a file exists. The byte cap is what
|
||||
# keeps a vendored 2 MB bundle from being tokenised for no reason.
|
||||
HEAD_LINES = 60
|
||||
MAX_BYTES = 400_000
|
||||
|
||||
|
||||
# ── the file set ───────────────────────────────────────────────────────────
|
||||
#
|
||||
# Never parse .gitignore. It has negations, directory semantics, precedence
|
||||
# across nested files and a global excludes file, and a hand-rolled parser that
|
||||
# gets 95% of that right is worse than none: it is wrong silently, on exactly
|
||||
# the files someone took care to exclude. Ask git, which is always installed
|
||||
# here because the output of this tool is a git repository.
|
||||
|
||||
def _git(args, **kw):
|
||||
return subprocess.run(["git", *args], capture_output=True, text=True, **kw)
|
||||
|
||||
|
||||
def is_git(path: Path) -> bool:
|
||||
r = _git(["-C", str(path), "rev-parse", "--git-dir"])
|
||||
return r.returncode == 0
|
||||
|
||||
|
||||
def file_set(source: Path):
|
||||
"""
|
||||
The paths a history would contain, relative to the source, sorted.
|
||||
|
||||
A source with git is asked what it tracks — the same question handover.sh
|
||||
asks, and for the same reason: a hand-maintained list drifts, and the drift
|
||||
shows up as a file that silently never got committed.
|
||||
|
||||
A source with NO git is the general case, and the interesting one. Rather
|
||||
than reimplementing the ignore rules, git is pointed at the tree with its
|
||||
own directory kept in a temporary path: `ls-files -o --exclude-standard`
|
||||
then means untracked-and-not-ignored, which is exactly the candidate set.
|
||||
Nothing is written inside the tree, so a dry run leaves no .git behind to
|
||||
explain later.
|
||||
"""
|
||||
if is_git(source):
|
||||
listed = _git(["-C", str(source), "ls-files", "-z"]).stdout
|
||||
paths = [p for p in listed.split("\0") if p]
|
||||
# A tracked file that has been deleted but not committed is still in
|
||||
# ls-files. It would abort `apply` partway through with a missing path,
|
||||
# so drop it here and let `verify` be the thing that complains.
|
||||
return sorted(p for p in paths if (source / p).is_file())
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-git-") as tmp:
|
||||
env = dict(os.environ, GIT_DIR=str(Path(tmp) / "git"), GIT_WORK_TREE=str(source))
|
||||
subprocess.run(["git", "init", "-q"], env=env, capture_output=True)
|
||||
listed = subprocess.run(
|
||||
["git", "ls-files", "-o", "--exclude-standard", "-z"],
|
||||
env=env, capture_output=True, text=True,
|
||||
).stdout
|
||||
return sorted(p for p in listed.split("\0")
|
||||
if p and (source / p).is_file())
|
||||
|
||||
|
||||
# ── roles ──────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# A file's role is what decides where it lands when nothing references it, and
|
||||
# most files reference nothing. The ranks are the ordering heuristic itself,
|
||||
# read bottom-up off rig's log: ignore rules and README first, then the config
|
||||
# layer, then the things that source it, the front door late because it only
|
||||
# dispatches, and the bootstrap account last because it narrates the rest.
|
||||
|
||||
ROLE_RANK = {
|
||||
"skeleton": 0, # .gitignore, .gitattributes — the rules before the files
|
||||
"readme": 10, # what this is, and the one prerequisite
|
||||
"pin": 20, # versions, dependency manifests
|
||||
"config": 30, # the layer everything else reads
|
||||
"profile": 40, # named variants of that config
|
||||
"template": 50, # shapes rendered later
|
||||
"source": 60, # the work
|
||||
"test": 70,
|
||||
"asset": 76,
|
||||
"lock": 78, # generated from a pin; never interesting, never first
|
||||
"doc": 80,
|
||||
"frontdoor": 90, # Makefile, Tiltfile — a dispatcher, so it comes after
|
||||
"bootstrap": 100, # BOOTSTRAP/INSTALL — the account of everything above
|
||||
}
|
||||
|
||||
_SKELETON = {".gitignore", ".gitattributes", ".editorconfig", ".dockerignore",
|
||||
"license", "license.md", "license.txt", "copying", "notice"}
|
||||
_FRONTDOOR = {"makefile", "gnumakefile", "tiltfile", "justfile", "taskfile.yml",
|
||||
"dockerfile", "docker-compose.yml", "docker-compose.yaml"}
|
||||
_LOCK = {"package-lock.json", "poetry.lock", "pnpm-lock.yaml", "yarn.lock",
|
||||
"cargo.lock", "go.sum", "composer.lock", "gemfile.lock", "uv.lock"}
|
||||
_PIN = {"requirements.txt", "pyproject.toml", "package.json", "go.mod",
|
||||
"cargo.toml", "gemfile", "versions.env", "setup.py", "setup.cfg"}
|
||||
_ASSET_EXT = {".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".webp", ".pdf",
|
||||
".woff", ".woff2", ".ttf", ".eot", ".mp4", ".zip", ".ods", ".xlsx",
|
||||
".bundle", ".so", ".dylib", ".dll", ".wasm"}
|
||||
_DOC_EXT = {".md", ".rst", ".txt", ".adoc"}
|
||||
|
||||
|
||||
def ignored_but_tracked(source: Path, paths):
|
||||
"""
|
||||
Files git tracks that the ignore rules say it should not.
|
||||
|
||||
`git add -f` is how they get there, and it is not always a mistake — a
|
||||
built artifact committed on purpose looks exactly like this. But so does a
|
||||
dump, a credentials file or a data directory that someone forced in once
|
||||
and nobody noticed since, and those are the ones worth catching.
|
||||
|
||||
`--no-index` is the whole trick: without it check-ignore stays quiet about
|
||||
anything already tracked, which is precisely the set being asked about.
|
||||
"""
|
||||
if not paths or not is_git(source):
|
||||
return set()
|
||||
r = subprocess.run(
|
||||
["git", "-C", str(source), "check-ignore", "--no-index", "--stdin", "-z"],
|
||||
input="\0".join(paths) + "\0", text=True, capture_output=True)
|
||||
# 0 = some matched, 1 = none matched, anything else is a real failure and
|
||||
# not a reason to refuse to copy.
|
||||
if r.returncode not in (0, 1):
|
||||
return set()
|
||||
return {p for p in r.stdout.split("\0") if p}
|
||||
|
||||
|
||||
def role_of(path: str) -> str:
|
||||
p = Path(path)
|
||||
name, low = p.name, p.name.lower()
|
||||
parts = [s.lower() for s in p.parts]
|
||||
stem = p.stem.lower()
|
||||
|
||||
if low in _SKELETON:
|
||||
return "skeleton"
|
||||
if low in _LOCK:
|
||||
return "lock"
|
||||
if low in _FRONTDOOR or low.startswith("dockerfile"):
|
||||
return "frontdoor"
|
||||
if stem in ("bootstrap", "install", "installing", "getting-started", "quickstart"):
|
||||
return "bootstrap"
|
||||
if stem == "readme":
|
||||
# Only the repo's own README opens the history. A README inside a
|
||||
# subdirectory documents that subdirectory and travels with it.
|
||||
return "readme" if len(p.parts) == 1 else "doc"
|
||||
if low in _PIN or low.endswith(".lock"):
|
||||
return "lock" if low.endswith(".lock") else "pin"
|
||||
if p.suffix.lower() in _ASSET_EXT:
|
||||
return "asset"
|
||||
if "test" in parts or "tests" in parts or stem.startswith("test_") or stem.endswith("_test"):
|
||||
return "test"
|
||||
if p.suffix in (".tpl", ".tmpl", ".j2", ".mustache") or low.endswith((".yaml.tpl", ".tmpl")):
|
||||
return "template"
|
||||
if "templates" in parts:
|
||||
return "template"
|
||||
# A profile is a named variant sitting in a directory of siblings: env.d/,
|
||||
# profiles/, overlays/. The directory is the signal, not the extension.
|
||||
if any(d in parts for d in ("env.d", "profiles", "environments")):
|
||||
return "profile"
|
||||
if stem in ("config", "settings", "conf", "defaults") or low in (".env.example", "env.example"):
|
||||
return "config"
|
||||
if low.endswith(".env") or low.endswith(".env.example"):
|
||||
return "profile"
|
||||
if p.suffix.lower() in _DOC_EXT or "docs" in parts or "doc" in parts:
|
||||
return "doc"
|
||||
return "source"
|
||||
|
||||
|
||||
# ── what a file says about itself ──────────────────────────────────────────
|
||||
|
||||
_COMMENT = {
|
||||
"#": (".sh", ".bash", ".py", ".yaml", ".yml", ".toml", ".env", ".cfg", ".conf", ".tf", ""),
|
||||
"//": (".js", ".ts", ".jsx", ".tsx", ".go", ".java", ".c", ".h", ".cpp", ".rs", ".scala"),
|
||||
}
|
||||
|
||||
|
||||
def opening_comment(text: str, path: str) -> str:
|
||||
"""
|
||||
The comment block at the top of the file, or the module docstring.
|
||||
|
||||
The shebang and any editor modeline are skipped — they are not prose. The
|
||||
block ends at the first line that is not a comment, which is what makes it
|
||||
the file's own statement of purpose rather than a running commentary.
|
||||
"""
|
||||
lines = text.split("\n")[:HEAD_LINES]
|
||||
i = 0
|
||||
while i < len(lines) and (
|
||||
lines[i].startswith("#!") or not lines[i].strip()
|
||||
or lines[i].lstrip().startswith(("# -*-", "# vim:", "# shellcheck"))
|
||||
):
|
||||
i += 1
|
||||
|
||||
# A docstring: take it whole, it is the same statement in another syntax.
|
||||
rest = "\n".join(lines[i:]).lstrip()
|
||||
for quote in ('"""', "'''"):
|
||||
if rest.startswith(quote):
|
||||
end = rest.find(quote, len(quote))
|
||||
if end != -1:
|
||||
return rest[len(quote):end].strip()
|
||||
|
||||
suffix = Path(path).suffix.lower()
|
||||
markers = [m for m, exts in _COMMENT.items() if suffix in exts] or ["#"]
|
||||
block = []
|
||||
for line in lines[i:]:
|
||||
stripped = line.strip()
|
||||
if not any(stripped.startswith(m) for m in markers):
|
||||
break
|
||||
for m in markers:
|
||||
if stripped.startswith(m):
|
||||
block.append(stripped[len(m):].strip())
|
||||
break
|
||||
return "\n".join(block).strip()
|
||||
|
||||
|
||||
_TOKEN = re.compile(r"[A-Za-z0-9_./+-]{4,}")
|
||||
|
||||
# What opens a comment, by language. Used to tell a reference apart from a
|
||||
# mention, which is the difference between an edge and a footnote.
|
||||
_COMMENT_PREFIX = ("#", "//", "--", ";", "*", "/*")
|
||||
|
||||
|
||||
def _strip_comment(line: str) -> str:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith(_COMMENT_PREFIX):
|
||||
return ""
|
||||
# A trailing comment on a real line: keep the code, drop the aside.
|
||||
for marker in (" #", " //"):
|
||||
cut = line.find(marker)
|
||||
if cut != -1:
|
||||
line = line[:cut]
|
||||
return line
|
||||
|
||||
|
||||
def referenced_paths(text: str, path: str, by_path, by_base):
|
||||
"""
|
||||
Which other files in this repo this one names, split by how it names them.
|
||||
|
||||
Deliberately textual rather than per-language. A shell `source
|
||||
"$DIR/lib/config.sh"`, a Makefile's `ctrl/cluster.sh`, a kustomization's
|
||||
`- namespace.yaml` and a Dockerfile's `COPY run.py .` are all the same fact
|
||||
— this file needs that one — and four parsers would find it four ways and
|
||||
disagree at the edges.
|
||||
|
||||
The split matters more than the matching does. A reference in code is a
|
||||
dependency: config.sh has to exist before the script that sources it. A
|
||||
reference in a comment is a footnote — .gitignore names `ctrl/wizard.sh`
|
||||
to say the opposite of "I need this", and README names every file in the
|
||||
repo. Counting those as edges puts the ignore rules after the code they
|
||||
exclude, which is exactly backwards. So prose informs the brief and never
|
||||
the order.
|
||||
|
||||
Bare filenames only count when they are unique in the repo, and only with
|
||||
their extension. Without both, every `config.py` in a tree of them becomes
|
||||
an edge to all the others and the ordering collapses into one cycle.
|
||||
"""
|
||||
def hits(blob):
|
||||
found = set()
|
||||
for token in set(_TOKEN.findall(blob)):
|
||||
token = token.strip("./")
|
||||
if not token:
|
||||
continue
|
||||
if token in by_path:
|
||||
found.update(by_path[token])
|
||||
continue
|
||||
# A bare name is only a reference if it carries an extension and
|
||||
# names exactly one file. "cluster" is a word; "cluster.sh" is not.
|
||||
if "." in token:
|
||||
candidates = by_base.get(token)
|
||||
if candidates and len(candidates) == 1:
|
||||
found.update(candidates)
|
||||
found.discard(path)
|
||||
return found
|
||||
|
||||
lines = text.split("\n")
|
||||
code = "\n".join(_strip_comment(l) for l in lines)
|
||||
strong = hits(code)
|
||||
mentions = hits(text) - strong
|
||||
return sorted(strong), sorted(mentions)
|
||||
|
||||
|
||||
def read_text(full: Path):
|
||||
"""Text, or None if this is not text. Size is checked before reading."""
|
||||
try:
|
||||
if full.stat().st_size > MAX_BYTES:
|
||||
return None
|
||||
raw = full.read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
if b"\0" in raw[:8000]:
|
||||
return None
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
# ── the index ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _digest(full: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with full.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(65536), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def state_dir(out) -> Path:
|
||||
"""
|
||||
Where this run's index, plan, briefs and messages live.
|
||||
|
||||
Always `out`, never the source. The source is opened read-only and nothing
|
||||
is written into it — which is what removed a whole class of special cases
|
||||
that used to be here: skipping the state directory during its own census,
|
||||
exempting it from `git status`, and writing a .git/info/exclude entry to
|
||||
keep it quiet. None of that has anywhere to happen now.
|
||||
"""
|
||||
return Path(out)
|
||||
|
||||
|
||||
def state_path(out) -> Path:
|
||||
return state_dir(out) / INDEX_FILE
|
||||
|
||||
|
||||
def current_file_set(source: Path, index) -> set:
|
||||
"""
|
||||
What the source holds right now, filtered the way the census was.
|
||||
|
||||
Recomputed rather than read back, because the guard it feeds exists to
|
||||
catch a file added after the scan. Reading the stored list would answer the
|
||||
easy question — "did the plan cover what we saw?" — instead of the one
|
||||
worth asking, which is "does the plan cover what is there?".
|
||||
"""
|
||||
from .sift import sift
|
||||
settings = (index or {}).get("filter", {})
|
||||
tracked = file_set(source)
|
||||
kept, _ = sift(source, tracked,
|
||||
keep_noise=True,
|
||||
keep_secrets=settings.get("keep_secrets", False),
|
||||
exclude=settings.get("exclude", ()),
|
||||
include=settings.get("include", ()),
|
||||
ignored=ignored_but_tracked(source, tracked))
|
||||
return set(kept)
|
||||
|
||||
|
||||
def load_index(out) -> dict:
|
||||
p = state_path(out)
|
||||
if not p.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except (OSError, json.JSONDecodeError):
|
||||
# A half-written cache is a cache miss, not a crash. Rebuilding costs
|
||||
# one run; refusing to start costs an explanation.
|
||||
return {}
|
||||
|
||||
|
||||
def scan(source: Path, out, keep_secrets=False, exclude=(), include=(),
|
||||
quiet=False) -> dict:
|
||||
"""
|
||||
Census the tree, reusing everything whose content has not changed.
|
||||
|
||||
The reuse is per file and keyed on content, not on mtime: a checkout, a
|
||||
branch switch or a `touch` all move mtimes without changing a byte, and
|
||||
re-reading the whole tree because git rewrote it is the cost this exists to
|
||||
avoid.
|
||||
"""
|
||||
from .sift import REASONS, sift
|
||||
|
||||
# The filter runs here, not at export time, so a key never reaches a plan
|
||||
# in the first place. Secrets and files the repo's own ignore rules
|
||||
# contradict are dropped; what a build regenerates is NOT — a lockfile is
|
||||
# content in a repo somebody is going to use, however little it says.
|
||||
# `copy` is the one that drops those, because a snapshot is for reading.
|
||||
tracked = file_set(source)
|
||||
paths, left_out = sift(source, tracked, keep_noise=True,
|
||||
keep_secrets=keep_secrets, exclude=exclude,
|
||||
include=include,
|
||||
ignored=ignored_but_tracked(source, tracked))
|
||||
previous = load_index(out).get("files", {})
|
||||
|
||||
by_path, by_base = {}, {}
|
||||
for p in paths:
|
||||
by_path.setdefault(p, []).append(p)
|
||||
by_base.setdefault(Path(p).name, []).append(p)
|
||||
|
||||
files, reused = {}, 0
|
||||
for rel in paths:
|
||||
full = source / rel
|
||||
try:
|
||||
digest = _digest(full)
|
||||
except OSError:
|
||||
continue
|
||||
old = previous.get(rel)
|
||||
if old and old.get("hash") == digest and "mentions" in old:
|
||||
files[rel] = old
|
||||
reused += 1
|
||||
continue
|
||||
|
||||
text = read_text(full)
|
||||
refs, mentions = ([], []) if text is None else referenced_paths(text, rel, by_path, by_base)
|
||||
entry = {
|
||||
"hash": digest,
|
||||
"size": full.stat().st_size,
|
||||
"role": role_of(rel),
|
||||
"binary": text is None,
|
||||
"why": "" if text is None else opening_comment(text, rel),
|
||||
"refs": refs, # in code: a dependency, and an edge
|
||||
"mentions": mentions, # in prose: context for the brief, never an edge
|
||||
}
|
||||
entry["lines"] = 0 if text is None else text.count("\n") + 1
|
||||
files[rel] = entry
|
||||
|
||||
# The filter settings travel with the index so the guards can re-derive
|
||||
# the same set later. Without them `check_plan` has to choose between
|
||||
# trusting a stale list and flagging every filtered file as missing.
|
||||
index = {"version": 1, "source": str(source), "files": files,
|
||||
"filter": {"keep_secrets": bool(keep_secrets),
|
||||
"exclude": list(exclude), "include": list(include)},
|
||||
"left_out": [{"path": p, "why": w} for p, w in left_out]}
|
||||
destination = state_path(out)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(index, indent=2, sort_keys=True))
|
||||
|
||||
if not quiet:
|
||||
fresh = len(files) - reused
|
||||
print(f"Scanned {len(files)} files: {fresh} read, {reused} reused from cache.")
|
||||
if left_out:
|
||||
# Never silent. A key that was tracked is a thing to know about,
|
||||
# and it stays true after the file stops travelling.
|
||||
print(f" {len(left_out)} left out of the history:")
|
||||
for rel, why in left_out:
|
||||
print(f" {rel} ({REASONS[why]})")
|
||||
print(f" -> {destination}")
|
||||
return index
|
||||
132
soleprint/station/tools/histgen/cli.py
Normal file
132
soleprint/station/tools/histgen/cli.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
The shape a station tool's command line has.
|
||||
|
||||
Every tool here grew its own copy of the same three things: argparse subcommands
|
||||
wired to `cmd_*` functions, a flag vocabulary that is nearly but not quite the
|
||||
same between tools, and an error convention. shuntgen spells `-s` as --spec in
|
||||
one subcommand and --source in another; modelgen calls the same idea --source
|
||||
everywhere; tester has neither. The differences are accidents, not decisions.
|
||||
|
||||
This is that shared shape, factored out. histgen is the first user. Nothing else
|
||||
is rewritten to use it — a scaffold earns adoption by being there when the next
|
||||
tool is written, not by a flag-day.
|
||||
|
||||
from .cli import Tool, fail
|
||||
|
||||
tool = Tool("histgen", __doc__)
|
||||
tool.command("scan", cmd_scan, "Read the repo and cache what was read.")
|
||||
tool.argument("scan", "repo", help="The tree to read.")
|
||||
tool.run()
|
||||
|
||||
Conventions it encodes, so they stop being re-decided:
|
||||
|
||||
--source/-s what to read --output/-o where to write
|
||||
--name/-n what to call it --force/-f write anyway
|
||||
--dry-run print, do not do
|
||||
|
||||
Progress goes to stdout as plain print(). Errors go to stderr prefixed
|
||||
'Error: ' and exit 1 — never a traceback, which tells a user nothing they
|
||||
can act on. `fail()` is the only exit path.
|
||||
|
||||
Heavy imports live inside the cmd_* function, never at module top, so
|
||||
`--help` stays instant and an optional dependency only costs the one
|
||||
subcommand that needs it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
# The flags worth having exactly one spelling of. A tool adds its own on top;
|
||||
# it does not redefine these.
|
||||
COMMON = {
|
||||
"source": dict(flags=("--source", "-s"), help="What to read."),
|
||||
"output": dict(flags=("--output", "-o"), help="Where to write."),
|
||||
"name": dict(flags=("--name", "-n"), help="What to call it."),
|
||||
"force": dict(flags=("--force", "-f"), action="store_true",
|
||||
help="Write even if the destination is occupied."),
|
||||
"dry_run": dict(flags=("--dry-run",), action="store_true",
|
||||
help="Print what would happen; change nothing."),
|
||||
}
|
||||
|
||||
|
||||
def fail(message, hint=None):
|
||||
"""The only way out on error: a line a user can act on, never a traceback."""
|
||||
print(f"Error: {message}", file=sys.stderr)
|
||||
if hint:
|
||||
print(f" {hint}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def refuse_to_clobber(path, force, marker, what):
|
||||
"""
|
||||
Regenerating is fine; overwriting something we did not write is not.
|
||||
|
||||
Lifted from shuntgen, which refuses a non-empty output directory unless it
|
||||
carries the file its own generator leaves behind. The check is cheap and the
|
||||
failure it prevents — silently eating a directory someone hand-wrote — is
|
||||
not recoverable from.
|
||||
"""
|
||||
if force or not path.exists():
|
||||
return
|
||||
if not any(path.iterdir()):
|
||||
return
|
||||
if (path / marker).exists():
|
||||
return
|
||||
fail(f"{path} already exists and was not written by {what}.",
|
||||
"Pick another path, or pass --force to write into it anyway.")
|
||||
|
||||
|
||||
def _prog(package, name):
|
||||
"""
|
||||
How this tool was actually invoked, for the usage line.
|
||||
|
||||
The folder is meant to be copied out and run on its own, so a usage line
|
||||
hardcoding `python -m station.tools.histgen` is wrong the moment it is —
|
||||
it names a path that does not exist on the machine reading it.
|
||||
"""
|
||||
return f"python -m {package or name}"
|
||||
|
||||
|
||||
class Tool:
|
||||
"""A tool's whole command line: subcommands, shared flags, one exit path."""
|
||||
|
||||
def __init__(self, name, description, package=None):
|
||||
self.name = name
|
||||
self.parser = argparse.ArgumentParser(
|
||||
prog=_prog(package, name),
|
||||
description=(description or "").strip().split("\n\n")[0],
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
self._subparsers = self.parser.add_subparsers(dest="command", required=True)
|
||||
self._commands = {}
|
||||
|
||||
def command(self, verb, func, help_text):
|
||||
"""Register a subcommand. `func` takes parsed args and returns None."""
|
||||
sub = self._subparsers.add_parser(verb, help=help_text, description=help_text)
|
||||
sub.set_defaults(func=func)
|
||||
self._commands[verb] = sub
|
||||
return sub
|
||||
|
||||
def argument(self, verb, *args, **kwargs):
|
||||
"""Add a positional or flag to one subcommand."""
|
||||
self._commands[verb].add_argument(*args, **kwargs)
|
||||
|
||||
def common(self, verb, *names, **overrides):
|
||||
"""Add shared flags by name, so their spelling is decided in one place."""
|
||||
for key in names:
|
||||
spec = dict(COMMON[key])
|
||||
flags = spec.pop("flags")
|
||||
spec.update(overrides.get(key, {}))
|
||||
self._commands[verb].add_argument(*flags, **spec)
|
||||
|
||||
def run(self, argv=None):
|
||||
args = self.parser.parse_args(argv)
|
||||
try:
|
||||
args.func(args)
|
||||
except KeyboardInterrupt:
|
||||
# Ctrl-C is a decision, not a crash. Say so and leave quietly.
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except BrokenPipeError:
|
||||
# `... | head` closes the pipe early; that is the caller's business.
|
||||
sys.exit(0)
|
||||
153
soleprint/station/tools/histgen/config.py
Normal file
153
soleprint/station/tools/histgen/config.py
Normal file
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Where to read from and where to write to.
|
||||
|
||||
Two directories, and the whole tool hangs off the difference between them:
|
||||
|
||||
source the tree to read. Opened read-only, always. Nothing is written
|
||||
into it, ever — not a commit, not a .git, not a state file. It
|
||||
can be a checkout you do not own or a read-only mount.
|
||||
|
||||
out everything this produces. The index, the plan, the briefs, and
|
||||
`out/<name>/` — a copy of the source with the designed history
|
||||
committed into it. Delete the directory and you have lost
|
||||
nothing but time.
|
||||
|
||||
The same repo gets scanned, planned and re-planned a dozen times while its
|
||||
grouping is argued with, and passing the pair of paths to every one of five
|
||||
verbs gets old. So they can live in a file instead — the arrangement
|
||||
`ppl/ctrl/distill.sh` already uses, where the JSON beside the script is picked
|
||||
up when nothing else says otherwise.
|
||||
|
||||
{
|
||||
"source": "~/work/some-project",
|
||||
"out": "~/histories/some-project",
|
||||
"max_files": null
|
||||
}
|
||||
|
||||
That separation is what makes the thing safe to experiment with. The history
|
||||
is an argument you will have more than once, and every attempt is a directory
|
||||
you can throw away rather than a repo you have to put back.
|
||||
|
||||
Precedence is the usual one, most specific first:
|
||||
|
||||
the command line -> --config FILE -> histgen.json beside the tool
|
||||
-> the defaults
|
||||
|
||||
so a config file sets a starting point and never wins an argument with a flag
|
||||
that was typed deliberately.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
CONFIG_NAME = "histgen.json"
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
KEYS = ("source", "out", "max_files", "keep_history", "branch",
|
||||
"keep_secrets", "exclude", "include")
|
||||
|
||||
# `repo` was what `source` used to be called, back when the tool committed
|
||||
# into the tree it read. Accepted rather than rejected, because a config file
|
||||
# written last week should not be an error message.
|
||||
ALIASES = {"repo": "source"}
|
||||
|
||||
TEMPLATE = {
|
||||
"source": None,
|
||||
"out": None,
|
||||
"max_files": None,
|
||||
"keep_history": False,
|
||||
"branch": None,
|
||||
"keep_secrets": False,
|
||||
"exclude": [],
|
||||
"include": [],
|
||||
}
|
||||
|
||||
|
||||
def default_path() -> Path:
|
||||
"""The config beside the tool, used when nothing else is named."""
|
||||
return HERE / CONFIG_NAME
|
||||
|
||||
|
||||
def find(explicit=None):
|
||||
"""The config file to read, or None. An explicit one that is missing is an error."""
|
||||
if explicit:
|
||||
path = Path(explicit).expanduser()
|
||||
if not path.is_file():
|
||||
from .cli import fail
|
||||
fail(f"No such config file: {path}")
|
||||
return path
|
||||
beside = default_path()
|
||||
return beside if beside.is_file() else None
|
||||
|
||||
|
||||
def load(explicit=None) -> dict:
|
||||
"""Read the config, or return the defaults. Unknown keys are an error."""
|
||||
settings = dict(TEMPLATE)
|
||||
path = find(explicit)
|
||||
if not path:
|
||||
return settings
|
||||
|
||||
from .cli import fail
|
||||
try:
|
||||
raw = json.loads(path.read_text())
|
||||
except json.JSONDecodeError as e:
|
||||
fail(f"{path} is not valid JSON: {e}")
|
||||
if not isinstance(raw, dict):
|
||||
fail(f"{path} should hold an object, not a {type(raw).__name__}.")
|
||||
|
||||
# A typo in a key would otherwise be silent, and the symptom — the tool
|
||||
# ignoring a setting that is plainly written in the file — is a bad one to
|
||||
# debug. Naming the valid keys costs one line.
|
||||
raw = {ALIASES.get(k, k): v for k, v in raw.items()}
|
||||
unknown = sorted(set(raw) - set(KEYS))
|
||||
if unknown:
|
||||
fail(f"{path}: unknown key(s) {', '.join(unknown)}.",
|
||||
f"Known keys: {', '.join(KEYS)}.")
|
||||
|
||||
settings.update({k: v for k, v in raw.items() if v is not None})
|
||||
settings["_path"] = str(path)
|
||||
return settings
|
||||
|
||||
|
||||
def resolve(args, explicit=None):
|
||||
"""
|
||||
Fold the config under the command line and hand back what to actually use.
|
||||
|
||||
Paths are expanded and made absolute here rather than at each use, so
|
||||
everything downstream compares like with like — a `~` that survived into a
|
||||
path comparison is a bug that only shows up on someone else's machine.
|
||||
"""
|
||||
settings = load(explicit)
|
||||
|
||||
source = getattr(args, "source", None) or settings.get("source")
|
||||
out = getattr(args, "out", None) or settings.get("out")
|
||||
|
||||
return {
|
||||
"source": Path(source).expanduser().resolve() if source else None,
|
||||
"out": Path(out).expanduser().resolve() if out else None,
|
||||
"max_files": getattr(args, "max_files", None) or settings.get("max_files"),
|
||||
"keep_history": (getattr(args, "keep_history", False)
|
||||
or settings.get("keep_history", False)),
|
||||
"branch": getattr(args, "branch", None) or settings.get("branch"),
|
||||
"keep_secrets": (getattr(args, "keep_secrets", False)
|
||||
or settings.get("keep_secrets", False)),
|
||||
"exclude": list(getattr(args, "exclude", None) or [])
|
||||
+ list(settings.get("exclude") or []),
|
||||
"include": list(getattr(args, "include", None) or [])
|
||||
+ list(settings.get("include") or []),
|
||||
"config_path": settings.get("_path"),
|
||||
}
|
||||
|
||||
|
||||
def write_template(path: Path, source=None, out=None) -> Path:
|
||||
"""Write a starter config, never over one that already exists."""
|
||||
from .cli import fail
|
||||
if path.exists():
|
||||
fail(f"{path} already exists.", "Edit it, or name another path.")
|
||||
body = dict(TEMPLATE)
|
||||
if source:
|
||||
body["source"] = str(source)
|
||||
if out:
|
||||
body["out"] = str(out)
|
||||
path.write_text(json.dumps(body, indent=2) + "\n")
|
||||
return path
|
||||
613
soleprint/station/tools/histgen/export.py
Normal file
613
soleprint/station/tools/histgen/export.py
Normal file
@@ -0,0 +1,613 @@
|
||||
"""
|
||||
Materialise the source into the out directory and commit the designed history.
|
||||
|
||||
The source is never touched. What gets committed is a copy, made here, and the
|
||||
copy is the only thing that ends up with a history — so an attempt that goes
|
||||
wrong costs a `rm -rf` rather than an afternoon putting a real checkout back.
|
||||
|
||||
Everything up to this point is analysis and can be recomputed. This part writes,
|
||||
so it starts by working out what it is writing into. Four states, and they are
|
||||
genuinely different:
|
||||
|
||||
absent nothing there yet. Copy the tree, init, commit.
|
||||
|
||||
unfinished a copy is there with commits this tool made and a record of
|
||||
where it stopped. Something interrupted the run — a signal, a
|
||||
full disk, a hook that refused. Continue from the group after
|
||||
the last one recorded.
|
||||
|
||||
foreign a copy is there with commits this tool did not make. That is
|
||||
history someone else is entitled to, so nothing is rewritten,
|
||||
moved or deleted: the designed account is committed to its own
|
||||
orphan branch and the existing branch is left exactly as it
|
||||
was. Two tiers, which is what `all/ctrl/handover.sh` has been
|
||||
saying all along.
|
||||
|
||||
stale a copy is there that does not match the plan any more. Refuse,
|
||||
and say which of the two moved.
|
||||
|
||||
Telling the second apart from the third is the whole reason progress.json
|
||||
exists. Without it both read as "there are commits here", and the tool either
|
||||
destroys work it should have kept or refuses to finish work it started.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .census import current_file_set, is_git, load_index, state_dir
|
||||
from .cli import fail
|
||||
from .order import plan_path
|
||||
|
||||
MESSAGE_DIR = "messages"
|
||||
SCRIPT_FILE = "regen.sh"
|
||||
PROGRESS_FILE = "progress.json"
|
||||
|
||||
# Where the designed account goes when the copy already carries a history that
|
||||
# is not ours. A name, not a number, because it has to mean something in a
|
||||
# branch list six months from now.
|
||||
DEFAULT_BRANCH = "designed-history"
|
||||
|
||||
|
||||
# ── where the copy lives ───────────────────────────────────────────────────
|
||||
|
||||
def repo_dir(source: Path, out: Path) -> Path:
|
||||
"""
|
||||
The copy, named after the source.
|
||||
|
||||
Named rather than called `repo/`, because this directory gets `cd`-ed into,
|
||||
pushed from and looked at in a file manager, and "adapter" answers a
|
||||
question there that "repo" does not.
|
||||
"""
|
||||
return Path(out) / source.name
|
||||
|
||||
|
||||
def progress_path(out) -> Path:
|
||||
return state_dir(out) / PROGRESS_FILE
|
||||
|
||||
|
||||
def plan_fingerprint(plan) -> str:
|
||||
"""
|
||||
Identifies the plan a history was built from, by its groups and their paths.
|
||||
|
||||
Messages are deliberately not in it. Rewording a commit that has not been
|
||||
made yet must not invalidate the twelve that have — that is the normal way
|
||||
this tool gets used, one group's message at a time.
|
||||
"""
|
||||
shape = [[g["n"], sorted(g["paths"])] for g in plan["groups"]]
|
||||
return hashlib.sha256(json.dumps(shape, sort_keys=True).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def load_progress(out) -> dict:
|
||||
p = progress_path(out)
|
||||
if not p.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except (OSError, json.JSONDecodeError):
|
||||
# Unreadable progress means we cannot prove which commits are ours, and
|
||||
# guessing is exactly the thing this file exists to avoid.
|
||||
return {}
|
||||
|
||||
|
||||
def save_progress(out, data) -> None:
|
||||
progress_path(out).parent.mkdir(parents=True, exist_ok=True)
|
||||
progress_path(out).write_text(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
# ── git ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _git(repo: Path, *args, check=True):
|
||||
r = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True)
|
||||
if check and r.returncode != 0:
|
||||
fail(f"git {' '.join(args[:2])} failed: {r.stderr.strip() or r.stdout.strip()}")
|
||||
return r
|
||||
|
||||
|
||||
def _head(repo: Path):
|
||||
r = _git(repo, "rev-parse", "HEAD", check=False)
|
||||
return r.stdout.strip() if r.returncode == 0 else None
|
||||
|
||||
|
||||
def _commit_count(repo: Path) -> int:
|
||||
r = _git(repo, "rev-list", "--count", "HEAD", check=False)
|
||||
return int(r.stdout.strip()) if r.returncode == 0 and r.stdout.strip() else 0
|
||||
|
||||
|
||||
# ── what state is the out directory in ─────────────────────────────────────
|
||||
|
||||
def inspect(source: Path, out: Path, plan=None):
|
||||
"""
|
||||
Read the out directory and say what is there. Writes nothing.
|
||||
|
||||
`status` prints this; `export` branches on it. One function so the two can
|
||||
never disagree about what they are looking at, which they would within a
|
||||
week of being written separately.
|
||||
"""
|
||||
copy = repo_dir(source, out)
|
||||
progress = load_progress(out)
|
||||
report = {
|
||||
"copy": copy,
|
||||
"exists": copy.is_dir(),
|
||||
"git": copy.is_dir() and is_git(copy),
|
||||
"commits": 0,
|
||||
"state": "absent",
|
||||
"done": [],
|
||||
"remaining": [],
|
||||
"branch": None,
|
||||
"detail": "",
|
||||
}
|
||||
if not report["exists"]:
|
||||
report["detail"] = "nothing exported yet"
|
||||
if plan:
|
||||
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||
return report
|
||||
|
||||
if not report["git"]:
|
||||
report["state"] = "stale"
|
||||
report["detail"] = ("a directory is there but it is not a git repo — "
|
||||
"an export that died before `git init`")
|
||||
return report
|
||||
|
||||
report["commits"] = _commit_count(copy)
|
||||
report["branch"] = _git(copy, "rev-parse", "--abbrev-ref", "HEAD",
|
||||
check=False).stdout.strip() or None
|
||||
|
||||
if report["commits"] == 0:
|
||||
report["state"] = "absent"
|
||||
report["detail"] = "a repo with no commits"
|
||||
if plan:
|
||||
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||
return report
|
||||
|
||||
ours = progress.get("commits", [])
|
||||
head = _head(copy)
|
||||
|
||||
if not ours:
|
||||
report["state"] = "foreign"
|
||||
report["detail"] = (f"{report['commits']} commit(s) this tool did not make")
|
||||
if plan:
|
||||
report["remaining"] = [g["n"] for g in plan["groups"]]
|
||||
return report
|
||||
|
||||
if plan and progress.get("plan") != plan_fingerprint(plan):
|
||||
report["state"] = "stale"
|
||||
report["detail"] = ("the plan changed after this history was started — "
|
||||
"the groups are not the ones these commits were made from")
|
||||
report["done"] = [c["n"] for c in ours]
|
||||
return report
|
||||
|
||||
# The record has to still describe reality. If the branch moved underneath
|
||||
# us — a rebase, a reset, an amend — continuing would build on something
|
||||
# other than what was recorded, and quietly.
|
||||
if head != ours[-1]["sha"]:
|
||||
report["state"] = "stale"
|
||||
report["detail"] = ("the copy has moved since this tool last wrote to it "
|
||||
f"(expected {ours[-1]['sha'][:9]}, found "
|
||||
f"{(head or '-')[:9]})")
|
||||
report["done"] = [c["n"] for c in ours]
|
||||
return report
|
||||
|
||||
report["done"] = [c["n"] for c in ours]
|
||||
if plan:
|
||||
report["remaining"] = [g["n"] for g in plan["groups"]
|
||||
if g["n"] not in set(report["done"])]
|
||||
report["state"] = "complete" if plan and not report["remaining"] else "unfinished"
|
||||
report["detail"] = (f"{len(report['done'])} group(s) committed by this tool"
|
||||
+ (f", {len(report['remaining'])} to go" if report["remaining"] else ""))
|
||||
return report
|
||||
|
||||
|
||||
# ── checks ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_plan(out):
|
||||
p = plan_path(out)
|
||||
if not p.exists():
|
||||
fail(f"No plan at {p}.", "Run `scan` then `plan` first.")
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except json.JSONDecodeError as e:
|
||||
# Hand-editing plan.json is the expected workflow, so a trailing comma
|
||||
# is a normal event and deserves a line number rather than a traceback.
|
||||
fail(f"{p} is not valid JSON: {e}")
|
||||
|
||||
|
||||
def check_plan(source: Path, out, plan, require_messages=True):
|
||||
"""
|
||||
Refuse a plan that could not produce the tree it claims to.
|
||||
|
||||
What the plan is measured against is the source as it stands *now*, with
|
||||
the same filter the census used. Not the stored list: a file added after
|
||||
the scan is exactly what this is here to catch, and a stored list cannot
|
||||
see it. Not the unfiltered source either, or every deliberately dropped key
|
||||
comes back as a file no group covers.
|
||||
"""
|
||||
planned, dupes = [], []
|
||||
for g in plan["groups"]:
|
||||
for p in g["paths"]:
|
||||
(dupes if p in planned else planned).append(p)
|
||||
|
||||
problems = []
|
||||
if dupes:
|
||||
problems.append(f"{len(dupes)} path(s) appear in more than one group: "
|
||||
+ ", ".join(sorted(set(dupes))[:5]))
|
||||
|
||||
missing = [p for p in planned if not (source / p).is_file()]
|
||||
if missing:
|
||||
problems.append(f"{len(missing)} planned path(s) are not in the source: "
|
||||
+ ", ".join(missing[:5]))
|
||||
|
||||
present = current_file_set(source, load_index(out))
|
||||
unplanned = sorted(present - set(planned))
|
||||
if unplanned:
|
||||
problems.append(f"{len(unplanned)} file(s) are in the source but in no group: "
|
||||
+ ", ".join(unplanned[:5])
|
||||
+ "\n Re-run `scan` and `plan` if the source changed since.")
|
||||
|
||||
empty = [g["n"] for g in plan["groups"] if not g["paths"]]
|
||||
if empty:
|
||||
problems.append(f"group(s) {empty} have no paths")
|
||||
|
||||
if require_messages:
|
||||
unwritten = [g["n"] for g in plan["groups"] if not (g.get("title") or "").strip()]
|
||||
if unwritten:
|
||||
problems.append(
|
||||
f"{len(unwritten)} group(s) have no title: "
|
||||
+ ", ".join(str(n) for n in unwritten[:8])
|
||||
+ f"\n Read {state_dir(out) / 'briefs'}/ and fill them in, "
|
||||
"or pass --allow-untitled.")
|
||||
|
||||
if problems:
|
||||
for p in problems:
|
||||
print(f"Error: {p}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return planned
|
||||
|
||||
|
||||
def _message(group):
|
||||
title = (group.get("title") or "").strip() or f"{group['slug']} ({len(group['paths'])} files)"
|
||||
body = (group.get("body") or "").strip()
|
||||
return f"{title}\n\n{body}\n" if body else f"{title}\n"
|
||||
|
||||
|
||||
def source_tree_hash(source: Path, paths):
|
||||
"""
|
||||
The tree hash the source files would produce, without committing anything.
|
||||
|
||||
Runs against a temporary index and, when the source has no git, a temporary
|
||||
git directory too. The source's own index is never touched: someone running
|
||||
this mid-edit must not lose their staging area to a verification step.
|
||||
"""
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-idx-") as tmp:
|
||||
env = dict(os.environ, GIT_INDEX_FILE=str(Path(tmp) / "index"))
|
||||
if not (source / ".git").exists():
|
||||
env["GIT_DIR"] = str(Path(tmp) / "git")
|
||||
env["GIT_WORK_TREE"] = str(source)
|
||||
subprocess.run(["git", "init", "-q"], env=env, capture_output=True)
|
||||
proc = subprocess.run(
|
||||
["git", "-C", str(source), "update-index", "--add", "--stdin"],
|
||||
input="\n".join(paths) + "\n", text=True, capture_output=True, env=env)
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
r = subprocess.run(["git", "-C", str(source), "write-tree"],
|
||||
capture_output=True, text=True, env=env)
|
||||
return r.stdout.strip() if r.returncode == 0 else None
|
||||
|
||||
|
||||
# ── the guards ─────────────────────────────────────────────────────────────
|
||||
|
||||
def verify(copy: Path, expected_tree=None, quiet=False):
|
||||
"""
|
||||
Nothing left untracked, and the tree still matches the source.
|
||||
|
||||
Nothing is exempt from the first check. The state directory lives in `out`
|
||||
and the copy lives inside it, so there is genuinely nothing of ours in the
|
||||
tree being checked — which is stricter than the version that had to forgive
|
||||
its own scaffolding.
|
||||
"""
|
||||
ok = True
|
||||
status = _git(copy, "status", "--porcelain").stdout.strip()
|
||||
if status:
|
||||
ok = False
|
||||
print("Error: the tree is not clean — these never made it into a commit:",
|
||||
file=sys.stderr)
|
||||
for line in status.split("\n")[:20]:
|
||||
print(f" {line}", file=sys.stderr)
|
||||
if len(status.split("\n")) > 20:
|
||||
print(f" ... and {len(status.split(chr(10))) - 20} more", file=sys.stderr)
|
||||
elif not quiet:
|
||||
print(" nothing left untracked: ok")
|
||||
|
||||
if expected_tree:
|
||||
head = _git(copy, "rev-parse", "HEAD^{tree}", check=False).stdout.strip()
|
||||
if head != expected_tree:
|
||||
ok = False
|
||||
print(f"Error: the exported tree does not match the source.\n"
|
||||
f" source {expected_tree}\n HEAD {head}", file=sys.stderr)
|
||||
elif not quiet:
|
||||
print(f" tree matches source ({head[:12]}): ok")
|
||||
return ok
|
||||
|
||||
|
||||
# ── materialising the copy ─────────────────────────────────────────────────
|
||||
|
||||
def materialise(source: Path, copy: Path, paths, keep_history=False, quiet=False):
|
||||
"""
|
||||
Put the planned files into the copy, and nothing else.
|
||||
|
||||
Copied file by file from the plan rather than with `cp -r`, because the
|
||||
plan is the definition of what belongs in the history: anything gitignored,
|
||||
anything untracked, and the source's own .git are all things the source has
|
||||
and the export must not.
|
||||
|
||||
`keep_history` is the exception, and the only reason the source's .git ever
|
||||
comes across: it is what lets an existing history be carried into the copy
|
||||
so the designed account can sit beside it instead of replacing it.
|
||||
"""
|
||||
copy.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if keep_history and (source / ".git").is_dir() and not (copy / ".git").exists():
|
||||
shutil.copytree(source / ".git", copy / ".git", symlinks=True)
|
||||
# A copied .git still points its index at files that are about to be
|
||||
# rewritten underneath it; reset so status reflects the copy, not the
|
||||
# source's staging area at the moment it was cloned.
|
||||
_git(copy, "reset", "-q", check=False)
|
||||
|
||||
for rel in paths:
|
||||
target = copy / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source / rel, target)
|
||||
|
||||
if not quiet:
|
||||
print(f" copied {len(paths)} files -> {copy}")
|
||||
|
||||
|
||||
def _start_branch(copy: Path, report, branch, quiet=False):
|
||||
"""
|
||||
Decide which ref the designed history goes on, and get there.
|
||||
|
||||
A foreign history is not ours to move, so the designed account starts from
|
||||
an orphan — no parent, nothing shared — and the branch that was there keeps
|
||||
pointing exactly where it did.
|
||||
"""
|
||||
if report["state"] != "foreign":
|
||||
return None
|
||||
name = branch or DEFAULT_BRANCH
|
||||
if _git(copy, "rev-parse", "--verify", name, check=False).returncode == 0:
|
||||
fail(f"Branch '{name}' already exists in {copy}.",
|
||||
"Pick another with --branch, or delete it if it was a failed attempt.")
|
||||
kept = report["branch"] or "the existing branch"
|
||||
if not quiet:
|
||||
print(f" {report['commits']} existing commit(s) on {kept}: kept, untouched")
|
||||
print(f" the designed history goes on a new orphan branch '{name}'")
|
||||
_git(copy, "checkout", "-q", "--orphan", name)
|
||||
# --orphan keeps the index, which would make the first designed commit
|
||||
# carry every file the old branch had staged.
|
||||
_git(copy, "rm", "-rq", "--cached", ".", check=False)
|
||||
return name
|
||||
|
||||
|
||||
def export(source: Path, out, dry_run=False, commands=False, allow_untitled=False,
|
||||
keep_history=False, branch=None, force=False, quiet=False):
|
||||
plan = load_plan(out)
|
||||
planned = check_plan(source, out, plan, require_messages=not allow_untitled)
|
||||
expected = source_tree_hash(source, planned)
|
||||
if not expected:
|
||||
fail("Could not compute the source tree hash.",
|
||||
"Without it the export cannot be checked, and an unchecked export "
|
||||
"is the thing this refuses to produce.")
|
||||
|
||||
copy = repo_dir(source, out)
|
||||
report = inspect(source, out, plan)
|
||||
|
||||
if dry_run:
|
||||
return _emit_script(source, out, plan, planned, expected, report,
|
||||
keep_history, branch, quiet)
|
||||
|
||||
if commands:
|
||||
return _emit_commands(source, out, plan, planned, expected, force, quiet)
|
||||
|
||||
if report["state"] == "stale" and not force:
|
||||
fail(f"{copy}: {report['detail']}.",
|
||||
"Pass --force to discard what is there and export again, or point "
|
||||
"--out somewhere else to keep it.")
|
||||
if report["state"] == "complete":
|
||||
print(f"Already exported: {len(report['done'])} groups committed in {copy}.")
|
||||
print("Nothing to do. Re-plan, or use --force to start over.")
|
||||
return True
|
||||
|
||||
if report["state"] == "stale" and force:
|
||||
if not quiet:
|
||||
print(f" discarding {copy}")
|
||||
shutil.rmtree(copy)
|
||||
save_progress(out, {})
|
||||
report = inspect(source, out, plan)
|
||||
|
||||
progress = load_progress(out)
|
||||
resuming = report["state"] == "unfinished"
|
||||
|
||||
if resuming:
|
||||
done = set(report["done"])
|
||||
if not quiet:
|
||||
print(f"Resuming: {len(done)} of {len(plan['groups'])} groups already "
|
||||
f"committed in {copy}.")
|
||||
# The files are already there from the interrupted run, but a source
|
||||
# edited since would otherwise be silently ignored.
|
||||
materialise(source, copy, planned, quiet=quiet)
|
||||
else:
|
||||
done = set()
|
||||
materialise(source, copy, planned, keep_history=keep_history, quiet=quiet)
|
||||
if not is_git(copy):
|
||||
_git(copy, "init", "-q")
|
||||
# Re-read the copy. --keep-history has only just put a history into it,
|
||||
# so the state worked out before the directory existed cannot have seen
|
||||
# it — and acting on the stale answer commits the designed account on
|
||||
# top of the history it was supposed to sit beside.
|
||||
report = inspect(source, out, plan)
|
||||
active = _start_branch(copy, report, branch, quiet)
|
||||
progress = {"plan": plan_fingerprint(plan), "source": str(source),
|
||||
"branch": active, "commits": []}
|
||||
save_progress(out, progress)
|
||||
|
||||
for g in plan["groups"]:
|
||||
if g["n"] in done:
|
||||
continue
|
||||
_git(copy, "add", "--", *g["paths"])
|
||||
msg = copy / ".git" / "HISTGEN_MSG"
|
||||
msg.write_text(_message(g))
|
||||
_git(copy, "commit", "-q", "-F", str(msg))
|
||||
msg.unlink(missing_ok=True)
|
||||
# Recorded after each commit, not at the end. The whole point is to
|
||||
# survive the run not reaching the end.
|
||||
progress.setdefault("commits", []).append({"n": g["n"], "sha": _head(copy)})
|
||||
progress["plan"] = plan_fingerprint(plan)
|
||||
save_progress(out, progress)
|
||||
if not quiet:
|
||||
print(f" {g['n']:02d} {_message(g).splitlines()[0]}")
|
||||
|
||||
if not quiet:
|
||||
# The repo's own count, not the plan's. With a kept history the two
|
||||
# differ, and the number a reader wants is what is actually in there.
|
||||
where = _git(copy, "rev-parse", "--abbrev-ref", "HEAD",
|
||||
check=False).stdout.strip()
|
||||
print(f"\n{_commit_count(copy)} commits on {where} in {copy}. "
|
||||
"Checking:", flush=True)
|
||||
if not verify(copy, expected, quiet=quiet):
|
||||
sys.exit(1)
|
||||
return True
|
||||
|
||||
|
||||
def _emit_script(source, out, plan, planned, expected, report,
|
||||
keep_history, branch, quiet):
|
||||
"""
|
||||
Write the export as a shell script instead of running it.
|
||||
|
||||
Reviewing plain git commands before they run is worth more here than
|
||||
anywhere else: this is the one operation whose mistakes are baked into
|
||||
every commit that follows.
|
||||
"""
|
||||
state = state_dir(out)
|
||||
copy = repo_dir(source, out)
|
||||
msg_dir = state / MESSAGE_DIR
|
||||
msg_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stale in msg_dir.glob("*.txt"):
|
||||
stale.unlink()
|
||||
|
||||
q = shlex.quote
|
||||
lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"# Generated by histgen. Review, then run from anywhere.",
|
||||
"set -euo pipefail",
|
||||
"",
|
||||
f"SOURCE={q(str(source))}",
|
||||
f"COPY={q(str(copy))}",
|
||||
"",
|
||||
'mkdir -p "$COPY"',
|
||||
]
|
||||
if keep_history and (source / ".git").is_dir():
|
||||
lines.append('test -d "$COPY/.git" || cp -a "$SOURCE/.git" "$COPY/.git"')
|
||||
lines += [
|
||||
"# Only the planned files: not the source's .git, not anything ignored.",
|
||||
'while IFS= read -r f; do mkdir -p "$COPY/$(dirname "$f")"; '
|
||||
'cp -p "$SOURCE/$f" "$COPY/$f"; done <<\'PATHS\'',
|
||||
*planned,
|
||||
"PATHS",
|
||||
"",
|
||||
'cd "$COPY"',
|
||||
"test -d .git || git init -q",
|
||||
]
|
||||
if report["state"] == "foreign":
|
||||
name = branch or DEFAULT_BRANCH
|
||||
lines += [f"# {report['commits']} existing commit(s) stay where they are.",
|
||||
f"git checkout -q --orphan {q(name)}",
|
||||
"git rm -rq --cached . || true", ""]
|
||||
|
||||
for g in plan["groups"]:
|
||||
name = f"{g['n']:02d}-{g['slug']}.txt"
|
||||
(msg_dir / name).write_text(_message(g))
|
||||
lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}",
|
||||
"git add -- " + " ".join(q(p) for p in g["paths"]),
|
||||
f"git commit -q -F {q(str(msg_dir / name))}",
|
||||
""]
|
||||
|
||||
lines += [
|
||||
"# The two guards. A history that fails these is not worth keeping.",
|
||||
'test -z "$(git status --porcelain)" || '
|
||||
'{ echo "FAILED: files left untracked" >&2; exit 1; }',
|
||||
f'test "$(git rev-parse HEAD^{{tree}})" = "{expected}" || '
|
||||
'{ echo "FAILED: exported tree does not match source" >&2; exit 1; }',
|
||||
'echo "ok: $(git rev-list --count HEAD) commits, tree verified"',
|
||||
"",
|
||||
]
|
||||
|
||||
script = state / SCRIPT_FILE
|
||||
script.write_text("\n".join(lines))
|
||||
script.chmod(0o755)
|
||||
if not quiet:
|
||||
print(f"Wrote {len(plan['groups'])} commits as commands -> {script}")
|
||||
print(f" messages -> {msg_dir}")
|
||||
return True
|
||||
|
||||
|
||||
def _emit_commands(source: Path, out, plan, planned, expected, force, quiet):
|
||||
"""
|
||||
Copy the files, create no repo, and print the commands to make the history.
|
||||
|
||||
The other two modes each decide something for you: `export` runs the whole
|
||||
thing, `--dry-run` writes a script that would. This one does the half that
|
||||
is tedious and gets the other half out of the way — the copy is made, and
|
||||
what comes back is a list you read, edit and run yourself.
|
||||
|
||||
Nothing here creates a .git. `git init` is the first line of the list rather
|
||||
than something already done, because a repo that appeared without you asking
|
||||
is exactly what someone reaching for this mode does not want.
|
||||
"""
|
||||
copy = repo_dir(source, out)
|
||||
|
||||
# Asked for no repo, so an existing one is a contradiction worth stopping
|
||||
# for: the commands below would commit into it rather than into a fresh one.
|
||||
if (copy / ".git").exists() and not force:
|
||||
fail(f"{copy} already contains a git repo.",
|
||||
"This mode creates none and the commands assume none. Delete it, "
|
||||
"point --out elsewhere, or pass --force to copy the files in anyway.")
|
||||
|
||||
materialise(source, copy, planned, quiet=quiet)
|
||||
|
||||
state = state_dir(out)
|
||||
msg_dir = state / MESSAGE_DIR
|
||||
msg_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stale in msg_dir.glob("*.txt"):
|
||||
stale.unlink()
|
||||
|
||||
q = shlex.quote
|
||||
lines = [f"cd {q(str(copy))}", "git init", ""]
|
||||
for g in plan["groups"]:
|
||||
name = f"{g['n']:02d}-{g['slug']}.txt"
|
||||
(msg_dir / name).write_text(_message(g))
|
||||
lines += [f"# {g['n']:02d} {_message(g).splitlines()[0]}",
|
||||
"git add -- " + " ".join(q(p) for p in g["paths"]),
|
||||
f"git commit -F {q(str(msg_dir / name))}",
|
||||
""]
|
||||
|
||||
lines += [
|
||||
"# Worth running afterwards. The first says no file was silently",
|
||||
"# missed; the second says the result is byte-identical to the source.",
|
||||
"git status --porcelain",
|
||||
f"git rev-parse HEAD^{{tree}} # expect {expected}",
|
||||
"",
|
||||
]
|
||||
|
||||
listing = "\n".join(lines)
|
||||
(state / "commands.sh").write_text(listing)
|
||||
|
||||
if not quiet:
|
||||
print(f" messages -> {msg_dir}")
|
||||
print(f" this list -> {state / 'commands.sh'}\n")
|
||||
print(listing)
|
||||
return True
|
||||
116
soleprint/station/tools/histgen/history.py
Normal file
116
soleprint/station/tools/histgen/history.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
What the history already there says, next to what the plan proposes.
|
||||
|
||||
Reads and reports. It never rewrites: published history is someone else's
|
||||
clone, and the useful output here is an argument about ordering, not a
|
||||
force-push.
|
||||
|
||||
Most repos this gets pointed at are in the checkpoint tier — "updates 33.1
|
||||
139", "working state", "debugging" — where the honest finding is that the log
|
||||
records when work was saved and nothing about how the thing is built. That is
|
||||
worth printing plainly, because it is the case for keeping a second, designed
|
||||
history rather than trying to repair this one.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
|
||||
# Subjects that carry no information about what changed. Matching is on the
|
||||
# whole subject, lowercased, with trailing numbers dropped — "updates 33.1 139"
|
||||
# and "updates 33.1 84" are the same non-statement.
|
||||
NOISE = {"update", "updates", "wip", "fix", "fixes", "changes", "some changes",
|
||||
"working state", "debugging", "init commit", "initial commit", "misc",
|
||||
"checkpoint", "save", "final", "for final test", "cleanup", "tmp"}
|
||||
|
||||
|
||||
def _subject_is_noise(subject):
|
||||
s = subject.strip().lower().rstrip("0123456789. ")
|
||||
return s in NOISE or not s
|
||||
|
||||
|
||||
def read_history(repo):
|
||||
"""[(sha, subject, [paths])] oldest first, or [] if there is no history."""
|
||||
r = subprocess.run(
|
||||
["git", "-C", str(repo), "log", "--reverse", "--name-only",
|
||||
"--format=%x00%h%x1f%s"],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
commits = []
|
||||
for chunk in r.stdout.split("\0"):
|
||||
if not chunk.strip():
|
||||
continue
|
||||
head, _, rest = chunk.partition("\n")
|
||||
sha, _, subject = head.partition("\x1f")
|
||||
paths = [l for l in rest.split("\n") if l.strip()]
|
||||
commits.append((sha, subject, paths))
|
||||
return commits
|
||||
|
||||
|
||||
def compare(source, plan, quiet=False):
|
||||
"""Print how the existing history lines up with the proposed one."""
|
||||
commits = read_history(source)
|
||||
groups = plan["groups"]
|
||||
if not commits:
|
||||
print(f"{len(groups)} groups proposed; the source has no history to compare.")
|
||||
return {"commits": 0, "groups": len(groups)}
|
||||
|
||||
where = {p: g["n"] for g in groups for p in g["paths"]}
|
||||
|
||||
# A commit maps to the group holding most of the files it touched. Files
|
||||
# that no longer exist are dropped rather than counted against it — a
|
||||
# commit that deleted something is not disagreeing about order.
|
||||
mapped, noise, touches = {}, [], defaultdict(list)
|
||||
for sha, subject, paths in commits:
|
||||
hits = [where[p] for p in paths if p in where]
|
||||
if not hits:
|
||||
noise.append((sha, subject, "touches nothing that still exists"))
|
||||
continue
|
||||
best = max(set(hits), key=lambda n: (hits.count(n), -n))
|
||||
spread = len(set(hits))
|
||||
mapped[sha] = (best, subject, spread, len(hits))
|
||||
touches[best].append(sha)
|
||||
if _subject_is_noise(subject):
|
||||
noise.append((sha, subject, f"says nothing; touches {spread} group(s)"))
|
||||
|
||||
print(f"{len(groups)} groups proposed, {len(commits)} existing commits.\n")
|
||||
|
||||
# Order disagreement: walking the real history, does the group number ever
|
||||
# go backwards? That is the concrete "this was built in a different order".
|
||||
seen_max, inversions = 0, []
|
||||
for sha, subject, _ in commits:
|
||||
if sha not in mapped:
|
||||
continue
|
||||
n = mapped[sha][0]
|
||||
if n < seen_max:
|
||||
inversions.append((sha, n, seen_max, subject))
|
||||
seen_max = max(seen_max, n)
|
||||
|
||||
for g in groups:
|
||||
shas = touches.get(g["n"], [])
|
||||
title = g.get("title") or g["slug"]
|
||||
if not shas:
|
||||
mark, note = "+", "no existing commit builds this"
|
||||
elif len(shas) == 1:
|
||||
mark, note = "=", f"{shas[0]}"
|
||||
else:
|
||||
mark, note = "~", f"split across {len(shas)} commits ({', '.join(shas[:4])})"
|
||||
print(f" {mark} {g['n']:02d} {title[:44]:46} {note}")
|
||||
|
||||
if inversions:
|
||||
print(f"\n ! {len(inversions)} commit(s) land earlier in the proposed order "
|
||||
f"than work already done:")
|
||||
for sha, n, high, subject in inversions[:10]:
|
||||
print(f" {sha} group {n:02d} after group {high:02d} {subject[:44]}")
|
||||
|
||||
if noise:
|
||||
print(f"\n ? {len(noise)} commit(s) carry no usable account of the change:")
|
||||
for sha, subject, why in noise[:10]:
|
||||
print(f" {sha} {subject[:44]:46} {why}")
|
||||
if len(noise) > 10:
|
||||
print(f" ... and {len(noise) - 10} more")
|
||||
|
||||
print("\n = matched one commit ~ split + not in history "
|
||||
"! out of order ? uninformative")
|
||||
return {"commits": len(commits), "groups": len(groups),
|
||||
"inversions": len(inversions), "noise": len(noise)}
|
||||
450
soleprint/station/tools/histgen/order.py
Normal file
450
soleprint/station/tools/histgen/order.py
Normal file
@@ -0,0 +1,450 @@
|
||||
"""
|
||||
The order the files go in, and where one commit stops and the next begins.
|
||||
|
||||
Two decisions, and they are not the same decision. Order answers "what can be
|
||||
understood before what"; grouping answers "what is one idea". Getting the first
|
||||
right and the second wrong gives you 64 correct commits nobody wants to read.
|
||||
|
||||
The order is role first, references second. Roles carry the heuristic — ignore
|
||||
rules and README, then the config layer, then the things that source it, the
|
||||
front door late because it only dispatches, the bootstrap account last because
|
||||
it narrates everything above it. References refine within that, so a config
|
||||
lands before the script that sources it.
|
||||
|
||||
References never override roles. A reference in code is a dependency, but roles
|
||||
already encode dependencies that no reference states: nothing in the repo
|
||||
*refers to* .gitignore, and the README is named by nothing while naming
|
||||
everything. Letting edges win produces the ignore rules committed after the
|
||||
code they exclude — technically consistent, and unreadable.
|
||||
|
||||
Nothing here is authoritative. plan.json is a file, and moving a path from one
|
||||
group to another is the expected way to use it: this gets the shape right so
|
||||
the argument is about two or three groups, not sixty-four paths.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from .census import ROLE_RANK, state_dir
|
||||
|
||||
PLAN_FILE = "plan.json"
|
||||
|
||||
# Roles that name other files without depending on them. Their outgoing edges
|
||||
# are dropped: a README mentioning every script in the tree is a table of
|
||||
# contents, not a build order.
|
||||
NARRATIVE = {"skeleton", "readme", "doc", "bootstrap", "asset", "lock"}
|
||||
|
||||
# A directory that carries its own README, Makefile or package manifest is a
|
||||
# project in its own right. It is committed whole and late — it stands on the
|
||||
# repo around it, so it cannot be read before it. rig's sample-rig is the case:
|
||||
# sixteen files, one idea, and it calls rig's own addon script rather than
|
||||
# reimplementing it.
|
||||
# A README is NOT one of these. Any directory worth having explains itself, and
|
||||
# ctrl/k8s/README.md documenting four manifests does not make them a project —
|
||||
# it made them sort after the Makefile, which is where this rule came from. What
|
||||
# marks a project is something that builds or resolves it.
|
||||
SUBPROJECT_MARKERS = {"makefile", "package.json", "pyproject.toml",
|
||||
"go.mod", "cargo.toml", "gemfile", "build.gradle"}
|
||||
SUBPROJECT_RANK = 95 # after the front door, before the bootstrap account
|
||||
|
||||
# How many files one commit may hold before it stops being one idea. Soft: a
|
||||
# subproject and a hub with its satellites are exempt, because splitting those
|
||||
# produces a commit that does not build.
|
||||
DEFAULT_MAX_FILES = 8
|
||||
|
||||
# Roles whose files earn their way into a commit by referring to each other,
|
||||
# rather than by sitting in the same directory.
|
||||
CODE_ROLES = {"source", "test", "frontdoor"}
|
||||
|
||||
|
||||
def _subprojects(paths):
|
||||
"""Directories that are their own project -> the files beneath them."""
|
||||
marked = set()
|
||||
for p in paths:
|
||||
parent = str(Path(p).parent)
|
||||
if parent not in (".", "") and Path(p).name.lower() in SUBPROJECT_MARKERS:
|
||||
marked.add(parent)
|
||||
# A subproject inside a subproject belongs to the outer one; one commit,
|
||||
# not two nested ones.
|
||||
roots = {d for d in marked
|
||||
if not any(d != o and d.startswith(o + "/") for o in marked)}
|
||||
owned = {}
|
||||
for p in paths:
|
||||
for root in roots:
|
||||
if p == root or p.startswith(root + "/"):
|
||||
owned[p] = root
|
||||
break
|
||||
return owned
|
||||
|
||||
|
||||
# Roles that are one idea when they sit side by side. A diagram and the source
|
||||
# it renders from belong in the same commit; so do a pin and the config that
|
||||
# reads it. Grouping only — the ordering still keeps them apart.
|
||||
GROUP_TIER = {"asset": "doc", "lock": "pin", "pin": "pin", "config": "pin"}
|
||||
|
||||
|
||||
def _cluster(path, owner, dirs=()):
|
||||
"""
|
||||
The directory a file is grouped under: its subproject, else its own parent.
|
||||
|
||||
The parent, not the top-level directory. Under `ctrl` everything in a tree
|
||||
this shape lands in one bucket — eleven scripts, four profiles and a k8s
|
||||
tree — and the split has to be reconstructed afterwards from references
|
||||
that were never going to describe it.
|
||||
"""
|
||||
if owner:
|
||||
return owner
|
||||
# `ctrl/addons.sh` belongs with `ctrl/addons/`, not with its own siblings.
|
||||
# Clustering is what decides which files are even considered together, so a
|
||||
# hub parted from its satellites here can never be rejoined later.
|
||||
hub = _hub_of(path)
|
||||
if hub and hub in dirs:
|
||||
return hub
|
||||
return str(Path(path).parent)
|
||||
|
||||
|
||||
def adaptive_cap(count, requested=None):
|
||||
"""
|
||||
How many files one commit may hold, for a repo this size.
|
||||
|
||||
A fixed cap does not survive the range. rig is 64 files and wants commits
|
||||
of three or four; spr is 507 and at a flat eight plans a hundred and
|
||||
forty-three, which is the same failure as one commit from the other end.
|
||||
|
||||
A twentieth of the tree, floored at the default, lands close to what these
|
||||
repos were actually built as: rig plans 18 against a real 18, spr 77
|
||||
against a real 78. It is a starting point, not a claim — --max-files
|
||||
overrides it and plan.json is editable either way.
|
||||
"""
|
||||
if requested:
|
||||
return requested
|
||||
return max(DEFAULT_MAX_FILES, -(-count // 20))
|
||||
|
||||
|
||||
def order_files(index, max_files=None):
|
||||
"""Return an ordered list of groups, each a list of repo-relative paths."""
|
||||
files = index["files"]
|
||||
paths = sorted(files)
|
||||
max_files = adaptive_cap(len(paths), max_files)
|
||||
owner = _subprojects(paths)
|
||||
|
||||
def rank(p):
|
||||
return SUBPROJECT_RANK if p in owner else ROLE_RANK.get(files[p]["role"], 60)
|
||||
|
||||
# ── edges ──────────────────────────────────────────────────────────────
|
||||
# An edge b -> a means "b must come after a". Only real dependencies count,
|
||||
# and only between files whose roles do not already disagree: a reference
|
||||
# pointing backwards up the role order is a mention the extractor could not
|
||||
# tell from a dependency, and honouring it inverts the tier.
|
||||
after = defaultdict(set)
|
||||
for p in paths:
|
||||
if files[p]["role"] in NARRATIVE or p in owner:
|
||||
continue
|
||||
for dep in files[p]["refs"]:
|
||||
if dep in files and dep != p and rank(dep) <= rank(p):
|
||||
after[p].add(dep)
|
||||
|
||||
# ── ordering ───────────────────────────────────────────────────────────
|
||||
# Kahn's algorithm, taking the lowest (rank, path) that is ready. Ties are
|
||||
# broken by path so two runs on the same tree produce the same history.
|
||||
blockers = {p: set(after[p]) for p in paths}
|
||||
dependents = defaultdict(set)
|
||||
for p, deps in blockers.items():
|
||||
for d in deps:
|
||||
dependents[d].add(p)
|
||||
|
||||
ready = sorted((p for p in paths if not blockers[p]), key=lambda p: (rank(p), p))
|
||||
ordered = []
|
||||
while ready:
|
||||
p = ready.pop(0)
|
||||
ordered.append(p)
|
||||
for d in sorted(dependents[p]):
|
||||
blockers[d].discard(p)
|
||||
if not blockers[d]:
|
||||
ready.append(d)
|
||||
ready.sort(key=lambda q: (rank(q), q))
|
||||
|
||||
# A cycle leaves files unplaced. Two shell scripts that source each other is
|
||||
# a real thing and not an error here, so append them in role order rather
|
||||
# than refusing to produce a plan at all.
|
||||
if len(ordered) < len(paths):
|
||||
ordered += sorted(set(paths) - set(ordered), key=lambda p: (rank(p), p))
|
||||
|
||||
return _coalesce(_group(ordered, files, owner, after, max_files), owner, max_files)
|
||||
|
||||
|
||||
def _common_dir(paths):
|
||||
"""The deepest directory every path in the group sits under."""
|
||||
parts = list(Path(paths[0]).parent.parts)
|
||||
for p in paths[1:]:
|
||||
other = Path(p).parent.parts
|
||||
keep = []
|
||||
for a, b in zip(parts, other):
|
||||
if a != b:
|
||||
break
|
||||
keep.append(a)
|
||||
parts = keep
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def _coalesce(groups, owner, max_files):
|
||||
"""
|
||||
Merge neighbouring groups that are really one idea in one place.
|
||||
|
||||
Cutting on directory is right for a shallow tree and wrong for a deep one:
|
||||
a directory holding a single file is not an idea, and a repo of five
|
||||
hundred files has a lot of them. Left alone spr planned three hundred and
|
||||
thirty-seven commits, which is the same failure as one commit, from the
|
||||
other end.
|
||||
|
||||
Only neighbours already adjacent in the order merge, only when they share
|
||||
a top-level directory, only under the cap, and only when BOTH are small.
|
||||
|
||||
Both, not either. Letting a lone file join whatever it happened to sit next
|
||||
to put `dockerhost.sh` inside the addons commit — a coherent group of seven
|
||||
with an eighth file that has nothing to do with it. A run of scattered
|
||||
singletons is fragmentation and should close up; a group that already says
|
||||
something should not absorb a stray because it had room.
|
||||
|
||||
What counts as small scales with the cap, so raising --max-files actually
|
||||
buys fewer commits. Gated at a flat two it did not: on a 500-file repo the
|
||||
cap could be raised from 8 to 40 and the count moved by four, because
|
||||
nothing was ever allowed to merge. Asking for bigger commits should widen
|
||||
what is considered fragmentation, not just what is allowed to survive.
|
||||
"""
|
||||
loose = max(2, max_files // 4)
|
||||
out = []
|
||||
for group in groups:
|
||||
if not out:
|
||||
out.append(group)
|
||||
continue
|
||||
previous = out[-1]
|
||||
if (len(previous) + len(group) <= max_files
|
||||
and max(len(previous), len(group)) <= loose
|
||||
and not any(p in owner for p in previous + group)
|
||||
and _shares_ancestor(_common_dir(previous), _common_dir(group))):
|
||||
out[-1] = previous + group
|
||||
else:
|
||||
out.append(group)
|
||||
return out
|
||||
|
||||
|
||||
def _shares_ancestor(a, b):
|
||||
"""Both at the root, or under a common top-level directory."""
|
||||
if not a and not b:
|
||||
return True
|
||||
return bool(a) and bool(b) and a[0] == b[0]
|
||||
|
||||
|
||||
def _group(ordered, files, owner, after, max_files):
|
||||
"""
|
||||
Cut the ordered list into commits.
|
||||
|
||||
One coherent idea per commit, not one directory per commit. What holds a
|
||||
group together is that its files refer to each other — ports.sh and the
|
||||
hosts template it renders, a kustomization and the manifests it lists. What
|
||||
separates two groups in the same directory is that neither names the other.
|
||||
|
||||
A hub and the directory named after it (addons.sh and addons/) travel
|
||||
together whatever their references say: committing the loader without the
|
||||
things it loads produces a commit that cannot run.
|
||||
"""
|
||||
dirs = {str(Path(p).parent) for p in ordered}
|
||||
groups, buf = [], []
|
||||
seen_cluster = seen_role = None
|
||||
|
||||
def flush():
|
||||
nonlocal buf
|
||||
if buf:
|
||||
groups.append(buf)
|
||||
buf = []
|
||||
|
||||
for path in ordered:
|
||||
own = owner.get(path)
|
||||
cluster = _cluster(path, own, dirs)
|
||||
role = "subproject" if own else GROUP_TIER.get(
|
||||
files[path]["role"], files[path]["role"])
|
||||
if cluster != seen_cluster or role != seen_role:
|
||||
flush()
|
||||
seen_cluster, seen_role = cluster, role
|
||||
buf.append(path)
|
||||
flush()
|
||||
|
||||
out = []
|
||||
for group in groups:
|
||||
# A subproject is one commit no matter how many files it holds.
|
||||
if any(p in owner for p in group):
|
||||
out.append(group)
|
||||
continue
|
||||
out.extend(_split(group, files, after, max_files))
|
||||
return out
|
||||
|
||||
|
||||
def _hub_of(path):
|
||||
"""`ctrl/addons.sh` is the hub of `ctrl/addons/`; returns that directory."""
|
||||
p = Path(path)
|
||||
return str(p.parent / p.stem) if p.suffix else None
|
||||
|
||||
|
||||
def _pure_hub(members, hubs):
|
||||
"""True when the group is exactly one hub and things inside its directory."""
|
||||
for hub, owner_file in hubs.items():
|
||||
if owner_file in members and all(
|
||||
m == owner_file or m.startswith(hub + "/") for m in members
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _split(group, files, after, max_files):
|
||||
"""
|
||||
Break one cluster into commits.
|
||||
|
||||
A run of siblings of the same kind in the same directory is left alone —
|
||||
four profile files under env.d/ are one idea, and nothing in them refers to
|
||||
anything, so splitting on references turns them into four commits saying
|
||||
the same thing four times. References are only asked about a group that is
|
||||
already too big or already spans directories.
|
||||
|
||||
Past that: connected components, because what holds a commit together is
|
||||
that its files name each other. A hub and the directory named after it
|
||||
(addons.sh and addons/) survive the cap, since committing the loader
|
||||
without the things it loads produces a commit that cannot run. Anything
|
||||
else over the cap gets its hub peeled off into its own commit and the
|
||||
remainder reconsidered — a cut at a named seam rather than at a count.
|
||||
"""
|
||||
if len(group) <= 1:
|
||||
return [group]
|
||||
|
||||
# ...but only for roles where sitting side by side IS the relationship.
|
||||
# Four env.d profiles are one idea. Seven scripts that happen to share a
|
||||
# directory are seven ideas, and `ctrl/` is full of them, so code always
|
||||
# gets asked about its references.
|
||||
parents = {str(Path(p).parent) for p in group}
|
||||
kinds = {files[p]["role"] for p in group}
|
||||
if len(parents) == 1 and not (kinds & CODE_ROLES):
|
||||
if len(group) <= max_files:
|
||||
return [group]
|
||||
# Over the cap and still one kind in one directory: cut it into runs.
|
||||
# Nothing here refers to anything, so asking references to find the
|
||||
# seam yields one commit per file — thirteen commits each saying "a
|
||||
# project note", which is worse than an admitted arbitrary cut.
|
||||
return [group[i:i + max_files] for i in range(0, len(group), max_files)]
|
||||
|
||||
index = {p: i for i, p in enumerate(group)}
|
||||
parent = list(range(len(group)))
|
||||
|
||||
def find(i):
|
||||
while parent[i] != i:
|
||||
parent[i] = parent[parent[i]]
|
||||
i = parent[i]
|
||||
return i
|
||||
|
||||
def union(a, b):
|
||||
ra, rb = find(a), find(b)
|
||||
if ra != rb:
|
||||
parent[max(ra, rb)] = min(ra, rb)
|
||||
|
||||
hubs = {}
|
||||
for p in group:
|
||||
hub = _hub_of(p)
|
||||
if hub:
|
||||
hubs[hub] = p
|
||||
for p in group:
|
||||
for hub, owner_file in hubs.items():
|
||||
if p.startswith(hub + "/"):
|
||||
union(index[p], index[owner_file])
|
||||
for dep in after.get(p, ()):
|
||||
if dep in index:
|
||||
union(index[p], index[dep])
|
||||
|
||||
components = defaultdict(list)
|
||||
for p in group:
|
||||
components[find(index[p])].append(p)
|
||||
|
||||
out = []
|
||||
for key in sorted(components):
|
||||
members = components[key]
|
||||
while len(members) > max_files and not _pure_hub(members, hubs):
|
||||
degree = {m: sum(1 for o in members if m in after.get(o, ())) for m in members}
|
||||
hub = max(degree, key=lambda m: (degree[m], m))
|
||||
if degree[hub] == 0:
|
||||
# Nothing holds this together and nothing names anything: an
|
||||
# arbitrary cut is the honest answer, so cut on the order we
|
||||
# already have rather than inventing a reason.
|
||||
out.extend([members[i:i + max_files]
|
||||
for i in range(0, len(members), max_files)])
|
||||
members = []
|
||||
break
|
||||
members.remove(hub)
|
||||
out.append([hub])
|
||||
if members:
|
||||
out.append(members)
|
||||
return [g for g in out if g]
|
||||
|
||||
|
||||
# ── the plan ───────────────────────────────────────────────────────────────
|
||||
|
||||
def plan_path(out) -> Path:
|
||||
return state_dir(out) / PLAN_FILE
|
||||
|
||||
|
||||
def build_plan(index, out, max_files=None, quiet=False):
|
||||
"""
|
||||
Write plan.json: ordered groups of paths with empty message slots.
|
||||
|
||||
This is the seam. Everything above is analysis and can be recomputed from
|
||||
the tree; everything below is git commands. An existing plan's messages are
|
||||
carried over when the group's paths still match exactly, so re-planning
|
||||
after editing three files does not throw away sixty written messages.
|
||||
"""
|
||||
groups = order_files(index, max_files)
|
||||
|
||||
written = {}
|
||||
existing = plan_path(out)
|
||||
if existing.exists():
|
||||
try:
|
||||
for g in json.loads(existing.read_text()).get("groups", []):
|
||||
if g.get("title"):
|
||||
written[tuple(sorted(g["paths"]))] = (g.get("title"), g.get("body", ""))
|
||||
except (OSError, json.JSONDecodeError, KeyError, TypeError):
|
||||
pass
|
||||
|
||||
document = {"version": 1, "groups": []}
|
||||
for i, paths in enumerate(groups, 1):
|
||||
title, body = written.get(tuple(sorted(paths)), ("", ""))
|
||||
document["groups"].append({
|
||||
"n": i,
|
||||
"slug": _slug(paths, index),
|
||||
"paths": paths,
|
||||
"title": title,
|
||||
"body": body,
|
||||
})
|
||||
|
||||
destination = plan_path(out)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(document, indent=2))
|
||||
if not quiet:
|
||||
kept = sum(1 for g in document["groups"] if g["title"])
|
||||
print(f"Planned {len(groups)} commits over "
|
||||
f"{sum(len(g) for g in groups)} files"
|
||||
+ (f", {kept} messages carried over" if kept else "") + ".")
|
||||
print(f" -> {destination}")
|
||||
return document
|
||||
|
||||
|
||||
def _slug(paths, index):
|
||||
"""A stable handle for a group, for filenames and for talking about it."""
|
||||
common = Path(paths[0]).parent
|
||||
for p in paths[1:]:
|
||||
parts = []
|
||||
for a, b in zip(common.parts, Path(p).parent.parts):
|
||||
if a != b:
|
||||
break
|
||||
parts.append(a)
|
||||
common = Path(*parts) if parts else Path(".")
|
||||
base = str(common).strip("./").replace("/", "-")
|
||||
if not base:
|
||||
base = Path(paths[0]).stem if len(paths) == 1 else index["files"][paths[0]]["role"]
|
||||
return base.lower().replace("_", "-").replace(".", "")[:40] or "root"
|
||||
413
soleprint/station/tools/histgen/selftest.py
Normal file
413
soleprint/station/tools/histgen/selftest.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""
|
||||
Prove the whole pipeline on a tree this builds itself.
|
||||
|
||||
`make check` after copying the folder somewhere new, with no repo to point at
|
||||
and nothing installed. It builds a small tree with the shapes that matter —
|
||||
ignore rules, a README, a pin, a config that sources it, a directory of
|
||||
profiles, a hub with satellites, a front door, an ignored directory — runs all
|
||||
five verbs over it, and asserts what has to be true.
|
||||
|
||||
The guards get the same treatment as the happy path. A check that only ever
|
||||
proves things work would have missed the one real bug found while writing this:
|
||||
the tree-hash guard returned None on exactly the trees it was written for, and
|
||||
reported success anyway.
|
||||
|
||||
python3 selftest.py # or: make check
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
TREE = {
|
||||
".gitignore": "# Generated output; regenerate rather than commit.\nbuild/\n*.log\n",
|
||||
".gitattributes": "# LF everywhere: a CRLF checkout breaks the shebang.\n* text=auto eol=lf\n",
|
||||
"README.md": "# demo\n\nWhat this is, and the one prerequisite.\n",
|
||||
"ctrl/versions.env": "# Pinned toolchain — one manifest, one answer.\nKUBECTL=1.31.0\n",
|
||||
"ctrl/lib/config.sh": "# Shared config loading. Sourced, never executed.\n"
|
||||
'. "$(dirname "$0")/../versions.env"\n',
|
||||
"ctrl/env.d/minimal.env": "# The smallest profile that still starts.\nADDONS=\n",
|
||||
"ctrl/env.d/full.env": "# Everything on, for a demo machine.\nADDONS=redis\n",
|
||||
"ctrl/addons.sh": "# Install the addons the profile asked for.\n"
|
||||
"# Adding one is adding a file, not editing a dispatcher.\n"
|
||||
'for a in ctrl/addons/*.sh; do sh "$a"; done\n',
|
||||
"ctrl/addons/redis.sh": "# Redis — the broker half, nothing else uses it.\necho redis\n",
|
||||
"ctrl/addons/postgres.sh": "# Postgres — the metadata store.\necho postgres\n",
|
||||
"Makefile": "# Thin front door: one target per ctrl/ script.\nup:\n\tsh ctrl/addons.sh\n",
|
||||
"BOOTSTRAP.md": "# From a bare machine to something running.\n",
|
||||
"build/generated.txt": "this is ignored and must never be committed",
|
||||
"noisy.log": "also ignored",
|
||||
}
|
||||
|
||||
IGNORED = {"build/generated.txt", "noisy.log"}
|
||||
|
||||
# Things a copy should leave behind, and the two lookalikes it must not. Added
|
||||
# to the tree only for the `copy` checks, and force-added so the ignored-but-
|
||||
# tracked case is real rather than described.
|
||||
SIFTABLE = {
|
||||
"package-lock.json": "lockfile contents",
|
||||
"dist/app.min.js": "minified",
|
||||
".env": "API_KEY=real-secret-value",
|
||||
".env.example": "API_KEY=",
|
||||
"certs/server.key": "-----BEGIN PRIVATE KEY-----",
|
||||
"certs/server.key.pub": "ssh-rsa AAAA",
|
||||
"assets/logo.png": "PNG",
|
||||
"build/forced.bin": "ignored yet tracked",
|
||||
}
|
||||
DROPPED = {"package-lock.json", "dist/app.min.js", ".env",
|
||||
"certs/server.key", "build/forced.bin"}
|
||||
KEPT_LOOKALIKES = {".env.example", "certs/server.key.pub", "assets/logo.png"}
|
||||
|
||||
|
||||
def run(pkg, parent, *args):
|
||||
return subprocess.run([sys.executable, "-m", pkg, *args],
|
||||
capture_output=True, text=True,
|
||||
env={"PYTHONPATH": str(parent), "PATH": "/usr/bin:/bin",
|
||||
"HOME": str(Path.home())})
|
||||
|
||||
|
||||
def build(root: Path):
|
||||
for rel, body in TREE.items():
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(body)
|
||||
|
||||
|
||||
def main():
|
||||
here = Path(__file__).resolve().parent
|
||||
pkg, parent = here.name, here.parent
|
||||
failures = []
|
||||
|
||||
def check(label, condition, detail=""):
|
||||
print(f" {'ok ' if condition else 'FAIL'} {label}")
|
||||
if not condition:
|
||||
failures.append(f"{label}{': ' + detail if detail else ''}")
|
||||
|
||||
def hg(*args):
|
||||
return run(pkg, parent, *args)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-selftest-") as tmp:
|
||||
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||
build(source)
|
||||
W = ["--source", str(source), "--out", str(out)]
|
||||
copy = out / "demo"
|
||||
|
||||
r = hg("run", *W)
|
||||
check("run: scan + plan + brief", r.returncode == 0, r.stderr.strip())
|
||||
if r.returncode != 0:
|
||||
print(r.stderr)
|
||||
return 1
|
||||
|
||||
index = json.loads((out / "index.json").read_text())
|
||||
plan = json.loads((out / "plan.json").read_text())
|
||||
planned = [p for g in plan["groups"] for p in g["paths"]]
|
||||
|
||||
check("the ignore rules were honoured", not (set(planned) & IGNORED),
|
||||
f"ignored files got planned: {sorted(set(planned) & IGNORED)}")
|
||||
check("every other file is in exactly one group",
|
||||
sorted(planned) == sorted(set(TREE) - IGNORED) and len(planned) == len(set(planned)))
|
||||
check("the ignore rules are committed first",
|
||||
plan["groups"][0]["paths"][0].startswith(".git"))
|
||||
check("the front door is not", "Makefile" not in plan["groups"][0]["paths"])
|
||||
check("the hub travels with its satellites",
|
||||
any({"ctrl/addons.sh", "ctrl/addons/redis.sh", "ctrl/addons/postgres.sh"}
|
||||
<= set(g["paths"]) for g in plan["groups"]))
|
||||
check("a comment is not mistaken for a dependency",
|
||||
index["files"][".gitignore"]["refs"] == [])
|
||||
check("a real dependency is found",
|
||||
"ctrl/versions.env" in index["files"]["ctrl/lib/config.sh"]["refs"])
|
||||
check("the reasoning was extracted for the message",
|
||||
"not editing a dispatcher" in index["files"]["ctrl/addons.sh"]["why"])
|
||||
check("a brief exists per commit",
|
||||
len(list((out / "briefs").glob("*.md"))) == len(plan["groups"]))
|
||||
check("list prints one line per commit",
|
||||
all(f"{g['n']:3}." in hg("list", *W).stdout for g in plan["groups"]))
|
||||
|
||||
# The guards refuse before they approve.
|
||||
check("export refuses a plan with no messages",
|
||||
"no title" in hg("export", *W).stderr)
|
||||
(source / "appeared-late.sh").write_text("# added after planning\n")
|
||||
r = hg("export", *W, "--allow-untitled")
|
||||
check("export refuses a file no group covers",
|
||||
r.returncode != 0 and "no group" in r.stderr)
|
||||
(source / "appeared-late.sh").unlink()
|
||||
|
||||
# --- commands: copy the files, make no repo, hand back the list ---
|
||||
# Its own out directory, so it needs its own plan in it: the plan is a
|
||||
# fact about an out directory, not about the source.
|
||||
hands = Path(str(out) + "-byhand")
|
||||
H = ["--source", str(source), "--out", str(hands)]
|
||||
hg("run", *H)
|
||||
r = hg("export", *H, "--allow-untitled", "--commands")
|
||||
check("commands: succeeds", r.returncode == 0, r.stderr.strip())
|
||||
made = hands / "demo"
|
||||
check("commands: the files were copied", (made / "README.md").is_file())
|
||||
check("commands: NO repo was created", not (made / ".git").exists())
|
||||
check("commands: gitignored files did not come across",
|
||||
not any((made / i).exists() for i in IGNORED))
|
||||
check("commands: git init is the first thing offered, not done for you",
|
||||
"git init" in r.stdout)
|
||||
check("commands: one add and one commit per group",
|
||||
r.stdout.count("git add -- ") == len(plan["groups"])
|
||||
and r.stdout.count("git commit -F ") == len(plan["groups"]))
|
||||
check("commands: the list is saved too", (hands / "commands.sh").is_file())
|
||||
|
||||
# The list has to actually work, which is the only claim that matters.
|
||||
subprocess.run(["bash", str(hands / "commands.sh")], capture_output=True,
|
||||
env={**os.environ, "GIT_AUTHOR_NAME": "t",
|
||||
"GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@t"})
|
||||
check("commands: running the list builds the history",
|
||||
subprocess.run(["git", "-C", str(made), "rev-list", "--count", "HEAD"],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
== str(len(plan["groups"])))
|
||||
check("commands: it leaves nothing untracked",
|
||||
not subprocess.run(["git", "-C", str(made), "status", "--porcelain"],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
expected = [l for l in r.stdout.split("\n") if "expect " in l][0].split("expect ")[1].strip()
|
||||
check("commands: the tree matches the hash it told you to expect",
|
||||
subprocess.run(["git", "-C", str(made), "rev-parse", "HEAD^{tree}"],
|
||||
capture_output=True, text=True).stdout.strip() == expected)
|
||||
check("commands: refuses when a repo is already there",
|
||||
hg("export", *H, "--allow-untitled", "--commands").returncode != 0)
|
||||
|
||||
# --- absent -> exported ---
|
||||
check("status says absent before anything is exported",
|
||||
"absent" in hg("status", *W).stdout)
|
||||
r = hg("export", *W, "--allow-untitled")
|
||||
check("export writes the history", r.returncode == 0, r.stderr.strip())
|
||||
check("both guards ran",
|
||||
"nothing left untracked: ok" in r.stdout and "tree matches source" in r.stdout)
|
||||
check("the copy is named after the source", copy.is_dir())
|
||||
check("one commit per group",
|
||||
subprocess.run(["git", "-C", str(copy), "rev-list", "--count", "HEAD"],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
== str(len(plan["groups"])))
|
||||
check("the ignored files never entered the history",
|
||||
not (set(subprocess.run(["git", "-C", str(copy), "ls-files"],
|
||||
capture_output=True, text=True).stdout.split()) & IGNORED))
|
||||
|
||||
# The invariant the whole design rests on.
|
||||
check("THE SOURCE WAS NEVER WRITTEN TO",
|
||||
not (source / ".git").exists() and not (source / ".histgen").exists()
|
||||
and sorted(p.relative_to(source).as_posix()
|
||||
for p in source.rglob("*") if p.is_file()) == sorted(TREE))
|
||||
|
||||
check("a finished export says so and stops",
|
||||
"Nothing to do" in hg("export", *W, "--allow-untitled").stdout)
|
||||
check("verify passes on its own", hg("verify", *W).returncode == 0)
|
||||
|
||||
# --- unfinished: interrupt, then resume ---
|
||||
progress = json.loads((out / "progress.json").read_text())
|
||||
# Half of them, whatever the fixture happens to plan. Hardcoding four
|
||||
# made this pass vacuously the moment the fixture planned exactly four:
|
||||
# nothing was left to resume, so "complete" was the honest answer and
|
||||
# the resume path was never entered.
|
||||
keep = progress["commits"][:max(1, len(progress["commits"]) // 2)]
|
||||
subprocess.run(["git", "-C", str(copy), "reset", "-q", "--hard", keep[-1]["sha"]])
|
||||
progress["commits"] = keep
|
||||
(out / "progress.json").write_text(json.dumps(progress))
|
||||
|
||||
check("status spots a half-finished history",
|
||||
"unfinished" in hg("status", *W).stdout)
|
||||
r = hg("export", *W, "--allow-untitled")
|
||||
check("export resumes rather than restarting",
|
||||
r.returncode == 0 and "Resuming" in r.stdout, r.stdout + r.stderr)
|
||||
check("resuming did not redo the commits already made",
|
||||
f"{len(keep)} of {len(plan['groups'])}" in r.stdout, r.stdout)
|
||||
check("the resumed history is complete and verified",
|
||||
"tree matches source" in r.stdout)
|
||||
|
||||
# --- stale: the plan moved ---
|
||||
plan["groups"][1]["paths"].append(plan["groups"][2]["paths"].pop())
|
||||
(out / "plan.json").write_text(json.dumps(plan))
|
||||
check("status spots a plan that no longer matches",
|
||||
"stale" in hg("status", *W).stdout)
|
||||
check("export refuses a stale export",
|
||||
hg("export", *W, "--allow-untitled").returncode != 0)
|
||||
check("--force starts over",
|
||||
hg("export", *W, "--allow-untitled", "--force").returncode == 0)
|
||||
|
||||
# --- foreign: a history that has to be kept ---
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-selftest-keep-") as tmp:
|
||||
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||
build(source)
|
||||
for cmd in (["init", "-q"], ["add", "-A"], ["commit", "-qm", "init commit"]):
|
||||
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True,
|
||||
env={**os.environ, "GIT_AUTHOR_NAME": "t",
|
||||
"GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
|
||||
"GIT_COMMITTER_EMAIL": "t@t"})
|
||||
before = subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
W = ["--source", str(source), "--out", str(out)]
|
||||
copy = out / "demo"
|
||||
|
||||
hg("run", *W)
|
||||
r = hg("export", *W, "--allow-untitled", "--keep-history")
|
||||
check("keep: export succeeds", r.returncode == 0, r.stderr.strip())
|
||||
check("keep: the old history is reported as kept", "kept, untouched" in r.stdout)
|
||||
|
||||
branches = subprocess.run(["git", "-C", str(copy), "branch", "--format=%(refname:short)"],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
check("keep: the designed history is on its own branch",
|
||||
"designed-history" in branches and len(branches) >= 2, str(branches))
|
||||
main = [b for b in branches if b != "designed-history"][0]
|
||||
check("keep: the old branch still points where it did",
|
||||
subprocess.run(["git", "-C", str(copy), "rev-parse", main],
|
||||
capture_output=True, text=True).stdout.strip() == before)
|
||||
check("keep: the two histories share no commit",
|
||||
subprocess.run(["git", "-C", str(copy), "merge-base", main, "designed-history"],
|
||||
capture_output=True, text=True).returncode != 0)
|
||||
check("keep: the source's own history is untouched",
|
||||
subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True).stdout.strip() == before)
|
||||
|
||||
# --- the filter has to hold on the HISTORY path, not just on copy ---
|
||||
# This is the case that was wrong: `copy` left a tracked key behind while
|
||||
# `scan` walked straight past it, so the key stayed out of the snapshot and
|
||||
# went into the commits.
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-selftest-secret-") as tmp:
|
||||
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||
build(source)
|
||||
for rel, body in SIFTABLE.items():
|
||||
(source / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
(source / rel).write_text(body)
|
||||
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||
for cmd in (["init", "-q"], ["add", "-A", "-f", "."], ["commit", "-qm", "init"]):
|
||||
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env)
|
||||
|
||||
W = ["--source", str(source), "--out", str(out)]
|
||||
r = hg("run", *W)
|
||||
check("filter: scan says what it left out", "left out of the history" in r.stdout)
|
||||
|
||||
index = json.loads((out / "index.json").read_text())
|
||||
plan = json.loads((out / "plan.json").read_text())
|
||||
planned = {p for g in plan["groups"] for p in g["paths"]}
|
||||
secrets = {".env", "certs/server.key"}
|
||||
check("filter: no secret reached the census", not (set(index["files"]) & secrets))
|
||||
check("filter: no secret reached the plan", not (planned & secrets))
|
||||
check("filter: the ignored-but-tracked file did not either",
|
||||
"build/forced.bin" not in planned)
|
||||
check("filter: the lookalikes are still in the plan",
|
||||
{".env.example", "certs/server.key.pub"} <= planned)
|
||||
check("filter: a lockfile IS kept — a repo wants its lockfile",
|
||||
"package-lock.json" in planned)
|
||||
|
||||
r = hg("export", *W, "--allow-untitled", "--commands")
|
||||
# Exact tokens, not substrings: ".env" is inside ".env.example", so a
|
||||
# substring test reports a leak every time the lookalike is kept —
|
||||
# which is exactly the behaviour that is wanted.
|
||||
added = {tok for line in r.stdout.split("\n") if line.startswith("git add -- ")
|
||||
for tok in line[len("git add -- "):].split()}
|
||||
check("filter: no secret reached the commands",
|
||||
not (added & secrets), str(sorted(added & secrets)))
|
||||
check("filter: the commands cover exactly the planned files",
|
||||
added == planned, str(sorted(added ^ planned)))
|
||||
check("filter: no secret reached the copy",
|
||||
not any((out / "demo" / x).exists() for x in secrets))
|
||||
|
||||
out2 = Path(tmp) / "out2"
|
||||
hg("run", "--source", str(source), "--out", str(out2))
|
||||
r = hg("export", "--source", str(source), "--out", str(out2), "--allow-untitled")
|
||||
check("filter: the guards still pass on the filtered set",
|
||||
"tree matches source" in r.stdout, r.stdout + r.stderr)
|
||||
tracked = subprocess.run(["git", "-C", str(out2 / "demo"), "ls-files"],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
check("filter: the committed history holds no secret",
|
||||
not (set(tracked) & secrets), str(tracked))
|
||||
|
||||
out3 = Path(tmp) / "out3"
|
||||
hg("run", "--source", str(source), "--out", str(out3), "--keep-secrets")
|
||||
index3 = json.loads((out3 / "index.json").read_text())
|
||||
check("filter: --keep-secrets brings them back",
|
||||
secrets <= set(index3["files"]))
|
||||
|
||||
# --- copy: the plain utility, no history involved ---
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-selftest-copy-") as tmp:
|
||||
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||
build(source)
|
||||
for rel, body in SIFTABLE.items():
|
||||
(source / rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
(source / rel).write_text(body)
|
||||
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
||||
for cmd in (["init", "-q"], ["add", "-A", "-f", "."],
|
||||
["commit", "-qm", "init"]):
|
||||
subprocess.run(["git", "-C", str(source), *cmd], capture_output=True, env=env)
|
||||
|
||||
C = ["--source", str(source), "--out", str(out)]
|
||||
r = hg("copy", *C)
|
||||
check("copy: succeeds", r.returncode == 0, r.stderr.strip())
|
||||
made = out / "demo"
|
||||
check("copy: no .git came across", not (made / ".git").exists())
|
||||
check("copy: nothing gitignored came across",
|
||||
not any((made / i).exists() for i in IGNORED))
|
||||
for rel in sorted(DROPPED):
|
||||
check(f"copy: left behind {rel}", not (made / rel).exists())
|
||||
for rel in sorted(KEPT_LOOKALIKES):
|
||||
check(f"copy: kept {rel}", (made / rel).is_file())
|
||||
check("copy: every drop is named in the output",
|
||||
all(rel in r.stdout for rel in DROPPED), r.stdout)
|
||||
check("copy: a manifest records what was left behind",
|
||||
(out / "copied.md").is_file()
|
||||
and all(rel in (out / "copied.md").read_text() for rel in DROPPED))
|
||||
check("copy: refuses a destination that is not empty",
|
||||
hg("copy", *C).returncode != 0)
|
||||
|
||||
preview = Path(tmp) / "preview"
|
||||
r = hg("copy", "--source", str(source), "--out", str(preview), "--dry-run")
|
||||
check("copy: --dry-run writes nothing", not preview.exists() and r.returncode == 0)
|
||||
|
||||
full = Path(tmp) / "full"
|
||||
r = hg("copy", "--source", str(source), "--out", str(full),
|
||||
"--all", "--keep-secrets")
|
||||
check("copy: --all --keep-secrets keeps what it says",
|
||||
(full / "demo" / "package-lock.json").is_file()
|
||||
and (full / "demo" / ".env").is_file())
|
||||
|
||||
picky = Path(tmp) / "picky"
|
||||
r = hg("copy", "--source", str(source), "--out", str(picky),
|
||||
"--exclude", "*.png", "--include", "package-lock.json")
|
||||
check("copy: --exclude drops by glob at any depth",
|
||||
not (picky / "demo" / "assets" / "logo.png").exists())
|
||||
check("copy: --include overrides the filters",
|
||||
(picky / "demo" / "package-lock.json").is_file())
|
||||
|
||||
# --- config ---
|
||||
with tempfile.TemporaryDirectory(prefix="histgen-selftest-cfg-") as tmp:
|
||||
source, out = Path(tmp) / "demo", Path(tmp) / "out"
|
||||
build(source)
|
||||
cfg = Path(tmp) / "settings.json"
|
||||
cfg.write_text(json.dumps({"source": str(source), "out": str(out)}))
|
||||
r = hg("config", "--config", str(cfg))
|
||||
check("config: a file supplies source and out",
|
||||
r.returncode == 0 and str(out) in r.stdout, r.stderr.strip())
|
||||
check("config: the command line wins",
|
||||
"/tmp/override" in hg("config", "--config", str(cfg),
|
||||
"--out", "/tmp/override").stdout)
|
||||
cfg.write_text(json.dumps({"source": str(source), "outp": "typo"}))
|
||||
check("config: a mistyped key is refused",
|
||||
"unknown key" in hg("config", "--config", str(cfg)).stderr)
|
||||
cfg.write_text(json.dumps({"repo": str(source), "out": str(out)}))
|
||||
check("config: the old 'repo' key still works",
|
||||
hg("config", "--config", str(cfg)).returncode == 0)
|
||||
check("out inside source is refused",
|
||||
"inside source" in hg("status", "--source", str(source),
|
||||
"--out", str(source / "sub")).stderr)
|
||||
|
||||
print()
|
||||
if failures:
|
||||
print(f"{len(failures)} check(s) failed:")
|
||||
for f in failures:
|
||||
print(f" - {f}")
|
||||
return 1
|
||||
print("all checks passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
142
soleprint/station/tools/histgen/sift.py
Normal file
142
soleprint/station/tools/histgen/sift.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""
|
||||
What to leave behind when copying a repo out of itself.
|
||||
|
||||
Three kinds of file get dropped, and they are dropped for three different
|
||||
reasons. Keeping them apart matters, because only one of the three is safe to
|
||||
decide silently.
|
||||
|
||||
derived something a build regenerates. Lockfiles, source maps, minified
|
||||
output, compiled objects, cache directories. The list is ported
|
||||
from `ppl/ctrl/distill.sh`, including the lesson written into its
|
||||
comments: the line is **derived-vs-content, not text-vs-binary**.
|
||||
That distinction was wrong there once and cost real files — a
|
||||
logo, a font the site loads, a downloadable PDF — because none of
|
||||
those can be regenerated from what is left, which is the only
|
||||
thing that makes a file safe to drop. So images, fonts and
|
||||
spreadsheets are content and are kept.
|
||||
|
||||
secret a private key, a credential store, an .env holding real values.
|
||||
distill does not do this; .gitignore usually has, and where it has
|
||||
not, the file is tracked and travels. That is not hypothetical —
|
||||
soleprint's own notes record an API key that was tracked in a
|
||||
tool's .env, and untracking it did not unpublish it.
|
||||
|
||||
ignored tracked, and yet matched by the repo's own ignore rules. Someone
|
||||
ran `git add -f` once. Sometimes deliberate — a built artifact
|
||||
committed on purpose — and sometimes a dump or a credentials file
|
||||
that went in and was never noticed again. Reported by name either
|
||||
way, because the repo is already contradicting itself about them.
|
||||
|
||||
oversize whatever --max-bytes says. Size is its own worry and gets its own
|
||||
knob rather than being smuggled in as a guess about kind.
|
||||
|
||||
Every drop is reported. A file quietly missing from a copy is the same class of
|
||||
failure as a file quietly missing from a history, and this tool exists because
|
||||
that class of failure is expensive.
|
||||
"""
|
||||
|
||||
import fnmatch
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# ── derived ────────────────────────────────────────────────────────────────
|
||||
# One list, one place to edit. `--all` turns it off wholesale.
|
||||
NOISE = re.compile(
|
||||
r'(^|/)(package-lock\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|yarn\.lock'
|
||||
r'|bun\.lock|bun\.lockb|uv\.lock|poetry\.lock|Pipfile\.lock|Cargo\.lock'
|
||||
r'|composer\.lock|Gemfile\.lock|go\.sum|\.DS_Store|Thumbs\.db)$'
|
||||
r'|\.(map|min\.js|min\.css)$'
|
||||
r'|\.(pyc|pyo|pyd|class|o|obj|a|lib|so|dylib|dll|wasm|pack|idx)$'
|
||||
r'|(^|/)(__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|node_modules)/'
|
||||
)
|
||||
|
||||
# ── secret ─────────────────────────────────────────────────────────────────
|
||||
# Deliberately narrow. A pattern that catches a real key once a year and a
|
||||
# needed file once a week gets turned off, and then it catches nothing.
|
||||
SECRET = re.compile(
|
||||
r'(^|/)\.env(\.[A-Za-z0-9_-]+)?$'
|
||||
r'|(^|/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)$'
|
||||
r'|\.(pem|key|p12|pfx|jks|keystore|ppk|asc|gpg)$'
|
||||
r'|(^|/)(\.netrc|\.npmrc|\.pypirc|\.htpasswd|\.dockercfg)$'
|
||||
r'|(^|/)\.ssh/'
|
||||
r'|(^|/)credentials(\.json|\.yaml|\.yml)?$'
|
||||
r'|(^|/)service-account[-_A-Za-z0-9]*\.json$'
|
||||
)
|
||||
|
||||
# The exceptions matter more than the rule. A committed .env.example is the
|
||||
# documented way to say what the real one needs, and dropping it takes the
|
||||
# documentation with the secret.
|
||||
SECRET_OK = re.compile(
|
||||
r'\.(example|sample|template|dist|tmpl)$'
|
||||
r'|(^|/)\.env\.(example|sample|template)$'
|
||||
r'|\.pub$'
|
||||
)
|
||||
|
||||
|
||||
def is_derived(rel: str) -> bool:
|
||||
return bool(NOISE.search(rel))
|
||||
|
||||
|
||||
def is_secret(rel: str) -> bool:
|
||||
return bool(SECRET.search(rel)) and not SECRET_OK.search(rel)
|
||||
|
||||
|
||||
def matches(rel: str, patterns) -> bool:
|
||||
"""
|
||||
Glob match, with distill's rule: a pattern holding no `/` also matches
|
||||
basenames at any depth, so `--exclude '*.csv'` means what it looks like.
|
||||
"""
|
||||
name = Path(rel).name
|
||||
for pattern in patterns or ():
|
||||
if fnmatch.fnmatch(rel, pattern):
|
||||
return True
|
||||
if "/" not in pattern and fnmatch.fnmatch(name, pattern):
|
||||
return True
|
||||
if pattern.endswith("/") and (rel + "/").startswith(pattern.lstrip("/")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def sift(source: Path, paths, keep_noise=False, keep_secrets=False,
|
||||
max_bytes=None, exclude=(), include=(), ignored=()):
|
||||
"""
|
||||
Split the file list into what to copy and what to leave, with reasons.
|
||||
|
||||
`include` is checked first and wins outright: it is the way to say "yes, I
|
||||
do want that lockfile" without turning the whole filter off.
|
||||
"""
|
||||
kept, dropped = [], []
|
||||
for rel in paths:
|
||||
if include and matches(rel, include):
|
||||
kept.append(rel)
|
||||
continue
|
||||
if exclude and matches(rel, exclude):
|
||||
dropped.append((rel, "excluded"))
|
||||
continue
|
||||
if not keep_secrets and is_secret(rel):
|
||||
dropped.append((rel, "secret"))
|
||||
continue
|
||||
if rel in ignored:
|
||||
dropped.append((rel, "ignored"))
|
||||
continue
|
||||
if not keep_noise and is_derived(rel):
|
||||
dropped.append((rel, "derived"))
|
||||
continue
|
||||
if max_bytes:
|
||||
try:
|
||||
if (source / rel).stat().st_size > max_bytes:
|
||||
dropped.append((rel, "oversize"))
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
kept.append(rel)
|
||||
return kept, dropped
|
||||
|
||||
|
||||
REASONS = {
|
||||
"ignored": "tracked, but the ignore rules say they should not be",
|
||||
"derived": "a build regenerates these",
|
||||
"secret": "looks like a key or a credential",
|
||||
"oversize": "larger than --max-bytes",
|
||||
"excluded": "matched --exclude",
|
||||
}
|
||||
121
soleprint/station/tools/histgen/snapshot.py
Normal file
121
soleprint/station/tools/histgen/snapshot.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
Copy a repo's files out of it, without the repo.
|
||||
|
||||
Named snapshot.py and not copy.py, which is what it was for about ten minutes.
|
||||
A module called `copy` beside the code shadows the standard library's, and the
|
||||
directory lands on sys.path whenever anything is run from inside it — so
|
||||
`dataclasses` imported this file instead, and every command died on an import
|
||||
error before parsing a single argument. The verb is still `copy`; the file
|
||||
cannot be.
|
||||
|
||||
The plain utility underneath everything else: point it at a tree, get a folder
|
||||
holding what the project actually is — no `.git`, nothing gitignored, nothing a
|
||||
build regenerates, and nothing that looks like a key.
|
||||
|
||||
It is the thing to reach for when the history is not the point. Handing a
|
||||
snapshot to someone, feeding a tree to something that should not see the
|
||||
history, or getting a clean starting tree before planning one.
|
||||
|
||||
What is dropped is reported and written to a manifest, never assumed. A file
|
||||
missing from a copy without a line saying so is the same failure this whole
|
||||
tool exists to prevent, one directory earlier.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from .census import file_set, ignored_but_tracked, is_git, state_dir
|
||||
from .cli import fail
|
||||
from .sift import REASONS, sift
|
||||
|
||||
MANIFEST = "copied.md"
|
||||
|
||||
|
||||
def destination(source: Path, out: Path) -> Path:
|
||||
"""`out/<name>`, so the folder keeps the name the thing already had."""
|
||||
return Path(out) / source.name
|
||||
|
||||
|
||||
def take(source: Path, out, keep_noise=False, keep_secrets=False, max_bytes=None,
|
||||
exclude=(), include=(), force=False, dry_run=False, quiet=False):
|
||||
dest = destination(source, out)
|
||||
|
||||
if dest.exists() and any(dest.iterdir()) and not force and not dry_run:
|
||||
fail(f"{dest} already exists and is not empty.",
|
||||
"Pass --force to write into it anyway, or point --out elsewhere.")
|
||||
|
||||
paths = file_set(source)
|
||||
kept, dropped = sift(source, paths, keep_noise=keep_noise,
|
||||
keep_secrets=keep_secrets, max_bytes=max_bytes,
|
||||
exclude=exclude, include=include,
|
||||
ignored=ignored_but_tracked(source, paths))
|
||||
|
||||
if not quiet:
|
||||
print(f"{len(paths)} files tracked, {len(kept)} to copy, {len(dropped)} left behind.")
|
||||
by_reason = {}
|
||||
for rel, why in dropped:
|
||||
by_reason.setdefault(why, []).append(rel)
|
||||
for why in ("secret", "ignored", "derived", "oversize", "excluded"):
|
||||
hits = by_reason.get(why)
|
||||
if not hits:
|
||||
continue
|
||||
# Secrets are listed in full however many there are. The others are
|
||||
# bulk and a count is enough; a key that got dropped is a thing you
|
||||
# want to see the name of, because it means it was tracked.
|
||||
shown = hits if why in ("secret", "ignored") else hits[:5]
|
||||
print(f"\n {why} — {REASONS[why]} ({len(hits)}):")
|
||||
for rel in shown:
|
||||
print(f" {rel}")
|
||||
if len(hits) > len(shown):
|
||||
print(f" ... and {len(hits) - len(shown)} more")
|
||||
|
||||
if dry_run:
|
||||
if not quiet:
|
||||
print(f"\nNothing written. Would copy to {dest}.")
|
||||
return {"kept": kept, "dropped": dropped, "dest": dest}
|
||||
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for rel in kept:
|
||||
target = dest / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source / rel, target)
|
||||
|
||||
_manifest(source, out, dest, paths, kept, dropped)
|
||||
|
||||
if not quiet:
|
||||
print(f"\nCopied to {dest}")
|
||||
print(f" no .git — {'the source has one and it was not copied'
|
||||
if is_git(source) else 'the source has none either'}")
|
||||
print(f" what was left behind: {state_dir(out) / MANIFEST}")
|
||||
return {"kept": kept, "dropped": dropped, "dest": dest}
|
||||
|
||||
|
||||
def _manifest(source, out, dest, paths, kept, dropped):
|
||||
"""A record of the decision, beside the copy rather than inside it."""
|
||||
lines = [
|
||||
f"# Copied from `{source}`", "",
|
||||
f"- source: `{source}`",
|
||||
f"- copy: `{dest}`",
|
||||
f"- {len(paths)} files tracked, {len(kept)} copied, {len(dropped)} left behind",
|
||||
"",
|
||||
"No `.git` was copied. The file list is what git tracks, so nothing "
|
||||
"untracked or ignored came across — except where a file was tracked "
|
||||
"*despite* the ignore rules, which is listed below rather than assumed.",
|
||||
"",
|
||||
]
|
||||
by_reason = {}
|
||||
for rel, why in dropped:
|
||||
by_reason.setdefault(why, []).append(rel)
|
||||
for why in ("secret", "ignored", "derived", "oversize", "excluded"):
|
||||
hits = by_reason.get(why)
|
||||
if not hits:
|
||||
continue
|
||||
lines += [f"## {why} — {REASONS[why]}", ""]
|
||||
lines += [f"- `{rel}`" for rel in hits]
|
||||
lines.append("")
|
||||
if not dropped:
|
||||
lines += ["Nothing was left behind.", ""]
|
||||
|
||||
path = state_dir(out) / MANIFEST
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(lines))
|
||||
77
soleprint/station/tools/histgen/templates/index.html
Normal file
77
soleprint/station/tools/histgen/templates/index.html
Normal file
@@ -0,0 +1,77 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>histgen — station</title>
|
||||
<link rel="stylesheet" href="/theme.css">
|
||||
<style>
|
||||
body { font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
background: var(--surface-0, #101418); color: var(--text-0, #d8dee4);
|
||||
margin: 0; padding: 2rem; line-height: 1.55; }
|
||||
h1 { margin: 0 0 .25rem; font-size: 1.4rem; }
|
||||
p.lede { margin: 0 0 1.5rem; color: var(--text-1, #8b98a5); }
|
||||
form { display: flex; gap: .5rem; margin-bottom: 1.5rem; }
|
||||
input, button { font: inherit; padding: .45rem .7rem;
|
||||
background: var(--surface-1, #161c22); color: inherit;
|
||||
border: 1px solid var(--border, #2a333d); border-radius: 4px; }
|
||||
button { cursor: pointer; }
|
||||
.group { border: 1px solid var(--border, #2a333d); border-radius: 4px;
|
||||
padding: .6rem .9rem; margin-bottom: .5rem;
|
||||
background: var(--surface-1, #161c22); }
|
||||
.n { color: var(--accent, #4fb3a6); }
|
||||
.title { font-weight: 600; }
|
||||
.untitled { color: var(--text-1, #8b98a5); font-style: italic; }
|
||||
.paths { margin: .4rem 0 0; padding-left: 1.1rem; color: var(--text-1, #8b98a5);
|
||||
font-size: .87rem; }
|
||||
#summary { color: var(--text-1, #8b98a5); margin-bottom: 1rem; }
|
||||
.err { color: #e08a5c; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>histgen</h1>
|
||||
<p class="lede">A proposed history, before it is a history. Read-only —
|
||||
<code>python -m station.tools.histgen apply <repo></code> is what commits.</p>
|
||||
|
||||
<form onsubmit="load(event)">
|
||||
<input id="repo" placeholder="path to a repo, e.g. rig" size="40" autofocus>
|
||||
<button type="submit">Show plan</button>
|
||||
</form>
|
||||
|
||||
<div id="summary"></div>
|
||||
<div id="groups"></div>
|
||||
|
||||
<script>
|
||||
async function load(event) {
|
||||
event.preventDefault();
|
||||
const repo = document.getElementById('repo').value.trim();
|
||||
const summary = document.getElementById('summary');
|
||||
const groups = document.getElementById('groups');
|
||||
groups.innerHTML = ''; summary.textContent = 'Loading…';
|
||||
try {
|
||||
const res = await fetch(`/station/tools/histgen/api/plan?repo=${encodeURIComponent(repo)}`);
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(data.detail || res.statusText);
|
||||
summary.textContent =
|
||||
`${data.commits} commits over ${data.files} files` +
|
||||
(data.untitled.length ? ` — ${data.untitled.length} still without a message` : '');
|
||||
for (const g of data.groups) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'group';
|
||||
const title = g.title
|
||||
? `<span class="title">${escapeHtml(g.title)}</span>`
|
||||
: `<span class="untitled">${escapeHtml(g.slug)} — no message yet</span>`;
|
||||
el.innerHTML = `<span class="n">${String(g.n).padStart(2,'0')}</span> ${title}
|
||||
<ul class="paths">${g.paths.map(p => `<li>${escapeHtml(p)}</li>`).join('')}</ul>`;
|
||||
groups.appendChild(el);
|
||||
}
|
||||
} catch (e) {
|
||||
summary.innerHTML = `<span class="err">${escapeHtml(e.message)}</span>`;
|
||||
}
|
||||
}
|
||||
function escapeHtml(s) {
|
||||
return String(s).replace(/[&<>"']/g, c =>
|
||||
({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user