Merge branch 'main' into docgen-graphgen
This commit is contained in:
@@ -129,7 +129,7 @@ Every script stays runnable on its own — the standalone rule holds:
|
|||||||
```bash
|
```bash
|
||||||
python build.py --cfg amar # -> gen/amar/
|
python build.py --cfg amar # -> gen/amar/
|
||||||
cd gen/standalone && python run.py # bare-metal
|
cd gen/standalone && python run.py # bare-metal
|
||||||
./ctrl/kind-up.sh # still works directly
|
./ctrl/cluster.sh up # still runs directly; rig builds the cluster
|
||||||
cd gen/<room> && ./ctrl/start.sh # each room owns its lifecycle scripts
|
cd gen/<room> && ./ctrl/start.sh # each room owns its lifecycle scripts
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
2
Makefile
2
Makefile
@@ -13,7 +13,7 @@
|
|||||||
# make component ARGS="publish soleprint-ui /tmp/out --dist"
|
# make component ARGS="publish soleprint-ui /tmp/out --dist"
|
||||||
# make deploy ARGS="--build"
|
# make deploy ARGS="--build"
|
||||||
#
|
#
|
||||||
# Every script stays runnable on its own (./ctrl/kind-up.sh still works, and each
|
# Every script stays runnable on its own (./ctrl/cluster.sh up still works, and each
|
||||||
# built room keeps its own gen/<room>/ctrl/*.sh) — the standalone rule holds, and
|
# built room keeps its own gen/<room>/ctrl/*.sh) — the standalone rule holds, and
|
||||||
# this only saves typing.
|
# this only saves typing.
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -6,16 +6,49 @@
|
|||||||
# ./ctrl/cluster.sh down # delete it (drops every room's namespace)
|
# ./ctrl/cluster.sh down # delete it (drops every room's namespace)
|
||||||
# ./ctrl/cluster.sh status # what's running on it
|
# ./ctrl/cluster.sh status # what's running on it
|
||||||
#
|
#
|
||||||
# One target, one script — the variants live here. The kind-*.sh files stay
|
# spr depends on rig, never the other way round. Building and deleting a cluster
|
||||||
# exactly as they are and remain runnable on their own; this only dispatches.
|
# is rig's job, so up and down hand straight to rig/ctrl/cluster.sh, carrying the
|
||||||
|
# four things that make this cluster spr's rather than rig's defaults:
|
||||||
|
#
|
||||||
|
# CLUSTER=spr rooms deploy into the kind-spr context
|
||||||
|
# KIND_CONFIG spr's own shape, which maps the rooms' gateway NodePorts
|
||||||
|
# REGISTRY_MODE=none rooms load images straight into the node
|
||||||
|
# PROFILE=minimal pinned here, so a change to rig's own ctrl/.env can never
|
||||||
|
# quietly add addons to spr's cluster
|
||||||
|
#
|
||||||
|
# status stays here: it answers a question about rooms, not about the cluster.
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RIG_CTRL="$SCRIPT_DIR/../rig/ctrl"
|
||||||
|
|
||||||
|
rig() {
|
||||||
|
CLUSTER=spr \
|
||||||
|
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml" \
|
||||||
|
REGISTRY_MODE=none \
|
||||||
|
PROFILE=minimal \
|
||||||
|
bash "$RIG_CTRL/cluster.sh" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
case "${1:-status}" in
|
case "${1:-status}" in
|
||||||
up) exec "$SCRIPT_DIR/kind-up.sh" ;;
|
up)
|
||||||
down) exec "$SCRIPT_DIR/kind-down.sh" ;;
|
rig up
|
||||||
status) exec "$SCRIPT_DIR/kind-status.sh" ;;
|
echo
|
||||||
|
echo "Per-room deploy:"
|
||||||
|
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
||||||
|
;;
|
||||||
|
down)
|
||||||
|
rig down
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
if ! kind get clusters 2>/dev/null | grep -qx spr; then
|
||||||
|
echo "No 'spr' kind cluster — run: make cluster up"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
kubectl --context kind-spr get namespaces -l soleprint-room
|
||||||
|
echo
|
||||||
|
kubectl --context kind-spr get pods -A -l soleprint-room
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unknown subcommand: $1" >&2
|
echo "Unknown subcommand: $1" >&2
|
||||||
echo "Usage: cluster.sh [up|down|status]" >&2
|
echo "Usage: cluster.sh [up|down|status]" >&2
|
||||||
|
|||||||
@@ -3,9 +3,15 @@ apiVersion: kind.x-k8s.io/v1alpha4
|
|||||||
# Single shared cluster for all soleprint rooms.
|
# Single shared cluster for all soleprint rooms.
|
||||||
# Each room deploys into its own namespace; gateway Services pick a
|
# Each room deploys into its own namespace; gateway Services pick a
|
||||||
# NodePort from the 30080-30099 range mapped here.
|
# NodePort from the 30080-30099 range mapped here.
|
||||||
name: spr
|
#
|
||||||
|
# Built by rig, not by spr: ctrl/cluster.sh hands this file to
|
||||||
|
# rig/ctrl/cluster.sh, which substitutes CLUSTER and NODE_IMAGE (named without
|
||||||
|
# braces here so this comment survives the substitution). The shape is spr's —
|
||||||
|
# what its cluster needs is spr's business. Building it is rig's.
|
||||||
|
name: ${CLUSTER}
|
||||||
nodes:
|
nodes:
|
||||||
- role: control-plane
|
- role: control-plane
|
||||||
|
image: ${NODE_IMAGE}
|
||||||
extraPortMappings:
|
extraPortMappings:
|
||||||
# Room gateway NodePorts (one per active room).
|
# Room gateway NodePorts (one per active room).
|
||||||
- {containerPort: 30080, hostPort: 30080, protocol: TCP}
|
- {containerPort: 30080, hostPort: 30080, protocol: TCP}
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Delete the shared `spr` kind cluster (drops every room's namespace too).
|
|
||||||
# Use `gen/<room>/ctrl/k8s-down.sh` instead if you only want to remove
|
|
||||||
# a single room's namespace.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "Deleting kind cluster 'spr'..."
|
|
||||||
kind delete cluster --name spr
|
|
||||||
else
|
|
||||||
echo "No kind cluster 'spr' to delete."
|
|
||||||
fi
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Show what's running on the shared `spr` cluster.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if ! kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "No 'spr' kind cluster — run ctrl/kind-up.sh"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
kubectl --context kind-spr get namespaces -l soleprint-room
|
|
||||||
echo
|
|
||||||
kubectl --context kind-spr get pods -A -l soleprint-room
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Create (or no-op) the single shared `spr` kind cluster used by every
|
|
||||||
# soleprint room. Per-room work happens inside namespaces — see
|
|
||||||
# `gen/<room>/ctrl/k8s-up.sh`.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
KIND_CONFIG="$SCRIPT_DIR/k8s/kind-config.yaml"
|
|
||||||
|
|
||||||
if kind get clusters 2>/dev/null | grep -q '^spr$'; then
|
|
||||||
echo "Kind cluster 'spr' already exists."
|
|
||||||
else
|
|
||||||
echo "Creating kind cluster 'spr'..."
|
|
||||||
kind create cluster --config "$KIND_CONFIG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
kubectl config use-context kind-spr >/dev/null
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "Cluster ready. Per-room deploy:"
|
|
||||||
echo " cd gen/<room> && ./ctrl/k8s-up.sh"
|
|
||||||
@@ -194,8 +194,8 @@ follows is only the mechanical part.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
SLUG=<slug> # short, lowercase, no separators
|
SLUG=<slug> # short, lowercase, no separators
|
||||||
cp -r ~/wdir/all/projects/templates/broad ~/wdir/"$SLUG"
|
cp -r ~/wdir/semester/all/projects/templates/broad ~/wdir/semester/"$SLUG"
|
||||||
cd ~/wdir/"$SLUG"
|
cd ~/wdir/semester/"$SLUG"
|
||||||
grep -rl '<slug>' ctrl | xargs sed -i "s/<slug>/$SLUG/g"
|
grep -rl '<slug>' ctrl | xargs sed -i "s/<slug>/$SLUG/g"
|
||||||
cp ctrl/k8s/.env.example ctrl/k8s/.env
|
cp ctrl/k8s/.env.example ctrl/k8s/.env
|
||||||
git init && git add -A && git commit -m "scaffold $SLUG from broad"
|
git init && git add -A && git commit -m "scaffold $SLUG from broad"
|
||||||
@@ -213,7 +213,7 @@ scaffold's Makefile from the directory, so there is nothing to edit for either.
|
|||||||
is already in use, so copying it unchanged puts two projects on one port:
|
is already in use, so copying it unchanged puts two projects on one port:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
grep -h '^TILT_PORT=' ~/wdir/*/ctrl/k8s/.env 2>/dev/null | sort
|
grep -h '^TILT_PORT=' ~/wdir/semester/*/ctrl/k8s/.env 2>/dev/null | sort
|
||||||
```
|
```
|
||||||
|
|
||||||
Choose a free one in `10300–10399` — the range ALL reserves in
|
Choose a free one in `10300–10399` — the range ALL reserves in
|
||||||
@@ -243,7 +243,7 @@ The workload is an nginx placeholder so a fresh copy reaches something that
|
|||||||
answers; replace it. Keep `30080` in step between the overlay patch and
|
answers; replace it. Keep `30080` in step between the overlay patch and
|
||||||
`kind-config.yaml`'s `containerPort` — the hostPort is this project's to pick.
|
`kind-config.yaml`'s `containerPort` — the hostPort is this project's to pick.
|
||||||
Reachability is a plain kind port mapping: no ingress controller and no MetalLB.
|
Reachability is a plain kind port mapping: no ingress controller and no MetalLB.
|
||||||
Caddy maps `<slug>.local.ar` onto the host port (`~/wdir/ppl/local/Caddyfile`),
|
Caddy maps `<slug>.local.ar` onto the host port (`~/wdir/semester/ppl/local/Caddyfile`),
|
||||||
with `*.local.ar` resolving to 127.0.0.1 through dnsmasq. That is the whole chain.
|
with `*.local.ar` resolving to 127.0.0.1 through dnsmasq. That is the whole chain.
|
||||||
|
|
||||||
**The one file the scaffold still does not ship is `ctrl/Tiltfile`** — `make
|
**The one file the scaffold still does not ship is `ctrl/Tiltfile`** — `make
|
||||||
@@ -269,9 +269,9 @@ delete-and-recreate for when a cluster wedges.
|
|||||||
## Register it
|
## Register it
|
||||||
|
|
||||||
The project exists; now it is findable. Add an entry to
|
The project exists; now it is findable. Add an entry to
|
||||||
`~/wdir/all/projects/index.json` and write its `projects/<slug>.md` beside the
|
`~/wdir/semester/all/projects/index.json` and write its `projects/<slug>.md` beside the
|
||||||
others. Structured fields in the index, prose in the markdown.
|
others. Structured fields in the index, prose in the markdown.
|
||||||
|
|
||||||
Putting it on the CI server and deploying it is `ppl`'s half, and it starts at
|
Putting it on the CI server and deploying it is `ppl`'s half, and it starts at
|
||||||
`~/wdir/ppl/ctrl/init-repo.sh` — gitea remote, then Woodpecker. That is a
|
`~/wdir/semester/ppl/ctrl/init-repo.sh` — gitea remote, then Woodpecker. That is a
|
||||||
different document.
|
different document.
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ $(eval $(ARGS):;@:)
|
|||||||
.PHONY: $(ARGS)
|
.PHONY: $(ARGS)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: help setup check mem deps deps-image cluster registry addons ports \
|
.PHONY: help setup check mem deps deps-image pins cluster registry addons ports \
|
||||||
newbox dockerhost docs tilt \
|
newbox dockerhost docs tilt \
|
||||||
kind-up kind-down kind-reset tilt-up tilt-down
|
kind-up kind-down kind-reset tilt-up tilt-down
|
||||||
|
|
||||||
@@ -56,6 +56,9 @@ mem: ## memory, and any cap holding it [status|backup
|
|||||||
deps: ## install the toolchain [core|dev] (default dev)
|
deps: ## install the toolchain [core|dev] (default dev)
|
||||||
bash ctrl/deps.sh install $(or $(ARGS),dev)
|
bash ctrl/deps.sh install $(or $(ARGS),dev)
|
||||||
|
|
||||||
|
pins: ## standalone/rigdeps.sh still installs what rig pins?
|
||||||
|
bash ctrl/pins.sh
|
||||||
|
|
||||||
deps-image: ## build the installer image [full]
|
deps-image: ## build the installer image [full]
|
||||||
docker build -f ctrl/Dockerfile.deps \
|
docker build -f ctrl/Dockerfile.deps \
|
||||||
--target $(if $(filter full,$(ARGS)),deps-full,deps) \
|
--target $(if $(filter full,$(ARGS)),deps-full,deps) \
|
||||||
|
|||||||
@@ -32,13 +32,128 @@ if [ ! -f ./.env ]; then
|
|||||||
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
echo " ! ctrl/.env missing — copy it: cp ctrl/.env.example ctrl/.env"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# A 3-node profile on a box that's already full is the most common first
|
# ── memory ─────────────────────────────────────────────────────────────────
|
||||||
# failure, and it presents as pods stuck Pending rather than anything obvious.
|
#
|
||||||
avail=$(awk '/^MemAvailable:/{printf "%d", $2/1024/1024}' /proc/meminfo)
|
# A profile on a box that is already full is the most common first failure, and
|
||||||
need=$((NODES * 2))
|
# it presents as pods stuck Pending rather than anything that says "memory".
|
||||||
if [ "$avail" -lt "$need" ]; then
|
# Warns; never blocks. Whether to try anyway is the user's call.
|
||||||
echo " ! profile '${PROFILE_NAME}' wants ~${need} GB, ${avail} GB available"
|
|
||||||
echo " 'make cluster list' shows what else is running; 'make cluster free' stops it"
|
# A /proc/meminfo field in MB, 0 if absent. MEMINFO and OVERCOMMIT_FILE exist
|
||||||
|
# only so the tight and does-not-fit branches can be exercised against another
|
||||||
|
# machine's real numbers; in normal use they are the kernel's own files.
|
||||||
|
mb_of() {
|
||||||
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||||
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# What one node costs, measured rather than guessed. On 2026-09-11 a minimal
|
||||||
|
# control-plane node ran at 620 MiB idle and ~728 MiB with a small mock, plus
|
||||||
|
# 16 MiB for the local registry — ~745 MiB of working set. 800 rounds that up,
|
||||||
|
# and agrees with the 800 MB observed independently on a larger rig. Worker
|
||||||
|
# nodes carry no etcd or apiserver and are lighter, so for a multi-node shape
|
||||||
|
# this errs high. It is the cluster alone: whatever you deploy comes on top.
|
||||||
|
NODE_MB=800
|
||||||
|
|
||||||
|
# Every running container's working set in MB, tagged with the kind cluster it
|
||||||
|
# belongs to ('-' when it is not kind). docker stats reports usage minus page
|
||||||
|
# cache, which is what actually competes — cache is handed back under pressure.
|
||||||
|
# Counting only kind would hide the usual culprit on a managed workspace, where
|
||||||
|
# the memory is held by other containers entirely.
|
||||||
|
container_mb() {
|
||||||
|
docker info >/dev/null 2>&1 || return 0
|
||||||
|
awk -F'\t' '
|
||||||
|
FILENAME == ARGV[1] { cl[$1] = ($2 == "" ? "-" : $2); if ($2 != "") isc[$2] = 1; next }
|
||||||
|
{
|
||||||
|
grp = ($1 in cl ? cl[$1] : "-")
|
||||||
|
# A kind cluster'"'"'s local registry is a plain container with no kind
|
||||||
|
# label, named <cluster>-registry, so on its own it would read as a
|
||||||
|
# stranger. It belongs to its cluster — but only if that cluster exists:
|
||||||
|
# a registry whose cluster is gone is a genuine stray, and says so.
|
||||||
|
if (grp == "-" && $1 ~ /-registry$/) {
|
||||||
|
base = $1; sub(/-registry$/, "", base)
|
||||||
|
if (base in isc) grp = base
|
||||||
|
}
|
||||||
|
split($2, u, " "); v = u[1]; mb = 0
|
||||||
|
if (v ~ /GiB$/) { sub(/GiB$/, "", v); mb = v * 1024 }
|
||||||
|
else if (v ~ /MiB$/) { sub(/MiB$/, "", v); mb = v }
|
||||||
|
else if (v ~ /KiB$/) { sub(/KiB$/, "", v); mb = v / 1024 }
|
||||||
|
else if (v ~ /B$/) { sub(/B$/, "", v); mb = v / 1048576 }
|
||||||
|
printf "%d\t%s\t%s\n", mb, grp, $1
|
||||||
|
}
|
||||||
|
' <(docker ps --format '{{.Names}}\t{{.Label "io.x-k8s.kind.cluster"}}' 2>/dev/null) \
|
||||||
|
<(docker stats --no-stream --format '{{.Name}}\t{{.MemUsage}}' 2>/dev/null)
|
||||||
|
}
|
||||||
|
|
||||||
|
total_mb=$(mb_of MemTotal)
|
||||||
|
avail_mb=$(mb_of MemAvailable)
|
||||||
|
swap_used_mb=$(( $(mb_of SwapTotal) - $(mb_of SwapFree) ))
|
||||||
|
overcommit=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||||
|
need_mb=$(( NODES * NODE_MB ))
|
||||||
|
|
||||||
|
rows=$(container_mb)
|
||||||
|
# Once this environment's own cluster is running, its real footprint is already
|
||||||
|
# out of MemAvailable and the per-node estimate stops being relevant. Subtracting
|
||||||
|
# the measurement from the estimate would count the same memory twice, and a
|
||||||
|
# running cluster that happens to sit under 800 MB would still "need" the gap.
|
||||||
|
ours_mb=$(awk -F'\t' -v c="$CLUSTER" '$2 == c { s += $1 } END { print s + 0 }' <<< "$rows")
|
||||||
|
still_mb=$(( ours_mb > 0 ? 0 : need_mb ))
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "memory"
|
||||||
|
printf " this profile ~%d MB %s node(s) x %d MB — the cluster alone, your workload on top\n" \
|
||||||
|
"$need_mb" "$NODES" "$NODE_MB"
|
||||||
|
if [ "$ours_mb" -gt 0 ]; then
|
||||||
|
printf " already held %d MB by '%s', which is up\n" "$ours_mb" "$CLUSTER"
|
||||||
|
fi
|
||||||
|
printf " available %d MB of %d MB\n" "$avail_mb" "$total_mb"
|
||||||
|
|
||||||
|
# The biggest things holding memory right now, other than this cluster: kind
|
||||||
|
# clusters summed per cluster, everything else by container name.
|
||||||
|
others=$(awk -F'\t' -v c="$CLUSTER" '
|
||||||
|
$2 != c && $2 != "-" && $2 != "" { k["kind cluster \x27" $2 "\x27"] += $1 }
|
||||||
|
$2 == "-" { k["container \x27" $3 "\x27"] += $1 }
|
||||||
|
END { for (n in k) printf "%d\t%s\n", k[n], n }' <<< "$rows" | sort -rn)
|
||||||
|
if [ -n "$others" ]; then
|
||||||
|
echo " held elsewhere:"
|
||||||
|
head -6 <<< "$others" | awk -F'\t' '{ printf " %6d MB %s\n", $1, $2 }'
|
||||||
|
n_others=$(wc -l <<< "$others")
|
||||||
|
if [ "$n_others" -gt 6 ]; then
|
||||||
|
echo " ... and $((n_others - 6)) more"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
headroom=$(( avail_mb - still_mb ))
|
||||||
|
if [ "$still_mb" -eq 0 ]; then
|
||||||
|
if [ "$headroom" -ge 512 ]; then
|
||||||
|
printf " fits — already up; %d MB headroom for what you deploy\n" "$headroom"
|
||||||
|
else
|
||||||
|
printf " ! already up, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||||
|
fi
|
||||||
|
elif [ "$headroom" -ge 512 ]; then
|
||||||
|
printf " fits — %d MB headroom for what you deploy\n" "$headroom"
|
||||||
|
elif [ "$headroom" -ge 0 ]; then
|
||||||
|
printf " ! fits, but only %d MB headroom for anything you deploy\n" "$headroom"
|
||||||
|
else
|
||||||
|
printf " ! does not fit right now: ~%d MB needed, %d MB available\n" "$still_mb" "$avail_mb"
|
||||||
|
# Two failures with opposite fixes, and telling them apart is the point.
|
||||||
|
if [ "$still_mb" -le "$total_mb" ]; then
|
||||||
|
echo " The machine is big enough; something else is holding memory (above)."
|
||||||
|
echo " Stopping that is what helps — a bigger VM would not."
|
||||||
|
if grep -q 'kind cluster' <<< "$others"; then
|
||||||
|
echo " 'make cluster free' stops the other kind clusters. It stops, never deletes."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " The machine itself is too small: ~${still_mb} MB needed, ${total_mb} MB total."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$swap_used_mb" -gt 0 ]; then
|
||||||
|
printf " ! %d MB already in swap — available memory does not count it, so expect a\n" "$swap_used_mb"
|
||||||
|
echo " cluster here to be slow well before it fails"
|
||||||
|
fi
|
||||||
|
if [ "$overcommit" = "1" ]; then
|
||||||
|
echo " ! overcommit=1: allocations never fail here, so read 'fits' as a ceiling."
|
||||||
|
echo " A cluster that starts cleanly can still lose processes to the OOM killer."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# The CA reaches three places and only one of them is ours. Report the other two.
|
# The CA reaches three places and only one of them is ours. Report the other two.
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ up() {
|
|||||||
# Say what this profile locks in BEFORE spending minutes building it:
|
# Say what this profile locks in BEFORE spending minutes building it:
|
||||||
# the audit policy is an apiserver flag and cannot be changed later.
|
# the audit policy is an apiserver flag and cannot be changed later.
|
||||||
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
|
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
|
||||||
echo " shape ctrl/k8s/$KIND_CONFIG"
|
echo " shape ${KIND_CONFIG_SHOWN}"
|
||||||
echo " nodes $NODES"
|
echo " nodes $NODES"
|
||||||
echo " image $NODE_IMAGE"
|
echo " image $NODE_IMAGE"
|
||||||
echo " audit $AUDIT"
|
echo " audit $AUDIT"
|
||||||
|
|||||||
180
rig/ctrl/deps.sh
180
rig/ctrl/deps.sh
@@ -49,6 +49,14 @@ MANUAL=()
|
|||||||
# Host FILES (/etc/..., /mnt/c/...) must be read through the mount. Kernel-level
|
# Host FILES (/etc/..., /mnt/c/...) must be read through the mount. Kernel-level
|
||||||
# facts (kernel version, meminfo, inotify) are shared with the container, so the
|
# facts (kernel version, meminfo, inotify) are shared with the container, so the
|
||||||
# container's own view is already the host's.
|
# container's own view is already the host's.
|
||||||
|
# A /proc/meminfo field in MB, 0 if the field is absent. MEMINFO exists so the
|
||||||
|
# tight and does-not-fit branches can be exercised against a real machine's
|
||||||
|
# numbers from somewhere else; in normal use it is always /proc/meminfo.
|
||||||
|
mb_of() {
|
||||||
|
awk -v k="$1:" '$1 == k { printf "%d", $2 / 1024; found = 1 }
|
||||||
|
END { if (!found) printf "0" }' "${MEMINFO:-/proc/meminfo}"
|
||||||
|
}
|
||||||
|
|
||||||
host_file() {
|
host_file() {
|
||||||
local p="${1#/}"
|
local p="${1#/}"
|
||||||
if [ "$HOST_ROOT" != "/" ] && [ -e "$HOST_ROOT/$p" ]; then
|
if [ "$HOST_ROOT" != "/" ] && [ -e "$HOST_ROOT/$p" ]; then
|
||||||
@@ -92,21 +100,34 @@ detect() {
|
|||||||
local osr; osr=$(host_file /etc/os-release)
|
local osr; osr=$(host_file /etc/os-release)
|
||||||
[ -r "$osr" ] && echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' "$osr")"
|
[ -r "$osr" ] && echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' "$osr")"
|
||||||
|
|
||||||
local total_kb avail_kb
|
# In MB. Whole gigabytes lose nearly half a GB on exactly the machines where
|
||||||
total_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo)
|
# it matters: 1874 MB available used to print as "1 GB". Facts only — whether
|
||||||
avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
|
# that is enough depends on the profile, which check.sh knows and this does not.
|
||||||
printf " memory %d GB total, %d GB available\n" \
|
local total_mb avail_mb swap_total_mb swap_used_mb om
|
||||||
$((total_kb / 1024 / 1024)) $((avail_kb / 1024 / 1024))
|
total_mb=$(mb_of MemTotal)
|
||||||
|
avail_mb=$(mb_of MemAvailable)
|
||||||
if [ $((avail_kb / 1024 / 1024)) -lt 4 ]; then
|
swap_total_mb=$(mb_of SwapTotal)
|
||||||
echo " ! under 4 GB available — a multi-node profile will struggle."
|
swap_used_mb=$(( swap_total_mb - $(mb_of SwapFree) ))
|
||||||
echo " 'make cluster list' shows the others; 'make cluster free' stops them."
|
printf " memory %d MB total, %d MB available\n" "$total_mb" "$avail_mb"
|
||||||
|
if [ "$swap_total_mb" -gt 0 ]; then
|
||||||
|
printf " swap %d MB used of %d MB\n" "$swap_used_mb" "$swap_total_mb"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# How the kernel answers an allocation it cannot really satisfy. With 1 it
|
||||||
|
# always says yes and settles up later with the OOM killer, so a cluster that
|
||||||
|
# starts cleanly can still lose processes afterwards.
|
||||||
|
om=$(cat "${OVERCOMMIT_FILE:-/proc/sys/vm/overcommit_memory}" 2>/dev/null || echo '?')
|
||||||
|
case "$om" in
|
||||||
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess" ;;
|
||||||
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit" ;;
|
||||||
|
2) echo " overcommit 2 strict — an allocation fails honestly instead of killing later" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
detect_wsl
|
detect_wsl
|
||||||
detect_filesystem
|
detect_filesystem
|
||||||
detect_docker
|
detect_docker
|
||||||
detect_inotify
|
detect_inotify
|
||||||
|
detect_toolchain
|
||||||
}
|
}
|
||||||
|
|
||||||
detect_wsl() {
|
detect_wsl() {
|
||||||
@@ -322,6 +343,87 @@ CORE_TOOLS="kubectl jq"
|
|||||||
# nothing structural stopping a push there.
|
# nothing structural stopping a push there.
|
||||||
DEV_TOOLS="kind tilt ctlptl"
|
DEV_TOOLS="kind tilt ctlptl"
|
||||||
|
|
||||||
|
# ── what is already on this machine ───────────────────────────────────────
|
||||||
|
#
|
||||||
|
# A tool already on PATH at its pinned version is left where it is. Without
|
||||||
|
# this, install downloads a second copy into OUT_BIN and then reports the first
|
||||||
|
# one as shadowed — noise, and wrong, when both are the same version. That is
|
||||||
|
# the normal state of any machine someone set up by hand: the AWS Workspace
|
||||||
|
# keeps its toolchain in ~/wdir/bin, all five at exactly these pins.
|
||||||
|
|
||||||
|
pin_of() {
|
||||||
|
case "$1" in
|
||||||
|
kubectl) echo "$KUBECTL_VERSION" ;;
|
||||||
|
jq) echo "$JQ_VERSION" ;;
|
||||||
|
kind) echo "$KIND_VERSION" ;;
|
||||||
|
tilt) echo "$TILT_VERSION" ;;
|
||||||
|
ctlptl) echo "$CTLPTL_VERSION" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# The version string a binary reports. Each tool spells the question
|
||||||
|
# differently, and kubectl has to be told --client or it goes looking for a
|
||||||
|
# server to ask.
|
||||||
|
reported_version() {
|
||||||
|
local tool="$1" path="$2"
|
||||||
|
case "$tool" in
|
||||||
|
kubectl) "$path" version --client 2>/dev/null ;;
|
||||||
|
jq) "$path" --version 2>/dev/null ;;
|
||||||
|
*) "$path" version 2>/dev/null ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# Does the binary at PATH report PIN? Matched as a whole version token, so
|
||||||
|
# 0.37.6 never matches 10.37.60, with the leading v optional either side: kind
|
||||||
|
# says v0.32.0, jq says jq-1.8.2, and tilt says v0.37.6 against a pin of 0.37.6.
|
||||||
|
#
|
||||||
|
# Bash's own regex rather than grep, deliberately. grep is not the same program
|
||||||
|
# on every machine — some builds reject patterns that others accept — and a
|
||||||
|
# failed grep inside a count reads exactly like a zero.
|
||||||
|
version_matches() {
|
||||||
|
local tool="$1" path="$2" pin="$3" out v re
|
||||||
|
out=$(reported_version "$tool" "$path") || return 1
|
||||||
|
v="${pin#v}"
|
||||||
|
v="${v//./\\.}"
|
||||||
|
re="(^|[^0-9.])v?${v}([^0-9.]|\$)"
|
||||||
|
[[ $out =~ $re ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# DEPS_ONLY narrows a fetch to the tools it names. Unset means the whole tier,
|
||||||
|
# which is what an explicit `deps.sh fetch` always gets: "download these into
|
||||||
|
# DIR" must not quietly skip something because this machine happens to have it.
|
||||||
|
# Only install() sets it, to what detect_toolchain found missing or mismatched.
|
||||||
|
want() { [ -z "${DEPS_ONLY:-}" ] || [[ " $DEPS_ONLY " == *" $1 "* ]]; }
|
||||||
|
|
||||||
|
# Every tool in the tier with its state, probed once and reported once. What
|
||||||
|
# still needs fetching is left in TOOLCHAIN_NEED for install() to act on.
|
||||||
|
TOOLCHAIN_NEED=""
|
||||||
|
detect_toolchain() {
|
||||||
|
local tier="${TIER:-dev}" b pin path found
|
||||||
|
TOOLCHAIN_NEED=""
|
||||||
|
echo
|
||||||
|
echo "toolchain (pinned, tier '$tier')"
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
pin=$(pin_of "$b")
|
||||||
|
path=$(command -v "$b" 2>/dev/null || true)
|
||||||
|
if [ -z "$path" ]; then
|
||||||
|
printf " - %-8s %-9s not found\n" "$b" "$pin"
|
||||||
|
TOOLCHAIN_NEED+="$b "
|
||||||
|
elif version_matches "$b" "$path" "$pin"; then
|
||||||
|
printf " %-8s %-9s %s\n" "$b" "$pin" "$path"
|
||||||
|
else
|
||||||
|
found=$(reported_version "$b" "$path" 2>/dev/null | head -1 || true)
|
||||||
|
printf " ! %-8s wants %s, %s reports '%s'\n" "$b" "$pin" "$path" "$found"
|
||||||
|
TOOLCHAIN_NEED+="$b "
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -z "$TOOLCHAIN_NEED" ]; then
|
||||||
|
echo " every pinned tool is already on PATH — nothing to fetch"
|
||||||
|
else
|
||||||
|
echo " 'make deps' fetches only: ${TOOLCHAIN_NEED% }"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
fetch() {
|
fetch() {
|
||||||
local dest="$OUT_BIN" tier="${TIER:-dev}"
|
local dest="$OUT_BIN" tier="${TIER:-dev}"
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
@@ -342,13 +444,17 @@ fetch() {
|
|||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "fetching '$tier' toolchain (source: $DEPS_SOURCE)"
|
if [ -n "${DEPS_ONLY:-}" ]; then
|
||||||
fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"
|
echo "fetching ${DEPS_ONLY% } (source: $DEPS_SOURCE)"
|
||||||
fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"
|
else
|
||||||
|
echo "fetching '$tier' toolchain (source: $DEPS_SOURCE)"
|
||||||
|
fi
|
||||||
|
if want kubectl; then fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"; fi
|
||||||
|
if want jq; then fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"; fi
|
||||||
if [ "$tier" = "dev" ]; then
|
if [ "$tier" = "dev" ]; then
|
||||||
fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"
|
if want kind; then fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"; fi
|
||||||
fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0
|
if want tilt; then fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0; fi
|
||||||
fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0
|
if want ctlptl; then fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0; fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
fix_ownership "$dest"
|
fix_ownership "$dest"
|
||||||
@@ -390,6 +496,9 @@ warn_shadowing() {
|
|||||||
command -v "$b" 2>/dev/null || true)
|
command -v "$b" 2>/dev/null || true)
|
||||||
[ -n "$existing" ] || continue
|
[ -n "$existing" ] || continue
|
||||||
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||||
|
# The same version in both places is not a conflict: nothing changes for
|
||||||
|
# any other project whichever copy PATH happens to find first.
|
||||||
|
if version_matches "$b" "$existing" "$(pin_of "$b")"; then continue; fi
|
||||||
shadowed+=" $b $existing"$'\n'
|
shadowed+=" $b $existing"$'\n'
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -412,25 +521,34 @@ warn_shadowing() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
install() {
|
install() {
|
||||||
local tier="${1:-dev}"
|
local tier="${1:-dev}" b
|
||||||
|
TIER="$tier"
|
||||||
detect
|
detect
|
||||||
echo
|
|
||||||
fetch "$tier"
|
|
||||||
echo
|
|
||||||
echo "installed to $OUT_BIN ($tier):"
|
|
||||||
for b in $(tier_tools "$tier"); do
|
|
||||||
[ -x "$OUT_BIN/$b" ] && echo " $b"
|
|
||||||
done
|
|
||||||
if [ "$tier" = "core" ]; then
|
|
||||||
echo " (no kind/tilt — 'make deps dev' adds them)"
|
|
||||||
fi
|
|
||||||
warn_shadowing "$tier"
|
|
||||||
|
|
||||||
case ":${PATH}:" in
|
# detect_toolchain has already probed PATH. Fetch only what it found missing
|
||||||
*":$OUT_BIN:"*) ;;
|
# or at the wrong version; a tool already present at its pin stays where it is.
|
||||||
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
if [ -n "$TOOLCHAIN_NEED" ]; then
|
||||||
|
echo
|
||||||
|
DEPS_ONLY="$TOOLCHAIN_NEED" fetch "$tier"
|
||||||
|
echo
|
||||||
|
echo "installed to $OUT_BIN ($tier):"
|
||||||
|
for b in $TOOLCHAIN_NEED; do
|
||||||
|
if [ -x "$OUT_BIN/$b" ]; then echo " $b"; fi
|
||||||
|
done
|
||||||
|
if [ "$tier" = "core" ]; then
|
||||||
|
echo " (no kind/tilt — 'make deps dev' adds them)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Only worth saying when something actually landed in OUT_BIN. When every
|
||||||
|
# tool was satisfied elsewhere, OUT_BIN may reasonably be off PATH, and
|
||||||
|
# telling the user to add it would be advice to fix nothing.
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||||
export PATH=\"${OUT_BIN}:\$PATH\"") ;;
|
export PATH=\"${OUT_BIN}:\$PATH\"") ;;
|
||||||
esac
|
esac
|
||||||
|
fi
|
||||||
|
warn_shadowing "$tier"
|
||||||
|
|
||||||
report_manual
|
report_manual
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -103,11 +103,21 @@ load_config() {
|
|||||||
|
|
||||||
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
|
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
|
||||||
# shape is adding a file; there is no dispatcher to edit.
|
# shape is adding a file; there is no dispatcher to edit.
|
||||||
|
#
|
||||||
|
# A host that needs its own shape — extra port mappings, more nodes — passes
|
||||||
|
# an absolute path instead, and rig renders it exactly like one of its own:
|
||||||
|
# ${CLUSTER} and ${NODE_IMAGE} are substituted either way. The shape stays in
|
||||||
|
# the host's tree, because what a host's cluster needs is the host's business;
|
||||||
|
# rig only knows how to build whatever it is handed.
|
||||||
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
|
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
|
||||||
KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"
|
case "$KIND_CONFIG" in
|
||||||
|
/*) KIND_CONFIG_PATH="$KIND_CONFIG"; KIND_CONFIG_SHOWN="$KIND_CONFIG" ;;
|
||||||
|
*) KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"; KIND_CONFIG_SHOWN="ctrl/k8s/${KIND_CONFIG}" ;;
|
||||||
|
esac
|
||||||
if [ ! -f "$KIND_CONFIG_PATH" ]; then
|
if [ ! -f "$KIND_CONFIG_PATH" ]; then
|
||||||
echo "no such cluster shape: ctrl/k8s/${KIND_CONFIG}" >&2
|
echo "no such cluster shape: ${KIND_CONFIG_SHOWN}" >&2
|
||||||
echo "available: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
|
echo "rig's own: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
|
||||||
|
echo "or pass an absolute path to a shape of your own" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
57
rig/ctrl/pins.sh
Executable file
57
rig/ctrl/pins.sh
Executable file
@@ -0,0 +1,57 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Do the standalone scripts still install what rig pins?
|
||||||
|
#
|
||||||
|
# standalone/rigdeps.sh carries its toolchain pins inline, because it exists for
|
||||||
|
# a machine that will never have ctrl/versions.env. That makes two copies of the
|
||||||
|
# same versions and checksums, and two copies drift the day one is edited and
|
||||||
|
# the other forgotten. This is the check that notices.
|
||||||
|
#
|
||||||
|
# ctrl/versions.env is the source of truth. Only the keys rigdeps.sh itself
|
||||||
|
# defines are compared: versions.env also pins addon images (cert-manager,
|
||||||
|
# metallb, metrics-server) that rigdeps.sh never installs, and demanding those
|
||||||
|
# would make this fail forever for no reason.
|
||||||
|
#
|
||||||
|
# Exits non-zero on any mismatch — unlike the host checks, this one is a test.
|
||||||
|
#
|
||||||
|
# Usage: pins.sh
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
SOURCE=./versions.env
|
||||||
|
COPY=../standalone/rigdeps.sh
|
||||||
|
|
||||||
|
[ -r "$COPY" ] || { echo "no $COPY to compare" >&2; exit 1; }
|
||||||
|
|
||||||
|
# KEY=value for the pin keys a file defines, quotes stripped. awk rather than a
|
||||||
|
# grep regex, which is not the same program everywhere.
|
||||||
|
pins() {
|
||||||
|
awk -F= '/^[A-Z_]+_(VERSION|SHA256)=/ {
|
||||||
|
v = substr($0, index($0, "=") + 1); gsub(/^["\x27]|["\x27]$/, "", v)
|
||||||
|
print $1 "=" v }' "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "pins: standalone/rigdeps.sh against ctrl/versions.env"
|
||||||
|
bad=0
|
||||||
|
while IFS='=' read -r key copy_val; do
|
||||||
|
[ -n "$key" ] || continue
|
||||||
|
src_val=$(pins "$SOURCE" | sed -n "s/^${key}=//p" | head -1)
|
||||||
|
if [ -z "$src_val" ]; then
|
||||||
|
printf " ! %-16s in rigdeps.sh but not in versions.env\n" "$key"
|
||||||
|
bad=1
|
||||||
|
elif [ "$src_val" = "$copy_val" ]; then
|
||||||
|
printf " %-16s %s\n" "$key" "$( [ ${#src_val} -gt 20 ] && echo "${src_val:0:12}…" || echo "$src_val" )"
|
||||||
|
else
|
||||||
|
printf " ! %-16s versions.env %s\n" "$key" "$src_val"
|
||||||
|
printf " %-16s rigdeps.sh %s\n" "" "$copy_val"
|
||||||
|
bad=1
|
||||||
|
fi
|
||||||
|
done < <(pins "$COPY")
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$bad" -eq 0 ]; then
|
||||||
|
echo "in step — rigdeps.sh installs exactly what rig pins."
|
||||||
|
else
|
||||||
|
echo "DRIFT. versions.env is the source of truth: copy the differing lines from it"
|
||||||
|
echo "into standalone/rigdeps.sh, taking checksums from the publisher's release list."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
37
rig/standalone/README.md
Normal file
37
rig/standalone/README.md
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# standalone — single files for a machine the full rig is not going to
|
||||||
|
|
||||||
|
Each script here does one of rig's jobs without the rest of the tree. Copy one
|
||||||
|
file onto a machine, run it, read the output. Nothing to clone, nothing to
|
||||||
|
install first.
|
||||||
|
|
||||||
|
| file | does | full-rig equivalent |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `rigdeps.sh` | installs kind, kubectl, tilt, ctlptl and jq at rig's pins, checksum-verified, no sudo | `make deps` (`ctrl/deps.sh`) |
|
||||||
|
| `rigmini.sh` | reports how much memory the machine *advertises* and what caps it; `push` measures what it will actually *survive* | `make mem`, and the memory section of `make check` |
|
||||||
|
|
||||||
|
**These are transitional.** Where the full rig is installed, use its own
|
||||||
|
targets instead; they read `ctrl/versions.env` and the profile, which these
|
||||||
|
cannot.
|
||||||
|
|
||||||
|
## Why single files
|
||||||
|
|
||||||
|
`rigdeps.sh` carries its pins inline, because `ctrl/versions.env` is not on the
|
||||||
|
machine it is for. That makes two copies of the same versions and checksums.
|
||||||
|
`make pins` compares them and fails on any difference — `ctrl/versions.env` is
|
||||||
|
the source of truth.
|
||||||
|
|
||||||
|
`rigmini.sh` exists because on a container or managed workspace `/proc/meminfo`
|
||||||
|
reports the *host's* memory while a cgroup cap kills processes at a fraction of
|
||||||
|
it. `status` reads the caps; `push` allocates until something stops it.
|
||||||
|
|
||||||
|
## Use
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash rigdeps.sh detect # report, change nothing
|
||||||
|
bash rigdeps.sh install dev # install into ~/.local/bin
|
||||||
|
bash rigmini.sh status # advertised memory and caps; safe
|
||||||
|
bash rigmini.sh push # allocates until it stops — not on a machine you need
|
||||||
|
```
|
||||||
|
|
||||||
|
`rigmini.sh push` deliberately consumes memory. Run `status` first, and only run
|
||||||
|
`push` somewhere it is acceptable for other processes to be squeezed.
|
||||||
574
rig/standalone/rigdeps.sh
Executable file
574
rig/standalone/rigdeps.sh
Executable file
@@ -0,0 +1,574 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Put kind, tilt and kubectl on a machine that has none of them.
|
||||||
|
#
|
||||||
|
# The single file companion to rigmini.sh, for the same reason: rig installs its
|
||||||
|
# toolchain from ctrl/deps.sh reading ctrl/versions.env, and neither of those is
|
||||||
|
# going to a fresh AWS WorkSpace. The pins live inline here instead.
|
||||||
|
#
|
||||||
|
# What it will not do, deliberately:
|
||||||
|
#
|
||||||
|
# * no sudo, no apt, no yum. It writes ONLY into $OUT_BIN (default
|
||||||
|
# ~/.local/bin). Everything needing root — installing Docker, joining the
|
||||||
|
# docker group, raising inotify limits — is REPORTED for you to decide on.
|
||||||
|
# That is what makes it safe to run on a machine that already works.
|
||||||
|
# * no unverified download. Every artifact is checked against a SHA256 taken
|
||||||
|
# from the publisher's own release list. A mismatch aborts.
|
||||||
|
# * no guessing at another architecture. See ARCHITECTURE below.
|
||||||
|
#
|
||||||
|
# Two tiers, because "install the toolchain" is not one decision:
|
||||||
|
#
|
||||||
|
# core kubectl, jq — talk to a cluster someone else runs. Nothing that
|
||||||
|
# creates one. The right answer on a managed or corporate machine.
|
||||||
|
# dev core plus kind, tilt and ctlptl — build clusters and hot-reload
|
||||||
|
# into them. The default, and what you want on a workspace of your own.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# rigdeps.sh detect report the host, change nothing
|
||||||
|
# rigdeps.sh list the pinned versions and where they come from
|
||||||
|
# rigdeps.sh install [core|dev] detect, download, verify, install, report
|
||||||
|
# rigdeps.sh fetch [core|dev] [--to DIR] download + verify only
|
||||||
|
# rigdeps.sh verify run what is installed and see if it works
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
OUT_BIN="${OUT_BIN:-$HOME/.local/bin}"
|
||||||
|
|
||||||
|
# ── the pinned toolchain ───────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# ARCHITECTURE. These checksums are the upstream-published SHA256 of the
|
||||||
|
# **linux/amd64** artifact and of nothing else. An arm64 WorkSpace bundle needs
|
||||||
|
# a different binary with a different checksum, and this script refuses rather
|
||||||
|
# than reusing these — a checksum that is merely plausible is worse than none,
|
||||||
|
# because it turns a verified download into a ceremony.
|
||||||
|
#
|
||||||
|
# To bump a version, or to add arm64: take the checksum from the release's own
|
||||||
|
# published list, never from a download you did.
|
||||||
|
#
|
||||||
|
# curl -sSL https://github.com/<org>/<repo>/releases/download/<tag>/checksums.txt
|
||||||
|
#
|
||||||
|
# kubectl publishes its own instead, at <KUBECTL_URL>.sha256.
|
||||||
|
|
||||||
|
KIND_VERSION=v0.32.0
|
||||||
|
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||||
|
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
|
||||||
|
|
||||||
|
KUBECTL_VERSION=v1.36.3
|
||||||
|
KUBECTL_SHA256=ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336
|
||||||
|
KUBECTL_URL="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
|
||||||
|
|
||||||
|
TILT_VERSION=0.37.6
|
||||||
|
TILT_SHA256=e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6
|
||||||
|
TILT_URL="https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz"
|
||||||
|
|
||||||
|
# ctlptl creates a kind cluster WITH a local registry wired in, which is what
|
||||||
|
# keeps images off docker.io — an unqualified image name resolves to
|
||||||
|
# docker.io/library/<name>, and there is nothing structural stopping a push there.
|
||||||
|
CTLPTL_VERSION=0.9.4
|
||||||
|
CTLPTL_SHA256=c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e
|
||||||
|
CTLPTL_URL="https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.x86_64.tar.gz"
|
||||||
|
|
||||||
|
# Upstream's static build. Debian's jq is linked against libjq/libonig, which is
|
||||||
|
# fine on Debian and not portable anywhere else.
|
||||||
|
JQ_VERSION=1.8.2
|
||||||
|
JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
|
||||||
|
JQ_URL="https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
|
||||||
|
|
||||||
|
CORE_TOOLS="kubectl jq"
|
||||||
|
DEV_TOOLS="kind tilt ctlptl"
|
||||||
|
|
||||||
|
# No helm: every rig addon installs with `kubectl apply -f`, so nothing has ever
|
||||||
|
# invoked it. Add it the day something actually needs a chart.
|
||||||
|
|
||||||
|
# Collected as we go, printed by report_manual() at the very end. Anything that
|
||||||
|
# needs root or a decision lands here instead of being done.
|
||||||
|
MANUAL=()
|
||||||
|
|
||||||
|
# ── platform ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
Linux) ;;
|
||||||
|
*) echo "$(uname -s) is not Linux. These are linux binaries; nothing here" >&2
|
||||||
|
echo "would run even if it downloaded." >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
arch() {
|
||||||
|
case "$(uname -m)" in
|
||||||
|
x86_64|amd64) echo amd64 ;;
|
||||||
|
aarch64|arm64) echo arm64 ;;
|
||||||
|
*) uname -m ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# The pins above are amd64. Rather than download something that cannot execute
|
||||||
|
# and let it fail as "cannot execute binary file: Exec format error", say so
|
||||||
|
# here and hand over the commands that produce the right checksums.
|
||||||
|
require_amd64() {
|
||||||
|
local a; a=$(arch)
|
||||||
|
[ "$a" = "amd64" ] && return 0
|
||||||
|
cat >&2 <<EOF
|
||||||
|
This machine is ${a} ($(uname -m)); every pin in this script is linux/amd64.
|
||||||
|
|
||||||
|
Nothing here would run, so it does not download. To make an ${a} version, the
|
||||||
|
URLs need the ${a} artifact and the checksums need to come from each project's
|
||||||
|
own published list — not from these values, and not from a download you did:
|
||||||
|
|
||||||
|
curl -sSL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${a}/kubectl.sha256
|
||||||
|
curl -sSL https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/checksums.txt
|
||||||
|
curl -sSL https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt
|
||||||
|
|
||||||
|
Edit the pinned block at the top of this file with what those print.
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
|
# ── the tools this script itself needs ─────────────────────────────────────
|
||||||
|
|
||||||
|
# A fresh minimal image may genuinely have neither curl nor wget. Find out once,
|
||||||
|
# up front, rather than half way through the first download.
|
||||||
|
DL=""
|
||||||
|
pick_downloader() {
|
||||||
|
if command -v curl >/dev/null 2>&1; then DL=curl
|
||||||
|
elif command -v wget >/dev/null 2>&1; then DL=wget
|
||||||
|
else
|
||||||
|
echo "neither curl nor wget is installed, so nothing can be downloaded." >&2
|
||||||
|
echo "Install one first: $(pkg_install_cmd curl)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
download() {
|
||||||
|
local url="$1" out="$2"
|
||||||
|
case "$DL" in
|
||||||
|
curl) curl -fsSL --retry 3 -o "$out" "$url" ;;
|
||||||
|
wget) wget -q --tries=3 -O "$out" "$url" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# sha256sum is coreutils; shasum is the perl one that turns up on stripped
|
||||||
|
# images. Verification is not optional, so if neither exists that is fatal.
|
||||||
|
SHA=""
|
||||||
|
pick_sha() {
|
||||||
|
if command -v sha256sum >/dev/null 2>&1; then SHA=sha256sum
|
||||||
|
elif command -v shasum >/dev/null 2>&1; then SHA="shasum -a 256"
|
||||||
|
else
|
||||||
|
echo "no sha256sum and no shasum — downloads could not be verified." >&2
|
||||||
|
echo "Refusing to install unverified binaries." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── package manager, for the instructions only ─────────────────────────────
|
||||||
|
# This never runs a package manager. It names one so the reported action is
|
||||||
|
# something you can paste, on the distro you are actually on — an apt line on
|
||||||
|
# Amazon Linux 2 is a wrong answer dressed up as help.
|
||||||
|
|
||||||
|
pkg_install_cmd() {
|
||||||
|
local pkg="$1"
|
||||||
|
if command -v apt-get >/dev/null 2>&1; then echo "sudo apt-get update && sudo apt-get install -y $pkg"
|
||||||
|
elif command -v dnf >/dev/null 2>&1; then echo "sudo dnf install -y $pkg"
|
||||||
|
elif command -v yum >/dev/null 2>&1; then echo "sudo yum install -y $pkg"
|
||||||
|
elif command -v zypper >/dev/null 2>&1; then echo "sudo zypper install -y $pkg"
|
||||||
|
elif command -v apk >/dev/null 2>&1; then echo "sudo apk add $pkg"
|
||||||
|
else echo "install '$pkg' with this system's package manager"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_pkg() {
|
||||||
|
# Debian and Ubuntu call it docker.io; the RPM distros call it docker.
|
||||||
|
if command -v apt-get >/dev/null 2>&1; then echo docker.io; else echo docker; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── detect ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
detect() {
|
||||||
|
echo "host"
|
||||||
|
echo " kernel $(uname -r)"
|
||||||
|
echo " arch $(arch) ($(uname -m))"
|
||||||
|
[ -r /etc/os-release ] && \
|
||||||
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||||
|
if is_wsl; then echo " platform WSL"; else echo " platform native linux"; fi
|
||||||
|
|
||||||
|
local total_kb avail_kb
|
||||||
|
total_kb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo)
|
||||||
|
avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
|
||||||
|
printf " memory %d GB total, %d GB available\n" \
|
||||||
|
$((total_kb / 1024 / 1024)) $((avail_kb / 1024 / 1024))
|
||||||
|
if [ $((avail_kb / 1024 / 1024)) -lt 4 ]; then
|
||||||
|
echo " ! under 4 GB available — a cluster will struggle here."
|
||||||
|
echo " rigmini.sh says how much this box will actually give you."
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " install to $OUT_BIN"
|
||||||
|
detect_libc
|
||||||
|
detect_prereqs
|
||||||
|
detect_docker
|
||||||
|
detect_inotify
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# tilt is the one binary here that needs a recent glibc. MEASURED, not guessed:
|
||||||
|
# tilt 0.37.6 on Amazon Linux 2 (glibc 2.26) fails with
|
||||||
|
#
|
||||||
|
# /lib64/libc.so.6: version `GLIBC_2.34' not found (required by .../tilt)
|
||||||
|
#
|
||||||
|
# which names a symbol rather than the problem. Amazon Linux 2 is a stock
|
||||||
|
# WorkSpaces bundle, so this is the likely case, not an exotic one. Report the
|
||||||
|
# version now; `verify` catches the actual failure after installing.
|
||||||
|
detect_libc() {
|
||||||
|
local v=""
|
||||||
|
if command -v ldd >/dev/null 2>&1; then
|
||||||
|
v=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$' || true)
|
||||||
|
fi
|
||||||
|
if [ -z "$v" ]; then
|
||||||
|
echo " libc unknown (no ldd) — 'verify' is the real test"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " libc glibc $v"
|
||||||
|
if [ "$(printf '%s\n2.34\n' "$v" | sort -V | head -1)" != "2.34" ]; then
|
||||||
|
echo " ! older than glibc 2.34, which tilt needs. kubectl, kind, jq and"
|
||||||
|
echo " ctlptl are static or libc-only and work here; tilt will not start."
|
||||||
|
echo " Install the core tier, or run tilt from a container."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# What this script needs to do its own job. Reported here so `detect` answers
|
||||||
|
# "will install work?" instead of leaving you to find out one download in.
|
||||||
|
# Amazon Linux 2 ships without tar, which is exactly the surprise this catches.
|
||||||
|
detect_prereqs() {
|
||||||
|
local missing=""
|
||||||
|
if command -v curl >/dev/null 2>&1; then echo " download curl"
|
||||||
|
elif command -v wget >/dev/null 2>&1; then echo " download wget"
|
||||||
|
else echo " ! no curl and no wget — nothing can be downloaded"; missing+=" curl"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1; then
|
||||||
|
echo " checksums ok"
|
||||||
|
else
|
||||||
|
echo " ! no sha256sum or shasum — downloads could not be verified"
|
||||||
|
missing+=" coreutils"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v tar >/dev/null 2>&1 && command -v gzip >/dev/null 2>&1; then
|
||||||
|
echo " archives tar + gzip"
|
||||||
|
else
|
||||||
|
echo " ! no tar/gzip — tilt and ctlptl ship as tarballs, so the dev tier"
|
||||||
|
echo " cannot be unpacked. The core tier is two bare binaries and is fine."
|
||||||
|
missing+=" tar gzip"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$missing" ]; then
|
||||||
|
MANUAL+=("Install what this script needs to run at all:
|
||||||
|
$(pkg_install_cmd "${missing# }")")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_docker() {
|
||||||
|
# kind builds a cluster out of containers. Without a reachable daemon,
|
||||||
|
# everything here installs perfectly and then does nothing.
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
if [ -S /var/run/docker.sock ]; then
|
||||||
|
echo " docker socket present, no cli"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo " ! docker not installed — kind has nothing to build a cluster in"
|
||||||
|
MANUAL+=("Install Docker. It is the one real prerequisite, and the only
|
||||||
|
thing here that needs root:
|
||||||
|
$(pkg_install_cmd "$(docker_pkg)")
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
sudo usermod -aG docker \"\$USER\"
|
||||||
|
then log out and back in, so the new group applies to your shell.")
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if docker info >/dev/null 2>&1; then
|
||||||
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null)"
|
||||||
|
local n
|
||||||
|
n=$(docker ps --filter "label=io.x-k8s.kind.cluster" --format '{{.Names}}' 2>/dev/null | wc -l)
|
||||||
|
# Must be an `if`, not `[ ] && echo`: as the last statement here the
|
||||||
|
# latter returns 1 when the count is zero, and `set -e` kills the
|
||||||
|
# caller. That is the fresh-machine case, where it does most harm.
|
||||||
|
if [ "$n" -gt 0 ]; then
|
||||||
|
echo " - $n kind node container(s) already running"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ! docker cli present but the daemon is unreachable"
|
||||||
|
MANUAL+=("Start Docker, or add yourself to the docker group:
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
sudo usermod -aG docker \"\$USER\" # then log out and back in")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# kind and tilt both watch large trees. Distro defaults are far too low and the
|
||||||
|
# failure mode is silent: tilt simply stops noticing that files changed.
|
||||||
|
detect_inotify() {
|
||||||
|
local w i
|
||||||
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||||
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||||
|
echo " inotify watches=$w instances=$i"
|
||||||
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||||
|
echo " ! low — tilt will silently stop seeing file changes"
|
||||||
|
MANUAL+=("Raise the inotify limits (needs root):
|
||||||
|
echo -e 'fs.inotify.max_user_watches=524288\\nfs.inotify.max_user_instances=512' \\
|
||||||
|
| sudo tee /etc/sysctl.d/99-rig.conf
|
||||||
|
sudo sysctl --system")
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── fetch ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
verify_sha() {
|
||||||
|
local file="$1" want="$2" name="$3" got
|
||||||
|
got=$($SHA "$file" | awk '{print $1}')
|
||||||
|
if [ "$got" != "$want" ]; then
|
||||||
|
echo >&2
|
||||||
|
echo "CHECKSUM MISMATCH for $name — not installing it." >&2
|
||||||
|
echo " expected $want" >&2
|
||||||
|
echo " got $got" >&2
|
||||||
|
echo >&2
|
||||||
|
echo "Either the pin in this script is stale, or what arrived is not what" >&2
|
||||||
|
echo "the publisher released. Neither is worth guessing about." >&2
|
||||||
|
rm -f "$file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# fetch_bin <name> <url> <sha256> <dest-dir> — a bare binary
|
||||||
|
fetch_bin() {
|
||||||
|
local name="$1" url="$2" sha="$3" dest="$4"
|
||||||
|
local tmp="$dest/.$name.tmp"
|
||||||
|
printf ' %-8s ' "$name"
|
||||||
|
download "$url" "$tmp"
|
||||||
|
verify_sha "$tmp" "$sha" "$name"
|
||||||
|
mv "$tmp" "$dest/$name"
|
||||||
|
chmod +x "$dest/$name"
|
||||||
|
echo "ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
# fetch_tgz <name> <url> <sha256> <dest-dir> <path-inside> <strip>
|
||||||
|
# Archive layouts differ, so the caller says which. tilt and ctlptl both ship
|
||||||
|
# the binary at the archive root, hence strip=0.
|
||||||
|
fetch_tgz() {
|
||||||
|
local name="$1" url="$2" sha="$3" dest="$4" inner="$5" strip="$6"
|
||||||
|
local tmp="$dest/.$name.tgz"
|
||||||
|
printf ' %-8s ' "$name"
|
||||||
|
download "$url" "$tmp"
|
||||||
|
verify_sha "$tmp" "$sha" "$name"
|
||||||
|
# --no-same-owner: some archives ship as uid 1001, and extracting as root
|
||||||
|
# would otherwise restore an owner that is not you.
|
||||||
|
tar -xzf "$tmp" -C "$dest" --strip-components="$strip" --no-same-owner "$inner"
|
||||||
|
rm -f "$tmp"
|
||||||
|
chmod +x "$dest/$name"
|
||||||
|
echo "ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch() {
|
||||||
|
local dest="$OUT_BIN" tier="dev"
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--to) dest="${2:?--to needs a directory}"; shift 2 ;;
|
||||||
|
core|dev) tier="$1"; shift ;;
|
||||||
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
mkdir -p "$dest"
|
||||||
|
|
||||||
|
if ! command -v tar >/dev/null 2>&1 && [ "$tier" = "dev" ]; then
|
||||||
|
echo "tar is missing, and tilt and ctlptl ship as tarballs." >&2
|
||||||
|
echo " $(pkg_install_cmd tar)" >&2
|
||||||
|
echo "Or install the core tier, which is two bare binaries: $0 install core" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "fetching '$tier' into $dest (verifying every checksum)"
|
||||||
|
fetch_bin kubectl "$KUBECTL_URL" "$KUBECTL_SHA256" "$dest"
|
||||||
|
fetch_bin jq "$JQ_URL" "$JQ_SHA256" "$dest"
|
||||||
|
if [ "$tier" = "dev" ]; then
|
||||||
|
fetch_bin kind "$KIND_URL" "$KIND_SHA256" "$dest"
|
||||||
|
fetch_tgz tilt "$TILT_URL" "$TILT_SHA256" "$dest" tilt 0
|
||||||
|
fetch_tgz ctlptl "$CTLPTL_URL" "$CTLPTL_SHA256" "$dest" ctlptl 0
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── verify ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
tier_tools() { [ "$1" = "core" ] && echo "$CORE_TOOLS" || echo "$CORE_TOOLS $DEV_TOOLS"; }
|
||||||
|
|
||||||
|
# Downloading a verified binary proves it is the right file, not that this
|
||||||
|
# machine can run it. On an old distro tilt fails here, with a linker error
|
||||||
|
# about a missing symbol, and finding that out now beats finding out during a
|
||||||
|
# first cluster build.
|
||||||
|
verify_tools() {
|
||||||
|
local tier="${1:-dev}" b bin out rc broke=0
|
||||||
|
echo "checking that each one actually runs"
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
bin="$OUT_BIN/$b"
|
||||||
|
if [ ! -x "$bin" ]; then
|
||||||
|
printf ' %-8s not installed\n' "$b"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
rc=0
|
||||||
|
case "$b" in
|
||||||
|
kubectl) out=$("$bin" version --client 2>&1 | head -1) || rc=$? ;;
|
||||||
|
jq) out=$("$bin" --version 2>&1 | head -1) || rc=$? ;;
|
||||||
|
*) out=$("$bin" version 2>&1 | head -1) || rc=$? ;;
|
||||||
|
esac
|
||||||
|
if [ "$rc" -eq 0 ]; then
|
||||||
|
printf ' %-8s %s\n' "$b" "$out"
|
||||||
|
else
|
||||||
|
printf ' ! %-6s does not run here: %s\n' "$b" "$out"
|
||||||
|
broke=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$broke" -eq 1 ]; then
|
||||||
|
echo
|
||||||
|
echo " A binary that downloads and verifies but will not start is almost"
|
||||||
|
echo " always this distro's libc being older than the release needs."
|
||||||
|
echo " 'detect' prints the glibc version. The core tier (kubectl + jq)"
|
||||||
|
echo " has no such dependency and will work regardless."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── install ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Installing into a directory early in PATH silently replaces whatever the
|
||||||
|
# machine was already using, which on a shared or corporate machine can break
|
||||||
|
# unrelated work — kubectl more than one minor away from its cluster is the
|
||||||
|
# common one. Say so; never decide it.
|
||||||
|
warn_shadowing() {
|
||||||
|
local b existing shadowed="" tier="${1:-dev}"
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) return 0 ;; # not on PATH, so nothing is being shadowed yet
|
||||||
|
esac
|
||||||
|
for b in $(tier_tools "$tier"); do
|
||||||
|
[ -x "$OUT_BIN/$b" ] || continue
|
||||||
|
existing=$(PATH=$(echo "$PATH" | tr ':' '\n' | grep -vx "$OUT_BIN" | paste -sd:) \
|
||||||
|
command -v "$b" 2>/dev/null || true)
|
||||||
|
[ -n "$existing" ] || continue
|
||||||
|
[ "$existing" = "$OUT_BIN/$b" ] && continue
|
||||||
|
shadowed+=" $b $existing"$'\n'
|
||||||
|
done
|
||||||
|
[ -n "$shadowed" ] || return 0
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo " ! these were already installed elsewhere and are now shadowed:"
|
||||||
|
printf '%s' "$shadowed"
|
||||||
|
MANUAL+=("Decide which toolchain wins. To keep the previous one:
|
||||||
|
rm -f $(for b in $(tier_tools "$tier"); do printf '%s ' "$OUT_BIN/$b"; done)
|
||||||
|
Or install somewhere private instead:
|
||||||
|
OUT_BIN=\$PWD/bin $0 install")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
report_manual() {
|
||||||
|
echo
|
||||||
|
if [ ${#MANUAL[@]} -eq 0 ]; then
|
||||||
|
echo "nothing left to do by hand."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "host actions this cannot perform (${#MANUAL[@]}):"
|
||||||
|
echo
|
||||||
|
local n=1 m
|
||||||
|
for m in "${MANUAL[@]}"; do
|
||||||
|
echo " $n. $m"
|
||||||
|
echo
|
||||||
|
n=$((n + 1))
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
install() {
|
||||||
|
local tier="${1:-dev}"
|
||||||
|
detect
|
||||||
|
echo
|
||||||
|
fetch "$tier"
|
||||||
|
echo
|
||||||
|
verify_tools "$tier"
|
||||||
|
warn_shadowing "$tier"
|
||||||
|
|
||||||
|
if [ "$tier" = "core" ]; then
|
||||||
|
echo
|
||||||
|
echo " core tier: no kind, tilt or ctlptl. '$0 install dev' adds them."
|
||||||
|
fi
|
||||||
|
|
||||||
|
case ":${PATH}:" in
|
||||||
|
*":$OUT_BIN:"*) ;;
|
||||||
|
*) MANUAL+=("Put the toolchain on your PATH — add to ~/.bashrc:
|
||||||
|
export PATH=\"${OUT_BIN}:\$PATH\"
|
||||||
|
then: source ~/.bashrc") ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
report_manual
|
||||||
|
|
||||||
|
if [ "$tier" = "dev" ]; then
|
||||||
|
echo "Once Docker is reachable and this is on PATH:"
|
||||||
|
echo
|
||||||
|
echo " kind create cluster --name scratch"
|
||||||
|
echo " kubectl cluster-info --context kind-scratch"
|
||||||
|
echo " kind delete cluster --name scratch"
|
||||||
|
echo
|
||||||
|
echo "That round trip is the real test that this machine can host a rig."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
list() {
|
||||||
|
echo "pinned, linux/amd64 only:"
|
||||||
|
printf ' %-8s %s\n' kubectl "$KUBECTL_VERSION"
|
||||||
|
printf ' %-8s %s\n' jq "$JQ_VERSION"
|
||||||
|
printf ' %-8s %s\n' kind "$KIND_VERSION"
|
||||||
|
printf ' %-8s %s\n' tilt "$TILT_VERSION"
|
||||||
|
printf ' %-8s %s\n' ctlptl "$CTLPTL_VERSION"
|
||||||
|
echo
|
||||||
|
echo " core = $CORE_TOOLS"
|
||||||
|
echo " dev = $CORE_TOOLS $DEV_TOOLS"
|
||||||
|
echo
|
||||||
|
echo "Checksums are pinned in the block at the top of this file. To bump one,"
|
||||||
|
echo "take the new checksum from the publisher's own release list — the header"
|
||||||
|
echo "comment has the exact commands."
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── main ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
|
||||||
|
case "${1:-install}" in
|
||||||
|
detect) detect; report_manual ;;
|
||||||
|
list) list ;;
|
||||||
|
verify) verify_tools "${2:-dev}" ;;
|
||||||
|
fetch) shift; require_amd64; pick_downloader; pick_sha; fetch "$@" ;;
|
||||||
|
install) shift; require_amd64; pick_downloader; pick_sha; install "${1:-dev}" ;;
|
||||||
|
*) echo "usage: $0 [detect|list|install|fetch|verify]" >&2
|
||||||
|
echo " install [core|dev] (default dev)" >&2
|
||||||
|
echo " fetch [core|dev] [--to DIR]" >&2
|
||||||
|
echo " OUT_BIN=<dir> overrides the install directory" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
635
rig/standalone/rigmini.sh
Executable file
635
rig/standalone/rigmini.sh
Executable file
@@ -0,0 +1,635 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# How much memory this box will actually give you before something dies.
|
||||||
|
#
|
||||||
|
# rig answers this for a machine it is installed on. This is the single file
|
||||||
|
# version, for a machine rig is not going to: paste it onto a fresh AWS
|
||||||
|
# WorkSpace, an EC2 box or a container, run it, and get the same numbers in the
|
||||||
|
# same order so two machines can be read side by side.
|
||||||
|
#
|
||||||
|
# There are two numbers and they are rarely the same. `status` reports what the
|
||||||
|
# machine ADVERTISES and what is quietly capping it. `push` finds what it will
|
||||||
|
# SURVIVE, by allocating until it stops.
|
||||||
|
#
|
||||||
|
# The gap between them is the whole reason this exists. Under WSL the cap lives
|
||||||
|
# in .wslconfig; in a container or a managed workspace it is a cgroup limit, and
|
||||||
|
# there /proc/meminfo reports the HOST's memory while the kernel kills you at a
|
||||||
|
# fraction of it. A script that only read MemTotal would confidently report 32 GB
|
||||||
|
# on a box that OOMs at 2.
|
||||||
|
#
|
||||||
|
# Reports and instructs. It never raises a limit, frees anything, writes a
|
||||||
|
# config or installs a package — on a machine you are still evaluating, a probe
|
||||||
|
# that changes what it is measuring is worse than no probe.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# rigmini.sh status what it has, what caps it
|
||||||
|
# rigmini.sh push [--to GB] [--to-oom] climb until it stops
|
||||||
|
# rigmini.sh all [--budget GB] both, then the verdict
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# ── defaults ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push().
|
||||||
|
STEP_EXPLICIT=no # whether --step was given, which turns the scaling off.
|
||||||
|
TO_MB="" # --to: stop here regardless. Empty means no hard cap.
|
||||||
|
TO_OOM=no # --to-oom: opt in to running until the kernel intervenes.
|
||||||
|
BUDGET_GB=6 # what the rig data profile is assumed to want; see all().
|
||||||
|
BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below.
|
||||||
|
|
||||||
|
# ── platform ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
||||||
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
||||||
|
# the tooling. Detectable, so name it instead.
|
||||||
|
require_linux() {
|
||||||
|
case "$(uname -s)" in
|
||||||
|
MINGW*|MSYS*|CYGWIN*)
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
||||||
|
|
||||||
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
||||||
|
|
||||||
|
wsl --install
|
||||||
|
|
||||||
|
That enables Windows features and needs a reboot, so it is not something this
|
||||||
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
||||||
|
this from there.
|
||||||
|
EOF
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Everything below reads /proc. Without it there is nothing to measure, and
|
||||||
|
# failing here beats printing a page of empty fields.
|
||||||
|
if [ ! -r /proc/meminfo ]; then
|
||||||
|
echo "no readable /proc/meminfo — this needs a Linux kernel." >&2
|
||||||
|
echo "On macOS or a BSD none of the numbers below exist." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
||||||
|
|
||||||
|
is_container() {
|
||||||
|
[ -f /.dockerenv ] && return 0
|
||||||
|
grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
platform() {
|
||||||
|
if is_wsl; then echo WSL
|
||||||
|
elif is_container; then echo container
|
||||||
|
else echo "native linux"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── reading memory ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
||||||
|
|
||||||
|
# MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on
|
||||||
|
# corporate images — need the estimate it replaced, which is worse but not wrong.
|
||||||
|
avail_meminfo_mb() {
|
||||||
|
if grep -q '^MemAvailable:' /proc/meminfo; then
|
||||||
|
mb MemAvailable
|
||||||
|
else
|
||||||
|
awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Where a cgroup records this cgroup's own limit and usage. Set once by
|
||||||
|
# find_cgroup, because every later reading needs both and hunting for the files
|
||||||
|
# on each call would be the slow part of the poll loop.
|
||||||
|
CG_MAX_FILE=""
|
||||||
|
CG_CUR_FILE=""
|
||||||
|
CG_VERSION=""
|
||||||
|
|
||||||
|
find_cgroup() {
|
||||||
|
local rel
|
||||||
|
|
||||||
|
# Inside a container the cgroup namespace makes the top of the tree BE the
|
||||||
|
# container's own cgroup, so the unqualified path is already the right one.
|
||||||
|
# On a host it is the root cgroup, which is never limited — hence the second
|
||||||
|
# attempt via /proc/self/cgroup, which names the slice this shell is in.
|
||||||
|
if [ -r /sys/fs/cgroup/memory.max ]; then
|
||||||
|
CG_VERSION=v2
|
||||||
|
CG_MAX_FILE=/sys/fs/cgroup/memory.max
|
||||||
|
CG_CUR_FILE=/sys/fs/cgroup/memory.current
|
||||||
|
elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
|
||||||
|
CG_VERSION=v1
|
||||||
|
CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes
|
||||||
|
CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes
|
||||||
|
fi
|
||||||
|
|
||||||
|
rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||||
|
if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then
|
||||||
|
CG_VERSION=v2
|
||||||
|
CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max"
|
||||||
|
CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
||||||
|
if [ -n "$rel" ] && [ "$rel" != "/" ] \
|
||||||
|
&& [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then
|
||||||
|
CG_VERSION=v1
|
||||||
|
CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes"
|
||||||
|
CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes"
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# The cap in MB, or "" when there is none worth reporting. v2 spells unlimited
|
||||||
|
# "max"; v1 spells it as a number near 2^63, which is why this compares against
|
||||||
|
# MemTotal rather than testing for a magic value — a "limit" above the machine's
|
||||||
|
# own memory is not a limit, however it is written.
|
||||||
|
cgroup_cap_mb() {
|
||||||
|
local raw cap
|
||||||
|
[ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; }
|
||||||
|
raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max)
|
||||||
|
[ "$raw" = "max" ] && { echo ""; return 0; }
|
||||||
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
cap=$((raw / 1024 / 1024))
|
||||||
|
[ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; }
|
||||||
|
echo "$cap"
|
||||||
|
}
|
||||||
|
|
||||||
|
cgroup_used_mb() {
|
||||||
|
local raw
|
||||||
|
[ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; }
|
||||||
|
raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "")
|
||||||
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
echo $((raw / 1024 / 1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
# ulimit -v is a per-process address-space cap. It stops YOU long before the box
|
||||||
|
# does, and because it is inherited from a login shell it is easy to hit without
|
||||||
|
# knowing it is set.
|
||||||
|
ulimit_v_mb() {
|
||||||
|
local v; v=$(ulimit -v 2>/dev/null || echo unlimited)
|
||||||
|
[ "$v" = "unlimited" ] && { echo ""; return 0; }
|
||||||
|
case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
||||||
|
echo $((v / 1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
# The number everything else is about: the lowest of the things that can stop
|
||||||
|
# you. Printed at the end of `status` and used as the sanity bound in `push`.
|
||||||
|
effective_ceiling_mb() {
|
||||||
|
local c; c=$(mb MemTotal)
|
||||||
|
local cap; cap=$(cgroup_cap_mb)
|
||||||
|
local ul; ul=$(ulimit_v_mb)
|
||||||
|
[ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap"
|
||||||
|
[ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul"
|
||||||
|
echo "$c"
|
||||||
|
}
|
||||||
|
|
||||||
|
# How much room is left RIGHT NOW, from whichever accounting actually governs.
|
||||||
|
# In a capped container /proc/meminfo describes the host and is worse than
|
||||||
|
# useless for this — it would report tens of gigabytes free on a box that is one
|
||||||
|
# allocation from being killed.
|
||||||
|
headroom_mb() {
|
||||||
|
local cap used
|
||||||
|
cap=$(cgroup_cap_mb)
|
||||||
|
used=$(cgroup_used_mb)
|
||||||
|
if [ -n "$cap" ] && [ -n "$used" ]; then
|
||||||
|
echo $(( cap - used ))
|
||||||
|
else
|
||||||
|
avail_meminfo_mb
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── status ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
||||||
|
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
||||||
|
# Windows, then fall back to whichever profile actually owns a config.
|
||||||
|
wslconfig_path() {
|
||||||
|
local profile winpath found
|
||||||
|
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
||||||
|
case "$profile" in
|
||||||
|
""|*%*) ;;
|
||||||
|
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
||||||
|
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
||||||
|
echo "$winpath/.wslconfig"; return 0
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
||||||
|
[ -n "$found" ] && echo "$found"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
hogs() {
|
||||||
|
echo " holding the most:"
|
||||||
|
ps -eo rss,comm --sort=-rss 2>/dev/null \
|
||||||
|
| awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
status() {
|
||||||
|
local total avail swap_total swap_free cap ul cur
|
||||||
|
|
||||||
|
echo "host"
|
||||||
|
echo " platform $(platform)"
|
||||||
|
echo " kernel $(uname -r)"
|
||||||
|
[ -r /etc/os-release ] && \
|
||||||
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
||||||
|
echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)"
|
||||||
|
|
||||||
|
# ── the caps first, because they decide what the totals below are worth ──
|
||||||
|
echo
|
||||||
|
echo "caps"
|
||||||
|
cap=$(cgroup_cap_mb)
|
||||||
|
if [ -n "$cap" ]; then
|
||||||
|
cur=$(cgroup_used_mb)
|
||||||
|
echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)"
|
||||||
|
echo " ! /proc/meminfo below describes the HOST, not this cgroup."
|
||||||
|
echo " $(mb MemTotal) MB total is not yours; ${cap} MB is."
|
||||||
|
elif [ -n "$CG_VERSION" ]; then
|
||||||
|
echo " cgroup none (${CG_VERSION} present, no memory limit set)"
|
||||||
|
else
|
||||||
|
echo " cgroup no memory controller found"
|
||||||
|
fi
|
||||||
|
|
||||||
|
ul=$(ulimit_v_mb)
|
||||||
|
if [ -n "$ul" ]; then
|
||||||
|
echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell"
|
||||||
|
echo " it stops this process long before the machine runs out"
|
||||||
|
else
|
||||||
|
echo " ulimit -v unlimited"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# overcommit_memory=0 is the default heuristic: a large allocation is
|
||||||
|
# granted on a guess, and the reckoning arrives later as an OOM kill rather
|
||||||
|
# than as a failed malloc. It is why `push` touches every page it asks for.
|
||||||
|
local om or_
|
||||||
|
om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?')
|
||||||
|
or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?')
|
||||||
|
case "$om" in
|
||||||
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;;
|
||||||
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;;
|
||||||
|
2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;;
|
||||||
|
*) echo " overcommit ${om}" ;;
|
||||||
|
esac
|
||||||
|
[ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for"
|
||||||
|
|
||||||
|
# ── what it says it has ──
|
||||||
|
total=$(mb MemTotal); avail=$(avail_meminfo_mb)
|
||||||
|
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
||||||
|
echo
|
||||||
|
echo "memory"
|
||||||
|
echo " total ${total} MB"
|
||||||
|
echo " available ${avail} MB"
|
||||||
|
echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
||||||
|
if [ "$swap_total" -eq 0 ]; then
|
||||||
|
echo " - no swap: this box has no cushion. It goes from fine to OOM-killed"
|
||||||
|
echo " with nothing in between, which is the abrupt failure you get in a VM."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB,
|
||||||
|
# and the resulting failure names neither shm nor the size.
|
||||||
|
if [ -d /dev/shm ]; then
|
||||||
|
local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}')
|
||||||
|
if [ -n "$shm" ]; then
|
||||||
|
if [ "$shm" -le 64 ]; then
|
||||||
|
echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB"
|
||||||
|
echo " is docker's default. Raise it with --shm-size when the cabinet fails."
|
||||||
|
else
|
||||||
|
echo " /dev/shm ${shm} MB"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "disk"
|
||||||
|
local d
|
||||||
|
for d in / /tmp /var/lib/docker; do
|
||||||
|
[ -d "$d" ] || continue
|
||||||
|
df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}'
|
||||||
|
done
|
||||||
|
|
||||||
|
# kind and Tilt both watch large trees, and the failure mode is silent:
|
||||||
|
# they simply stop noticing file changes. Cheap to report while we are here.
|
||||||
|
local w i
|
||||||
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
||||||
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
||||||
|
echo
|
||||||
|
echo "tooling"
|
||||||
|
echo " inotify watches=$w instances=$i"
|
||||||
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
||||||
|
echo " ! low — anything watching files will silently stop seeing changes"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
if [ -S /var/run/docker.sock ]; then
|
||||||
|
echo " docker socket present, no cli"
|
||||||
|
else
|
||||||
|
echo " docker not installed"
|
||||||
|
fi
|
||||||
|
elif docker info >/dev/null 2>&1; then
|
||||||
|
local n
|
||||||
|
n=$(docker ps -q 2>/dev/null | wc -l)
|
||||||
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running"
|
||||||
|
else
|
||||||
|
echo " ! docker cli present but the daemon is unreachable"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# WSL keeps its cap on the Windows side, in a file this shell can read but
|
||||||
|
# not usefully apply — the change costs a full VM restart. Report it, and
|
||||||
|
# report the commonest mistake, which is editing it and not restarting.
|
||||||
|
if is_wsl; then
|
||||||
|
local cfg conf
|
||||||
|
cfg=$(wslconfig_path)
|
||||||
|
echo
|
||||||
|
echo "wsl"
|
||||||
|
if [ -z "$cfg" ]; then
|
||||||
|
echo " ! cannot tell which Windows profile owns .wslconfig"
|
||||||
|
else
|
||||||
|
echo " config $cfg"
|
||||||
|
conf=$(sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$cfg" 2>/dev/null \
|
||||||
|
| tail -1 | tr -d '[:space:]')
|
||||||
|
if [ -n "$conf" ]; then
|
||||||
|
echo " configured $conf (booted ${total} MB)"
|
||||||
|
echo " - if those disagree the edit has not been applied."
|
||||||
|
echo " From a WINDOWS terminal: wsl --shutdown"
|
||||||
|
else
|
||||||
|
echo " configured no memory= set (WSL defaults to half the host RAM, or 8 GB,"
|
||||||
|
echo " whichever is less — which is where your Airflow ceiling comes from)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "effective ceiling $(effective_ceiling_mb) MB"
|
||||||
|
echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box"
|
||||||
|
echo " claims. 'push' measures what it will actually hand over."
|
||||||
|
|
||||||
|
[ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; }
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── push ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
STATE=""
|
||||||
|
CHILD=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then
|
||||||
|
kill -KILL "$CHILD" 2>/dev/null || true
|
||||||
|
wait "$CHILD" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
[ -n "$STATE" ] && rm -f "$STATE"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# The child allocates and stops itself; the parent only watches. That split is
|
||||||
|
# the point: under --to-oom the allocating process is expected to be killed, and
|
||||||
|
# something has to survive to say how far it got.
|
||||||
|
allocator() {
|
||||||
|
# Raise our own OOM score to the maximum so the kernel picks THIS process
|
||||||
|
# first. Raising needs no privilege (only lowering does). Without it, the
|
||||||
|
# kernel is free to choose your shell, your ssh session or dockerd — on a
|
||||||
|
# box you are still using, that is not an acceptable coin toss.
|
||||||
|
echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true
|
||||||
|
|
||||||
|
local arr=() held=0 i=0 rss swapped avail first_swap=0
|
||||||
|
local bytes=$((STEP_MB * 1024 * 1024))
|
||||||
|
local swap_used_start
|
||||||
|
swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) ))
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
# Written STRAIGHT INTO the array element. The obvious spelling —
|
||||||
|
# build one chunk and `arr+=("$chunk")` — costs three copies per step,
|
||||||
|
# not one: the template stays resident, expanding "$chunk" makes a
|
||||||
|
# temporary word, and the append makes the element. A 128 MB step then
|
||||||
|
# needs 384 MB transiently, and on a small box it is killed on the
|
||||||
|
# first append while reporting a third of the true ceiling.
|
||||||
|
#
|
||||||
|
# printf -v into a subscript also means every page is written, so it is
|
||||||
|
# resident rather than merely promised — the only kind of allocation
|
||||||
|
# that measures anything under heuristic overcommit.
|
||||||
|
printf -v "arr[$i]" '%*s' "$bytes" ''
|
||||||
|
i=$((i + 1)); held=$((held + STEP_MB))
|
||||||
|
|
||||||
|
rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0)
|
||||||
|
avail=$(headroom_mb)
|
||||||
|
swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start ))
|
||||||
|
[ "$swapped" -lt 0 ] && swapped=0
|
||||||
|
|
||||||
|
printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \
|
||||||
|
"$held" "$rss" "$avail" "$swapped"
|
||||||
|
printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE"
|
||||||
|
|
||||||
|
# Worth calling out separately from the ceiling: this is where the box
|
||||||
|
# stops being fast and starts being unusable, which for a scheduler is
|
||||||
|
# a different and earlier problem than being killed.
|
||||||
|
if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then
|
||||||
|
first_swap=$held
|
||||||
|
echo " - first swap page at ${held} MB — past here it works but crawls"
|
||||||
|
echo "swapat $held" >> "$STATE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then
|
||||||
|
echo "stop reached-the-cap" >> "$STATE"; return 0
|
||||||
|
fi
|
||||||
|
if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then
|
||||||
|
echo "stop floor" >> "$STATE"; return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
push() {
|
||||||
|
local total ceiling rc=0 last held rss swapat stop
|
||||||
|
total=$(mb MemTotal)
|
||||||
|
ceiling=$(effective_ceiling_mb)
|
||||||
|
|
||||||
|
# A step is worth about a sixty-fourth of the ceiling: enough resolution to
|
||||||
|
# find the edge, few enough lines to read, and small enough that the
|
||||||
|
# transient cost of one allocation never dominates a small box. A fixed
|
||||||
|
# size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB.
|
||||||
|
if [ "$STEP_EXPLICIT" = no ]; then
|
||||||
|
STEP_MB=$(( ceiling / 64 ))
|
||||||
|
[ "$STEP_MB" -lt 4 ] && STEP_MB=4
|
||||||
|
[ "$STEP_MB" -gt 256 ] && STEP_MB=256
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Stop with a cushion rather than riding it to the kill. How big a cushion
|
||||||
|
# depends on what it is protecting. Under a cgroup cap, running out kills
|
||||||
|
# only this container's own processes, so it need cover no more than the
|
||||||
|
# shell that prints the result — and a 512 MB cushion on a 1 GB box would
|
||||||
|
# halve the answer. On a host there is everything else to protect, and the
|
||||||
|
# OOM killer does not promise to pick the process that caused the problem.
|
||||||
|
if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi
|
||||||
|
[ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 ))
|
||||||
|
|
||||||
|
STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX")
|
||||||
|
trap cleanup EXIT
|
||||||
|
# INT kills the child and lets the summary below print anyway, so an
|
||||||
|
# impatient Ctrl-C still tells you how far it got — and, more importantly,
|
||||||
|
# still gives the memory back.
|
||||||
|
trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT
|
||||||
|
|
||||||
|
echo "push"
|
||||||
|
echo " step ${STEP_MB} MB per allocation, every page touched"
|
||||||
|
echo " ceiling ${ceiling} MB claimed"
|
||||||
|
if [ -n "$TO_MB" ]; then
|
||||||
|
echo " stopping at ${TO_MB} MB (--to)"
|
||||||
|
elif [ "$TO_OOM" = yes ]; then
|
||||||
|
echo " ! stopping only when the kernel stops it (--to-oom)"
|
||||||
|
echo " the allocating child is marked as the preferred OOM victim,"
|
||||||
|
echo " but nothing about an OOM kill is entirely polite. Not on a box"
|
||||||
|
echo " running anything you mind losing."
|
||||||
|
else
|
||||||
|
echo " stopping when headroom drops below ${FLOOR_MB} MB"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
|
allocator &
|
||||||
|
CHILD=$!
|
||||||
|
wait "$CHILD" || rc=$?
|
||||||
|
CHILD=""
|
||||||
|
trap - INT
|
||||||
|
|
||||||
|
last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true)
|
||||||
|
held=$(echo "$last" | awk '{print $1}')
|
||||||
|
rss=$(echo "$last" | awk '{print $2}')
|
||||||
|
swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||||
|
stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ -z "$held" ]; then
|
||||||
|
echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —"
|
||||||
|
echo " try a smaller --step, or check ulimit -v in 'status'."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " reached ${rss:-$held} MB resident"
|
||||||
|
[ -n "$swapat" ] && echo " swapping from ${swapat} MB"
|
||||||
|
|
||||||
|
case "$stop" in
|
||||||
|
reached-the-cap)
|
||||||
|
echo " outcome stopped at the --to cap, not at a limit."
|
||||||
|
echo " The box held ${TO_MB} MB without complaint; there is more." ;;
|
||||||
|
floor)
|
||||||
|
echo " outcome stopped with a cushion intact, by choice."
|
||||||
|
echo " The real ceiling is higher — --to-oom finds it, at the"
|
||||||
|
echo " cost of an actual OOM kill." ;;
|
||||||
|
interrupted)
|
||||||
|
echo " outcome interrupted at ${rss:-$held} MB — where you stopped it,"
|
||||||
|
echo " not where the box did." ;;
|
||||||
|
*)
|
||||||
|
# No stop line means the child did not decide to stop: it was ended.
|
||||||
|
if [ "$rc" -ge 128 ]; then
|
||||||
|
echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB."
|
||||||
|
elif [ "$rc" -ne 0 ]; then
|
||||||
|
echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})."
|
||||||
|
echo " bash could not get the next chunk — an honest malloc"
|
||||||
|
echo " failure rather than a kill. That is the strict-overcommit"
|
||||||
|
echo " or ulimit path."
|
||||||
|
else
|
||||||
|
echo " outcome ended at ${rss:-$held} MB."
|
||||||
|
fi
|
||||||
|
local ev
|
||||||
|
ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true)
|
||||||
|
if [ -n "$ev" ]; then
|
||||||
|
echo " kernel ${ev#*] }"
|
||||||
|
else
|
||||||
|
echo " - dmesg is unreadable here (dmesg_restrict, or no privilege),"
|
||||||
|
echo " so the kill cannot be confirmed from this side. The number stands."
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# The gap between the claim and the measurement is the finding — but only
|
||||||
|
# when the BOX chose where to stop. An empty $stop means the child was ended
|
||||||
|
# rather than deciding to end; anything else (--to, the floor) is a stop we
|
||||||
|
# asked for, and flagging those as short of the ceiling would put a warning
|
||||||
|
# on every deliberately small run.
|
||||||
|
local got="${rss:-$held}"
|
||||||
|
echo
|
||||||
|
if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then
|
||||||
|
echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it."
|
||||||
|
echo " Something is taking the difference. 'status' names the candidates:"
|
||||||
|
echo " a cgroup cap, ulimit -v, or memory already resident."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── all ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
all() {
|
||||||
|
status
|
||||||
|
echo
|
||||||
|
echo "────────────────────────────────────────────────────────────"
|
||||||
|
echo
|
||||||
|
push
|
||||||
|
|
||||||
|
local got budget_mb ceiling
|
||||||
|
budget_mb=$(( BUDGET_GB * 1024 ))
|
||||||
|
ceiling=$(effective_ceiling_mb)
|
||||||
|
got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true)
|
||||||
|
[ -n "$got" ] || got=0
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "verdict"
|
||||||
|
echo " budget ${BUDGET_GB} GB for kind + postgres + redis + airflow"
|
||||||
|
# Only worth explaining while it is still a guess. Once --budget is given
|
||||||
|
# the number came from somewhere better than this reasoning, and repeating
|
||||||
|
# the derivation would describe a figure that is no longer in use.
|
||||||
|
if [ "$BUDGET_EXPLICIT" = no ]; then
|
||||||
|
echo " - that is 2 GB per kind node, which is rig's own figure, plus about"
|
||||||
|
echo " 4 GB for the three cabinets. THE 4 GB IS AN ESTIMATE, not something"
|
||||||
|
echo " measured. Re-run with --budget once you have watched the real thing."
|
||||||
|
fi
|
||||||
|
echo " measured ${got} MB handed over"
|
||||||
|
|
||||||
|
if [ "$got" -ge "$budget_mb" ]; then
|
||||||
|
echo " fits, with $(( got - budget_mb )) MB spare."
|
||||||
|
if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then
|
||||||
|
echo " - under 30% spare is thin for a scheduler. Airflow's memory use"
|
||||||
|
echo " is spiky, and the spikes are what get killed."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ! short by $(( budget_mb - got )) MB."
|
||||||
|
if [ "$ceiling" -ge "$budget_mb" ]; then
|
||||||
|
echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it."
|
||||||
|
echo " Free something, or read the caps section again."
|
||||||
|
else
|
||||||
|
echo " The box does not have it to give. A bigger bundle, or a smaller"
|
||||||
|
echo " profile: PROFILE=minimal drops the cabinets entirely."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── main ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
parse_flags() {
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;;
|
||||||
|
--to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;;
|
||||||
|
--step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;;
|
||||||
|
--to-oom) TO_OOM=yes; shift ;;
|
||||||
|
--budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;;
|
||||||
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then
|
||||||
|
echo "--to and --to-oom contradict each other: one stops early, the other" >&2
|
||||||
|
echo "refuses to stop at all. Pick one." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
require_linux
|
||||||
|
find_cgroup
|
||||||
|
|
||||||
|
cmd="${1:-status}"
|
||||||
|
[ $# -gt 0 ] && shift
|
||||||
|
|
||||||
|
case "$cmd" in
|
||||||
|
status) parse_flags "$@"; status ;;
|
||||||
|
push) parse_flags "$@"; push ;;
|
||||||
|
all) parse_flags "$@"; all ;;
|
||||||
|
*) echo "usage: $0 [status|push|all]" >&2
|
||||||
|
echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2
|
||||||
|
echo " all [--budget GB]" >&2
|
||||||
|
exit 1 ;;
|
||||||
|
esac
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Render per-room manifests for deploy into the shared `spr` kind cluster.
|
"""Render per-room manifests for deploy into the shared `spr` kind cluster.
|
||||||
|
|
||||||
The `spr` cluster itself is created via `ctrl/kind-up.sh` at the repo root
|
The `spr` cluster itself is created by `make cluster up` at the repo root,
|
||||||
(one cluster, all rooms). Each room becomes a namespace inside it.
|
which hands spr's shape to rig to build (one cluster, all rooms). Each room becomes a namespace inside it.
|
||||||
|
|
||||||
Called from build.py when a room opts in to k8s output. Emits:
|
Called from build.py when a room opts in to k8s output. Emits:
|
||||||
|
|
||||||
|
|||||||
@@ -395,21 +395,21 @@ resources:
|
|||||||
|
|
||||||
|
|
||||||
# ─── Lifecycle scripts ──────────────────────────────────────────────
|
# ─── Lifecycle scripts ──────────────────────────────────────────────
|
||||||
# These target the shared `spr` kind cluster (created via repo-root
|
# These target the shared `spr` kind cluster (created by `make cluster up`
|
||||||
# ctrl/kind-up.sh). Each room owns a namespace inside that cluster.
|
# at the repo root, which builds it with rig). Each room owns a namespace inside that cluster.
|
||||||
|
|
||||||
def k8s_up_sh(*, room: str, cluster: str, nodeport: int) -> str:
|
def k8s_up_sh(*, room: str, cluster: str, nodeport: int) -> str:
|
||||||
return f"""\
|
return f"""\
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Apply the "{room}" room into the shared `{cluster}` kind cluster.
|
# Apply the "{room}" room into the shared `{cluster}` kind cluster.
|
||||||
# (Run repo-root ctrl/kind-up.sh first if the cluster doesn't exist.)
|
# (Run 'make cluster up' at the repo root first if the cluster doesn't exist.)
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)"
|
||||||
K8S_DIR="$SCRIPT_DIR/k8s"
|
K8S_DIR="$SCRIPT_DIR/k8s"
|
||||||
|
|
||||||
if ! kind get clusters 2>/dev/null | grep -q '^{cluster}$'; then
|
if ! kind get clusters 2>/dev/null | grep -q '^{cluster}$'; then
|
||||||
echo "Kind cluster '{cluster}' not found — run ctrl/kind-up.sh from the repo root first."
|
echo "Kind cluster '{cluster}' not found — run 'make cluster up' from the repo root first."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -432,7 +432,7 @@ def k8s_down_sh(*, room: str, cluster: str) -> str:
|
|||||||
return f"""\
|
return f"""\
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Remove the "{room}" namespace from the shared `{cluster}` cluster.
|
# Remove the "{room}" namespace from the shared `{cluster}` cluster.
|
||||||
# Leaves the cluster itself running (use repo-root ctrl/kind-down.sh to drop everything).
|
# Leaves the cluster itself running (use 'make cluster down' at the repo root to drop everything).
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
CTX="kind-{cluster}"
|
CTX="kind-{cluster}"
|
||||||
|
|||||||
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
4
soleprint/station/tools/distill/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# Local job: which repos this machine distills is not a fact about the tool, and
|
||||||
|
# the list names whatever those repos are. Copy distill-example.json to
|
||||||
|
# distill.json and edit that; it stays here.
|
||||||
|
distill.json
|
||||||
26
soleprint/station/tools/distill/distill-example.json
Normal file
26
soleprint/station/tools/distill/distill-example.json
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that. Everything that shapes the run is at the top; 'repos' is just a list of paths. Preview any change with: ./distill.sh list",
|
||||||
|
|
||||||
|
"_command": "tree = a directory per repo. digest = one .md per repo, flattened into a single readable file. both = each repo as a directory AND a .md. list = write nothing, just report what would be kept.",
|
||||||
|
"command": "list",
|
||||||
|
|
||||||
|
"out": "distilled",
|
||||||
|
|
||||||
|
"_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base (it gets a _PARTIAL.md saying so).",
|
||||||
|
"branch_mode": "full",
|
||||||
|
"diff_base": "main",
|
||||||
|
|
||||||
|
"_filters": "Applied to every repo. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, images, minified, maps). max_bytes skips anything larger and names it in MANIFEST.md.",
|
||||||
|
"exclude": [],
|
||||||
|
"include": [],
|
||||||
|
"all": false,
|
||||||
|
"max_bytes": null,
|
||||||
|
"skip_unchanged": true,
|
||||||
|
"prune": true,
|
||||||
|
|
||||||
|
"_repos": "'path' is absolute, or a name resolved under --root. 'branches' is optional — leave it out for the working tree as it stands, uncommitted changes included. A path may appear more than once.",
|
||||||
|
"repos": [
|
||||||
|
{ "path": "/path/to/some-repo" },
|
||||||
|
{ "path": "/path/to/another-repo", "branches": ["origin/main", "origin/feature/example"] }
|
||||||
|
]
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,75 @@
|
|||||||
|
# explode — one file back into the tree it describes
|
||||||
|
|
||||||
|
The other half of `distill.sh`. Together they are a round trip:
|
||||||
|
|
||||||
|
```
|
||||||
|
distill.sh digest -> one file -> paste into a chat -> the reply -> explode.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
`distill` flattens repos into a single readable file so they fit somewhere that
|
||||||
|
only takes text. `explode` takes the answer and writes it back onto disk. Neither
|
||||||
|
is much use without the other.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./explode.sh --list reply.md # what is in there; writes nothing
|
||||||
|
./explode.sh -o ./restored reply.md # write the tree
|
||||||
|
./explode.sh -o ./restored --force x.md # overwrite what is already there
|
||||||
|
./explode.sh --contract > contract.txt # the format to hand to the model
|
||||||
|
./explode.sh --selftest # check this copy against known input
|
||||||
|
```
|
||||||
|
|
||||||
|
## Ask for `@@`, and attach it
|
||||||
|
|
||||||
|
`--contract` prints the output format to give whatever writes the reply. **Attach
|
||||||
|
that file; do not paste it into the message.** A chat box renders markdown before
|
||||||
|
the model sees it, and `===` alone under a line of text is setext syntax for a
|
||||||
|
heading — so a pasted spec gets rendered as a title and the model is told nothing.
|
||||||
|
`---` is worse, `##` is a heading, backticks open a fence. `@@` means nothing in
|
||||||
|
markdown, which is exactly why it is the marker to ask for.
|
||||||
|
|
||||||
|
Keeping the wording in `--contract` rather than in a note somewhere means what you
|
||||||
|
ask for cannot drift from what the parser accepts.
|
||||||
|
|
||||||
|
## Layouts
|
||||||
|
|
||||||
|
Four shapes are recognised, picked automatically; `--format` overrides the guess.
|
||||||
|
|
||||||
|
| shape | when |
|
||||||
|
| --- | --- |
|
||||||
|
| `@@ FILE: path` … `@@ END` | **ask for this** — explicit, and invisible to markdown |
|
||||||
|
| `=== FILE: path` … `=== END` | the same thing, still read; do not ask for it |
|
||||||
|
| `=== path` marker | a marker line, then the file until the next one |
|
||||||
|
| `## path` + fenced block | `distill.sh`'s own digest |
|
||||||
|
|
||||||
|
Explicit open and close is worth insisting on: a writer emitting plain three-backtick
|
||||||
|
fences silently truncates any file that itself contains a fence — every README with a
|
||||||
|
shell example — because the nested fence looks exactly like the closing one.
|
||||||
|
|
||||||
|
## One reply, several projects
|
||||||
|
|
||||||
|
A thread usually touches more than one repo, and produces one file regardless. The
|
||||||
|
contract asks for paths that begin with the project name, so point `-o` at the
|
||||||
|
directory those projects sit in and each file lands in its own worktree:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./ctrl/explode.sh --list bundle.txt # what is in there, write nothing
|
./explode.sh -o ~/wdir ~/Downloads/reply.md
|
||||||
./ctrl/explode.sh -o ./restored bundle.txt # write the tree
|
|
||||||
./ctrl/explode.sh -o ./restored --force x.md # overwrite what is already there
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
No mapping table to maintain, and a new project needs no change here.
|
||||||
|
|
||||||
|
## Refusals
|
||||||
|
|
||||||
|
Paths come out of a text file, so they are untrusted. Anything absolute or reaching
|
||||||
|
upward with `..` is refused and **nothing** is written — the check runs over the whole
|
||||||
|
input before the first file is created. An unclosed block is refused too, rather than
|
||||||
|
writing the file short. Existing files are never overwritten without `--force`.
|
||||||
|
|
||||||
|
A digest whose files `distill` **clipped** to fit a token budget is refused for the
|
||||||
|
same reason: the document holds only their head and tail, and a truncated file that
|
||||||
|
reads complete is the failure the whole format exists to prevent. The tree copy beside
|
||||||
|
the digest has them whole — take them from there.
|
||||||
|
|
||||||
|
Two limits worth knowing: a file whose last line had no trailing newline comes back
|
||||||
|
with one, and in the bare `=== path` layout a line starting with `=== ` inside a file's
|
||||||
|
own content cannot be told from a real marker. The digest and `@@` layouts have no such
|
||||||
|
ambiguity.
|
||||||
|
|||||||
@@ -1,18 +1,22 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Explode one file back into the tree of files it describes.
|
# Explode one file back into the tree of files it describes.
|
||||||
#
|
#
|
||||||
# The inverse of ctrl/distill.sh's digest: something hands you a single text
|
# The inverse of distill.sh's digest: something hands you a single text
|
||||||
# file with many files inside it, each introduced by its path, and you want the
|
# file with many files inside it, each introduced by its path, and you want the
|
||||||
# directory back.
|
# directory back.
|
||||||
#
|
#
|
||||||
# Three layouts are understood, picked automatically. Prefer the first if you
|
# Three layouts are understood, picked automatically. Prefer the first if you
|
||||||
# control what writes the file:
|
# control what writes the file:
|
||||||
#
|
#
|
||||||
# === FILE: pkg/models/domain.py explicit open and close. Nothing has to be
|
# @@ FILE: pkg/models/domain.py explicit open and close. Nothing has to be
|
||||||
# <the file> counted or inferred, and a block that is
|
# <the file> counted or inferred, and a block that is
|
||||||
# === END never closed is an error rather than a
|
# @@ END never closed is an error rather than a
|
||||||
# file quietly missing its tail.
|
# file quietly missing its tail.
|
||||||
#
|
#
|
||||||
|
# === FILE: pkg/models/domain.py the same thing with '===' instead of '@@'.
|
||||||
|
# <the file> Still read, but do not ask for it: see the
|
||||||
|
# === END note on markdown below.
|
||||||
|
#
|
||||||
# === ./pkg/models/domain.py a marker line, then the file, until the
|
# === ./pkg/models/domain.py a marker line, then the file, until the
|
||||||
# <the file> next marker or the end
|
# <the file> next marker or the end
|
||||||
#
|
#
|
||||||
@@ -30,12 +34,15 @@
|
|||||||
# -n same as --list
|
# -n same as --list
|
||||||
# --force overwrite files that already exist
|
# --force overwrite files that already exist
|
||||||
# --format F fenced | marker | digest | auto (default: auto)
|
# --format F fenced | marker | digest | auto (default: auto)
|
||||||
|
# --contract print the output format to hand to whatever generates the file
|
||||||
# --selftest check this copy of the script against known input and exit
|
# --selftest check this copy of the script against known input and exit
|
||||||
#
|
#
|
||||||
# Examples:
|
# Examples:
|
||||||
# explode.sh --list bundle.txt
|
# explode.sh --list bundle.txt
|
||||||
# explode.sh -o ./restored bundle.txt
|
# explode.sh -o ./restored bundle.txt
|
||||||
# explode.sh -o ./restored --force repo.md
|
# explode.sh -o ./restored --force repo.md
|
||||||
|
# explode.sh --contract > /tmp/contract.txt # attach this, do not paste it
|
||||||
|
# explode.sh -o ~/wdir ~/Downloads/reply.md # one reply, every project
|
||||||
#
|
#
|
||||||
# Why the explicit form is worth asking for: a writer that emits plain three-
|
# Why the explicit form is worth asking for: a writer that emits plain three-
|
||||||
# backtick fences truncates any file that itself contains a fence — every README
|
# backtick fences truncates any file that itself contains a fence — every README
|
||||||
@@ -43,6 +50,24 @@
|
|||||||
# exactly like the closing one. distill.sh avoids that by making its fences
|
# exactly like the closing one. distill.sh avoids that by making its fences
|
||||||
# longer than anything inside the file, but nothing else will bother.
|
# longer than anything inside the file, but nothing else will bother.
|
||||||
#
|
#
|
||||||
|
# Ask for '@@', not '==='. A chat box renders markdown before the model sees the
|
||||||
|
# message, and '===' alone on a line directly under text is setext syntax for a
|
||||||
|
# level-one heading — so the format spec you paste gets swallowed and rendered as
|
||||||
|
# a title, and the model is told nothing. '---' is worse (heading AND horizontal
|
||||||
|
# rule), '##' is a heading, backticks open a fence. '@@' has no meaning in
|
||||||
|
# markdown at all, which is the whole reason to use it. Both are parsed here, so
|
||||||
|
# nothing already written stops working.
|
||||||
|
#
|
||||||
|
# The other half of that: put the spec in an attached file rather than the
|
||||||
|
# message body. Attachments are not rendered. '--contract' prints the exact text
|
||||||
|
# to attach, so the wording cannot drift from the parser that reads the reply
|
||||||
|
# back.
|
||||||
|
#
|
||||||
|
# One reply, several projects. A thread produces one file however many repos it
|
||||||
|
# touched, so --contract asks for paths that start with the project name. Point
|
||||||
|
# -o at the directory those projects are siblings in and each file lands in its
|
||||||
|
# own worktree — no table to maintain, and a new worktree needs no change here.
|
||||||
|
#
|
||||||
# Two limits worth knowing. A file whose last line has no trailing newline comes
|
# Two limits worth knowing. A file whose last line has no trailing newline comes
|
||||||
# back with one: the digest has to put a newline before the closing fence, so the
|
# back with one: the digest has to put a newline before the closing fence, so the
|
||||||
# distinction is not in the input to recover. And in marker layout a line
|
# distinction is not in the input to recover. And in marker layout a line
|
||||||
@@ -66,6 +91,7 @@ FORCE=""
|
|||||||
FORMAT="auto"
|
FORMAT="auto"
|
||||||
SRC=""
|
SRC=""
|
||||||
SELFTEST=""
|
SELFTEST=""
|
||||||
|
CONTRACT=""
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
@@ -73,6 +99,7 @@ while [ $# -gt 0 ]; do
|
|||||||
--list|-n) LIST=1 ;;
|
--list|-n) LIST=1 ;;
|
||||||
--force) FORCE=1 ;;
|
--force) FORCE=1 ;;
|
||||||
--format) shift; FORMAT="${1:-}" ;;
|
--format) shift; FORMAT="${1:-}" ;;
|
||||||
|
--contract) CONTRACT=1 ;;
|
||||||
--selftest) SELFTEST=1 ;;
|
--selftest) SELFTEST=1 ;;
|
||||||
-h|--help) usage; exit 0 ;;
|
-h|--help) usage; exit 0 ;;
|
||||||
-*) die "unknown option: $1" ;;
|
-*) die "unknown option: $1" ;;
|
||||||
@@ -81,6 +108,43 @@ while [ $# -gt 0 ]; do
|
|||||||
shift
|
shift
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# ── the contract ───────────────────────────────────────────────────────────
|
||||||
|
# One copy of the wording, printed rather than remembered, so what you ask for
|
||||||
|
# and what this parses cannot drift apart. Attach it; do not paste it into the
|
||||||
|
# message body, where markdown gets a say first.
|
||||||
|
contract() {
|
||||||
|
cat <<'CONTRACT'
|
||||||
|
OUTPUT FORMAT
|
||||||
|
|
||||||
|
Return every file you changed or created in full, one after another, using
|
||||||
|
exactly this shape and nothing else:
|
||||||
|
|
||||||
|
@@ FILE: <project>/relative/path/to/file.py
|
||||||
|
<the complete contents of the file>
|
||||||
|
@@ END
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- One @@ FILE: line per file, and a matching @@ END line after its last line.
|
||||||
|
- Start every path with the project it belongs to, spelled exactly as the
|
||||||
|
heading of the document it came from, then the path relative to that
|
||||||
|
project's root. One reply covers every project we touched; the prefix is
|
||||||
|
the only thing that says which file goes where, so it is never optional
|
||||||
|
and never abbreviated.
|
||||||
|
- No leading ./ or /.
|
||||||
|
- Between @@ FILE: and @@ END, emit the file verbatim. Do not wrap it in
|
||||||
|
markdown fences, do not add line numbers, do not elide anything as
|
||||||
|
"unchanged" or "...". A partial file is worse than no file.
|
||||||
|
- Anything you want to say to me goes outside the blocks, before the first
|
||||||
|
@@ FILE: or after the last @@ END. Text between blocks is ignored.
|
||||||
|
- Return whole files only. No diffs, no patches, no hunks.
|
||||||
|
- If a file's own content happens to contain a line starting with @@, say so
|
||||||
|
in your prose so I know to check that block by hand.
|
||||||
|
CONTRACT
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ -n "$CONTRACT" ]; then contract; exit 0; fi
|
||||||
|
|
||||||
# ── self-test ──────────────────────────────────────────────────────────────
|
# ── self-test ──────────────────────────────────────────────────────────────
|
||||||
# So a copy of this script on another machine can be checked without any real
|
# So a copy of this script on another machine can be checked without any real
|
||||||
# input, and without asking whether it is the version that knows a given format.
|
# input, and without asking whether it is the version that knows a given format.
|
||||||
@@ -137,6 +201,21 @@ FIXTURE
|
|||||||
check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)"
|
check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
# The other two layouts still work.
|
# The other two layouts still work.
|
||||||
|
# The @@ markers, which are the ones to ask a chat model for.
|
||||||
|
printf '@@ FILE: pkg/a.py\nx = 1\n@@ END\n@@ FILE: b.md\n# t\n\n```sh\nls\n```\n@@ END\n' > "$t/g.txt"
|
||||||
|
"$0" -o "$t/g" "$t/g.txt" >/dev/null 2>&1 || true
|
||||||
|
check "at-markers: file count" "2" "$(find "$t/g" -type f 2>/dev/null | wc -l)"
|
||||||
|
check "at-markers: fenced body" "2" "$(grep -c '```' "$t/g/b.md" 2>/dev/null || echo 0)"
|
||||||
|
printf '@@ FILE: a.py\nx = 1\n' > "$t/h.txt"
|
||||||
|
"$0" -o "$t/h" "$t/h.txt" >/dev/null 2>&1 || true
|
||||||
|
check "at-markers: unterminated" "1" "$([ -e "$t/h" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
|
# One reply, several projects: the prefix is just the first directory.
|
||||||
|
printf '@@ FILE: projA/a.py\nx = 1\n@@ END\n@@ FILE: projB/deep/b.py\ny = 2\n@@ END\n' > "$t/i.txt"
|
||||||
|
"$0" -o "$t/i" "$t/i.txt" >/dev/null 2>&1 || true
|
||||||
|
check "multi-project: first" "x = 1" "$(cat "$t/i/projA/a.py" 2>/dev/null)"
|
||||||
|
check "multi-project: nested" "y = 2" "$(cat "$t/i/projB/deep/b.py" 2>/dev/null)"
|
||||||
|
|
||||||
printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt"
|
printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt"
|
||||||
"$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true
|
"$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true
|
||||||
check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)"
|
check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)"
|
||||||
@@ -145,6 +224,19 @@ FIXTURE
|
|||||||
"$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true
|
"$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true
|
||||||
check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)"
|
check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)"
|
||||||
|
|
||||||
|
# What distill actually writes: a metadata line between the heading and the
|
||||||
|
# fence, and prose sections whose heading is followed by no fence at all.
|
||||||
|
printf '# d\n\n## Tree\n\nx/\n y.py\n\n## x/y.py\n\n_1 lines · 6 bytes_\n\n```python\nz = 1\n```\n' > "$t/j.txt"
|
||||||
|
"$0" -o "$t/j" "$t/j.txt" >/dev/null 2>&1 || true
|
||||||
|
check "digest: metadata line" "z = 1" "$(cat "$t/j/x/y.py" 2>/dev/null)"
|
||||||
|
check "digest: prose skipped" "1" "$(find "$t/j" -type f 2>/dev/null | wc -l)"
|
||||||
|
|
||||||
|
# A clipped file is head and tail only. Writing it would truncate the real
|
||||||
|
# one, so the whole run is refused.
|
||||||
|
printf '# d\n\n## x/y.py\n\n_900 lines · 60000 bytes · CLIPPED — head and tail only_\n\n```python\nz = 1\n```\n' > "$t/k.txt"
|
||||||
|
"$0" -o "$t/k" "$t/k.txt" >/dev/null 2>&1 || true
|
||||||
|
check "digest: clipped refused" "1" "$([ -e "$t/k" ] && echo 0 || echo 1)"
|
||||||
|
|
||||||
echo
|
echo
|
||||||
if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current"
|
if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current"
|
||||||
else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2
|
else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2
|
||||||
@@ -163,9 +255,9 @@ case "$FORMAT" in fenced|marker|digest|auto) ;; *) die "--format must be fenced,
|
|||||||
# markdown will contain plenty of '=== ' inside its own fenced content, and a
|
# markdown will contain plenty of '=== ' inside its own fenced content, and a
|
||||||
# marker file can quote a '## ' heading just as easily.
|
# marker file can quote a '## ' heading just as easily.
|
||||||
if [ "$FORMAT" = auto ]; then
|
if [ "$FORMAT" = auto ]; then
|
||||||
n_fenced=$(grep -cE '^=== +FILE: +[^ ]' "$SRC" || true)
|
n_fenced=$(grep -cE '^(===|@@) +FILE: +[^ ]' "$SRC" || true)
|
||||||
n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true)
|
n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true)
|
||||||
n_marker=$((n_marker - n_fenced - $(grep -cE '^=== +END[ \t]*$' "$SRC" || true)))
|
n_marker=$((n_marker - n_fenced - $(grep -cE '^(===|@@) +END[ \t]*$' "$SRC" || true)))
|
||||||
[ "$n_marker" -lt 0 ] && n_marker=0
|
[ "$n_marker" -lt 0 ] && n_marker=0
|
||||||
n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true)
|
n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true)
|
||||||
if [ "$n_fenced" -gt 0 ]; then
|
if [ "$n_fenced" -gt 0 ]; then
|
||||||
@@ -220,7 +312,7 @@ parse() {
|
|||||||
|
|
||||||
# Explicit open/close. The whole point is that nothing is inferred:
|
# Explicit open/close. The whole point is that nothing is inferred:
|
||||||
# content is content until the END line, whatever it looks like.
|
# content is content until the END line, whatever it looks like.
|
||||||
fmt == "fenced" && path == "" && /^=== +FILE: +/ {
|
fmt == "fenced" && path == "" && /^(===|@@) +FILE: +/ {
|
||||||
p = substr($0, index($0, "FILE:") + 5)
|
p = substr($0, index($0, "FILE:") + 5)
|
||||||
sub(/^[ \t]+/, "", p)
|
sub(/^[ \t]+/, "", p)
|
||||||
p = clean(p)
|
p = clean(p)
|
||||||
@@ -228,7 +320,7 @@ parse() {
|
|||||||
open_file(p)
|
open_file(p)
|
||||||
next
|
next
|
||||||
}
|
}
|
||||||
fmt == "fenced" && path != "" && /^=== +END[ \t]*$/ { flush(); next }
|
fmt == "fenced" && path != "" && /^(===|@@) +END[ \t]*$/ { flush(); next }
|
||||||
fmt == "fenced" && path == "" { next } # anything between blocks is prose
|
fmt == "fenced" && path == "" { next } # anything between blocks is prose
|
||||||
|
|
||||||
fmt == "marker" && /^=== +/ {
|
fmt == "marker" && /^=== +/ {
|
||||||
@@ -256,6 +348,22 @@ parse() {
|
|||||||
}
|
}
|
||||||
fmt == "digest" && expect == 1 {
|
fmt == "digest" && expect == 1 {
|
||||||
if ($0 ~ /^[ \t]*$/) next # blank line between the two
|
if ($0 ~ /^[ \t]*$/) next # blank line between the two
|
||||||
|
# distill.sh puts an italic "N lines, B bytes" line under each heading,
|
||||||
|
# so the fence is no longer the next thing after it. Step over that
|
||||||
|
# line rather than reading it as prose — without this, every file in
|
||||||
|
# a current digest is skipped and the whole document looks empty.
|
||||||
|
if ($0 ~ /^_.*_[ \t]*$/) {
|
||||||
|
# Unless it says the file was clipped. A clipped body is head
|
||||||
|
# and tail with a marker in between; writing it out would
|
||||||
|
# replace a real file with a truncated one that reads complete,
|
||||||
|
# which is the exact failure the unterminated check exists for.
|
||||||
|
if ($0 ~ /CLIPPED/) {
|
||||||
|
print "CLIPPED\t" pending
|
||||||
|
bad = 1; expect = 0; pending = ""
|
||||||
|
next
|
||||||
|
}
|
||||||
|
next
|
||||||
|
}
|
||||||
if ($0 ~ /^`{3,}/) { # a fence: this is a file
|
if ($0 ~ /^`{3,}/) { # a fence: this is a file
|
||||||
match($0, /^`+/)
|
match($0, /^`+/)
|
||||||
fence = substr($0, 1, RLENGTH)
|
fence = substr($0, 1, RLENGTH)
|
||||||
@@ -302,7 +410,7 @@ fi
|
|||||||
# looks complete is the failure this format exists to prevent.
|
# looks complete is the failure this format exists to prevent.
|
||||||
wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)"
|
wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)"
|
||||||
if [ -n "$wrongfmt" ]; then
|
if [ -n "$wrongfmt" ]; then
|
||||||
echo "$SELF: this file uses '=== FILE: path' / '=== END', but it was read as" >&2
|
echo "$SELF: this file uses 'FILE: path' / 'END' markers, but it was read as" >&2
|
||||||
echo "the plain marker format, which would create a directory called 'FILE: .'" >&2
|
echo "the plain marker format, which would create a directory called 'FILE: .'" >&2
|
||||||
echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2
|
echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2
|
||||||
exit 1
|
exit 1
|
||||||
@@ -310,13 +418,25 @@ fi
|
|||||||
|
|
||||||
unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)"
|
unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)"
|
||||||
if [ -n "$unterminated" ]; then
|
if [ -n "$unterminated" ]; then
|
||||||
echo "$SELF: refusing — this block was never closed with '=== END':" >&2
|
echo "$SELF: refusing — this block was never closed with '@@ END' or '=== END':" >&2
|
||||||
printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2
|
printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2
|
||||||
echo "the file it describes would be silently truncated" >&2
|
echo "the file it describes would be silently truncated" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT)' || true)"
|
# distill clips the largest files to fit a budget, and says so. The tree copy
|
||||||
|
# beside the digest has them whole, so the fix is to take them from there — not
|
||||||
|
# to write out the head and tail under the real name.
|
||||||
|
clipped="$(printf '%s\n' "$scan" | grep '^CLIPPED' || true)"
|
||||||
|
if [ -n "$clipped" ]; then
|
||||||
|
echo "$SELF: refusing — distill clipped these, so the digest has only their" >&2
|
||||||
|
echo "head and tail:" >&2
|
||||||
|
printf '%s\n' "$clipped" | sed 's/^CLIPPED\t/ /' >&2
|
||||||
|
echo "take them from the tree copy instead; writing these would truncate them" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT|CLIPPED)' || true)"
|
||||||
[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)"
|
[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)"
|
||||||
count=$(printf '%s\n' "$listing" | grep -c . )
|
count=$(printf '%s\n' "$listing" | grep -c . )
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user