mariano/soleprint
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rig updates

Browse Source
This commit is contained in:
buenosairesam
2026-09-17 00:39:00 -03:00
parent 730ebaff2f
commit de5b1b7ea8
21 changed files with 69 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rig/ctrl/check.sh
View File

@@ -23,7 +23,7 @@ load_config
echo
echo "config"
echo " profile ${PROFILE_NAME} (nodes=${NODES} audit=${AUDIT})"
echo " profile ${PROFILE_NAME} (nodes=${NODES})"
echo " cluster ${CLUSTER} (context ${KUBECONTEXT})"
echo " registry ${REGISTRY_MODE}"
echo " ingress ${INGRESS_MODE}"

9
rig/ctrl/cluster.sh
View File

@@ -23,15 +23,14 @@ up() {
echo "cluster '$CLUSTER' exists — converging"
else
# Say what this profile locks in BEFORE spending minutes building it:
# the audit policy is an apiserver flag and cannot be changed later.
# the kind config is fixed at creation and cannot be changed later.
echo "creating cluster '$CLUSTER' from profile '$PROFILE_NAME'"
echo " shape ${KIND_CONFIG_SHOWN}"
echo " kind config ${KIND_CONFIG}"
echo " nodes $NODES"
echo " image $NODE_IMAGE"
echo " audit $AUDIT"
echo " ingress $INGRESS_MODE"
echo " registry $REGISTRY_MODE"
echo " (audit is fixed at creation — 'make cluster reset' to change it)"
echo " (fixed at creation — edit the kind config, then 'make cluster reset')"
echo
render_kind_config | kind create cluster --config -
@@ -69,7 +68,7 @@ down() {
}
# The escape hatch for a wedged cluster, and the only way to change a
# creation-time setting such as the audit policy.
# creation-time setting such as the node count or port mappings.
reset() {
down
echo

9
rig/ctrl/env.d/client.env.example
View File

@@ -3,16 +3,11 @@
# directory and name it — PROFILE=client in ctrl/.env, or on the command line. It
# then overlays the defaults; anything it does not set, they still supply.
#
# client — the regulated-estate shape. Multi-node so taints, affinity and
# topology are real; apiserver audit on; images through a pull-through cache of
# the corporate registry.
#
# Costs roughly 4-6 GB. Check `make cluster list` before starting this alongside
# other work — see the memory note in the README.
# client — images through a pull-through cache of the corporate registry, with
# TLS and metrics addons. More nodes or port mappings: edit k8s/kind-config.yaml.tpl.
PROFILE_NAME=client
K8S_VERSION=v1_36
KIND_CONFIG=kind-config.client.yaml.tpl
ADDONS="metallb cert-manager metrics-server"
REGISTRY_MODE=mirror
INGRESS_MODE=hostport

1
rig/ctrl/env.d/data.env.example
View File

@@ -14,7 +14,6 @@
PROFILE_NAME=data
K8S_VERSION=v1_36
KIND_CONFIG=kind-config.yaml.tpl
# Order matters: addons.sh installs in the order listed, and airflow refuses to
# start without a metadata database, so postgres comes first.
ADDONS="metallb postgres redis airflow"

1
rig/ctrl/env.d/offline.env.example
View File

@@ -11,7 +11,6 @@
PROFILE_NAME=offline
K8S_VERSION=v1_36
KIND_CONFIG=kind-config.audit.yaml.tpl
ADDONS="metallb"
REGISTRY_MODE=local
INGRESS_MODE=hostport

24
rig/ctrl/k8s/README.md
View File

@@ -1,13 +1,12 @@
# `ctrl/k8s` — cluster shape, and what runs on it
# `ctrl/k8s` — the cluster, and what runs on it
Same layout as every other project here: a kind config, a kustomize `base/`,
and an `overlays/dev/` that patches it.
```
kind-config*.yaml.tpl the cluster itself — nodes, ports, audit
kind-config.yaml.tpl the cluster itself — nodes, ports
base/ the components, as plain manifests
overlays/dev/ how this rig differs from the base
audit-policy.yaml mounted into the apiserver by the audit shapes
```
## Why the cluster config is a template
@@ -22,21 +21,12 @@ directory name — so a literal would make every copy collide on both.
`gettext-base`, which a minimal Debian does not have, and Docker being the only
prerequisite is the one promise rig makes.
**The chosen file is the source of truth for node count and audit.**
`lib/config.sh` reads both back out of it, so a profile names a shape and does
not restate what the YAML already says.
**To change the cluster, edit this file** — more nodes, other port mappings —
then `make cluster reset`: a kind config is fixed at creation, not re-applied.
`lib/config.sh` reads the node count back out of it, so nothing restates it.
| file | nodes | audit | profiles |
| --- | --- | --- | --- |
| `kind-config.yaml.tpl` | 1 | off | the default; the `data` example |
| `kind-config.audit.yaml.tpl` | 1 | on | the `offline` example |
| `kind-config.client.yaml.tpl` | 3 | on | the `client` example |
With no profile the default shape is used; a profile picks another with
`KIND_CONFIG`. Adding a shape is adding a file — there is no dispatcher to edit.
Audit is an apiserver flag and therefore fixed at creation: changing it is
`make cluster reset`, not a re-apply.
A project that builds its own cluster through rig passes its own file as
`KIND_CONFIG=<path>`; it is rendered the same way.
## `base/` — replace these

44
rig/ctrl/k8s/audit-policy.yaml
View File

@@ -1,44 +0,0 @@
# Apiserver audit policy. Mounted into the control plane at creation when a
# profile sets AUDIT=on — an apiserver flag, so it cannot be added to a running
# cluster without recreating it.
#
# Deliberately modest: enough to make "who changed what, and when" answerable
# during onboarding without filling the disk. Read the log with:
# docker exec <cluster>-control-plane cat /var/log/kubernetes/audit.log
apiVersion: audit.k8s.io/v1
kind: Policy
# Never log the request body for these — they contain credentials.
omitStages:
- RequestReceived
rules:
# Secrets/configmaps: record that access happened, never the contents.
- level: Metadata
resources:
- group: ""
resources: ["secrets", "configmaps"]
# Authn/authz decisions — the part an auditor actually asks about.
- level: Metadata
nonResourceURLs:
- /apis*
- /api*
# Mutations to workloads and policy: full request, so a diff is reconstructable.
- level: Request
verbs: ["create", "update", "patch", "delete"]
resources:
- group: ""
resources: ["pods", "services", "serviceaccounts", "namespaces"]
- group: "apps"
- group: "networking.k8s.io"
- group: "rbac.authorization.k8s.io"
# Everything else that changes state: metadata only.
- level: Metadata
verbs: ["create", "update", "patch", "delete"]
# Reads are dropped entirely — otherwise controller polling drowns the log.
- level: None
verbs: ["get", "list", "watch"]

57
rig/ctrl/k8s/kind-config.audit.yaml.tpl
View File

@@ -1,57 +0,0 @@
# Cluster shape: one node, apiserver audit ON. Used by the `offline` profile.
#
# Audit is an apiserver flag, so it is fixed when the cluster is created —
# changing it means `make cluster reset`, not a re-apply. That is why it is a
# property of the cluster file rather than something switched at runtime.
#
# k8s >= 1.31 uses kubeadm v1beta4, where extraArgs is a LIST of name/value
# pairs. The older map form is silently ignored — it does not error, audit
# simply never turns on.
#
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
# (named without the ${...} braces so this line survives the substitution)
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: ${CLUSTER}
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
apiServer:
extraArgs:
- name: audit-policy-file
value: /etc/kubernetes/audit/policy.yaml
- name: audit-log-path
value: /var/log/kubernetes/audit.log
- name: audit-log-maxage
value: "7"
extraVolumes:
- name: audit-policy
hostPath: /etc/kubernetes/audit
mountPath: /etc/kubernetes/audit
readOnly: true
- name: audit-log
hostPath: /var/log/kubernetes
mountPath: /var/log/kubernetes
readOnly: false
nodes:
- role: control-plane
image: ${NODE_IMAGE}
# hostPath is resolved by the HOST dockerd, so this must be a host path even
# when cluster.sh runs inside the installer container. HOST_WORKDIR says where
# this rig lives on the host; bare on a host it is just the repo root.
extraMounts:
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
containerPath: /etc/kubernetes/audit/policy.yaml
readOnly: true
extraPortMappings:
- containerPort: 30080
hostPort: ${HTTP_PORT}
listenAddress: "0.0.0.0"
protocol: TCP

55
rig/ctrl/k8s/kind-config.client.yaml.tpl
View File

@@ -1,55 +0,0 @@
# Cluster shape: three nodes, apiserver audit ON. Used by the `client` profile —
# the regulated-estate shape.
#
# Multi-node so taints, affinity and topology spread are real rather than
# vacuously satisfied by a single node. It costs roughly 4-6 GB; run
# `make cluster list` before starting this alongside other work.
#
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
# (named without the ${...} braces so this line survives the substitution)
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: ${CLUSTER}
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
apiServer:
extraArgs:
- name: audit-policy-file
value: /etc/kubernetes/audit/policy.yaml
- name: audit-log-path
value: /var/log/kubernetes/audit.log
- name: audit-log-maxage
value: "7"
extraVolumes:
- name: audit-policy
hostPath: /etc/kubernetes/audit
mountPath: /etc/kubernetes/audit
readOnly: true
- name: audit-log
hostPath: /var/log/kubernetes
mountPath: /var/log/kubernetes
readOnly: false
nodes:
- role: control-plane
image: ${NODE_IMAGE}
extraMounts:
- hostPath: ${HOST_WORKDIR}/ctrl/k8s/audit-policy.yaml
containerPath: /etc/kubernetes/audit/policy.yaml
readOnly: true
extraPortMappings:
- containerPort: 30080
hostPort: ${HTTP_PORT}
listenAddress: "0.0.0.0"
protocol: TCP
- role: worker
image: ${NODE_IMAGE}
- role: worker
image: ${NODE_IMAGE}

7
rig/ctrl/k8s/kind-config.yaml.tpl
View File

@@ -1,4 +1,5 @@
# Cluster shape: one node, no audit. Used by the `minimal` and `data` profiles.
# The cluster. One node by default — add nodes or port mappings by editing this
# file, then `make cluster reset`.
#
# A TEMPLATE rather than a plain kind-config.yaml because a rig is copied and
# renamed to make a second environment, and both the cluster name and the host
@@ -9,8 +10,8 @@
#
# Substituted by ctrl/cluster.sh: CLUSTER, NODE_IMAGE, HTTP_PORT, HOST_WORKDIR
# (named without the ${...} braces so this line survives the substitution)
# Node count and audit are READ BACK from this file by lib/config.sh, so this
# YAML is the source of truth for both — there is no second place to update.
# The node count is READ BACK from this file by lib/config.sh, so this YAML is
# the source of truth for it — there is no second place to update.
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: ${CLUSTER}

43
rig/ctrl/lib/config.sh
View File

@@ -7,7 +7,7 @@
#
# built-in defaults below; fill only what nothing else set
# ctrl/versions.env pinned toolchain + image digests (committed)
# ctrl/env.d/<profile> cluster shape — OPTIONAL, examples ship as *.env.example
# ctrl/env.d/<profile> addons, registry — OPTIONAL, examples ship as *.env.example
# ctrl/.env machine-local values and secrets (gitignored)
# the caller's env `make cluster up PROFILE=<name>` (always wins)
#
@@ -19,9 +19,8 @@
# Values a user can reasonably override per-invocation. Anything set in the
# environment when load_config runs is restored after the files are read.
# NODES and AUDIT are deliberately NOT here: they are properties of the chosen
# ctrl/k8s/kind-config*.yaml.tpl and are read back out of it below, so there is
# one place that decides the shape of the cluster rather than two that can drift.
# NODES is deliberately NOT here: it is read back out of the kind config below,
# so the file is the one place that decides it.
#
# REGISTRY_PORT and MANIFESTS_DIR were missing here while ctrl/.env set them, so
# the caller's env silently LOST to the file for those two — the one precedence
@@ -136,32 +135,18 @@ load_config() {
exit 1
fi
# The cluster's shape is a file in ctrl/k8s/, named by the profile. Adding a
# shape is adding a file; there is no dispatcher to edit.
#
# A host that needs its own shape — extra port mappings, more nodes — passes
# an absolute path instead, and rig renders it exactly like one of its own:
# ${CLUSTER} and ${NODE_IMAGE} are substituted either way. The shape stays in
# the host's tree, because what a host's cluster needs is the host's business;
# rig only knows how to build whatever it is handed.
KIND_CONFIG="${KIND_CONFIG:-kind-config.yaml.tpl}"
case "$KIND_CONFIG" in
/*) KIND_CONFIG_PATH="$KIND_CONFIG"; KIND_CONFIG_SHOWN="$KIND_CONFIG" ;;
*) KIND_CONFIG_PATH="./k8s/${KIND_CONFIG}"; KIND_CONFIG_SHOWN="ctrl/k8s/${KIND_CONFIG}" ;;
esac
if [ ! -f "$KIND_CONFIG_PATH" ]; then
echo "no such cluster shape: ${KIND_CONFIG_SHOWN}" >&2
echo "rig's own: $(ls k8s/kind-config*.yaml.tpl 2>/dev/null | xargs -n1 basename | tr '\n' ' ')" >&2
echo "or pass an absolute path to a shape of your own" >&2
# The cluster is one file: k8s/kind-config.yaml.tpl. To change it, edit it.
# KIND_CONFIG is only "use this file instead", for a project that builds its
# own cluster through rig (a path relative to ctrl/, or absolute).
KIND_CONFIG="${KIND_CONFIG:-./k8s/kind-config.yaml.tpl}"
if [ ! -f "$KIND_CONFIG" ]; then
echo "no kind config at KIND_CONFIG=${KIND_CONFIG}" >&2
exit 1
fi
# Read the shape back out of the YAML rather than trusting a profile to
# restate it. check.sh sizes the memory warning on NODES, and cluster.sh
# prints AUDIT before spending minutes building something that cannot be
# changed afterwards — both would mislead if the numbers drifted.
NODES=$(grep -c '^ - role:' "$KIND_CONFIG_PATH")
if grep -q 'audit-policy-file' "$KIND_CONFIG_PATH"; then AUDIT=on; else AUDIT=off; fi
# Read the node count back out of the file rather than restating it:
# check.sh and the memory tool size their budget on NODES.
NODES=$(grep -c '^ - role:' "$KIND_CONFIG")
# What one node costs, measured rather than guessed. On 2026-09-11 a minimal
# control-plane node ran at 620 MiB idle and ~728 MiB with a small mock, plus
@@ -175,7 +160,7 @@ load_config() {
NODE_MB=800
}
# Render a cluster shape to stdout. sed rather than envsubst: envsubst is
# Render the kind config to stdout. sed rather than envsubst: envsubst is
# gettext-base, absent from a minimal Debian, and Docker is meant to be the only
# prerequisite. The variable list is explicit so a template cannot quietly start
# depending on something the caller does not set.
@@ -188,7 +173,7 @@ render_kind_config() {
-e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" \
-e "s|\${HTTP_PORT}|${HTTP_PORT}|g" \
-e "s|\${HOST_WORKDIR}|${host_workdir}|g" \
"$KIND_CONFIG_PATH"
"$KIND_CONFIG"
}
_config_restore() {

4
rig/ctrl/selftest.sh
View File

@@ -99,7 +99,9 @@ test_value() {
# profile, template or pinned version to be present for this to run.
PROFILE) config_profiles | head -1 ;;
K8S_VERSION) (set -a; source ./versions.env; compgen -v NODE_IMAGE_v | sort -V | head -1 | sed 's/^NODE_IMAGE_//') ;;
KIND_CONFIG) ls k8s/kind-config*.yaml.tpl 2>/dev/null | sort | head -1 | xargs -r basename ;;
# An absolute path, as a project passing its own file does. Never equal
# to the default, so the check cannot pass by accident.
KIND_CONFIG) echo "$PWD/k8s/kind-config.yaml.tpl" ;;
*_PORT) echo "19999" ;;
CLUSTER) echo "selftest-name" ;;
MANIFESTS_DIR) echo "../elsewhere/overlays/dev" ;;