big update based on real wokr

rig/examples/data/addons/airflow.sh

@@ -5,6 +5,7 @@
 # Requires the postgres addon; refuses to install without it.
 # Notes: ../README.md
 set -euo pipefail
+ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -70,77 +71,7 @@ if [ -d "$dags" ]; then
 fi
 
 echo "  applying manifests"
-$K apply -n "$NS" -f - >/dev/null <<YAML
-apiVersion: v1
-kind: Service
-metadata:
-  name: airflow
-spec:
-  selector:
-    app: airflow
-  ports:
-    - port: 8080
-      targetPort: 8080
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: airflow
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: airflow
-  template:
-    metadata:
-      labels:
-        app: airflow
-    spec:
-      containers:
-        - name: airflow
-          image: ${AIRFLOW_IMAGE}
-          args: ["standalone"]
-          env:
-            - name: AIRFLOW__CORE__EXECUTOR
-              value: LocalExecutor
-            - name: AIRFLOW__CORE__LOAD_EXAMPLES
-              value: "false"
-            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
-              valueFrom:
-                secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
-            - name: AIRFLOW_CONN_APP_DB
-              valueFrom:
-                secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
-            - name: AIRFLOW__CORE__FERNET_KEY
-              valueFrom:
-                secretKeyRef: {name: airflow, key: FERNET_KEY}
-            - name: _AIRFLOW_WWW_USER_USERNAME
-              valueFrom:
-                secretKeyRef: {name: airflow, key: ADMIN_USER}
-            - name: _AIRFLOW_WWW_USER_PASSWORD
-              valueFrom:
-                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
-          ports:
-            - containerPort: 8080
-          volumeMounts:
-            - name: dags
-              mountPath: /opt/airflow/dags
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: 8080
-            # First boot runs the whole migration before it serves anything.
-            initialDelaySeconds: 60
-            periodSeconds: 15
-            failureThreshold: 20
-      volumes:
-        - name: dags
-          configMap:
-            name: airflow-dags
-            optional: true
-YAML
+render_manifest "$ADDON_DIR/airflow.yaml" | $K apply -n "$NS" -f - >/dev/null
 
 echo "  waiting for airflow (the first boot migrates the database, so this is slow)..."
 $K rollout status deployment/airflow -n "$NS" --timeout=600s

rig/examples/data/addons/airflow.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: airflow
+spec:
+  selector:
+    app: airflow
+  ports:
+    - port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airflow
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: airflow
+  template:
+    metadata:
+      labels:
+        app: airflow
+    spec:
+      containers:
+        - name: airflow
+          image: ${AIRFLOW_IMAGE}
+          args: ["standalone"]
+          env:
+            - name: AIRFLOW__CORE__EXECUTOR
+              value: LocalExecutor
+            - name: AIRFLOW__CORE__LOAD_EXAMPLES
+              value: "false"
+            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+              valueFrom:
+                secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
+            - name: AIRFLOW_CONN_APP_DB
+              valueFrom:
+                secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
+            - name: AIRFLOW__CORE__FERNET_KEY
+              valueFrom:
+                secretKeyRef: {name: airflow, key: FERNET_KEY}
+            - name: _AIRFLOW_WWW_USER_USERNAME
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_USER}
+            - name: _AIRFLOW_WWW_USER_PASSWORD
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: dags
+              mountPath: /opt/airflow/dags
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            # First boot runs the whole migration before it serves anything.
+            initialDelaySeconds: 60
+            periodSeconds: 15
+            failureThreshold: 20
+      volumes:
+        - name: dags
+          configMap:
+            name: airflow-dags
+            optional: true

rig/examples/data/addons/postgres.sh

@@ -2,6 +2,7 @@
 # PostgreSQL for this overlay: plain manifests, one replica on a PVC, password generated once and kept.
 # Notes: ../README.md
 set -euo pipefail
+ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -26,77 +27,7 @@ else
 fi
 
 echo "  applying manifests"
-$K apply -n "$NS" -f - >/dev/null <<YAML
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: postgres-data
-spec:
-  accessModes: [ReadWriteOnce]
-  resources:
-    requests:
-      storage: ${POSTGRES_STORAGE:-2Gi}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: postgres
-spec:
-  selector:
-    app: postgres
-  ports:
-    - port: 5432
-      targetPort: 5432
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: postgres
-spec:
-  replicas: 1
-  # One volume, one writer. Rolling would start a second pod against the same
-  # PVC before the first exits, and Postgres refuses to share a data directory.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: postgres
-  template:
-    metadata:
-      labels:
-        app: postgres
-    spec:
-      containers:
-        - name: postgres
-          image: ${POSTGRES_IMAGE}
-          envFrom:
-            - secretRef:
-                name: postgres
-          env:
-            # The image initialises into the volume root otherwise, and a
-            # lost+found from the PVC makes it refuse to initdb.
-            - name: PGDATA
-              value: /var/lib/postgresql/data/pgdata
-          ports:
-            - containerPort: 5432
-          volumeMounts:
-            - name: data
-              mountPath: /var/lib/postgresql/data
-          readinessProbe:
-            exec:
-              command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          livenessProbe:
-            exec:
-              command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
-            initialDelaySeconds: 30
-            periodSeconds: 15
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: postgres-data
-YAML
+render_manifest "$ADDON_DIR/postgres.yaml" | $K apply -n "$NS" -f - >/dev/null
 
 echo "  waiting for postgres..."
 $K rollout status deployment/postgres -n "$NS" --timeout=240s

rig/examples/data/addons/postgres.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-data
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests:
+      storage: ${POSTGRES_STORAGE:-2Gi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+spec:
+  replicas: 1
+  # One volume, one writer. Rolling would start a second pod against the same
+  # PVC before the first exits, and Postgres refuses to share a data directory.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: ${POSTGRES_IMAGE}
+          envFrom:
+            - secretRef:
+                name: postgres
+          env:
+            # The image initialises into the volume root otherwise, and a
+            # lost+found from the PVC makes it refuse to initdb.
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          ports:
+            - containerPort: 5432
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U $POSTGRES_USER -d $POSTGRES_DB"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["sh", "-c", "pg_isready -U $POSTGRES_USER -d $POSTGRES_DB"]
+            initialDelaySeconds: 30
+            periodSeconds: 15
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: postgres-data

rig/examples/data/addons/redis.sh

@@ -2,6 +2,7 @@
 # Redis for this overlay: cache and broker (Celery), no persistence.
 # Notes: ../README.md
 set -euo pipefail
+ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"   # its manifest sits beside it
 cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
 
 source ./lib/config.sh
@@ -13,43 +14,7 @@ NS="${DATA_NAMESPACE:-data}"
 $K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
 
 echo "  applying manifests"
-$K apply -n "$NS" -f - >/dev/null <<YAML
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-spec:
-  selector:
-    app: redis
-  ports:
-    - port: 6379
-      targetPort: 6379
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis
-  template:
-    metadata:
-      labels:
-        app: redis
-    spec:
-      containers:
-        - name: redis
-          image: ${REDIS_IMAGE}
-          ports:
-            - containerPort: 6379
-          readinessProbe:
-            exec:
-              command: ["redis-cli", "ping"]
-            initialDelaySeconds: 3
-            periodSeconds: 5
-YAML
+render_manifest "$ADDON_DIR/redis.yaml" | $K apply -n "$NS" -f - >/dev/null
 
 echo "  waiting for redis..."
 $K rollout status deployment/redis -n "$NS" --timeout=180s

rig/examples/data/addons/redis.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  selector:
+    app: redis
+  ports:
+    - port: 6379
+      targetPort: 6379
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: ${REDIS_IMAGE}
+          ports:
+            - containerPort: 6379
+          readinessProbe:
+            exec:
+              command: ["redis-cli", "ping"]
+            initialDelaySeconds: 3
+            periodSeconds: 5

rig/examples/data/rig.env

@@ -7,6 +7,9 @@
 # LocalExecutor and needs no broker: add redis (before airflow) only to switch to Celery.
 ADDONS="metallb postgres airflow"
 
+# What each part costs in MB, for `make check`'s memory budget (rig adds the kind node).
+MEMORY_BUDGET="airflow=1800 postgres=300"
+
 # Namespace for the dependency containers (k8s/base/kustomization.yaml names it too).
 DATA_NAMESPACE=data