init rig

rig/ctrl/addons/airflow.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Apache Airflow — the cluster half of soleprint's airflow cabinet.
+#
+# Airflow needs a metadata database before it will start at all, so this refuses
+# rather than rolls a pod that will CrashLoopBackOff while the real problem
+# (postgres missing from ADDONS) stays invisible in the logs.
+#
+# One pod on `standalone`, matching the compose cabinet: migration, admin user,
+# scheduler and webserver in a single container. The official chart's five
+# deployments model an installation; a room switching this on wants pipelines.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+source ./lib/config.sh
+load_config
+
+K="kubectl --context ${KUBECONTEXT}"
+NS="${DATA_NAMESPACE:-data}"
+
+if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
+    echo "  ! airflow needs the postgres addon, and it is not installed" >&2
+    echo "    add it before airflow in the profile's ADDONS:" >&2
+    echo "      ADDONS=\"... postgres airflow\"" >&2
+    exit 1
+fi
+
+# Reuse the credential postgres generated rather than storing a second copy.
+db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
+db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
+db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
+
+if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
+    echo "  secret exists, keeping the current admin password and fernet key"
+else
+    admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
+    # Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
+    # restart invalidates every stored connection.
+    fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
+    $K create secret generic airflow -n "$NS" \
+        --from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
+        --from-literal=ADMIN_PASSWORD="$admin_password" \
+        --from-literal=FERNET_KEY="$fernet_key" \
+        --from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/${db_name}" \
+        >/dev/null
+    echo "  generated an admin password (read it back with the command below)"
+fi
+
+echo "  applying manifests"
+$K apply -n "$NS" -f - >/dev/null <<YAML
+apiVersion: v1
+kind: Service
+metadata:
+  name: airflow
+spec:
+  selector:
+    app: airflow
+  ports:
+    - port: 8080
+      targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airflow
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: airflow
+  template:
+    metadata:
+      labels:
+        app: airflow
+    spec:
+      containers:
+        - name: airflow
+          image: ${AIRFLOW_IMAGE}
+          args: ["standalone"]
+          env:
+            - name: AIRFLOW__CORE__EXECUTOR
+              value: LocalExecutor
+            - name: AIRFLOW__CORE__LOAD_EXAMPLES
+              value: "false"
+            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+              valueFrom:
+                secretKeyRef: {name: airflow, key: SQL_ALCHEMY_CONN}
+            - name: AIRFLOW__CORE__FERNET_KEY
+              valueFrom:
+                secretKeyRef: {name: airflow, key: FERNET_KEY}
+            - name: _AIRFLOW_WWW_USER_USERNAME
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_USER}
+            - name: _AIRFLOW_WWW_USER_PASSWORD
+              valueFrom:
+                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
+          ports:
+            - containerPort: 8080
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            # First boot runs the whole migration before it serves anything.
+            initialDelaySeconds: 60
+            periodSeconds: 15
+            failureThreshold: 20
+YAML
+
+echo "  waiting for airflow (the first boot migrates the database, so this is slow)..."
+$K rollout status deployment/airflow -n "$NS" --timeout=600s
+
+echo "  in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
+echo "  reach it:   kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
+echo "  password:   kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"