soleprint/rig/examples/data/addons/airflow.sh

#!/usr/bin/env bash
# Apache Airflow for this overlay: one `standalone` pod (LocalExecutor — no broker).
# Its own `airflow` database on the postgres addon; the app's data reaches DAGs as the
# `app_db` connection; DAGs from this overlay's dags/, as a ConfigMap.
# Requires the postgres addon; refuses to install without it.
# Notes: ../README.md
set -euo pipefail
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"

source ./lib/config.sh
load_config

K="kubectl --context ${KUBECONTEXT}"
NS="${DATA_NAMESPACE:-data}"

if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
    echo "  ! airflow needs the postgres addon, and it is not installed" >&2
    echo "    add it before airflow in the overlay's ADDONS:" >&2
    echo "      ADDONS=\"... postgres airflow\"" >&2
    exit 1
fi

# ── metadata DB: airflow's own tables, kept out of the app's database ──────
# Same postgres instance, separate database, created once (idempotent).
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
psql() { $K exec -n "$NS" deploy/postgres -- psql -U "$db_user" -d "$db_name" -tAc "$1"; }
if [ "$(psql "SELECT 1 FROM pg_database WHERE datname = 'airflow'")" = 1 ]; then
    echo "  database 'airflow' exists"
else
    psql "CREATE DATABASE airflow" >/dev/null
    echo "  created database 'airflow' beside '${db_name}'"
fi

# ── what is generated once and kept: re-running never rotates these ────────
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
    echo "  secret exists, keeping the current admin password and fernet key"
else
    admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
    # Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
    # restart invalidates every stored connection.
    fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
    $K create secret generic airflow -n "$NS" \
        --from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
        --from-literal=ADMIN_PASSWORD="$admin_password" \
        --from-literal=FERNET_KEY="$fernet_key" \
        >/dev/null
    echo "  generated an admin password (read it back with the command below)"
fi

# ── connections: composed from what the postgres secret owns, every run ─────
# Nothing to drift: one password reaches the metadata DB and the data connection.
$K create secret generic airflow-connections -n "$NS" \
    --from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/airflow" \
    --from-literal=AIRFLOW_CONN_APP_DB="postgres://${db_user}:${db_pass}@postgres:5432/${db_name}" \
    --dry-run=client -o yaml | $K apply -f - >/dev/null

# ── DAG delivery: this overlay's dags/ as a ConfigMap ───────────────────────
# Edits land by re-running this addon (make cluster up). The later path — a kind
# extraMount of dags/ plus a Tilt sync — is noted in the README, not built.
# A ConfigMap volume is kubelet's ..data/..<timestamp> symlinks, and Airflow's DAG walker
# follows symlinks: without the .airflowignore it stops at "Detected recursive loop".
dags="$(_from_ctrl "$OVERLAY_DIR")/dags"
if [ -d "$dags" ]; then
    $K create configmap airflow-dags -n "$NS" --from-file="$dags" \
        --from-literal=.airflowignore='^\.\.' \
        --dry-run=client -o yaml | $K apply -f - >/dev/null
    echo "  dags: $(ls "$dags" | grep -c '\.py$') file(s) from $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")/dags"
fi

echo "  applying manifests"
$K apply -n "$NS" -f - >/dev/null <<YAML
apiVersion: v1
kind: Service
metadata:
  name: airflow
spec:
  selector:
    app: airflow
  ports:
    - port: 8080
      targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: airflow
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: airflow
  template:
    metadata:
      labels:
        app: airflow
    spec:
      containers:
        - name: airflow
          image: ${AIRFLOW_IMAGE}
          args: ["standalone"]
          env:
            - name: AIRFLOW__CORE__EXECUTOR
              value: LocalExecutor
            - name: AIRFLOW__CORE__LOAD_EXAMPLES
              value: "false"
            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
              valueFrom:
                secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
            - name: AIRFLOW_CONN_APP_DB
              valueFrom:
                secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
            - name: AIRFLOW__CORE__FERNET_KEY
              valueFrom:
                secretKeyRef: {name: airflow, key: FERNET_KEY}
            - name: _AIRFLOW_WWW_USER_USERNAME
              valueFrom:
                secretKeyRef: {name: airflow, key: ADMIN_USER}
            - name: _AIRFLOW_WWW_USER_PASSWORD
              valueFrom:
                secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
          ports:
            - containerPort: 8080
          volumeMounts:
            - name: dags
              mountPath: /opt/airflow/dags
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
            # First boot runs the whole migration before it serves anything.
            initialDelaySeconds: 60
            periodSeconds: 15
            failureThreshold: 20
      volumes:
        - name: dags
          configMap:
            name: airflow-dags
            optional: true
YAML

echo "  waiting for airflow (the first boot migrates the database, so this is slow)..."
$K rollout status deployment/airflow -n "$NS" --timeout=600s

echo "  in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
echo "  reach it:   kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
echo "  password:   kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"