#!/usr/bin/env bash # Apache Airflow for this overlay: one `standalone` pod (LocalExecutor — no broker). # Its own `airflow` database on the postgres addon; the app's data reaches DAGs as the # `app_db` connection; DAGs from this overlay's dags/, as a ConfigMap. # Requires the postgres addon; refuses to install without it. # Notes: ../README.md set -euo pipefail cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}" source ./lib/config.sh load_config K="kubectl --context ${KUBECONTEXT}" NS="${DATA_NAMESPACE:-data}" if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then echo " ! airflow needs the postgres addon, and it is not installed" >&2 echo " add it before airflow in the overlay's ADDONS:" >&2 echo " ADDONS=\"... postgres airflow\"" >&2 exit 1 fi # ── metadata DB: airflow's own tables, kept out of the app's database ────── # Same postgres instance, separate database, created once (idempotent). db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d) db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d) psql() { $K exec -n "$NS" deploy/postgres -- psql -U "$db_user" -d "$db_name" -tAc "$1"; } if [ "$(psql "SELECT 1 FROM pg_database WHERE datname = 'airflow'")" = 1 ]; then echo " database 'airflow' exists" else psql "CREATE DATABASE airflow" >/dev/null echo " created database 'airflow' beside '${db_name}'" fi # ── what is generated once and kept: re-running never rotates these ──────── if $K get secret -n "$NS" airflow >/dev/null 2>&1; then echo " secret exists, keeping the current admin password and fernet key" else admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24) # Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every # restart invalidates every stored connection. fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_') $K create secret generic airflow -n "$NS" \ --from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \ --from-literal=ADMIN_PASSWORD="$admin_password" \ --from-literal=FERNET_KEY="$fernet_key" \ >/dev/null echo " generated an admin password (read it back with the command below)" fi # ── connections: composed from what the postgres secret owns, every run ───── # Nothing to drift: one password reaches the metadata DB and the data connection. $K create secret generic airflow-connections -n "$NS" \ --from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/airflow" \ --from-literal=AIRFLOW_CONN_APP_DB="postgres://${db_user}:${db_pass}@postgres:5432/${db_name}" \ --dry-run=client -o yaml | $K apply -f - >/dev/null # ── DAG delivery: this overlay's dags/ as a ConfigMap ─────────────────────── # Edits land by re-running this addon (make cluster up). The later path — a kind # extraMount of dags/ plus a Tilt sync — is noted in the README, not built. # A ConfigMap volume is kubelet's ..data/.. symlinks, and Airflow's DAG walker # follows symlinks: without the .airflowignore it stops at "Detected recursive loop". dags="$(_from_ctrl "$OVERLAY_DIR")/dags" if [ -d "$dags" ]; then $K create configmap airflow-dags -n "$NS" --from-file="$dags" \ --from-literal=.airflowignore='^\.\.' \ --dry-run=client -o yaml | $K apply -f - >/dev/null echo " dags: $(ls "$dags" | grep -c '\.py$') file(s) from $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")/dags" fi echo " applying manifests" $K apply -n "$NS" -f - >/dev/null <