82 lines
4.3 KiB
Bash
Executable File
82 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Apache Airflow for this overlay: one `standalone` pod (LocalExecutor — no broker).
|
|
# Its own `airflow` database on the postgres addon; the app's data reaches DAGs as the
|
|
# `app_db` connection; DAGs from this overlay's dags/, as a ConfigMap.
|
|
# Requires the postgres addon; refuses to install without it.
|
|
# Notes: ../README.md
|
|
set -euo pipefail
|
|
ADDON_DIR="$(cd "$(dirname "$0")" && pwd)" # its manifest sits beside it
|
|
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
|
|
|
|
source ./lib/config.sh
|
|
load_config
|
|
|
|
K="kubectl --context ${KUBECONTEXT}"
|
|
NS="${DATA_NAMESPACE:-data}"
|
|
|
|
if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
|
|
echo " ! airflow needs the postgres addon, and it is not installed" >&2
|
|
echo " add it before airflow in the overlay's ADDONS:" >&2
|
|
echo " ADDONS=\"... postgres airflow\"" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# ── metadata DB: airflow's own tables, kept out of the app's database ──────
|
|
# Same postgres instance, separate database, created once (idempotent).
|
|
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
|
|
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
|
|
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
|
|
psql() { $K exec -n "$NS" deploy/postgres -- psql -U "$db_user" -d "$db_name" -tAc "$1"; }
|
|
if [ "$(psql "SELECT 1 FROM pg_database WHERE datname = 'airflow'")" = 1 ]; then
|
|
echo " database 'airflow' exists"
|
|
else
|
|
psql "CREATE DATABASE airflow" >/dev/null
|
|
echo " created database 'airflow' beside '${db_name}'"
|
|
fi
|
|
|
|
# ── what is generated once and kept: re-running never rotates these ────────
|
|
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
|
|
echo " secret exists, keeping the current admin password and fernet key"
|
|
else
|
|
admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
|
# Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
|
|
# restart invalidates every stored connection.
|
|
fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
|
|
$K create secret generic airflow -n "$NS" \
|
|
--from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
|
|
--from-literal=ADMIN_PASSWORD="$admin_password" \
|
|
--from-literal=FERNET_KEY="$fernet_key" \
|
|
>/dev/null
|
|
echo " generated an admin password (read it back with the command below)"
|
|
fi
|
|
|
|
# ── connections: composed from what the postgres secret owns, every run ─────
|
|
# Nothing to drift: one password reaches the metadata DB and the data connection.
|
|
$K create secret generic airflow-connections -n "$NS" \
|
|
--from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/airflow" \
|
|
--from-literal=AIRFLOW_CONN_APP_DB="postgres://${db_user}:${db_pass}@postgres:5432/${db_name}" \
|
|
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
|
|
# ── DAG delivery: this overlay's dags/ as a ConfigMap ───────────────────────
|
|
# Edits land by re-running this addon (make cluster up). The later path — a kind
|
|
# extraMount of dags/ plus a Tilt sync — is noted in the README, not built.
|
|
# A ConfigMap volume is kubelet's ..data/..<timestamp> symlinks, and Airflow's DAG walker
|
|
# follows symlinks: without the .airflowignore it stops at "Detected recursive loop".
|
|
dags="$(_from_ctrl "$OVERLAY_DIR")/dags"
|
|
if [ -d "$dags" ]; then
|
|
$K create configmap airflow-dags -n "$NS" --from-file="$dags" \
|
|
--from-literal=.airflowignore='^\.\.' \
|
|
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
echo " dags: $(ls "$dags" | grep -c '\.py$') file(s) from $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")/dags"
|
|
fi
|
|
|
|
echo " applying manifests"
|
|
render_manifest "$ADDON_DIR/airflow.yaml" | $K apply -n "$NS" -f - >/dev/null
|
|
|
|
echo " waiting for airflow (the first boot migrates the database, so this is slow)..."
|
|
$K rollout status deployment/airflow -n "$NS" --timeout=600s
|
|
|
|
echo " in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
|
|
echo " reach it: kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
|
|
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"
|