rig major updates
This commit is contained in:
150
rig/examples/data/addons/airflow.sh
Executable file
150
rig/examples/data/addons/airflow.sh
Executable file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# Apache Airflow for this overlay: one `standalone` pod (LocalExecutor — no broker).
|
||||
# Its own `airflow` database on the postgres addon; the app's data reaches DAGs as the
|
||||
# `app_db` connection; DAGs from this overlay's dags/, as a ConfigMap.
|
||||
# Requires the postgres addon; refuses to install without it.
|
||||
# Notes: ../README.md
|
||||
set -euo pipefail
|
||||
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
|
||||
echo " ! airflow needs the postgres addon, and it is not installed" >&2
|
||||
echo " add it before airflow in the overlay's ADDONS:" >&2
|
||||
echo " ADDONS=\"... postgres airflow\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── metadata DB: airflow's own tables, kept out of the app's database ──────
|
||||
# Same postgres instance, separate database, created once (idempotent).
|
||||
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
|
||||
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
|
||||
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
|
||||
psql() { $K exec -n "$NS" deploy/postgres -- psql -U "$db_user" -d "$db_name" -tAc "$1"; }
|
||||
if [ "$(psql "SELECT 1 FROM pg_database WHERE datname = 'airflow'")" = 1 ]; then
|
||||
echo " database 'airflow' exists"
|
||||
else
|
||||
psql "CREATE DATABASE airflow" >/dev/null
|
||||
echo " created database 'airflow' beside '${db_name}'"
|
||||
fi
|
||||
|
||||
# ── what is generated once and kept: re-running never rotates these ────────
|
||||
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
|
||||
echo " secret exists, keeping the current admin password and fernet key"
|
||||
else
|
||||
admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||
# Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
|
||||
# restart invalidates every stored connection.
|
||||
fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
|
||||
$K create secret generic airflow -n "$NS" \
|
||||
--from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
|
||||
--from-literal=ADMIN_PASSWORD="$admin_password" \
|
||||
--from-literal=FERNET_KEY="$fernet_key" \
|
||||
>/dev/null
|
||||
echo " generated an admin password (read it back with the command below)"
|
||||
fi
|
||||
|
||||
# ── connections: composed from what the postgres secret owns, every run ─────
|
||||
# Nothing to drift: one password reaches the metadata DB and the data connection.
|
||||
$K create secret generic airflow-connections -n "$NS" \
|
||||
--from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/airflow" \
|
||||
--from-literal=AIRFLOW_CONN_APP_DB="postgres://${db_user}:${db_pass}@postgres:5432/${db_name}" \
|
||||
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
||||
|
||||
# ── DAG delivery: this overlay's dags/ as a ConfigMap ───────────────────────
|
||||
# Edits land by re-running this addon (make cluster up). The later path — a kind
|
||||
# extraMount of dags/ plus a Tilt sync — is noted in the README, not built.
|
||||
# A ConfigMap volume is kubelet's ..data/..<timestamp> symlinks, and Airflow's DAG walker
|
||||
# follows symlinks: without the .airflowignore it stops at "Detected recursive loop".
|
||||
dags="$(_from_ctrl "$OVERLAY_DIR")/dags"
|
||||
if [ -d "$dags" ]; then
|
||||
$K create configmap airflow-dags -n "$NS" --from-file="$dags" \
|
||||
--from-literal=.airflowignore='^\.\.' \
|
||||
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
||||
echo " dags: $(ls "$dags" | grep -c '\.py$') file(s) from $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")/dags"
|
||||
fi
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
selector:
|
||||
app: airflow
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: airflow
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: airflow
|
||||
spec:
|
||||
containers:
|
||||
- name: airflow
|
||||
image: ${AIRFLOW_IMAGE}
|
||||
args: ["standalone"]
|
||||
env:
|
||||
- name: AIRFLOW__CORE__EXECUTOR
|
||||
value: LocalExecutor
|
||||
- name: AIRFLOW__CORE__LOAD_EXAMPLES
|
||||
value: "false"
|
||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
|
||||
- name: AIRFLOW_CONN_APP_DB
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
|
||||
- name: AIRFLOW__CORE__FERNET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: FERNET_KEY}
|
||||
- name: _AIRFLOW_WWW_USER_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_USER}
|
||||
- name: _AIRFLOW_WWW_USER_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: dags
|
||||
mountPath: /opt/airflow/dags
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
# First boot runs the whole migration before it serves anything.
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 15
|
||||
failureThreshold: 20
|
||||
volumes:
|
||||
- name: dags
|
||||
configMap:
|
||||
name: airflow-dags
|
||||
optional: true
|
||||
YAML
|
||||
|
||||
echo " waiting for airflow (the first boot migrates the database, so this is slow)..."
|
||||
$K rollout status deployment/airflow -n "$NS" --timeout=600s
|
||||
|
||||
echo " in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
|
||||
echo " reach it: kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
|
||||
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"
|
||||
Reference in New Issue
Block a user