110 lines
3.8 KiB
Bash
Executable File
110 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Apache Airflow — the cluster half of the airflow cabinet (one `standalone` pod).
|
|
# Requires the postgres addon; refuses to install without it.
|
|
# Notes: docs/notes/addons.md
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
source ./lib/config.sh
|
|
load_config
|
|
|
|
K="kubectl --context ${KUBECONTEXT}"
|
|
NS="${DATA_NAMESPACE:-data}"
|
|
|
|
if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
|
|
echo " ! airflow needs the postgres addon, and it is not installed" >&2
|
|
echo " add it before airflow in the profile's ADDONS:" >&2
|
|
echo " ADDONS=\"... postgres airflow\"" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Reuse the credential postgres generated rather than storing a second copy.
|
|
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
|
|
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
|
|
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
|
|
|
|
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
|
|
echo " secret exists, keeping the current admin password and fernet key"
|
|
else
|
|
admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
|
# Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
|
|
# restart invalidates every stored connection.
|
|
fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
|
|
$K create secret generic airflow -n "$NS" \
|
|
--from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
|
|
--from-literal=ADMIN_PASSWORD="$admin_password" \
|
|
--from-literal=FERNET_KEY="$fernet_key" \
|
|
--from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/${db_name}" \
|
|
>/dev/null
|
|
echo " generated an admin password (read it back with the command below)"
|
|
fi
|
|
|
|
echo " applying manifests"
|
|
$K apply -n "$NS" -f - >/dev/null <<YAML
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: airflow
|
|
spec:
|
|
selector:
|
|
app: airflow
|
|
ports:
|
|
- port: 8080
|
|
targetPort: 8080
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: airflow
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: airflow
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: airflow
|
|
spec:
|
|
containers:
|
|
- name: airflow
|
|
image: ${AIRFLOW_IMAGE}
|
|
args: ["standalone"]
|
|
env:
|
|
- name: AIRFLOW__CORE__EXECUTOR
|
|
value: LocalExecutor
|
|
- name: AIRFLOW__CORE__LOAD_EXAMPLES
|
|
value: "false"
|
|
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
|
valueFrom:
|
|
secretKeyRef: {name: airflow, key: SQL_ALCHEMY_CONN}
|
|
- name: AIRFLOW__CORE__FERNET_KEY
|
|
valueFrom:
|
|
secretKeyRef: {name: airflow, key: FERNET_KEY}
|
|
- name: _AIRFLOW_WWW_USER_USERNAME
|
|
valueFrom:
|
|
secretKeyRef: {name: airflow, key: ADMIN_USER}
|
|
- name: _AIRFLOW_WWW_USER_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
|
|
ports:
|
|
- containerPort: 8080
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
# First boot runs the whole migration before it serves anything.
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 15
|
|
failureThreshold: 20
|
|
YAML
|
|
|
|
echo " waiting for airflow (the first boot migrates the database, so this is slow)..."
|
|
$K rollout status deployment/airflow -n "$NS" --timeout=600s
|
|
|
|
echo " in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
|
|
echo " reach it: kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
|
|
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"
|