rig major updates
This commit is contained in:
150
rig/examples/data/addons/airflow.sh
Executable file
150
rig/examples/data/addons/airflow.sh
Executable file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# Apache Airflow for this overlay: one `standalone` pod (LocalExecutor — no broker).
|
||||
# Its own `airflow` database on the postgres addon; the app's data reaches DAGs as the
|
||||
# `app_db` connection; DAGs from this overlay's dags/, as a ConfigMap.
|
||||
# Requires the postgres addon; refuses to install without it.
|
||||
# Notes: ../README.md
|
||||
set -euo pipefail
|
||||
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
if ! $K get deployment -n "$NS" postgres >/dev/null 2>&1; then
|
||||
echo " ! airflow needs the postgres addon, and it is not installed" >&2
|
||||
echo " add it before airflow in the overlay's ADDONS:" >&2
|
||||
echo " ADDONS=\"... postgres airflow\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── metadata DB: airflow's own tables, kept out of the app's database ──────
|
||||
# Same postgres instance, separate database, created once (idempotent).
|
||||
db_user=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
|
||||
db_pass=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)
|
||||
db_name=$($K get secret -n "$NS" postgres -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
|
||||
psql() { $K exec -n "$NS" deploy/postgres -- psql -U "$db_user" -d "$db_name" -tAc "$1"; }
|
||||
if [ "$(psql "SELECT 1 FROM pg_database WHERE datname = 'airflow'")" = 1 ]; then
|
||||
echo " database 'airflow' exists"
|
||||
else
|
||||
psql "CREATE DATABASE airflow" >/dev/null
|
||||
echo " created database 'airflow' beside '${db_name}'"
|
||||
fi
|
||||
|
||||
# ── what is generated once and kept: re-running never rotates these ────────
|
||||
if $K get secret -n "$NS" airflow >/dev/null 2>&1; then
|
||||
echo " secret exists, keeping the current admin password and fernet key"
|
||||
else
|
||||
admin_password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||
# Airflow requires a 32-byte urlsafe-base64 key; without a fixed one every
|
||||
# restart invalidates every stored connection.
|
||||
fernet_key=$(head -c 32 /dev/urandom | base64 | tr '+/' '-_')
|
||||
$K create secret generic airflow -n "$NS" \
|
||||
--from-literal=ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}" \
|
||||
--from-literal=ADMIN_PASSWORD="$admin_password" \
|
||||
--from-literal=FERNET_KEY="$fernet_key" \
|
||||
>/dev/null
|
||||
echo " generated an admin password (read it back with the command below)"
|
||||
fi
|
||||
|
||||
# ── connections: composed from what the postgres secret owns, every run ─────
|
||||
# Nothing to drift: one password reaches the metadata DB and the data connection.
|
||||
$K create secret generic airflow-connections -n "$NS" \
|
||||
--from-literal=SQL_ALCHEMY_CONN="postgresql+psycopg2://${db_user}:${db_pass}@postgres:5432/airflow" \
|
||||
--from-literal=AIRFLOW_CONN_APP_DB="postgres://${db_user}:${db_pass}@postgres:5432/${db_name}" \
|
||||
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
||||
|
||||
# ── DAG delivery: this overlay's dags/ as a ConfigMap ───────────────────────
|
||||
# Edits land by re-running this addon (make cluster up). The later path — a kind
|
||||
# extraMount of dags/ plus a Tilt sync — is noted in the README, not built.
|
||||
# A ConfigMap volume is kubelet's ..data/..<timestamp> symlinks, and Airflow's DAG walker
|
||||
# follows symlinks: without the .airflowignore it stops at "Detected recursive loop".
|
||||
dags="$(_from_ctrl "$OVERLAY_DIR")/dags"
|
||||
if [ -d "$dags" ]; then
|
||||
$K create configmap airflow-dags -n "$NS" --from-file="$dags" \
|
||||
--from-literal=.airflowignore='^\.\.' \
|
||||
--dry-run=client -o yaml | $K apply -f - >/dev/null
|
||||
echo " dags: $(ls "$dags" | grep -c '\.py$') file(s) from $(basename "$(_abs_from_ctrl "$OVERLAY_DIR")")/dags"
|
||||
fi
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
selector:
|
||||
app: airflow
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: airflow
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: airflow
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: airflow
|
||||
spec:
|
||||
containers:
|
||||
- name: airflow
|
||||
image: ${AIRFLOW_IMAGE}
|
||||
args: ["standalone"]
|
||||
env:
|
||||
- name: AIRFLOW__CORE__EXECUTOR
|
||||
value: LocalExecutor
|
||||
- name: AIRFLOW__CORE__LOAD_EXAMPLES
|
||||
value: "false"
|
||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow-connections, key: SQL_ALCHEMY_CONN}
|
||||
- name: AIRFLOW_CONN_APP_DB
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow-connections, key: AIRFLOW_CONN_APP_DB}
|
||||
- name: AIRFLOW__CORE__FERNET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: FERNET_KEY}
|
||||
- name: _AIRFLOW_WWW_USER_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_USER}
|
||||
- name: _AIRFLOW_WWW_USER_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: {name: airflow, key: ADMIN_PASSWORD}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: dags
|
||||
mountPath: /opt/airflow/dags
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
# First boot runs the whole migration before it serves anything.
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 15
|
||||
failureThreshold: 20
|
||||
volumes:
|
||||
- name: dags
|
||||
configMap:
|
||||
name: airflow-dags
|
||||
optional: true
|
||||
YAML
|
||||
|
||||
echo " waiting for airflow (the first boot migrates the database, so this is slow)..."
|
||||
$K rollout status deployment/airflow -n "$NS" --timeout=600s
|
||||
|
||||
echo " in-cluster: http://airflow.${NS}.svc.cluster.local:8080"
|
||||
echo " reach it: kubectl --context ${KUBECONTEXT} -n ${NS} port-forward svc/airflow 8080:8080"
|
||||
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret airflow -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d"
|
||||
105
rig/examples/data/addons/postgres.sh
Executable file
105
rig/examples/data/addons/postgres.sh
Executable file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# PostgreSQL for this overlay: plain manifests, one replica on a PVC, password generated once and kept.
|
||||
# Notes: ../README.md
|
||||
set -euo pipefail
|
||||
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
$K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
|
||||
|
||||
# The password is generated once and then left alone, so re-running this does
|
||||
# not rotate the credential out from under whatever is already connected.
|
||||
if $K get secret -n "$NS" postgres >/dev/null 2>&1; then
|
||||
echo " secret exists, keeping the current password"
|
||||
else
|
||||
password=$(head -c 18 /dev/urandom | base64 | tr -d '/+=' | head -c 24)
|
||||
$K create secret generic postgres -n "$NS" \
|
||||
--from-literal=POSTGRES_DB="${POSTGRES_DB:-postgres}" \
|
||||
--from-literal=POSTGRES_USER="${POSTGRES_USER:-postgres}" \
|
||||
--from-literal=POSTGRES_PASSWORD="$password" >/dev/null
|
||||
echo " generated a password (read it back with the command printed below)"
|
||||
fi
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: postgres-data
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests:
|
||||
storage: ${POSTGRES_STORAGE:-2Gi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: postgres
|
||||
spec:
|
||||
selector:
|
||||
app: postgres
|
||||
ports:
|
||||
- port: 5432
|
||||
targetPort: 5432
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: postgres
|
||||
spec:
|
||||
replicas: 1
|
||||
# One volume, one writer. Rolling would start a second pod against the same
|
||||
# PVC before the first exits, and Postgres refuses to share a data directory.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: ${POSTGRES_IMAGE}
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: postgres
|
||||
env:
|
||||
# The image initialises into the volume root otherwise, and a
|
||||
# lost+found from the PVC makes it refuse to initdb.
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["sh", "-c", "pg_isready -U \$POSTGRES_USER -d \$POSTGRES_DB"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: postgres-data
|
||||
YAML
|
||||
|
||||
echo " waiting for postgres..."
|
||||
$K rollout status deployment/postgres -n "$NS" --timeout=240s
|
||||
|
||||
echo " in-cluster: postgres.${NS}.svc.cluster.local:5432"
|
||||
echo " password: kubectl --context ${KUBECONTEXT} -n ${NS} get secret postgres -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d"
|
||||
57
rig/examples/data/addons/redis.sh
Executable file
57
rig/examples/data/addons/redis.sh
Executable file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# Redis for this overlay: cache and broker (Celery), no persistence.
|
||||
# Notes: ../README.md
|
||||
set -euo pipefail
|
||||
cd "${RIG_CTRL:?run it through rig: bash ctrl/addons.sh install}"
|
||||
|
||||
source ./lib/config.sh
|
||||
load_config
|
||||
|
||||
K="kubectl --context ${KUBECONTEXT}"
|
||||
NS="${DATA_NAMESPACE:-data}"
|
||||
|
||||
$K get namespace "$NS" >/dev/null 2>&1 || $K create namespace "$NS"
|
||||
|
||||
echo " applying manifests"
|
||||
$K apply -n "$NS" -f - >/dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
selector:
|
||||
app: redis
|
||||
ports:
|
||||
- port: 6379
|
||||
targetPort: 6379
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: ${REDIS_IMAGE}
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 5
|
||||
YAML
|
||||
|
||||
echo " waiting for redis..."
|
||||
$K rollout status deployment/redis -n "$NS" --timeout=180s
|
||||
|
||||
echo " in-cluster: redis://redis.${NS}.svc.cluster.local:6379/0"
|
||||
Reference in New Issue
Block a user