865 lines
36 KiB
Bash
Executable File
865 lines
36 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# GENERATED by make standalone — do not edit
|
|
#
|
|
# rigmini.sh for profile 'data', flattened from:
|
|
# ctrl/mem.sh
|
|
# ctrl/lib/config.sh
|
|
# Edit those and run `make standalone`. Changes made here are lost, and
|
|
# `make selftest` fails while this file differs from what rig generates.
|
|
|
|
# ── from the libraries ──
|
|
declare -- CONFIG_OVERRIDABLE=$'PROFILE CLUSTER K8S_VERSION KIND_CONFIG ADDONS\n REGISTRY_MODE INGRESS_MODE DNS_MODE TILT_PORT\n SOURCE ARCH DEPS_SOURCE HTTP_PORT HTTPS_PORT\n REGISTRY_PORT MANIFESTS_DIR'
|
|
_config_restore ()
|
|
{
|
|
local line;
|
|
while IFS= read -r line; do
|
|
if [ -n "$line" ]; then
|
|
eval "export $line";
|
|
fi;
|
|
done <<< "$1";
|
|
return 0
|
|
}
|
|
default_cluster_name ()
|
|
{
|
|
local n;
|
|
n=$(basename "$(cd .. && pwd)");
|
|
n=$(echo "$n" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-');
|
|
n=$(echo "$n" | sed 's/^-*//; s/-*$//');
|
|
echo "${n:-rig}"
|
|
}
|
|
derive_port_base ()
|
|
{
|
|
local h;
|
|
h=$(printf '%s' "$1" | cksum | awk '{print $1}');
|
|
echo $((20000 + (h % 200) * 10))
|
|
}
|
|
render_kind_config ()
|
|
{
|
|
local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}";
|
|
sed -e "s|\${CLUSTER}|${CLUSTER}|g" -e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" -e "s|\${HTTP_PORT}|${HTTP_PORT}|g" -e "s|\${HOST_WORKDIR}|${host_workdir}|g" "$KIND_CONFIG_PATH"
|
|
}
|
|
|
|
# ── configuration, frozen for profile 'data' ──
|
|
load_config() {
|
|
local k saved=""
|
|
for k in $CONFIG_OVERRIDABLE; do
|
|
if [ -n "${!k+x}" ]; then saved+="$k=$(printf '%q' "${!k}")"$'\n'; fi
|
|
done
|
|
declare -gx ADDONS="metallb postgres redis airflow"
|
|
declare -gx AIRFLOW_ADMIN_USER="admin"
|
|
declare -gx AIRFLOW_IMAGE="apache/airflow:2.10.4"
|
|
declare -g AUDIT="off"
|
|
declare -gx CERT_MANAGER_VERSION="v1.21.1"
|
|
declare -g CLUSTER="rig"
|
|
declare -gx COMPOSE_SHA256="db1889184726840f75c4f9c001048430d4f25b3be3cb084d3ddd762bc0aed576"
|
|
declare -gx COMPOSE_URL="https://github.com/docker/compose/releases/download/v5.5.1/docker-compose-linux-x86_64"
|
|
declare -gx COMPOSE_VERSION="5.5.1"
|
|
declare -gx CTLPTL_SHA256="c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e"
|
|
declare -gx CTLPTL_URL="https://github.com/tilt-dev/ctlptl/releases/download/v0.9.4/ctlptl.0.9.4.linux.x86_64.tar.gz"
|
|
declare -gx CTLPTL_VERSION="0.9.4"
|
|
declare -gx DATA_NAMESPACE="data"
|
|
declare -gx DNS_MODE="hosts"
|
|
declare -g HTTPS_PORT="20311"
|
|
declare -g HTTP_PORT="20310"
|
|
declare -gx INGRESS_MODE="hostport"
|
|
declare -gx JQ_SHA256="b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f"
|
|
declare -gx JQ_URL="https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64"
|
|
declare -gx JQ_VERSION="1.8.2"
|
|
declare -gx K8S_VERSION="v1_36"
|
|
declare -gx KIND_CONFIG="kind-config.yaml.tpl"
|
|
declare -g KIND_CONFIG_PATH="./k8s/kind-config.yaml.tpl"
|
|
declare -g KIND_CONFIG_SHOWN="ctrl/k8s/kind-config.yaml.tpl"
|
|
declare -gx KIND_SHA256="50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54"
|
|
declare -gx KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/v0.32.0/kind-linux-amd64"
|
|
declare -gx KIND_VERSION="v0.32.0"
|
|
declare -g KUBECONTEXT="kind-rig"
|
|
declare -gx KUBECTL_SHA256="ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336"
|
|
declare -gx KUBECTL_URL="https://dl.k8s.io/release/v1.36.3/bin/linux/amd64/kubectl"
|
|
declare -gx KUBECTL_VERSION="v1.36.3"
|
|
declare -g MANIFESTS_DIR="ctrl/k8s/overlays/dev"
|
|
declare -gx METALLB_VERSION="v0.16.0"
|
|
declare -gx METRICS_SERVER_VERSION="v0.9.0"
|
|
declare -g NODES="1"
|
|
declare -g NODE_IMAGE="kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5"
|
|
declare -gx NODE_IMAGE_v1_33="kindest/node:v1.33.12@sha256:3f5c8443c620245e4d355cfe09e96a91ead32ceaa569d3f1ca9edf0cb2fe2ff4"
|
|
declare -gx NODE_IMAGE_v1_34="kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256"
|
|
declare -gx NODE_IMAGE_v1_35="kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95"
|
|
declare -gx NODE_IMAGE_v1_36="kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5"
|
|
declare -g NODE_MB="800"
|
|
declare -gx POSTGRES_DB="app"
|
|
declare -gx POSTGRES_IMAGE="postgres:16-alpine"
|
|
declare -gx POSTGRES_STORAGE="2Gi"
|
|
declare -gx POSTGRES_USER="app"
|
|
declare -gx PROFILE_NAME="data"
|
|
declare -gx REDIS_IMAGE="redis:7-alpine"
|
|
declare -gx REGISTRY_IMAGE="registry:2"
|
|
declare -gx REGISTRY_MODE="local"
|
|
declare -g REGISTRY_PORT="20313"
|
|
declare -gx STUB_IMAGE="python:3.12-slim"
|
|
declare -g TILT_PORT="20312"
|
|
declare -gx TILT_SHA256="e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6"
|
|
declare -gx TILT_URL="https://github.com/tilt-dev/tilt/releases/download/v0.37.6/tilt.0.37.6.linux.x86_64.tar.gz"
|
|
declare -gx TILT_VERSION="0.37.6"
|
|
_config_restore "$saved"
|
|
}
|
|
# ── end of frozen configuration ──
|
|
|
|
# ── ctrl/mem.sh ──
|
|
# How much memory this machine will actually give you before something dies —
|
|
# rig's memory tool, and (generated from this file) the standalone rigmini.sh.
|
|
#
|
|
# There are two numbers and they are rarely the same. `status` reports what the
|
|
# machine ADVERTISES and what is quietly capping it. `push` finds what it will
|
|
# SURVIVE, by allocating until it stops. `all` does both and weighs the result
|
|
# against what this profile's cluster needs.
|
|
#
|
|
# The gap between them is the whole reason this exists. Under WSL the cap lives
|
|
# in .wslconfig; in a container or a managed workspace it is a cgroup limit, and
|
|
# there /proc/meminfo reports the HOST's memory while the kernel kills you at a
|
|
# fraction of it. A script that only read MemTotal would confidently report 32 GB
|
|
# on a box that OOMs at 2.
|
|
#
|
|
# Runs on native Linux and under WSL. On WSL the memory you see is a VM
|
|
# allocation that can be raised, and the commonest failure is raising it without
|
|
# restarting — so status compares what .wslconfig says with what actually booted.
|
|
#
|
|
# Reports and instructs. It never raises a limit, frees anything or installs a
|
|
# package. The one write it can make is `backup`, which copies .wslconfig beside
|
|
# itself, so that `restore` has something to put back after a hand edit.
|
|
#
|
|
# Usage:
|
|
# mem.sh status what it has, what caps it
|
|
# mem.sh push [--to GB] [--to-oom] climb until it stops
|
|
# mem.sh all [--budget GB] both, then the verdict
|
|
# mem.sh backup | restore .wslconfig, WSL only
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
# (sourced library inlined above)
|
|
|
|
# ── defaults ───────────────────────────────────────────────────────────────
|
|
|
|
STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push().
|
|
STEP_EXPLICIT=no # whether --step was given, which turns the scaling off.
|
|
TO_MB="" # --to: stop here regardless. Empty means no hard cap.
|
|
TO_OOM=no # --to-oom: opt in to running until the kernel intervenes.
|
|
BUDGET_GB="" # --budget; empty means what this profile's cluster needs, from rig.
|
|
BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below.
|
|
|
|
# ── platform ───────────────────────────────────────────────────────────────
|
|
|
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
|
# the tooling. Detectable, so name it instead.
|
|
require_linux() {
|
|
case "$(uname -s)" in
|
|
MINGW*|MSYS*|CYGWIN*)
|
|
cat >&2 <<'EOF'
|
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
|
|
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
|
|
|
wsl --install
|
|
|
|
That enables Windows features and needs a reboot, so it is not something this
|
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
|
this from there.
|
|
EOF
|
|
exit 1 ;;
|
|
esac
|
|
|
|
# Everything below reads /proc. Without it there is nothing to measure, and
|
|
# failing here beats printing a page of empty fields.
|
|
if [ ! -r /proc/meminfo ]; then
|
|
echo "no readable /proc/meminfo — this needs a Linux kernel." >&2
|
|
echo "On macOS or a BSD none of the numbers below exist." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
|
|
|
is_container() {
|
|
[ -f /.dockerenv ] && return 0
|
|
grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null
|
|
}
|
|
|
|
platform() {
|
|
if is_wsl; then echo WSL
|
|
elif is_container; then echo container
|
|
else echo "native linux"
|
|
fi
|
|
}
|
|
|
|
# ── reading memory ─────────────────────────────────────────────────────────
|
|
|
|
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
|
|
|
# MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on
|
|
# corporate images — need the estimate it replaced, which is worse but not wrong.
|
|
avail_meminfo_mb() {
|
|
if grep -q '^MemAvailable:' /proc/meminfo; then
|
|
mb MemAvailable
|
|
else
|
|
awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo
|
|
fi
|
|
}
|
|
|
|
# Where a cgroup records this cgroup's own limit and usage. Set once by
|
|
# find_cgroup, because every later reading needs both and hunting for the files
|
|
# on each call would be the slow part of the poll loop.
|
|
CG_MAX_FILE=""
|
|
CG_CUR_FILE=""
|
|
CG_VERSION=""
|
|
|
|
find_cgroup() {
|
|
local rel
|
|
|
|
# Inside a container the cgroup namespace makes the top of the tree BE the
|
|
# container's own cgroup, so the unqualified path is already the right one.
|
|
# On a host it is the root cgroup, which is never limited — hence the second
|
|
# attempt via /proc/self/cgroup, which names the slice this shell is in.
|
|
if [ -r /sys/fs/cgroup/memory.max ]; then
|
|
CG_VERSION=v2
|
|
CG_MAX_FILE=/sys/fs/cgroup/memory.max
|
|
CG_CUR_FILE=/sys/fs/cgroup/memory.current
|
|
elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
|
|
CG_VERSION=v1
|
|
CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes
|
|
CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes
|
|
fi
|
|
|
|
rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
|
if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then
|
|
CG_VERSION=v2
|
|
CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max"
|
|
CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current"
|
|
return 0
|
|
fi
|
|
|
|
rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
|
if [ -n "$rel" ] && [ "$rel" != "/" ] \
|
|
&& [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then
|
|
CG_VERSION=v1
|
|
CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes"
|
|
CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# The cap in MB, or "" when there is none worth reporting. v2 spells unlimited
|
|
# "max"; v1 spells it as a number near 2^63, which is why this compares against
|
|
# MemTotal rather than testing for a magic value — a "limit" above the machine's
|
|
# own memory is not a limit, however it is written.
|
|
cgroup_cap_mb() {
|
|
local raw cap
|
|
[ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; }
|
|
raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max)
|
|
[ "$raw" = "max" ] && { echo ""; return 0; }
|
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
cap=$((raw / 1024 / 1024))
|
|
[ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; }
|
|
echo "$cap"
|
|
}
|
|
|
|
cgroup_used_mb() {
|
|
local raw
|
|
[ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; }
|
|
raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "")
|
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
echo $((raw / 1024 / 1024))
|
|
}
|
|
|
|
# ulimit -v is a per-process address-space cap. It stops YOU long before the box
|
|
# does, and because it is inherited from a login shell it is easy to hit without
|
|
# knowing it is set.
|
|
ulimit_v_mb() {
|
|
local v; v=$(ulimit -v 2>/dev/null || echo unlimited)
|
|
[ "$v" = "unlimited" ] && { echo ""; return 0; }
|
|
case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
echo $((v / 1024))
|
|
}
|
|
|
|
# The number everything else is about: the lowest of the things that can stop
|
|
# you. Printed at the end of `status` and used as the sanity bound in `push`.
|
|
effective_ceiling_mb() {
|
|
local c; c=$(mb MemTotal)
|
|
local cap; cap=$(cgroup_cap_mb)
|
|
local ul; ul=$(ulimit_v_mb)
|
|
[ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap"
|
|
[ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul"
|
|
echo "$c"
|
|
}
|
|
|
|
# How much room is left RIGHT NOW, from whichever accounting actually governs.
|
|
# In a capped container /proc/meminfo describes the host and is worse than
|
|
# useless for this — it would report tens of gigabytes free on a box that is one
|
|
# allocation from being killed.
|
|
headroom_mb() {
|
|
local cap used
|
|
cap=$(cgroup_cap_mb)
|
|
used=$(cgroup_used_mb)
|
|
if [ -n "$cap" ] && [ -n "$used" ]; then
|
|
echo $(( cap - used ))
|
|
else
|
|
avail_meminfo_mb
|
|
fi
|
|
}
|
|
|
|
# ── status ─────────────────────────────────────────────────────────────────
|
|
|
|
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
|
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
|
# Windows, then fall back to whichever profile actually owns a config.
|
|
wslconfig_path() {
|
|
local profile winpath found
|
|
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
|
case "$profile" in
|
|
""|*%*) ;;
|
|
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
|
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
|
echo "$winpath/.wslconfig"; return 0
|
|
fi ;;
|
|
esac
|
|
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
|
[ -n "$found" ] && echo "$found"
|
|
return 0
|
|
}
|
|
|
|
hogs() {
|
|
echo " holding the most:"
|
|
ps -eo rss,comm --sort=-rss 2>/dev/null \
|
|
| awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
|
return 0
|
|
}
|
|
|
|
status() {
|
|
local total avail swap_total swap_free cap ul cur
|
|
|
|
echo "host"
|
|
echo " platform $(platform)"
|
|
echo " kernel $(uname -r)"
|
|
[ -r /etc/os-release ] && \
|
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
|
echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)"
|
|
|
|
# ── the caps first, because they decide what the totals below are worth ──
|
|
echo
|
|
echo "caps"
|
|
cap=$(cgroup_cap_mb)
|
|
if [ -n "$cap" ]; then
|
|
cur=$(cgroup_used_mb)
|
|
echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)"
|
|
echo " ! /proc/meminfo below describes the HOST, not this cgroup."
|
|
echo " $(mb MemTotal) MB total is not yours; ${cap} MB is."
|
|
elif [ -n "$CG_VERSION" ]; then
|
|
echo " cgroup none (${CG_VERSION} present, no memory limit set)"
|
|
else
|
|
echo " cgroup no memory controller found"
|
|
fi
|
|
|
|
ul=$(ulimit_v_mb)
|
|
if [ -n "$ul" ]; then
|
|
echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell"
|
|
echo " it stops this process long before the machine runs out"
|
|
else
|
|
echo " ulimit -v unlimited"
|
|
fi
|
|
|
|
# overcommit_memory=0 is the default heuristic: a large allocation is
|
|
# granted on a guess, and the reckoning arrives later as an OOM kill rather
|
|
# than as a failed malloc. It is why `push` touches every page it asks for.
|
|
local om or_
|
|
om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?')
|
|
or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?')
|
|
case "$om" in
|
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;;
|
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;;
|
|
2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;;
|
|
*) echo " overcommit ${om}" ;;
|
|
esac
|
|
[ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for"
|
|
|
|
# ── what it says it has ──
|
|
total=$(mb MemTotal); avail=$(avail_meminfo_mb)
|
|
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
|
echo
|
|
echo "memory"
|
|
echo " total ${total} MB"
|
|
echo " available ${avail} MB"
|
|
echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
|
if [ "$swap_total" -eq 0 ]; then
|
|
echo " - no swap: this box has no cushion. It goes from fine to OOM-killed"
|
|
echo " with nothing in between, which is the abrupt failure you get in a VM."
|
|
fi
|
|
|
|
# postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB,
|
|
# and the resulting failure names neither shm nor the size.
|
|
if [ -d /dev/shm ]; then
|
|
local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}')
|
|
if [ -n "$shm" ]; then
|
|
if [ "$shm" -le 64 ]; then
|
|
echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB"
|
|
echo " is docker's default. Raise it with --shm-size when postgres fails."
|
|
else
|
|
echo " /dev/shm ${shm} MB"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "disk"
|
|
local d
|
|
for d in / /tmp /var/lib/docker; do
|
|
[ -d "$d" ] || continue
|
|
df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}'
|
|
done
|
|
|
|
# kind and Tilt both watch large trees, and the failure mode is silent:
|
|
# they simply stop noticing file changes. Cheap to report while we are here.
|
|
local w i
|
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
|
echo
|
|
echo "tooling"
|
|
echo " inotify watches=$w instances=$i"
|
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
|
echo " ! low — anything watching files will silently stop seeing changes"
|
|
fi
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
if [ -S /var/run/docker.sock ]; then
|
|
echo " docker socket present, no cli"
|
|
else
|
|
echo " docker not installed"
|
|
fi
|
|
elif docker info >/dev/null 2>&1; then
|
|
local n
|
|
n=$(docker ps -q 2>/dev/null | wc -l)
|
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running"
|
|
else
|
|
echo " ! docker cli present but the daemon is unreachable"
|
|
fi
|
|
|
|
# WSL keeps its cap on the Windows side, in a file this shell can read but
|
|
# not usefully apply — the change costs a full VM restart. Report it, and
|
|
# report the commonest mistake, which is editing it and not restarting.
|
|
if is_wsl; then
|
|
local cfg conf conf_mb n
|
|
cfg=$(wslconfig_path)
|
|
echo
|
|
echo "wsl"
|
|
if [ -z "$cfg" ]; then
|
|
echo " ! cannot tell which Windows profile owns .wslconfig"
|
|
else
|
|
echo " config $cfg"
|
|
conf=$(configured_memory "$cfg")
|
|
if [ -n "$conf" ]; then
|
|
conf_mb=$(to_mb "$conf")
|
|
echo " configured $conf (${conf_mb} MB), booted ${total} MB"
|
|
# The VM reports a little less than allocated; 15% covers the
|
|
# kernel without calling every healthy machine a mismatch.
|
|
if [ -n "$conf_mb" ] && [ "$total" -lt $(( conf_mb * 85 / 100 )) ]; then
|
|
echo " ! configured ${conf_mb} MB but booted ${total} MB — not applied yet."
|
|
echo " From a WINDOWS terminal: wsl --shutdown then start the distro again."
|
|
fi
|
|
else
|
|
echo " configured no memory= set (WSL defaults to 50% of host RAM, or 8 GB,"
|
|
echo " whichever is less). To raise it, add on the Windows side:"
|
|
echo " [wsl2]"
|
|
echo " memory=8GB"
|
|
echo " then from a WINDOWS terminal: wsl --shutdown"
|
|
fi
|
|
n=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
|
if [ "$n" -gt 0 ]; then
|
|
echo " backups $n (newest: $(ls -t "$cfg".*.bak 2>/dev/null | head -1))"
|
|
fi
|
|
fi
|
|
else
|
|
echo
|
|
echo " - native linux: no VM allocation to raise. If memory is tight the levers"
|
|
echo " are freeing something or adding swap."
|
|
fi
|
|
|
|
echo
|
|
echo "effective ceiling $(effective_ceiling_mb) MB"
|
|
echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box"
|
|
echo " claims. 'push' measures what it will actually hand over."
|
|
|
|
[ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; }
|
|
return 0
|
|
}
|
|
|
|
# ── .wslconfig ─────────────────────────────────────────────────────────────
|
|
|
|
require_wsl() {
|
|
if ! is_wsl; then
|
|
echo "$1 acts on .wslconfig, which only exists under WSL." >&2
|
|
echo "This is native Linux — there is no VM allocation to save or roll back." >&2
|
|
echo "Use 'status' to see what the machine actually has." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# backup and restore act on the file, so unlike status they must not guess.
|
|
wslconfig_required() {
|
|
local cfg; cfg=$(wslconfig_required)
|
|
if [ -z "$cfg" ]; then
|
|
echo "cannot tell which Windows profile owns .wslconfig. Candidates:" >&2
|
|
ls -d /mnt/c/Users/*/ 2>/dev/null \
|
|
| grep -viE "/(All Users|Default|Default User|Public)/$" | sed "s/^/ /" >&2
|
|
exit 1
|
|
fi
|
|
echo "$cfg"
|
|
}
|
|
|
|
configured_memory() {
|
|
[ -r "$1" ] || { echo ""; return; }
|
|
sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$1" | tail -1 | tr -d '[:space:]'
|
|
}
|
|
|
|
# "9GB" / "8192MB" / "9G" -> MB, so it can be compared with /proc/meminfo.
|
|
to_mb() {
|
|
local v="${1^^}" n
|
|
n=$(echo "$v" | tr -dc '0-9')
|
|
[ -n "$n" ] || { echo ""; return; }
|
|
case "$v" in
|
|
*GB|*G) echo $(( n * 1024 )) ;;
|
|
*MB|*M) echo "$n" ;;
|
|
*) echo $(( n / 1024 / 1024 )) ;;
|
|
esac
|
|
}
|
|
|
|
backup() {
|
|
require_wsl backup
|
|
local cfg dest
|
|
cfg=$(wslconfig_required)
|
|
[ -r "$cfg" ] || { echo "nothing to back up: $cfg does not exist" >&2; exit 1; }
|
|
# Timestamped and never overwritten: a backup that can destroy itself on a
|
|
# second run is not a backup.
|
|
dest="${cfg}.$(date +%Y%m%d-%H%M%S).bak"
|
|
cp "$cfg" "$dest"
|
|
echo "backed up $dest"
|
|
echo
|
|
echo "Edit $cfg by hand, then from a WINDOWS terminal: wsl --shutdown"
|
|
}
|
|
|
|
restore() {
|
|
require_wsl restore
|
|
local cfg newest count
|
|
cfg=$(wslconfig_required)
|
|
newest=$(ls -t "$cfg".*.bak 2>/dev/null | head -1 || true)
|
|
[ -n "$newest" ] || { echo "no backups found beside $cfg" >&2; exit 1; }
|
|
|
|
echo "restoring $newest"
|
|
echo " -> $cfg"
|
|
echo
|
|
|
|
# Newest is the right default — undo the last edit — but if you backed up
|
|
# *after* editing, the state you want is older. Show the rest so a no-op
|
|
# restore is obviously a no-op rather than a mystery.
|
|
count=$(ls "$cfg".*.bak 2>/dev/null | wc -l)
|
|
if [ "$count" -gt 1 ]; then
|
|
echo "$count backups exist, newest first:"
|
|
ls -t "$cfg".*.bak | sed 's/^/ /'
|
|
echo " (restoring the newest; copy another by hand to pick an older one)"
|
|
echo
|
|
fi
|
|
|
|
if [ -r "$cfg" ]; then
|
|
echo "what changes:"
|
|
if diff "$cfg" "$newest" > /tmp/mem.diff 2>&1 && [ ! -s /tmp/mem.diff ]; then
|
|
echo " nothing — that backup is identical to the current config"
|
|
else
|
|
sed 's/^/ /' /tmp/mem.diff
|
|
fi
|
|
rm -f /tmp/mem.diff
|
|
echo
|
|
fi
|
|
|
|
printf "proceed? [y/N] "
|
|
read -r reply
|
|
case "$reply" in
|
|
y|Y|yes|Yes) ;;
|
|
*) echo "left alone"; return 0 ;;
|
|
esac
|
|
cp "$newest" "$cfg"
|
|
echo "restored. From a WINDOWS terminal: wsl --shutdown"
|
|
}
|
|
|
|
# ── push ───────────────────────────────────────────────────────────────────
|
|
|
|
STATE=""
|
|
CHILD=""
|
|
|
|
cleanup() {
|
|
if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then
|
|
kill -KILL "$CHILD" 2>/dev/null || true
|
|
wait "$CHILD" 2>/dev/null || true
|
|
fi
|
|
[ -n "$STATE" ] && rm -f "$STATE"
|
|
return 0
|
|
}
|
|
|
|
# The child allocates and stops itself; the parent only watches. That split is
|
|
# the point: under --to-oom the allocating process is expected to be killed, and
|
|
# something has to survive to say how far it got.
|
|
allocator() {
|
|
# Raise our own OOM score to the maximum so the kernel picks THIS process
|
|
# first. Raising needs no privilege (only lowering does). Without it, the
|
|
# kernel is free to choose your shell, your ssh session or dockerd — on a
|
|
# box you are still using, that is not an acceptable coin toss.
|
|
echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true
|
|
|
|
local arr=() held=0 i=0 rss swapped avail first_swap=0
|
|
local bytes=$((STEP_MB * 1024 * 1024))
|
|
local swap_used_start
|
|
swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) ))
|
|
|
|
while :; do
|
|
# Written STRAIGHT INTO the array element. The obvious spelling —
|
|
# build one chunk and `arr+=("$chunk")` — costs three copies per step,
|
|
# not one: the template stays resident, expanding "$chunk" makes a
|
|
# temporary word, and the append makes the element. A 128 MB step then
|
|
# needs 384 MB transiently, and on a small box it is killed on the
|
|
# first append while reporting a third of the true ceiling.
|
|
#
|
|
# printf -v into a subscript also means every page is written, so it is
|
|
# resident rather than merely promised — the only kind of allocation
|
|
# that measures anything under heuristic overcommit.
|
|
printf -v "arr[$i]" '%*s' "$bytes" ''
|
|
i=$((i + 1)); held=$((held + STEP_MB))
|
|
|
|
rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0)
|
|
avail=$(headroom_mb)
|
|
swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start ))
|
|
[ "$swapped" -lt 0 ] && swapped=0
|
|
|
|
printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \
|
|
"$held" "$rss" "$avail" "$swapped"
|
|
printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE"
|
|
|
|
# Worth calling out separately from the ceiling: this is where the box
|
|
# stops being fast and starts being unusable, which for a scheduler is
|
|
# a different and earlier problem than being killed.
|
|
if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then
|
|
first_swap=$held
|
|
echo " - first swap page at ${held} MB — past here it works but crawls"
|
|
echo "swapat $held" >> "$STATE"
|
|
fi
|
|
|
|
if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then
|
|
echo "stop reached-the-cap" >> "$STATE"; return 0
|
|
fi
|
|
if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then
|
|
echo "stop floor" >> "$STATE"; return 0
|
|
fi
|
|
done
|
|
}
|
|
|
|
push() {
|
|
local total ceiling rc=0 last held rss swapat stop
|
|
total=$(mb MemTotal)
|
|
ceiling=$(effective_ceiling_mb)
|
|
|
|
# A step is worth about a sixty-fourth of the ceiling: enough resolution to
|
|
# find the edge, few enough lines to read, and small enough that the
|
|
# transient cost of one allocation never dominates a small box. A fixed
|
|
# size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB.
|
|
if [ "$STEP_EXPLICIT" = no ]; then
|
|
STEP_MB=$(( ceiling / 64 ))
|
|
[ "$STEP_MB" -lt 4 ] && STEP_MB=4
|
|
[ "$STEP_MB" -gt 256 ] && STEP_MB=256
|
|
fi
|
|
|
|
# Stop with a cushion rather than riding it to the kill. How big a cushion
|
|
# depends on what it is protecting. Under a cgroup cap, running out kills
|
|
# only this container's own processes, so it need cover no more than the
|
|
# shell that prints the result — and a 512 MB cushion on a 1 GB box would
|
|
# halve the answer. On a host there is everything else to protect, and the
|
|
# OOM killer does not promise to pick the process that caused the problem.
|
|
if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi
|
|
[ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 ))
|
|
|
|
STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX")
|
|
trap cleanup EXIT
|
|
# INT kills the child and lets the summary below print anyway, so an
|
|
# impatient Ctrl-C still tells you how far it got — and, more importantly,
|
|
# still gives the memory back.
|
|
trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT
|
|
|
|
echo "push"
|
|
echo " step ${STEP_MB} MB per allocation, every page touched"
|
|
echo " ceiling ${ceiling} MB claimed"
|
|
if [ -n "$TO_MB" ]; then
|
|
echo " stopping at ${TO_MB} MB (--to)"
|
|
elif [ "$TO_OOM" = yes ]; then
|
|
echo " ! stopping only when the kernel stops it (--to-oom)"
|
|
echo " the allocating child is marked as the preferred OOM victim,"
|
|
echo " but nothing about an OOM kill is entirely polite. Not on a box"
|
|
echo " running anything you mind losing."
|
|
else
|
|
echo " stopping when headroom drops below ${FLOOR_MB} MB"
|
|
fi
|
|
echo
|
|
|
|
allocator &
|
|
CHILD=$!
|
|
wait "$CHILD" || rc=$?
|
|
CHILD=""
|
|
trap - INT
|
|
|
|
last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true)
|
|
held=$(echo "$last" | awk '{print $1}')
|
|
rss=$(echo "$last" | awk '{print $2}')
|
|
swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
|
stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
|
|
|
echo
|
|
if [ -z "$held" ]; then
|
|
echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —"
|
|
echo " try a smaller --step, or check ulimit -v in 'status'."
|
|
return 1
|
|
fi
|
|
|
|
echo " reached ${rss:-$held} MB resident"
|
|
[ -n "$swapat" ] && echo " swapping from ${swapat} MB"
|
|
|
|
case "$stop" in
|
|
reached-the-cap)
|
|
echo " outcome stopped at the --to cap, not at a limit."
|
|
echo " The box held ${TO_MB} MB without complaint; there is more." ;;
|
|
floor)
|
|
echo " outcome stopped with a cushion intact, by choice."
|
|
echo " The real ceiling is higher — --to-oom finds it, at the"
|
|
echo " cost of an actual OOM kill." ;;
|
|
interrupted)
|
|
echo " outcome interrupted at ${rss:-$held} MB — where you stopped it,"
|
|
echo " not where the box did." ;;
|
|
*)
|
|
# No stop line means the child did not decide to stop: it was ended.
|
|
if [ "$rc" -ge 128 ]; then
|
|
echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB."
|
|
elif [ "$rc" -ne 0 ]; then
|
|
echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})."
|
|
echo " bash could not get the next chunk — an honest malloc"
|
|
echo " failure rather than a kill. That is the strict-overcommit"
|
|
echo " or ulimit path."
|
|
else
|
|
echo " outcome ended at ${rss:-$held} MB."
|
|
fi
|
|
local ev
|
|
ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true)
|
|
if [ -n "$ev" ]; then
|
|
echo " kernel ${ev#*] }"
|
|
else
|
|
echo " - dmesg is unreadable here (dmesg_restrict, or no privilege),"
|
|
echo " so the kill cannot be confirmed from this side. The number stands."
|
|
fi ;;
|
|
esac
|
|
|
|
# The gap between the claim and the measurement is the finding — but only
|
|
# when the BOX chose where to stop. An empty $stop means the child was ended
|
|
# rather than deciding to end; anything else (--to, the floor) is a stop we
|
|
# asked for, and flagging those as short of the ceiling would put a warning
|
|
# on every deliberately small run.
|
|
local got="${rss:-$held}"
|
|
echo
|
|
if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then
|
|
echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it."
|
|
echo " Something is taking the difference. 'status' names the candidates:"
|
|
echo " a cgroup cap, ulimit -v, or memory already resident."
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ── all ────────────────────────────────────────────────────────────────────
|
|
|
|
all() {
|
|
status
|
|
echo
|
|
echo "────────────────────────────────────────────────────────────"
|
|
echo
|
|
push
|
|
|
|
local got budget_mb ceiling
|
|
load_config
|
|
if [ -n "$BUDGET_GB" ]; then
|
|
budget_mb=$(( BUDGET_GB * 1024 ))
|
|
else
|
|
budget_mb=$(( NODES * NODE_MB ))
|
|
fi
|
|
ceiling=$(effective_ceiling_mb)
|
|
got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true)
|
|
[ -n "$got" ] || got=0
|
|
|
|
echo
|
|
echo "verdict"
|
|
if [ -n "$BUDGET_GB" ]; then
|
|
echo " budget ${budget_mb} MB (--budget)"
|
|
else
|
|
# rig's own figure for this profile: nodes times what one node costs.
|
|
# Addons carry no memory figure in rig yet, so this is the cluster alone
|
|
# and whatever you deploy comes on top. --budget once you know that too.
|
|
echo " budget ${budget_mb} MB — profile ${PROFILE_NAME}: ${NODES} node(s) x ${NODE_MB} MB,"
|
|
echo " the cluster alone; your workload comes on top (--budget GB)"
|
|
fi
|
|
echo " measured ${got} MB handed over"
|
|
|
|
if [ "$got" -ge "$budget_mb" ]; then
|
|
echo " fits, with $(( got - budget_mb )) MB spare."
|
|
if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then
|
|
echo " - under 30% spare is thin once a workload runs on top: memory use"
|
|
echo " is spiky, and the spikes are what get killed."
|
|
fi
|
|
else
|
|
echo " ! short by $(( budget_mb - got )) MB."
|
|
if [ "$ceiling" -ge "$budget_mb" ]; then
|
|
echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it."
|
|
echo " Free something, or read the caps section again."
|
|
else
|
|
echo " The box does not have it to give. A bigger machine, or a profile"
|
|
echo " with fewer nodes."
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ── main ───────────────────────────────────────────────────────────────────
|
|
|
|
parse_flags() {
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;;
|
|
--to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;;
|
|
--step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;;
|
|
--to-oom) TO_OOM=yes; shift ;;
|
|
--budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;;
|
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then
|
|
echo "--to and --to-oom contradict each other: one stops early, the other" >&2
|
|
echo "refuses to stop at all. Pick one." >&2
|
|
exit 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
require_linux
|
|
find_cgroup
|
|
|
|
cmd="${1:-status}"
|
|
[ $# -gt 0 ] && shift
|
|
|
|
case "$cmd" in
|
|
status) parse_flags "$@"; status ;;
|
|
push) parse_flags "$@"; push ;;
|
|
all) parse_flags "$@"; all ;;
|
|
backup) backup ;;
|
|
restore) restore ;;
|
|
*) echo "usage: $0 [status|push|all|backup|restore]" >&2
|
|
echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2
|
|
echo " all [--budget GB]" >&2
|
|
exit 1 ;;
|
|
esac
|