#!/usr/bin/env bash # GENERATED by make standalone — do not edit # # rigmini.sh for profile 'client', flattened from: # ctrl/mem.sh # ctrl/lib/config.sh # Edit those and run `make standalone`. Changes made here are lost, and # `make selftest` fails while this file differs from what rig generates. # ── from the libraries ── declare -- CONFIG_OVERRIDABLE=$'PROFILE CLUSTER K8S_VERSION KIND_CONFIG ADDONS\n REGISTRY_MODE INGRESS_MODE DNS_MODE TILT_PORT\n SOURCE ARCH DEPS_SOURCE HTTP_PORT HTTPS_PORT\n REGISTRY_PORT MANIFESTS_DIR' _config_restore () { local line; while IFS= read -r line; do if [ -n "$line" ]; then eval "export $line"; fi; done <<< "$1"; return 0 } default_cluster_name () { local n; n=$(basename "$(cd .. && pwd)"); n=$(echo "$n" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-'); n=$(echo "$n" | sed 's/^-*//; s/-*$//'); echo "${n:-rig}" } derive_port_base () { local h; h=$(printf '%s' "$1" | cksum | awk '{print $1}'); echo $((20000 + (h % 200) * 10)) } render_kind_config () { local host_workdir="${HOST_WORKDIR:-$(cd .. && pwd)}"; sed -e "s|\${CLUSTER}|${CLUSTER}|g" -e "s|\${NODE_IMAGE}|${NODE_IMAGE}|g" -e "s|\${HTTP_PORT}|${HTTP_PORT}|g" -e "s|\${HOST_WORKDIR}|${host_workdir}|g" "$KIND_CONFIG_PATH" } # ── configuration, frozen for profile 'client' ── load_config() { local k saved="" for k in $CONFIG_OVERRIDABLE; do if [ -n "${!k+x}" ]; then saved+="$k=$(printf '%q' "${!k}")"$'\n'; fi done declare -gx ADDONS="metallb cert-manager metrics-server" declare -gx AIRFLOW_IMAGE="apache/airflow:2.10.4" declare -g AUDIT="on" declare -gx CERT_MANAGER_VERSION="v1.21.1" declare -g CLUSTER="rig" declare -gx COMPOSE_SHA256="db1889184726840f75c4f9c001048430d4f25b3be3cb084d3ddd762bc0aed576" declare -gx COMPOSE_URL="https://github.com/docker/compose/releases/download/v5.5.1/docker-compose-linux-x86_64" declare -gx COMPOSE_VERSION="5.5.1" declare -gx CTLPTL_SHA256="c63a1ec28e60bc3faf6becb76f53355c5cf5e0143dafdd27ad85db5584fa6b1e" declare -gx CTLPTL_URL="https://github.com/tilt-dev/ctlptl/releases/download/v0.9.4/ctlptl.0.9.4.linux.x86_64.tar.gz" declare -gx CTLPTL_VERSION="0.9.4" declare -gx DNS_MODE="hosts" declare -g HTTPS_PORT="20311" declare -g HTTP_PORT="20310" declare -gx INGRESS_MODE="hostport" declare -gx JQ_SHA256="b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" declare -gx JQ_URL="https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64" declare -gx JQ_VERSION="1.8.2" declare -gx K8S_VERSION="v1_36" declare -gx KIND_CONFIG="kind-config.client.yaml.tpl" declare -g KIND_CONFIG_PATH="./k8s/kind-config.client.yaml.tpl" declare -g KIND_CONFIG_SHOWN="ctrl/k8s/kind-config.client.yaml.tpl" declare -gx KIND_SHA256="50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54" declare -gx KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/v0.32.0/kind-linux-amd64" declare -gx KIND_VERSION="v0.32.0" declare -g KUBECONTEXT="kind-rig" declare -gx KUBECTL_SHA256="ebbd080e7c2e275093b55915722043257eb24004363e20acb3c4d71919f88336" declare -gx KUBECTL_URL="https://dl.k8s.io/release/v1.36.3/bin/linux/amd64/kubectl" declare -gx KUBECTL_VERSION="v1.36.3" declare -g MANIFESTS_DIR="ctrl/k8s/overlays/dev" declare -gx METALLB_VERSION="v0.16.0" declare -gx METRICS_SERVER_VERSION="v0.9.0" declare -g NODES="3" declare -g NODE_IMAGE="kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5" declare -gx NODE_IMAGE_v1_33="kindest/node:v1.33.12@sha256:3f5c8443c620245e4d355cfe09e96a91ead32ceaa569d3f1ca9edf0cb2fe2ff4" declare -gx NODE_IMAGE_v1_34="kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256" declare -gx NODE_IMAGE_v1_35="kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95" declare -gx NODE_IMAGE_v1_36="kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5" declare -g NODE_MB="800" declare -gx POSTGRES_IMAGE="postgres:16-alpine" declare -gx PROFILE_NAME="client" declare -gx REDIS_IMAGE="redis:7-alpine" declare -gx REGISTRY_IMAGE="registry:2" declare -gx REGISTRY_MODE="mirror" declare -g REGISTRY_PORT="20313" declare -gx STUB_IMAGE="python:3.12-slim" declare -g TILT_PORT="20312" declare -gx TILT_SHA256="e9672b8a18d43501f35dcfe98465969a7db0e436b36cf0c50c7e6f8d40de5fe6" declare -gx TILT_URL="https://github.com/tilt-dev/tilt/releases/download/v0.37.6/tilt.0.37.6.linux.x86_64.tar.gz" declare -gx TILT_VERSION="0.37.6" _config_restore "$saved" } # ── end of frozen configuration ── # ── ctrl/mem.sh ── # How much memory this machine will actually give you before something dies — # rig's memory tool, and (generated from this file) the standalone rigmini.sh. # # There are two numbers and they are rarely the same. `status` reports what the # machine ADVERTISES and what is quietly capping it. `push` finds what it will # SURVIVE, by allocating until it stops. `all` does both and weighs the result # against what this profile's cluster needs. # # The gap between them is the whole reason this exists. Under WSL the cap lives # in .wslconfig; in a container or a managed workspace it is a cgroup limit, and # there /proc/meminfo reports the HOST's memory while the kernel kills you at a # fraction of it. A script that only read MemTotal would confidently report 32 GB # on a box that OOMs at 2. # # Runs on native Linux and under WSL. On WSL the memory you see is a VM # allocation that can be raised, and the commonest failure is raising it without # restarting — so status compares what .wslconfig says with what actually booted. # # Reports and instructs. It never raises a limit, frees anything or installs a # package. The one write it can make is `backup`, which copies .wslconfig beside # itself, so that `restore` has something to put back after a hand edit. # # Usage: # mem.sh status what it has, what caps it # mem.sh push [--to GB] [--to-oom] climb until it stops # mem.sh all [--budget GB] both, then the verdict # mem.sh backup | restore .wslconfig, WSL only set -euo pipefail cd "$(dirname "$0")" # (sourced library inlined above) # ── defaults ─────────────────────────────────────────────────────────────── STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push(). STEP_EXPLICIT=no # whether --step was given, which turns the scaling off. TO_MB="" # --to: stop here regardless. Empty means no hard cap. TO_OOM=no # --to-oom: opt in to running until the kernel intervenes. BUDGET_GB="" # --budget; empty means what this profile's cluster needs, from rig. BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below. # ── platform ─────────────────────────────────────────────────────────────── # Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and # then fails in a pile of confusing ways: no /proc, no docker socket, none of # the tooling. Detectable, so name it instead. require_linux() { case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) cat >&2 <<'EOF' This has to run inside WSL, not Git Bash / MSYS / Cygwin. If WSL is not installed yet, from an elevated PowerShell or Command Prompt: wsl --install That enables Windows features and needs a reboot, so it is not something this script will do for you. Afterwards, open the Linux shell it installs and run this from there. EOF exit 1 ;; esac # Everything below reads /proc. Without it there is nothing to measure, and # failing here beats printing a page of empty fields. if [ ! -r /proc/meminfo ]; then echo "no readable /proc/meminfo — this needs a Linux kernel." >&2 echo "On macOS or a BSD none of the numbers below exist." >&2 exit 1 fi } is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; } is_container() { [ -f /.dockerenv ] && return 0 grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null } platform() { if is_wsl; then echo WSL elif is_container; then echo container else echo "native linux" fi } # ── reading memory ───────────────────────────────────────────────────────── mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); } # MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on # corporate images — need the estimate it replaced, which is worse but not wrong. avail_meminfo_mb() { if grep -q '^MemAvailable:' /proc/meminfo; then mb MemAvailable else awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo fi } # Where a cgroup records this cgroup's own limit and usage. Set once by # find_cgroup, because every later reading needs both and hunting for the files # on each call would be the slow part of the poll loop. CG_MAX_FILE="" CG_CUR_FILE="" CG_VERSION="" find_cgroup() { local rel # Inside a container the cgroup namespace makes the top of the tree BE the # container's own cgroup, so the unqualified path is already the right one. # On a host it is the root cgroup, which is never limited — hence the second # attempt via /proc/self/cgroup, which names the slice this shell is in. if [ -r /sys/fs/cgroup/memory.max ]; then CG_VERSION=v2 CG_MAX_FILE=/sys/fs/cgroup/memory.max CG_CUR_FILE=/sys/fs/cgroup/memory.current elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then CG_VERSION=v1 CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes fi rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true) if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then CG_VERSION=v2 CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max" CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current" return 0 fi rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true) if [ -n "$rel" ] && [ "$rel" != "/" ] \ && [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then CG_VERSION=v1 CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes" fi return 0 } # The cap in MB, or "" when there is none worth reporting. v2 spells unlimited # "max"; v1 spells it as a number near 2^63, which is why this compares against # MemTotal rather than testing for a magic value — a "limit" above the machine's # own memory is not a limit, however it is written. cgroup_cap_mb() { local raw cap [ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; } raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max) [ "$raw" = "max" ] && { echo ""; return 0; } case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac cap=$((raw / 1024 / 1024)) [ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; } echo "$cap" } cgroup_used_mb() { local raw [ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; } raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "") case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac echo $((raw / 1024 / 1024)) } # ulimit -v is a per-process address-space cap. It stops YOU long before the box # does, and because it is inherited from a login shell it is easy to hit without # knowing it is set. ulimit_v_mb() { local v; v=$(ulimit -v 2>/dev/null || echo unlimited) [ "$v" = "unlimited" ] && { echo ""; return 0; } case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac echo $((v / 1024)) } # The number everything else is about: the lowest of the things that can stop # you. Printed at the end of `status` and used as the sanity bound in `push`. effective_ceiling_mb() { local c; c=$(mb MemTotal) local cap; cap=$(cgroup_cap_mb) local ul; ul=$(ulimit_v_mb) [ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap" [ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul" echo "$c" } # How much room is left RIGHT NOW, from whichever accounting actually governs. # In a capped container /proc/meminfo describes the host and is worse than # useless for this — it would report tens of gigabytes free on a box that is one # allocation from being killed. headroom_mb() { local cap used cap=$(cgroup_cap_mb) used=$(cgroup_used_mb) if [ -n "$cap" ] && [ -n "$used" ]; then echo $(( cap - used )) else avail_meminfo_mb fi } # ── status ───────────────────────────────────────────────────────────────── # /mnt/c/Users can hold several real accounts — a renamed login leaves the old # directory behind — so picking the first alphabetically is a coin toss. Ask # Windows, then fall back to whichever profile actually owns a config. wslconfig_path() { local profile winpath found profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true) case "$profile" in ""|*%*) ;; *) winpath=$(wslpath -u "$profile" 2>/dev/null || true) if [ -n "$winpath" ] && [ -d "$winpath" ]; then echo "$winpath/.wslconfig"; return 0 fi ;; esac found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true) [ -n "$found" ] && echo "$found" return 0 } hogs() { echo " holding the most:" ps -eo rss,comm --sort=-rss 2>/dev/null \ | awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}' return 0 } status() { local total avail swap_total swap_free cap ul cur echo "host" echo " platform $(platform)" echo " kernel $(uname -r)" [ -r /etc/os-release ] && \ echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)" echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)" # ── the caps first, because they decide what the totals below are worth ── echo echo "caps" cap=$(cgroup_cap_mb) if [ -n "$cap" ]; then cur=$(cgroup_used_mb) echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)" echo " ! /proc/meminfo below describes the HOST, not this cgroup." echo " $(mb MemTotal) MB total is not yours; ${cap} MB is." elif [ -n "$CG_VERSION" ]; then echo " cgroup none (${CG_VERSION} present, no memory limit set)" else echo " cgroup no memory controller found" fi ul=$(ulimit_v_mb) if [ -n "$ul" ]; then echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell" echo " it stops this process long before the machine runs out" else echo " ulimit -v unlimited" fi # overcommit_memory=0 is the default heuristic: a large allocation is # granted on a guess, and the reckoning arrives later as an OOM kill rather # than as a failed malloc. It is why `push` touches every page it asks for. local om or_ om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?') or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?') case "$om" in 0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;; 1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;; 2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;; *) echo " overcommit ${om}" ;; esac [ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for" # ── what it says it has ── total=$(mb MemTotal); avail=$(avail_meminfo_mb) swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree) echo echo "memory" echo " total ${total} MB" echo " available ${avail} MB" echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)" if [ "$swap_total" -eq 0 ]; then echo " - no swap: this box has no cushion. It goes from fine to OOM-killed" echo " with nothing in between, which is the abrupt failure you get in a VM." fi # postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB, # and the resulting failure names neither shm nor the size. if [ -d /dev/shm ]; then local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}') if [ -n "$shm" ]; then if [ "$shm" -le 64 ]; then echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB" echo " is docker's default. Raise it with --shm-size when postgres fails." else echo " /dev/shm ${shm} MB" fi fi fi echo echo "disk" local d for d in / /tmp /var/lib/docker; do [ -d "$d" ] || continue df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}' done # kind and Tilt both watch large trees, and the failure mode is silent: # they simply stop noticing file changes. Cheap to report while we are here. local w i w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0) i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0) echo echo "tooling" echo " inotify watches=$w instances=$i" if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then echo " ! low — anything watching files will silently stop seeing changes" fi if ! command -v docker >/dev/null 2>&1; then if [ -S /var/run/docker.sock ]; then echo " docker socket present, no cli" else echo " docker not installed" fi elif docker info >/dev/null 2>&1; then local n n=$(docker ps -q 2>/dev/null | wc -l) echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running" else echo " ! docker cli present but the daemon is unreachable" fi # WSL keeps its cap on the Windows side, in a file this shell can read but # not usefully apply — the change costs a full VM restart. Report it, and # report the commonest mistake, which is editing it and not restarting. if is_wsl; then local cfg conf conf_mb n cfg=$(wslconfig_path) echo echo "wsl" if [ -z "$cfg" ]; then echo " ! cannot tell which Windows profile owns .wslconfig" else echo " config $cfg" conf=$(configured_memory "$cfg") if [ -n "$conf" ]; then conf_mb=$(to_mb "$conf") echo " configured $conf (${conf_mb} MB), booted ${total} MB" # The VM reports a little less than allocated; 15% covers the # kernel without calling every healthy machine a mismatch. if [ -n "$conf_mb" ] && [ "$total" -lt $(( conf_mb * 85 / 100 )) ]; then echo " ! configured ${conf_mb} MB but booted ${total} MB — not applied yet." echo " From a WINDOWS terminal: wsl --shutdown then start the distro again." fi else echo " configured no memory= set (WSL defaults to 50% of host RAM, or 8 GB," echo " whichever is less). To raise it, add on the Windows side:" echo " [wsl2]" echo " memory=8GB" echo " then from a WINDOWS terminal: wsl --shutdown" fi n=$(ls "$cfg".*.bak 2>/dev/null | wc -l) if [ "$n" -gt 0 ]; then echo " backups $n (newest: $(ls -t "$cfg".*.bak 2>/dev/null | head -1))" fi fi else echo echo " - native linux: no VM allocation to raise. If memory is tight the levers" echo " are freeing something or adding swap." fi echo echo "effective ceiling $(effective_ceiling_mb) MB" echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box" echo " claims. 'push' measures what it will actually hand over." [ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; } return 0 } # ── .wslconfig ───────────────────────────────────────────────────────────── require_wsl() { if ! is_wsl; then echo "$1 acts on .wslconfig, which only exists under WSL." >&2 echo "This is native Linux — there is no VM allocation to save or roll back." >&2 echo "Use 'status' to see what the machine actually has." >&2 exit 1 fi } # backup and restore act on the file, so unlike status they must not guess. wslconfig_required() { local cfg; cfg=$(wslconfig_required) if [ -z "$cfg" ]; then echo "cannot tell which Windows profile owns .wslconfig. Candidates:" >&2 ls -d /mnt/c/Users/*/ 2>/dev/null \ | grep -viE "/(All Users|Default|Default User|Public)/$" | sed "s/^/ /" >&2 exit 1 fi echo "$cfg" } configured_memory() { [ -r "$1" ] || { echo ""; return; } sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$1" | tail -1 | tr -d '[:space:]' } # "9GB" / "8192MB" / "9G" -> MB, so it can be compared with /proc/meminfo. to_mb() { local v="${1^^}" n n=$(echo "$v" | tr -dc '0-9') [ -n "$n" ] || { echo ""; return; } case "$v" in *GB|*G) echo $(( n * 1024 )) ;; *MB|*M) echo "$n" ;; *) echo $(( n / 1024 / 1024 )) ;; esac } backup() { require_wsl backup local cfg dest cfg=$(wslconfig_required) [ -r "$cfg" ] || { echo "nothing to back up: $cfg does not exist" >&2; exit 1; } # Timestamped and never overwritten: a backup that can destroy itself on a # second run is not a backup. dest="${cfg}.$(date +%Y%m%d-%H%M%S).bak" cp "$cfg" "$dest" echo "backed up $dest" echo echo "Edit $cfg by hand, then from a WINDOWS terminal: wsl --shutdown" } restore() { require_wsl restore local cfg newest count cfg=$(wslconfig_required) newest=$(ls -t "$cfg".*.bak 2>/dev/null | head -1 || true) [ -n "$newest" ] || { echo "no backups found beside $cfg" >&2; exit 1; } echo "restoring $newest" echo " -> $cfg" echo # Newest is the right default — undo the last edit — but if you backed up # *after* editing, the state you want is older. Show the rest so a no-op # restore is obviously a no-op rather than a mystery. count=$(ls "$cfg".*.bak 2>/dev/null | wc -l) if [ "$count" -gt 1 ]; then echo "$count backups exist, newest first:" ls -t "$cfg".*.bak | sed 's/^/ /' echo " (restoring the newest; copy another by hand to pick an older one)" echo fi if [ -r "$cfg" ]; then echo "what changes:" if diff "$cfg" "$newest" > /tmp/mem.diff 2>&1 && [ ! -s /tmp/mem.diff ]; then echo " nothing — that backup is identical to the current config" else sed 's/^/ /' /tmp/mem.diff fi rm -f /tmp/mem.diff echo fi printf "proceed? [y/N] " read -r reply case "$reply" in y|Y|yes|Yes) ;; *) echo "left alone"; return 0 ;; esac cp "$newest" "$cfg" echo "restored. From a WINDOWS terminal: wsl --shutdown" } # ── push ─────────────────────────────────────────────────────────────────── STATE="" CHILD="" cleanup() { if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then kill -KILL "$CHILD" 2>/dev/null || true wait "$CHILD" 2>/dev/null || true fi [ -n "$STATE" ] && rm -f "$STATE" return 0 } # The child allocates and stops itself; the parent only watches. That split is # the point: under --to-oom the allocating process is expected to be killed, and # something has to survive to say how far it got. allocator() { # Raise our own OOM score to the maximum so the kernel picks THIS process # first. Raising needs no privilege (only lowering does). Without it, the # kernel is free to choose your shell, your ssh session or dockerd — on a # box you are still using, that is not an acceptable coin toss. echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true local arr=() held=0 i=0 rss swapped avail first_swap=0 local bytes=$((STEP_MB * 1024 * 1024)) local swap_used_start swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) )) while :; do # Written STRAIGHT INTO the array element. The obvious spelling — # build one chunk and `arr+=("$chunk")` — costs three copies per step, # not one: the template stays resident, expanding "$chunk" makes a # temporary word, and the append makes the element. A 128 MB step then # needs 384 MB transiently, and on a small box it is killed on the # first append while reporting a third of the true ceiling. # # printf -v into a subscript also means every page is written, so it is # resident rather than merely promised — the only kind of allocation # that measures anything under heuristic overcommit. printf -v "arr[$i]" '%*s' "$bytes" '' i=$((i + 1)); held=$((held + STEP_MB)) rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0) avail=$(headroom_mb) swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start )) [ "$swapped" -lt 0 ] && swapped=0 printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \ "$held" "$rss" "$avail" "$swapped" printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE" # Worth calling out separately from the ceiling: this is where the box # stops being fast and starts being unusable, which for a scheduler is # a different and earlier problem than being killed. if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then first_swap=$held echo " - first swap page at ${held} MB — past here it works but crawls" echo "swapat $held" >> "$STATE" fi if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then echo "stop reached-the-cap" >> "$STATE"; return 0 fi if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then echo "stop floor" >> "$STATE"; return 0 fi done } push() { local total ceiling rc=0 last held rss swapat stop total=$(mb MemTotal) ceiling=$(effective_ceiling_mb) # A step is worth about a sixty-fourth of the ceiling: enough resolution to # find the edge, few enough lines to read, and small enough that the # transient cost of one allocation never dominates a small box. A fixed # size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB. if [ "$STEP_EXPLICIT" = no ]; then STEP_MB=$(( ceiling / 64 )) [ "$STEP_MB" -lt 4 ] && STEP_MB=4 [ "$STEP_MB" -gt 256 ] && STEP_MB=256 fi # Stop with a cushion rather than riding it to the kill. How big a cushion # depends on what it is protecting. Under a cgroup cap, running out kills # only this container's own processes, so it need cover no more than the # shell that prints the result — and a 512 MB cushion on a 1 GB box would # halve the answer. On a host there is everything else to protect, and the # OOM killer does not promise to pick the process that caused the problem. if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi [ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 )) STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX") trap cleanup EXIT # INT kills the child and lets the summary below print anyway, so an # impatient Ctrl-C still tells you how far it got — and, more importantly, # still gives the memory back. trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT echo "push" echo " step ${STEP_MB} MB per allocation, every page touched" echo " ceiling ${ceiling} MB claimed" if [ -n "$TO_MB" ]; then echo " stopping at ${TO_MB} MB (--to)" elif [ "$TO_OOM" = yes ]; then echo " ! stopping only when the kernel stops it (--to-oom)" echo " the allocating child is marked as the preferred OOM victim," echo " but nothing about an OOM kill is entirely polite. Not on a box" echo " running anything you mind losing." else echo " stopping when headroom drops below ${FLOOR_MB} MB" fi echo allocator & CHILD=$! wait "$CHILD" || rc=$? CHILD="" trap - INT last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true) held=$(echo "$last" | awk '{print $1}') rss=$(echo "$last" | awk '{print $2}') swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true) stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true) echo if [ -z "$held" ]; then echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —" echo " try a smaller --step, or check ulimit -v in 'status'." return 1 fi echo " reached ${rss:-$held} MB resident" [ -n "$swapat" ] && echo " swapping from ${swapat} MB" case "$stop" in reached-the-cap) echo " outcome stopped at the --to cap, not at a limit." echo " The box held ${TO_MB} MB without complaint; there is more." ;; floor) echo " outcome stopped with a cushion intact, by choice." echo " The real ceiling is higher — --to-oom finds it, at the" echo " cost of an actual OOM kill." ;; interrupted) echo " outcome interrupted at ${rss:-$held} MB — where you stopped it," echo " not where the box did." ;; *) # No stop line means the child did not decide to stop: it was ended. if [ "$rc" -ge 128 ]; then echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB." elif [ "$rc" -ne 0 ]; then echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})." echo " bash could not get the next chunk — an honest malloc" echo " failure rather than a kill. That is the strict-overcommit" echo " or ulimit path." else echo " outcome ended at ${rss:-$held} MB." fi local ev ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true) if [ -n "$ev" ]; then echo " kernel ${ev#*] }" else echo " - dmesg is unreadable here (dmesg_restrict, or no privilege)," echo " so the kill cannot be confirmed from this side. The number stands." fi ;; esac # The gap between the claim and the measurement is the finding — but only # when the BOX chose where to stop. An empty $stop means the child was ended # rather than deciding to end; anything else (--to, the floor) is a stop we # asked for, and flagging those as short of the ceiling would put a warning # on every deliberately small run. local got="${rss:-$held}" echo if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it." echo " Something is taking the difference. 'status' names the candidates:" echo " a cgroup cap, ulimit -v, or memory already resident." fi return 0 } # ── all ──────────────────────────────────────────────────────────────────── all() { status echo echo "────────────────────────────────────────────────────────────" echo push local got budget_mb ceiling load_config if [ -n "$BUDGET_GB" ]; then budget_mb=$(( BUDGET_GB * 1024 )) else budget_mb=$(( NODES * NODE_MB )) fi ceiling=$(effective_ceiling_mb) got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true) [ -n "$got" ] || got=0 echo echo "verdict" if [ -n "$BUDGET_GB" ]; then echo " budget ${budget_mb} MB (--budget)" else # rig's own figure for this profile: nodes times what one node costs. # Addons carry no memory figure in rig yet, so this is the cluster alone # and whatever you deploy comes on top. --budget once you know that too. echo " budget ${budget_mb} MB — profile ${PROFILE_NAME}: ${NODES} node(s) x ${NODE_MB} MB," echo " the cluster alone; your workload comes on top (--budget GB)" fi echo " measured ${got} MB handed over" if [ "$got" -ge "$budget_mb" ]; then echo " fits, with $(( got - budget_mb )) MB spare." if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then echo " - under 30% spare is thin once a workload runs on top: memory use" echo " is spiky, and the spikes are what get killed." fi else echo " ! short by $(( budget_mb - got )) MB." if [ "$ceiling" -ge "$budget_mb" ]; then echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it." echo " Free something, or read the caps section again." else echo " The box does not have it to give. A bigger machine, or a profile" echo " with fewer nodes." fi fi return 0 } # ── main ─────────────────────────────────────────────────────────────────── parse_flags() { while [ $# -gt 0 ]; do case "$1" in --to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;; --to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;; --step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;; --to-oom) TO_OOM=yes; shift ;; --budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;; *) echo "unknown argument: $1" >&2; exit 1 ;; esac done if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then echo "--to and --to-oom contradict each other: one stops early, the other" >&2 echo "refuses to stop at all. Pick one." >&2 exit 1 fi return 0 } require_linux find_cgroup cmd="${1:-status}" [ $# -gt 0 ] && shift case "$cmd" in status) parse_flags "$@"; status ;; push) parse_flags "$@"; push ;; all) parse_flags "$@"; all ;; backup) backup ;; restore) restore ;; *) echo "usage: $0 [status|push|all|backup|restore]" >&2 echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2 echo " all [--budget GB]" >&2 exit 1 ;; esac