636 lines
27 KiB
Bash
Executable File
636 lines
27 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# How much memory this box will actually give you before something dies.
|
|
#
|
|
# rig answers this for a machine it is installed on. This is the single file
|
|
# version, for a machine rig is not going to: paste it onto a fresh AWS
|
|
# WorkSpace, an EC2 box or a container, run it, and get the same numbers in the
|
|
# same order so two machines can be read side by side.
|
|
#
|
|
# There are two numbers and they are rarely the same. `status` reports what the
|
|
# machine ADVERTISES and what is quietly capping it. `push` finds what it will
|
|
# SURVIVE, by allocating until it stops.
|
|
#
|
|
# The gap between them is the whole reason this exists. Under WSL the cap lives
|
|
# in .wslconfig; in a container or a managed workspace it is a cgroup limit, and
|
|
# there /proc/meminfo reports the HOST's memory while the kernel kills you at a
|
|
# fraction of it. A script that only read MemTotal would confidently report 32 GB
|
|
# on a box that OOMs at 2.
|
|
#
|
|
# Reports and instructs. It never raises a limit, frees anything, writes a
|
|
# config or installs a package — on a machine you are still evaluating, a probe
|
|
# that changes what it is measuring is worse than no probe.
|
|
#
|
|
# Usage:
|
|
# rigmini.sh status what it has, what caps it
|
|
# rigmini.sh push [--to GB] [--to-oom] climb until it stops
|
|
# rigmini.sh all [--budget GB] both, then the verdict
|
|
set -euo pipefail
|
|
|
|
# ── defaults ───────────────────────────────────────────────────────────────
|
|
|
|
STEP_MB=0 # per allocation; 0 means scale it to the ceiling. See push().
|
|
STEP_EXPLICIT=no # whether --step was given, which turns the scaling off.
|
|
TO_MB="" # --to: stop here regardless. Empty means no hard cap.
|
|
TO_OOM=no # --to-oom: opt in to running until the kernel intervenes.
|
|
BUDGET_GB=6 # what the rig data profile is assumed to want; see all().
|
|
BUDGET_EXPLICIT=no # whether --budget was given, which retires the guess below.
|
|
|
|
# ── platform ───────────────────────────────────────────────────────────────
|
|
|
|
# Windows outside WSL — Git Bash, MSYS, Cygwin — looks close enough to work and
|
|
# then fails in a pile of confusing ways: no /proc, no docker socket, none of
|
|
# the tooling. Detectable, so name it instead.
|
|
require_linux() {
|
|
case "$(uname -s)" in
|
|
MINGW*|MSYS*|CYGWIN*)
|
|
cat >&2 <<'EOF'
|
|
This has to run inside WSL, not Git Bash / MSYS / Cygwin.
|
|
|
|
If WSL is not installed yet, from an elevated PowerShell or Command Prompt:
|
|
|
|
wsl --install
|
|
|
|
That enables Windows features and needs a reboot, so it is not something this
|
|
script will do for you. Afterwards, open the Linux shell it installs and run
|
|
this from there.
|
|
EOF
|
|
exit 1 ;;
|
|
esac
|
|
|
|
# Everything below reads /proc. Without it there is nothing to measure, and
|
|
# failing here beats printing a page of empty fields.
|
|
if [ ! -r /proc/meminfo ]; then
|
|
echo "no readable /proc/meminfo — this needs a Linux kernel." >&2
|
|
echo "On macOS or a BSD none of the numbers below exist." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
is_wsl() { grep -qi microsoft /proc/version 2>/dev/null; }
|
|
|
|
is_container() {
|
|
[ -f /.dockerenv ] && return 0
|
|
grep -qE '(docker|containerd|kubepods|lxc|podman)' /proc/1/cgroup 2>/dev/null
|
|
}
|
|
|
|
platform() {
|
|
if is_wsl; then echo WSL
|
|
elif is_container; then echo container
|
|
else echo "native linux"
|
|
fi
|
|
}
|
|
|
|
# ── reading memory ─────────────────────────────────────────────────────────
|
|
|
|
mb() { echo $(( $(awk "/^$1:/{print \$2}" /proc/meminfo) / 1024 )); }
|
|
|
|
# MemAvailable arrived in kernel 3.14. Older kernels — and they turn up on
|
|
# corporate images — need the estimate it replaced, which is worse but not wrong.
|
|
avail_meminfo_mb() {
|
|
if grep -q '^MemAvailable:' /proc/meminfo; then
|
|
mb MemAvailable
|
|
else
|
|
awk '/^(MemFree|Buffers|Cached):/{t+=$2} END{print int(t/1024)}' /proc/meminfo
|
|
fi
|
|
}
|
|
|
|
# Where a cgroup records this cgroup's own limit and usage. Set once by
|
|
# find_cgroup, because every later reading needs both and hunting for the files
|
|
# on each call would be the slow part of the poll loop.
|
|
CG_MAX_FILE=""
|
|
CG_CUR_FILE=""
|
|
CG_VERSION=""
|
|
|
|
find_cgroup() {
|
|
local rel
|
|
|
|
# Inside a container the cgroup namespace makes the top of the tree BE the
|
|
# container's own cgroup, so the unqualified path is already the right one.
|
|
# On a host it is the root cgroup, which is never limited — hence the second
|
|
# attempt via /proc/self/cgroup, which names the slice this shell is in.
|
|
if [ -r /sys/fs/cgroup/memory.max ]; then
|
|
CG_VERSION=v2
|
|
CG_MAX_FILE=/sys/fs/cgroup/memory.max
|
|
CG_CUR_FILE=/sys/fs/cgroup/memory.current
|
|
elif [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
|
|
CG_VERSION=v1
|
|
CG_MAX_FILE=/sys/fs/cgroup/memory/memory.limit_in_bytes
|
|
CG_CUR_FILE=/sys/fs/cgroup/memory/memory.usage_in_bytes
|
|
fi
|
|
|
|
rel=$(awk -F: '$1=="0"{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
|
if [ -n "$rel" ] && [ "$rel" != "/" ] && [ -r "/sys/fs/cgroup${rel}/memory.max" ]; then
|
|
CG_VERSION=v2
|
|
CG_MAX_FILE="/sys/fs/cgroup${rel}/memory.max"
|
|
CG_CUR_FILE="/sys/fs/cgroup${rel}/memory.current"
|
|
return 0
|
|
fi
|
|
|
|
rel=$(awk -F: '$2 ~ /(^|,)memory(,|$)/{print $3; exit}' /proc/self/cgroup 2>/dev/null || true)
|
|
if [ -n "$rel" ] && [ "$rel" != "/" ] \
|
|
&& [ -r "/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes" ]; then
|
|
CG_VERSION=v1
|
|
CG_MAX_FILE="/sys/fs/cgroup/memory${rel}/memory.limit_in_bytes"
|
|
CG_CUR_FILE="/sys/fs/cgroup/memory${rel}/memory.usage_in_bytes"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# The cap in MB, or "" when there is none worth reporting. v2 spells unlimited
|
|
# "max"; v1 spells it as a number near 2^63, which is why this compares against
|
|
# MemTotal rather than testing for a magic value — a "limit" above the machine's
|
|
# own memory is not a limit, however it is written.
|
|
cgroup_cap_mb() {
|
|
local raw cap
|
|
[ -n "$CG_MAX_FILE" ] && [ -r "$CG_MAX_FILE" ] || { echo ""; return 0; }
|
|
raw=$(cat "$CG_MAX_FILE" 2>/dev/null || echo max)
|
|
[ "$raw" = "max" ] && { echo ""; return 0; }
|
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
cap=$((raw / 1024 / 1024))
|
|
[ "$cap" -ge "$(mb MemTotal)" ] && { echo ""; return 0; }
|
|
echo "$cap"
|
|
}
|
|
|
|
cgroup_used_mb() {
|
|
local raw
|
|
[ -n "$CG_CUR_FILE" ] && [ -r "$CG_CUR_FILE" ] || { echo ""; return 0; }
|
|
raw=$(cat "$CG_CUR_FILE" 2>/dev/null || echo "")
|
|
case "$raw" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
echo $((raw / 1024 / 1024))
|
|
}
|
|
|
|
# ulimit -v is a per-process address-space cap. It stops YOU long before the box
|
|
# does, and because it is inherited from a login shell it is easy to hit without
|
|
# knowing it is set.
|
|
ulimit_v_mb() {
|
|
local v; v=$(ulimit -v 2>/dev/null || echo unlimited)
|
|
[ "$v" = "unlimited" ] && { echo ""; return 0; }
|
|
case "$v" in ''|*[!0-9]*) echo ""; return 0 ;; esac
|
|
echo $((v / 1024))
|
|
}
|
|
|
|
# The number everything else is about: the lowest of the things that can stop
|
|
# you. Printed at the end of `status` and used as the sanity bound in `push`.
|
|
effective_ceiling_mb() {
|
|
local c; c=$(mb MemTotal)
|
|
local cap; cap=$(cgroup_cap_mb)
|
|
local ul; ul=$(ulimit_v_mb)
|
|
[ -n "$cap" ] && [ "$cap" -lt "$c" ] && c="$cap"
|
|
[ -n "$ul" ] && [ "$ul" -lt "$c" ] && c="$ul"
|
|
echo "$c"
|
|
}
|
|
|
|
# How much room is left RIGHT NOW, from whichever accounting actually governs.
|
|
# In a capped container /proc/meminfo describes the host and is worse than
|
|
# useless for this — it would report tens of gigabytes free on a box that is one
|
|
# allocation from being killed.
|
|
headroom_mb() {
|
|
local cap used
|
|
cap=$(cgroup_cap_mb)
|
|
used=$(cgroup_used_mb)
|
|
if [ -n "$cap" ] && [ -n "$used" ]; then
|
|
echo $(( cap - used ))
|
|
else
|
|
avail_meminfo_mb
|
|
fi
|
|
}
|
|
|
|
# ── status ─────────────────────────────────────────────────────────────────
|
|
|
|
# /mnt/c/Users can hold several real accounts — a renamed login leaves the old
|
|
# directory behind — so picking the first alphabetically is a coin toss. Ask
|
|
# Windows, then fall back to whichever profile actually owns a config.
|
|
wslconfig_path() {
|
|
local profile winpath found
|
|
profile=$(cmd.exe /c "echo %USERPROFILE%" 2>/dev/null | tr -d "\r\n" || true)
|
|
case "$profile" in
|
|
""|*%*) ;;
|
|
*) winpath=$(wslpath -u "$profile" 2>/dev/null || true)
|
|
if [ -n "$winpath" ] && [ -d "$winpath" ]; then
|
|
echo "$winpath/.wslconfig"; return 0
|
|
fi ;;
|
|
esac
|
|
found=$(ls -d /mnt/c/Users/*/.wslconfig 2>/dev/null | head -1 || true)
|
|
[ -n "$found" ] && echo "$found"
|
|
return 0
|
|
}
|
|
|
|
hogs() {
|
|
echo " holding the most:"
|
|
ps -eo rss,comm --sort=-rss 2>/dev/null \
|
|
| awk 'NR>1 && NR<=6 {printf " %6.0f MB %s\n", $1/1024, $2}'
|
|
return 0
|
|
}
|
|
|
|
status() {
|
|
local total avail swap_total swap_free cap ul cur
|
|
|
|
echo "host"
|
|
echo " platform $(platform)"
|
|
echo " kernel $(uname -r)"
|
|
[ -r /etc/os-release ] && \
|
|
echo " distro $(sed -n 's/^PRETTY_NAME="\(.*\)"/\1/p' /etc/os-release)"
|
|
echo " cpu $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo '?') online, load $(cut -d' ' -f1-3 /proc/loadavg)"
|
|
|
|
# ── the caps first, because they decide what the totals below are worth ──
|
|
echo
|
|
echo "caps"
|
|
cap=$(cgroup_cap_mb)
|
|
if [ -n "$cap" ]; then
|
|
cur=$(cgroup_used_mb)
|
|
echo " cgroup ${cap} MB (${CG_VERSION}, ${CG_CUR_FILE##*/} says ${cur:-?} MB used)"
|
|
echo " ! /proc/meminfo below describes the HOST, not this cgroup."
|
|
echo " $(mb MemTotal) MB total is not yours; ${cap} MB is."
|
|
elif [ -n "$CG_VERSION" ]; then
|
|
echo " cgroup none (${CG_VERSION} present, no memory limit set)"
|
|
else
|
|
echo " cgroup no memory controller found"
|
|
fi
|
|
|
|
ul=$(ulimit_v_mb)
|
|
if [ -n "$ul" ]; then
|
|
echo " ! ulimit -v ${ul} MB — a per-process cap, inherited from your shell"
|
|
echo " it stops this process long before the machine runs out"
|
|
else
|
|
echo " ulimit -v unlimited"
|
|
fi
|
|
|
|
# overcommit_memory=0 is the default heuristic: a large allocation is
|
|
# granted on a guess, and the reckoning arrives later as an OOM kill rather
|
|
# than as a failed malloc. It is why `push` touches every page it asks for.
|
|
local om or_
|
|
om=$(cat /proc/sys/vm/overcommit_memory 2>/dev/null || echo '?')
|
|
or_=$(cat /proc/sys/vm/overcommit_ratio 2>/dev/null || echo '?')
|
|
case "$om" in
|
|
0) echo " overcommit 0 heuristic — allocations are granted on a guess," ;;
|
|
1) echo " overcommit 1 always — every allocation succeeds; the OOM killer is the only limit," ;;
|
|
2) echo " overcommit 2 strict (ratio ${or_}%) — allocation fails honestly instead of killing later," ;;
|
|
*) echo " overcommit ${om}" ;;
|
|
esac
|
|
[ "$om" != "?" ] && echo " so RSS is the number to trust, not what a process asked for"
|
|
|
|
# ── what it says it has ──
|
|
total=$(mb MemTotal); avail=$(avail_meminfo_mb)
|
|
swap_total=$(mb SwapTotal); swap_free=$(mb SwapFree)
|
|
echo
|
|
echo "memory"
|
|
echo " total ${total} MB"
|
|
echo " available ${avail} MB"
|
|
echo " swap ${swap_total} MB ($(( swap_total - swap_free )) MB used)"
|
|
if [ "$swap_total" -eq 0 ]; then
|
|
echo " - no swap: this box has no cushion. It goes from fine to OOM-killed"
|
|
echo " with nothing in between, which is the abrupt failure you get in a VM."
|
|
fi
|
|
|
|
# postgres puts its shared buffers in /dev/shm. Docker's default is 64 MB,
|
|
# and the resulting failure names neither shm nor the size.
|
|
if [ -d /dev/shm ]; then
|
|
local shm; shm=$(df -Pm /dev/shm 2>/dev/null | awk 'NR==2{print $2}')
|
|
if [ -n "$shm" ]; then
|
|
if [ "$shm" -le 64 ]; then
|
|
echo " ! /dev/shm ${shm} MB — postgres puts shared memory here and 64 MB"
|
|
echo " is docker's default. Raise it with --shm-size when the cabinet fails."
|
|
else
|
|
echo " /dev/shm ${shm} MB"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "disk"
|
|
local d
|
|
for d in / /tmp /var/lib/docker; do
|
|
[ -d "$d" ] || continue
|
|
df -Pm "$d" 2>/dev/null | awk -v p="$d" 'NR==2{printf " %-12s %s MB free of %s MB\n", p, $4, $2}'
|
|
done
|
|
|
|
# kind and Tilt both watch large trees, and the failure mode is silent:
|
|
# they simply stop noticing file changes. Cheap to report while we are here.
|
|
local w i
|
|
w=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null || echo 0)
|
|
i=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null || echo 0)
|
|
echo
|
|
echo "tooling"
|
|
echo " inotify watches=$w instances=$i"
|
|
if [ "$w" -lt 524288 ] || [ "$i" -lt 512 ]; then
|
|
echo " ! low — anything watching files will silently stop seeing changes"
|
|
fi
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
if [ -S /var/run/docker.sock ]; then
|
|
echo " docker socket present, no cli"
|
|
else
|
|
echo " docker not installed"
|
|
fi
|
|
elif docker info >/dev/null 2>&1; then
|
|
local n
|
|
n=$(docker ps -q 2>/dev/null | wc -l)
|
|
echo " docker $(docker version --format '{{.Server.Version}}' 2>/dev/null), ${n} container(s) running"
|
|
else
|
|
echo " ! docker cli present but the daemon is unreachable"
|
|
fi
|
|
|
|
# WSL keeps its cap on the Windows side, in a file this shell can read but
|
|
# not usefully apply — the change costs a full VM restart. Report it, and
|
|
# report the commonest mistake, which is editing it and not restarting.
|
|
if is_wsl; then
|
|
local cfg conf
|
|
cfg=$(wslconfig_path)
|
|
echo
|
|
echo "wsl"
|
|
if [ -z "$cfg" ]; then
|
|
echo " ! cannot tell which Windows profile owns .wslconfig"
|
|
else
|
|
echo " config $cfg"
|
|
conf=$(sed -n 's/^[[:space:]]*memory[[:space:]]*=[[:space:]]*//p' "$cfg" 2>/dev/null \
|
|
| tail -1 | tr -d '[:space:]')
|
|
if [ -n "$conf" ]; then
|
|
echo " configured $conf (booted ${total} MB)"
|
|
echo " - if those disagree the edit has not been applied."
|
|
echo " From a WINDOWS terminal: wsl --shutdown"
|
|
else
|
|
echo " configured no memory= set (WSL defaults to half the host RAM, or 8 GB,"
|
|
echo " whichever is less — which is where your Airflow ceiling comes from)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "effective ceiling $(effective_ceiling_mb) MB"
|
|
echo " the lowest of MemTotal, the cgroup cap and ulimit -v. What the box"
|
|
echo " claims. 'push' measures what it will actually hand over."
|
|
|
|
[ "$avail" -lt $(( total / 5 )) ] && { echo; hogs; }
|
|
return 0
|
|
}
|
|
|
|
# ── push ───────────────────────────────────────────────────────────────────
|
|
|
|
STATE=""
|
|
CHILD=""
|
|
|
|
cleanup() {
|
|
if [ -n "$CHILD" ] && kill -0 "$CHILD" 2>/dev/null; then
|
|
kill -KILL "$CHILD" 2>/dev/null || true
|
|
wait "$CHILD" 2>/dev/null || true
|
|
fi
|
|
[ -n "$STATE" ] && rm -f "$STATE"
|
|
return 0
|
|
}
|
|
|
|
# The child allocates and stops itself; the parent only watches. That split is
|
|
# the point: under --to-oom the allocating process is expected to be killed, and
|
|
# something has to survive to say how far it got.
|
|
allocator() {
|
|
# Raise our own OOM score to the maximum so the kernel picks THIS process
|
|
# first. Raising needs no privilege (only lowering does). Without it, the
|
|
# kernel is free to choose your shell, your ssh session or dockerd — on a
|
|
# box you are still using, that is not an acceptable coin toss.
|
|
echo 1000 > "/proc/$BASHPID/oom_score_adj" 2>/dev/null || true
|
|
|
|
local arr=() held=0 i=0 rss swapped avail first_swap=0
|
|
local bytes=$((STEP_MB * 1024 * 1024))
|
|
local swap_used_start
|
|
swap_used_start=$(( $(mb SwapTotal) - $(mb SwapFree) ))
|
|
|
|
while :; do
|
|
# Written STRAIGHT INTO the array element. The obvious spelling —
|
|
# build one chunk and `arr+=("$chunk")` — costs three copies per step,
|
|
# not one: the template stays resident, expanding "$chunk" makes a
|
|
# temporary word, and the append makes the element. A 128 MB step then
|
|
# needs 384 MB transiently, and on a small box it is killed on the
|
|
# first append while reporting a third of the true ceiling.
|
|
#
|
|
# printf -v into a subscript also means every page is written, so it is
|
|
# resident rather than merely promised — the only kind of allocation
|
|
# that measures anything under heuristic overcommit.
|
|
printf -v "arr[$i]" '%*s' "$bytes" ''
|
|
i=$((i + 1)); held=$((held + STEP_MB))
|
|
|
|
rss=$(awk '/^VmRSS:/{print int($2/1024)}' "/proc/$BASHPID/status" 2>/dev/null || echo 0)
|
|
avail=$(headroom_mb)
|
|
swapped=$(( $(mb SwapTotal) - $(mb SwapFree) - swap_used_start ))
|
|
[ "$swapped" -lt 0 ] && swapped=0
|
|
|
|
printf '%8s MB held rss %7s MB headroom %7s MB swap +%s MB\n' \
|
|
"$held" "$rss" "$avail" "$swapped"
|
|
printf '%s %s %s %s\n' "$held" "$rss" "$avail" "$swapped" >> "$STATE"
|
|
|
|
# Worth calling out separately from the ceiling: this is where the box
|
|
# stops being fast and starts being unusable, which for a scheduler is
|
|
# a different and earlier problem than being killed.
|
|
if [ "$swapped" -gt 0 ] && [ "$first_swap" -eq 0 ]; then
|
|
first_swap=$held
|
|
echo " - first swap page at ${held} MB — past here it works but crawls"
|
|
echo "swapat $held" >> "$STATE"
|
|
fi
|
|
|
|
if [ -n "$TO_MB" ] && [ "$held" -ge "$TO_MB" ]; then
|
|
echo "stop reached-the-cap" >> "$STATE"; return 0
|
|
fi
|
|
if [ "$TO_OOM" = no ] && [ "$avail" -lt "$FLOOR_MB" ]; then
|
|
echo "stop floor" >> "$STATE"; return 0
|
|
fi
|
|
done
|
|
}
|
|
|
|
push() {
|
|
local total ceiling rc=0 last held rss swapat stop
|
|
total=$(mb MemTotal)
|
|
ceiling=$(effective_ceiling_mb)
|
|
|
|
# A step is worth about a sixty-fourth of the ceiling: enough resolution to
|
|
# find the edge, few enough lines to read, and small enough that the
|
|
# transient cost of one allocation never dominates a small box. A fixed
|
|
# size cannot do all three — 128 MB is fine on 16 GB and absurd on 512 MB.
|
|
if [ "$STEP_EXPLICIT" = no ]; then
|
|
STEP_MB=$(( ceiling / 64 ))
|
|
[ "$STEP_MB" -lt 4 ] && STEP_MB=4
|
|
[ "$STEP_MB" -gt 256 ] && STEP_MB=256
|
|
fi
|
|
|
|
# Stop with a cushion rather than riding it to the kill. How big a cushion
|
|
# depends on what it is protecting. Under a cgroup cap, running out kills
|
|
# only this container's own processes, so it need cover no more than the
|
|
# shell that prints the result — and a 512 MB cushion on a 1 GB box would
|
|
# halve the answer. On a host there is everything else to protect, and the
|
|
# OOM killer does not promise to pick the process that caused the problem.
|
|
if [ -n "$(cgroup_cap_mb)" ]; then FLOOR_MB=64; else FLOOR_MB=512; fi
|
|
[ $(( ceiling / 20 )) -gt "$FLOOR_MB" ] && FLOOR_MB=$(( ceiling / 20 ))
|
|
|
|
STATE=$(mktemp "${TMPDIR:-/tmp}/rigmini.XXXXXX")
|
|
trap cleanup EXIT
|
|
# INT kills the child and lets the summary below print anyway, so an
|
|
# impatient Ctrl-C still tells you how far it got — and, more importantly,
|
|
# still gives the memory back.
|
|
trap 'echo; echo " interrupted"; echo "stop interrupted" >> "$STATE"; [ -n "$CHILD" ] && kill -KILL "$CHILD" 2>/dev/null || true' INT
|
|
|
|
echo "push"
|
|
echo " step ${STEP_MB} MB per allocation, every page touched"
|
|
echo " ceiling ${ceiling} MB claimed"
|
|
if [ -n "$TO_MB" ]; then
|
|
echo " stopping at ${TO_MB} MB (--to)"
|
|
elif [ "$TO_OOM" = yes ]; then
|
|
echo " ! stopping only when the kernel stops it (--to-oom)"
|
|
echo " the allocating child is marked as the preferred OOM victim,"
|
|
echo " but nothing about an OOM kill is entirely polite. Not on a box"
|
|
echo " running anything you mind losing."
|
|
else
|
|
echo " stopping when headroom drops below ${FLOOR_MB} MB"
|
|
fi
|
|
echo
|
|
|
|
allocator &
|
|
CHILD=$!
|
|
wait "$CHILD" || rc=$?
|
|
CHILD=""
|
|
trap - INT
|
|
|
|
last=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 || true)
|
|
held=$(echo "$last" | awk '{print $1}')
|
|
rss=$(echo "$last" | awk '{print $2}')
|
|
swapat=$(awk '/^swapat/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
|
stop=$(awk '/^stop/{print $2}' "$STATE" 2>/dev/null | head -1 || true)
|
|
|
|
echo
|
|
if [ -z "$held" ]; then
|
|
echo " ! nothing was allocated. Even one ${STEP_MB} MB chunk failed —"
|
|
echo " try a smaller --step, or check ulimit -v in 'status'."
|
|
return 1
|
|
fi
|
|
|
|
echo " reached ${rss:-$held} MB resident"
|
|
[ -n "$swapat" ] && echo " swapping from ${swapat} MB"
|
|
|
|
case "$stop" in
|
|
reached-the-cap)
|
|
echo " outcome stopped at the --to cap, not at a limit."
|
|
echo " The box held ${TO_MB} MB without complaint; there is more." ;;
|
|
floor)
|
|
echo " outcome stopped with a cushion intact, by choice."
|
|
echo " The real ceiling is higher — --to-oom finds it, at the"
|
|
echo " cost of an actual OOM kill." ;;
|
|
interrupted)
|
|
echo " outcome interrupted at ${rss:-$held} MB — where you stopped it,"
|
|
echo " not where the box did." ;;
|
|
*)
|
|
# No stop line means the child did not decide to stop: it was ended.
|
|
if [ "$rc" -ge 128 ]; then
|
|
echo " outcome the child was killed (signal $((rc - 128))) at ${rss:-$held} MB."
|
|
elif [ "$rc" -ne 0 ]; then
|
|
echo " outcome the allocation failed at ${rss:-$held} MB (exit ${rc})."
|
|
echo " bash could not get the next chunk — an honest malloc"
|
|
echo " failure rather than a kill. That is the strict-overcommit"
|
|
echo " or ulimit path."
|
|
else
|
|
echo " outcome ended at ${rss:-$held} MB."
|
|
fi
|
|
local ev
|
|
ev=$(dmesg 2>/dev/null | tail -80 | grep -iE 'oom-kill|killed process' | tail -1 || true)
|
|
if [ -n "$ev" ]; then
|
|
echo " kernel ${ev#*] }"
|
|
else
|
|
echo " - dmesg is unreadable here (dmesg_restrict, or no privilege),"
|
|
echo " so the kill cannot be confirmed from this side. The number stands."
|
|
fi ;;
|
|
esac
|
|
|
|
# The gap between the claim and the measurement is the finding — but only
|
|
# when the BOX chose where to stop. An empty $stop means the child was ended
|
|
# rather than deciding to end; anything else (--to, the floor) is a stop we
|
|
# asked for, and flagging those as short of the ceiling would put a warning
|
|
# on every deliberately small run.
|
|
local got="${rss:-$held}"
|
|
echo
|
|
if [ -z "$stop" ] && [ "$got" -lt $(( ceiling * 70 / 100 )) ]; then
|
|
echo " ! claimed ${ceiling} MB, gave up ${got} MB — under 70% of it."
|
|
echo " Something is taking the difference. 'status' names the candidates:"
|
|
echo " a cgroup cap, ulimit -v, or memory already resident."
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ── all ────────────────────────────────────────────────────────────────────
|
|
|
|
all() {
|
|
status
|
|
echo
|
|
echo "────────────────────────────────────────────────────────────"
|
|
echo
|
|
push
|
|
|
|
local got budget_mb ceiling
|
|
budget_mb=$(( BUDGET_GB * 1024 ))
|
|
ceiling=$(effective_ceiling_mb)
|
|
got=$(grep -E '^[0-9]' "$STATE" 2>/dev/null | tail -1 | awk '{print $2}' || true)
|
|
[ -n "$got" ] || got=0
|
|
|
|
echo
|
|
echo "verdict"
|
|
echo " budget ${BUDGET_GB} GB for kind + postgres + redis + airflow"
|
|
# Only worth explaining while it is still a guess. Once --budget is given
|
|
# the number came from somewhere better than this reasoning, and repeating
|
|
# the derivation would describe a figure that is no longer in use.
|
|
if [ "$BUDGET_EXPLICIT" = no ]; then
|
|
echo " - that is 2 GB per kind node, which is rig's own figure, plus about"
|
|
echo " 4 GB for the three cabinets. THE 4 GB IS AN ESTIMATE, not something"
|
|
echo " measured. Re-run with --budget once you have watched the real thing."
|
|
fi
|
|
echo " measured ${got} MB handed over"
|
|
|
|
if [ "$got" -ge "$budget_mb" ]; then
|
|
echo " fits, with $(( got - budget_mb )) MB spare."
|
|
if [ "$got" -lt $(( budget_mb * 130 / 100 )) ]; then
|
|
echo " - under 30% spare is thin for a scheduler. Airflow's memory use"
|
|
echo " is spiky, and the spikes are what get killed."
|
|
fi
|
|
else
|
|
echo " ! short by $(( budget_mb - got )) MB."
|
|
if [ "$ceiling" -ge "$budget_mb" ]; then
|
|
echo " The box CLAIMS enough (${ceiling} MB) but did not deliver it."
|
|
echo " Free something, or read the caps section again."
|
|
else
|
|
echo " The box does not have it to give. A bigger bundle, or a smaller"
|
|
echo " profile: PROFILE=minimal drops the cabinets entirely."
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# ── main ───────────────────────────────────────────────────────────────────
|
|
|
|
parse_flags() {
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--to) TO_MB=$(( ${2:?--to needs a value in GB} * 1024 )); shift 2 ;;
|
|
--to-mb) TO_MB="${2:?--to-mb needs a value in MB}"; shift 2 ;;
|
|
--step) STEP_MB="${2:?--step needs a value in MB}"; STEP_EXPLICIT=yes; shift 2 ;;
|
|
--to-oom) TO_OOM=yes; shift ;;
|
|
--budget) BUDGET_GB="${2:?--budget needs a value in GB}"; BUDGET_EXPLICIT=yes; shift 2 ;;
|
|
*) echo "unknown argument: $1" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
if [ "$TO_OOM" = yes ] && [ -n "$TO_MB" ]; then
|
|
echo "--to and --to-oom contradict each other: one stops early, the other" >&2
|
|
echo "refuses to stop at all. Pick one." >&2
|
|
exit 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
require_linux
|
|
find_cgroup
|
|
|
|
cmd="${1:-status}"
|
|
[ $# -gt 0 ] && shift
|
|
|
|
case "$cmd" in
|
|
status) parse_flags "$@"; status ;;
|
|
push) parse_flags "$@"; push ;;
|
|
all) parse_flags "$@"; all ;;
|
|
*) echo "usage: $0 [status|push|all]" >&2
|
|
echo " push [--to GB] [--to-mb MB] [--step MB] [--to-oom]" >&2
|
|
echo " all [--budget GB]" >&2
|
|
exit 1 ;;
|
|
esac
|