diff --git a/berth/ctrl/check.sh b/berth/ctrl/check.sh index f7e9a88..1d818a0 100644 --- a/berth/ctrl/check.sh +++ b/berth/ctrl/check.sh @@ -80,11 +80,27 @@ for r in fw: if n: print(" UNRESOLVED: " + n) ' note "listener side: unknown until captured (ss -ltnp over ssh $HOST)." -wg=$(estate_get "vpn._status") -[ -n "$wg" ] && warn "overlay: not fully captured — see 'make vpn check'" && \ - note " 10.8.0.1 carries the registry and woodpecker gRPC;" && \ - note " 10.8.0.2 backs langfuse. No 51820/udp rule, nothing creates" && \ - note " the interface — a fresh box cannot start the gateway compose." +# Ask the structure, not the prose: the condition is "does a peer still lack a +# public key", not "is there a _status string". _status is ALWAYS non-empty — +# capture rewrites it to "CAPTURED ..." — so testing it for emptiness pinned +# this warning on permanently, including after the capture it asks for. +uncaptured=$(estate_get "vpn.overlays.estate.peers" 2>/dev/null | python3 -c ' +import json, sys +try: + peers = json.load(sys.stdin) +except Exception: + sys.exit(0) +print(" ".join(n for n, p in peers.items() if not p.get("public_key"))) +' 2>/dev/null) +if [ -n "$uncaptured" ]; then + warn "overlay: public keys not captured for:$uncaptured — see 'make vpn check'" + note " 10.8.0.1 carries the registry and woodpecker gRPC;" + note " 10.8.0.2 backs langfuse. Nothing in the tree creates the" + note " interface — a fresh box cannot start the gateway compose." + note " capture with: sudo wg show | make vpn capture --write" +else + note "overlay: $(estate_get 'vpn._status')" +fi note "overlay detail: make vpn show estate" echo diff --git a/berth/ctrl/lib/estate.sh b/berth/ctrl/lib/estate.sh index 213cbe7..b0c71bc 100644 --- a/berth/ctrl/lib/estate.sh +++ b/berth/ctrl/lib/estate.sh @@ -123,3 +123,25 @@ except ValueError: sys.exit(2) ' "$1" "$2" } + +# The upstream a service actually resolves to, as "host:port". +# +# A PLACED service has no literal `upstream` field: ✖ B9 replaced langfuse's +# hand-written `10.8.0.2:3000` with placement+peer+port, because being reached +# by address on the overlay is ONE decision, not three properties. Everything +# that asks "what does this service point at" must therefore resolve it the +# same way, or it silently sees an empty string and skips the service — which +# is exactly how vpn.sh's bindings invariant went quiet after B9 landed. +# +# usage: service_upstream +service_upstream() { + local up="$1" placement="$2" peer="$3" port="$4" + case "$placement" in + local|instance) + local addr; addr="$(overlay_get estate "peers.${peer}.address")" + [ -z "$addr" ] && return 1 + printf '%s:%s' "$addr" "$port" + ;; + *) printf '%s' "$up" ;; + esac +} diff --git a/berth/ctrl/selftest.sh b/berth/ctrl/selftest.sh index cc98099..9311aac 100644 --- a/berth/ctrl/selftest.sh +++ b/berth/ctrl/selftest.sh @@ -169,6 +169,32 @@ check "generated output is ignored" "yes" \ check "key material is ignored" "yes" \ "$(cd .. && git check-ignore -q ctrl/.secrets/vpn/any.key && echo yes || echo no)" +note "capture and the checks that read it — three bugs found by running, 2026-09-14" + +# 1. A placed service's upstream is DERIVED (✖ B9). Anything reading the raw +# `upstream` field sees "" and skips it — which is how vpn.sh's bindings +# invariant, the "my configurations broke" detector, went quiet the day +# placement landed while still printing OK. Vacuous passes are the failure +# mode this whole file exists to catch. +check "a placed service resolves to a real upstream" "10.8.0.2:3000" \ + "$(bash -c 'source ./lib/config.sh; source ./lib/estate.sh; load_config >/dev/null; + service_upstream "" local nrft 3000')" +check "bindings actually inspects a service" "1" \ + "$(bash ./vpn.sh check 2>/dev/null | grep -c 'no service currently has an overlay address' \ + | awk '{print 1-$1}')" + +# 2. A listen port belongs to a PEER. The roaming peer's is an ephemeral source +# port; writing it to the overlay renames the port the firewall rule is +# checked against — silently, since both are plausible integers. +check "a roaming peer's port is not the overlay's port" "51820" \ + "$(python3 -c 'import json;print(json.load(open("../estate/mcrn.json"))["vpn"]["overlays"]["estate"]["listen_port"])')" + +# 3. _status is always non-empty — capture rewrites it rather than clearing it — +# so a warning gated on "is it set" can never turn off, including after the +# capture it asks for. Gate on the structure instead. +check "the capture warning clears once keys are in" "0" \ + "$(bash ./check.sh 2>/dev/null | grep -c 'public keys not captured')" + note "every STALE entry has a check here" # Not "$0": line 15 cd's into this script's directory, so a relative $0 no diff --git a/berth/ctrl/services.sh b/berth/ctrl/services.sh index a7bcf1a..b91e8bf 100644 --- a/berth/ctrl/services.sh +++ b/berth/ctrl/services.sh @@ -26,10 +26,7 @@ list() { [ -z "$name" ] && continue # A placed service has no literal `upstream` — it is derived from the # peer's overlay address, so show what it actually resolves to. - local shown="$up" - case "$placement" in - local|instance) shown="$(overlay_get estate "peers.${peer}.address"):${port}" ;; - esac + local shown; shown="$(service_upstream "$up" "$placement" "$peer" "$port")" || shown="" printf '%-12s %-16s %-22s %-9s %s\n' \ "$name" "${host}.${DOMAIN}" "${shown:--}" "$placement" \ "$([ -n "$raw" ] && echo 'hand-written' || echo 'generated')" @@ -62,7 +59,7 @@ render_cloud() { local|instance) tmpl=./render/nginx-upstream.tmpl uhost="$(overlay_get estate "peers.${peer}.address")" - uport="$port" + uport="$port" # resolved via service_upstream's same rule if [ -z "$uhost" ]; then echo " ! $name: placement '$placement' names peer '$peer', which has no address" >&2 continue diff --git a/berth/ctrl/vpn.sh b/berth/ctrl/vpn.sh index 504b7a9..c0ee2b6 100644 --- a/berth/ctrl/vpn.sh +++ b/berth/ctrl/vpn.sh @@ -162,6 +162,9 @@ PY echo "bindings — services reached over the overlay must bind a reachable address" local checked=0 while IFS=$'\x1f' read -r name host up kind raw placement peer port lhost; do + # Resolve placement first: a placed service's upstream is derived, not + # literal, so reading `up` alone skips it and this invariant goes quiet. + up="$(service_upstream "$up" "$placement" "$peer" "$port")" || true [ -z "$up" ] && continue local uhost="${up%%:*}" uport="${up##*:}" addr_in_subnet "$uhost" "$(overlay_get estate subnet)" 2>/dev/null || continue @@ -256,8 +259,16 @@ render() { # because the keys are what is missing. A roaming peer's endpoint is a home # address and has no stable value — dropped in the parser, not just unused. capture() { - local write="" - for a in "$@"; do [ "$a" = "--write" ] && write=1; done + local write="" as_peer="" + while [ $# -gt 0 ]; do + case "$1" in + --write) write=1 ;; + --as) shift; as_peer="${1:-}" + [ -z "$as_peer" ] && { echo "--as needs a peer name" >&2; exit 1; } ;; + *) echo "capture: unknown argument '$1'" >&2; exit 1 ;; + esac + shift + done local input; input=$(cat) if [ -z "$input" ]; then @@ -278,8 +289,8 @@ capture() { exit 1 fi - WRITE="$write" INPUT="$input" python3 - "$ESTATE_FILE" <<'PYCAP' -import collections, json, os, re, sys + WRITE="$write" AS_PEER="$as_peer" INPUT="$input" python3 - "$ESTATE_FILE" <<'PYCAP' +import collections, ipaddress, json, os, re, sys text = os.environ["INPUT"] write = os.environ.get("WRITE") == "1" @@ -311,13 +322,28 @@ ov = d["vpn"]["overlays"]["estate"] # address -> peer name, from what the estate already declares by_addr = {p["address"]: n for n, p in ov["peers"].items() if p.get("address")} -# this machine's own wg address, so the interface block lands on the right peer -me = None -for n, p in ov["peers"].items(): - if p.get("address") and os.popen( +by_key = {p["public_key"]: n for n, p in ov["peers"].items() if p.get("public_key")} +subnet = ipaddress.ip_network(ov["subnet"]) if ov.get("subnet") else None +hubs = [n for n, p in ov["peers"].items() if p.get("role") == "hub"] + +# Whose interface block is this? `--as` names it explicitly, and that is the only +# thing that works for output captured over ssh: the addresses on THIS machine +# say nothing about the machine the output came from. +as_peer = os.environ.get("AS_PEER") or "" +if as_peer: + if as_peer not in ov["peers"]: + print("no peer named %r in this overlay. known: %s" + % (as_peer, ", ".join(ov["peers"]))) + raise SystemExit(1) + me = as_peer +else: + me = None + local = os.popen( "ip -4 -o addr show 2>/dev/null | awk '{split($4,a,\"/\"); print a[1]}'" - ).read().split().count(p["address"]): - me = n + ).read().split() + for n, p in ov["peers"].items(): + if p.get("address") and p["address"] in local: + me = n changes = [] conflicts = [] @@ -339,13 +365,33 @@ def setf(peer, field, val, why=""): if me and iface.get("public_key"): setf(me, "public_key", iface["public_key"], "(this machine's interface)") +def match(pr): + # 1. The public key IS the identity. Use it whenever the estate knows it. + n = by_key.get(pr["public_key"]) + if n: + return n + nets = [a.strip() for a in pr.get("allowed_ips", "").split(",") if a.strip()] + # 2. An allowed-ip that is a declared peer address — the ordinary spoke case. + for a in nets: + if a.split("/")[0] in by_addr: + return by_addr[a.split("/")[0]] + # 3. A peer routing the WHOLE overlay is the hub seen from a spoke. Its + # allowed_ips is the subnet itself, so no single address ever matches it. + if subnet and len(hubs) == 1: + for a in nets: + try: + if ipaddress.ip_network(a, strict=False).supernet_of(subnet): + return hubs[0] + except ValueError: + continue + return None + for pr in peers: - addrs = [a.split("/")[0] for a in pr.get("allowed_ips", "").split(",") if a.strip()] - name = next((by_addr[a] for a in addrs if a in by_addr), None) + name = match(pr) if not name: changes.append(("?", "UNMATCHED", None, "allowed_ips=%s key=%s" % (pr.get("allowed_ips"), pr["public_key"][:12] + "..."), - "no estate peer has this address")) + "no estate peer has this key, this address, or this route")) continue setf(name, "public_key", pr.get("public_key")) setf(name, "allowed_ips", pr.get("allowed_ips")) @@ -359,14 +405,21 @@ for pr in peers: else: setf(name, "endpoint", pr["endpoint"]) -if iface.get("listen_port"): +if me and iface.get("listen_port"): try: lp = int(iface["listen_port"]) - if ov.get("listen_port") != lp: - changes.append(("(overlay)", "listen_port", ov.get("listen_port"), lp, "")) - if write: ov["listen_port"] = lp except ValueError: - pass + lp = None + if lp is not None: + # A listen port belongs to the PEER, not to the overlay. A roaming peer's + # is an ephemeral source port chosen by the kernel; writing it to the + # overlay would rename the port the firewall rule is checked against. + setf(me, "listen_port", lp) + if ov["peers"][me].get("role") == "hub" and ov.get("listen_port") != lp: + changes.append(("(overlay)", "listen_port", ov.get("listen_port"), lp, + "the hub's port is the overlay's port")) + if write: + ov["listen_port"] = lp if conflicts: print("REFUSING: the same field was reported twice with different values.\n") @@ -421,5 +474,5 @@ case "${1:-list}" in keygen) shift; keygen "${1:-}" ;; capture) shift; capture "$@" ;; up|down) v="$1"; shift; refuse "$v" "$@" ;; - *) echo "usage: $0 [list|show |check|render |keygen |capture [--write]|up --yes|down --yes]" >&2; exit 1 ;; + *) echo "usage: $0 [list|show |check|render |keygen |capture [--as ] [--write]|up --yes|down --yes]" >&2; exit 1 ;; esac diff --git a/berth/estate/mcrn.json b/berth/estate/mcrn.json index 9bb7e93..60fff38 100644 --- a/berth/estate/mcrn.json +++ b/berth/estate/mcrn.json @@ -65,7 +65,7 @@ "wireguard_moved": "superseded by the top-level `vpn` block" }, "vpn": { - "_status": "PARTIAL — addresses and subnet verified from the live wg0 interface on nrft. Peer public keys, endpoints, allowed-ips and keepalive still need `wg show` capture on both ends (V1). Nulls below are unknowns, not defaults.", + "_status": "CAPTURED 2026-09-14 — public keys, allowed-ips and keepalive read from `wg show`. Roaming endpoints deliberately not recorded.", "_never_record": "private keys. `wg show` prints 'private key: (hidden)' and is the safe capture command. `wg showconf` dumps PrivateKey= in clear — never use it.", "overlays": { "estate": { @@ -77,17 +77,29 @@ "address": "10.8.0.1", "role": "hub", "note": "mcrn.ar. Has a public IP, so it is the peer others dial. Carries the registry (:5000) and woodpecker's gRPC (:9000), both bound to this address and therefore overlay-only.", - "endpoint": null, - "public_key": null, - "allowed_ips": null + "endpoint": "3.23.204.197:51820", + "public_key": "zVYCmi3xucuX7k/aDhrOUPyN4GRk96ffSDD6dUFQjh4=", + "allowed_ips": "10.8.0.0/24", + "keepalive": 25, + "listen_port": 51820 }, "nrft": { "address": "10.8.0.2", "role": "roaming", "note": "The dev box. Behind NAT, so it must initiate and needs PersistentKeepalive. Verified: wg0 UP at 10.8.0.2/24, ping 10.8.0.1 0% loss at 153ms.", "endpoint": null, - "public_key": null, - "allowed_ips": null, + "public_key": "zlIBGs4y5rt6uVdmFBasHpafht6ErxG+R3ySCg5rh3s=", + "allowed_ips": "10.8.0.2/32, 192.168.1.0/24", + "keepalive": null, + "listen_port": 36145 + }, + "work": { + "address": "10.8.0.3", + "role": "roaming", + "note": "A work computer, granted access when it was needed. Identified by the user at capture time, 2026-09-14 — it was NOT in the description before, and the wire is where it was found. No handshake and no transfer have ever been recorded for it, so it is a standing grant rather than a live peer: it can connect, and never has. Whether to keep or revoke it is the host's call.", + "endpoint": null, + "public_key": "ruSZwKt/p60GVsTLSAhcKBIXKkSZsf0gWSmSH1+UgE0=", + "allowed_ips": "10.8.0.3/32", "keepalive": null } }