new conventions

soleprint/atlas2/docgen/ir/validate.py

@@ -22,6 +22,7 @@ that produce a broken diagram.
 """
 
 import json
+import re
 import sys
 from pathlib import Path
 
@@ -90,6 +91,8 @@ def check(data: dict, *, strict_visual: bool = True) -> list[str]:
     if version is not None and not isinstance(version, str):
         problems.append(f"meta.schema_version must be a string, got {type(version).__name__}")
 
+    problems.extend(_check_larder(data["meta"].get("larder"), meta_schema))
+
     # -- nodes ------------------------------------------------------------
     node_required, node_allowed = _fields(schema, "nodes")
     seen: set[str] = set()
@@ -169,6 +172,79 @@ def check(data: dict, *, strict_visual: bool = True) -> list[str]:
     return problems
 
 
+# A credential that survived redaction, detected independently of the code that
+# does the redacting. `book/larder.py` has its own key list; this has a second
+# one on purpose. If the two ever disagree the check fires, which is the point —
+# a scrubber that is graded by its own word is not graded. Same reason
+# VISUAL_KEYS lives here and not in schema.json.
+_DSN_PASSWORD = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^:/@\s]+:(?P<secret>[^@/\s]+)@")
+_SECRET_PARAM = re.compile(
+    r"(?i)\b(?:password|passwd|pwd|secret|token|access_token|refresh_token"
+    r"|api_key|apikey|sig|signature|credentials)\s*=\s*(?P<secret>[^&;\s]+)"
+)
+_MASKED = {"***", "xxx", "redacted", "[redacted]", "masked"}
+
+
+def _check_larder(larder, meta_schema: dict) -> list[str]:
+    """The input measure, checked — shape, arithmetic, and no leaked secret.
+
+    Absent is fine and means "not measured". Present and wrong is not, because
+    the whole value of the measure is that the number can be trusted against the
+    result; a larder claiming 47 reads when it read 45 is worse than no larder.
+    """
+    if larder is None:
+        return []
+    if not isinstance(larder, dict):
+        return [f"meta.larder must be an object, got {type(larder).__name__}"]
+
+    problems = []
+    spec = meta_schema["properties"]["larder"]
+    for key in spec["required"]:
+        if key not in larder:
+            problems.append(f"meta.larder is missing {key!r}")
+    for key in larder:
+        if key not in spec["properties"]:
+            problems.append(f"meta.larder has unknown key {key!r}")
+
+    failed = larder.get("failed")
+    if failed is not None and not isinstance(failed, list):
+        problems.append("meta.larder.failed must be an array")
+        failed = None
+    elif failed:
+        for i, f in enumerate(failed):
+            if not isinstance(f, dict) or "name" not in f or "error" not in f:
+                problems.append(f"meta.larder.failed[{i}] needs both 'name' and 'error'")
+
+    # read is derived. Stored, it can disagree with itself, and this is where
+    # that shows up rather than in a book measure nobody can reconcile.
+    seen, read = larder.get("seen"), larder.get("read")
+    if isinstance(seen, int) and isinstance(read, int) and isinstance(failed, list):
+        if read != max(0, seen - len(failed)):
+            problems.append(
+                f"meta.larder.read is {read} but seen={seen} with {len(failed)} failed "
+                f"implies {max(0, seen - len(failed))} — the measure disagrees with itself"
+            )
+    if isinstance(read, int) and isinstance(seen, int) and read > seen:
+        problems.append(f"meta.larder.read ({read}) exceeds seen ({seen})")
+
+    unit = larder.get("unit")
+    if unit is not None and not isinstance(unit, str):
+        problems.append(f"meta.larder.unit must be a string, got {type(unit).__name__}")
+
+    identity = larder.get("identity")
+    if isinstance(identity, str):
+        for pattern, what in ((_DSN_PASSWORD, "a DSN password"),
+                              (_SECRET_PARAM, "a secret parameter")):
+            m = pattern.search(identity)
+            if m and m.group("secret").lower() not in _MASKED:
+                problems.append(
+                    f"meta.larder.identity still contains {what} — "
+                    "redact() in book/larder.py did not mask it, and this document "
+                    "must not be written anywhere"
+                )
+    return problems
+
+
 def validate(data: dict, **kw) -> dict:
     """check(), but raises. For use at a boundary where carrying on is wrong."""
     problems = check(data, **kw)