275 lines
12 KiB
Python
275 lines
12 KiB
Python
"""
|
|
Does *this* book still hold — the third test level.
|
|
|
|
python3 -m docgen check out/book/station
|
|
|
|
docgen has three levels of test, and they differ by what they assert *about*.
|
|
The distinction is rig's, from `rig/ctrl/selftest.sh`, and it is worth keeping
|
|
because it decides what a failure means:
|
|
|
|
make doctor the MACHINE. Never fails; it reports.
|
|
make check DOCGEN. Exits 1. "docgen no longer does what it says."
|
|
make check BOOK=<dir>
|
|
THIS BOOK. Exits 1. "this book no longer holds."
|
|
|
|
The third is the one that reaches a project docgen has never seen. It is also
|
|
where **framework and custom checks live together**, at different levels:
|
|
|
|
generated the spine's own assertions, identical for every book. Both
|
|
measures present, the reconciliation holding, every artifact
|
|
where the ledger says it is, the notebook still executing.
|
|
custom hand-written, in the book's own `checks.py`, using the same
|
|
check/note/skip helpers — so a project's line and a framework
|
|
line read identically and fail identically.
|
|
|
|
Same split as the notebook's base and overlay, for the same reason: generation
|
|
alone cannot know what *this* project cares about, and hand-authoring alone rots.
|
|
|
|
## The discipline these are written in
|
|
|
|
Carried from rig verbatim, because it is the point of the whole level:
|
|
|
|
> Each check is ONE decision that has already been made, with the reason above
|
|
> it — not coverage, and deliberately not an exhaustive sweep of use cases. A
|
|
> rule without its reason gets overridden the first time it is inconvenient.
|
|
> Failing one should read as **"you are about to undo this"** rather than
|
|
> "something broke".
|
|
|
|
## Writing a book's own checks
|
|
|
|
Put a `checks.py` next to `book.json`:
|
|
|
|
def checks(book, check, note, skip):
|
|
note("what this project will not give up")
|
|
|
|
# Payments moved once already and the move broke three dashboards.
|
|
# If it is not here, something renamed it again.
|
|
check("payments is still a module", True,
|
|
any(n["id"] == "app.payments" for n in book.ir["nodes"]))
|
|
|
|
`book` carries `.data` (the parsed `book.json`), `.ir` (the extracted graph),
|
|
and `.dir`. `check` takes (name, expected, actual) — expected first, because a
|
|
failure report is only useful if it says what was wanted.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from . import FIRST, LAST
|
|
|
|
|
|
class Loaded:
|
|
"""A book read back off disk, for checking rather than building."""
|
|
|
|
def __init__(self, directory):
|
|
self.dir = Path(directory)
|
|
path = self.dir / "book.json"
|
|
if not path.exists():
|
|
raise FileNotFoundError(
|
|
f"{path} does not exist — is {self.dir} a book directory?\n"
|
|
"Build one with: python3 -m docgen book --root <src> -o <dir>"
|
|
)
|
|
self.data = json.loads(path.read_text())
|
|
|
|
ir_path = self.dir / "steps" / "ir.json"
|
|
self.ir = json.loads(ir_path.read_text()) if ir_path.exists() else None
|
|
|
|
nb_path = self.dir / (self.data.get("notebook") or "notebook.ipynb")
|
|
self.notebook = json.loads(nb_path.read_text()) if nb_path.exists() else None
|
|
|
|
@property
|
|
def larder(self) -> dict:
|
|
return self.data.get("larder") or {}
|
|
|
|
@property
|
|
def measure(self) -> dict:
|
|
return self.data.get("book") or {}
|
|
|
|
|
|
class Report:
|
|
"""rig's reporting shape: sections, one line per decision, nothing aborts."""
|
|
|
|
def __init__(self):
|
|
self.passed, self.failed, self.skipped = 0, [], 0
|
|
|
|
def note(self, text: str) -> None:
|
|
print(f"\n{text}")
|
|
|
|
def check(self, name: str, expected, actual) -> bool:
|
|
if expected == actual:
|
|
print(f" ok {name}")
|
|
self.passed += 1
|
|
return True
|
|
print(f" FAIL {name}\n expected: {expected!r}\n got: {actual!r}")
|
|
self.failed.append(name)
|
|
return False
|
|
|
|
def skip(self, name: str, why: str) -> None:
|
|
print(f" -- {name} ({why})")
|
|
self.skipped += 1
|
|
|
|
def total(self) -> int:
|
|
print()
|
|
# stdout is block-buffered when redirected and stderr is not, so the
|
|
# failure summary printed below would otherwise arrive BEFORE the checks
|
|
# it summarises — which is how this was found, piping to `tail`.
|
|
sys.stdout.flush()
|
|
if not self.failed:
|
|
print(f"{self.passed} checks passed — this book still holds"
|
|
+ (f", {self.skipped} skipped" if self.skipped else ""))
|
|
return 0
|
|
print(f"FAILED — {len(self.failed)} of {self.passed + len(self.failed)}: "
|
|
f"{', '.join(self.failed)}", file=sys.stderr)
|
|
print("Read the comment next to the check. A failure here means a decision "
|
|
"has drifted, not that a tool is broken.", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def generated(book: Loaded, r: Report) -> None:
|
|
"""The spine's own assertions. Identical for every book, by design."""
|
|
|
|
r.note("the book is bracketed — both ends present")
|
|
# The one structural promise the whole design makes: an operation is
|
|
# measured at both ends. A book missing either end is not a short book, it
|
|
# is a book whose numbers cannot be reconciled against anything.
|
|
ids = [s["id"] for s in book.data.get("steps") or []]
|
|
r.check("the first step is the larder measure", FIRST, ids[0] if ids else None)
|
|
r.check("the last step is the book measure", LAST, ids[-1] if ids else None)
|
|
r.check("nothing is bracketed twice", [1, 1],
|
|
[ids.count(FIRST), ids.count(LAST)])
|
|
|
|
r.note("what came in was measured, not guessed")
|
|
larder = book.larder
|
|
for key in ("kind", "identity", "unit", "seen", "read", "failed"):
|
|
r.check(f"larder records {key}", True, key in larder)
|
|
# read is derived, so a stored value that disagrees means somebody wrote it
|
|
# by hand. This is the same arithmetic ir/validate.py enforces; asserted
|
|
# again here because a book can be assembled without going through the IR.
|
|
r.check("read == seen - failed", larder.get("read"),
|
|
max(0, larder.get("seen", 0) - len(larder.get("failed") or [])))
|
|
# The one field that could carry a secret out of a database URL.
|
|
r.check("the identity carries no password", True,
|
|
":***@" in larder.get("identity", "") or "@" not in larder.get("identity", ""))
|
|
|
|
r.note("the two ends reconcile")
|
|
reconciled = book.data.get("reconciled") or []
|
|
r.check("something was reconciled", True, len(reconciled) > 0)
|
|
for item in reconciled:
|
|
# Each of these is a claim the book makes about itself. A false one
|
|
# means the document below is smaller than the source and does not say
|
|
# so, which is the single failure this level exists to catch.
|
|
r.check(item["claim"], True, item["ok"])
|
|
|
|
r.note("the ledger describes files that exist")
|
|
for step in book.data.get("steps") or []:
|
|
artifact = step.get("artifact")
|
|
if not artifact:
|
|
continue
|
|
path = book.dir / artifact
|
|
# A ledger naming a file that is not there is worse than no ledger: it
|
|
# is a manifest somebody will build tooling against.
|
|
r.check(f"{artifact} is where the ledger says", True, path.exists())
|
|
if path.is_file():
|
|
r.check(f"{artifact} is {step['bytes']} bytes", step["bytes"],
|
|
path.stat().st_size)
|
|
|
|
r.note("the notebook carries the sequence, with the measures at its ends")
|
|
if book.notebook is None:
|
|
r.skip("notebook", "no notebook.ipynb in this book")
|
|
else:
|
|
cells = book.notebook.get("cells") or []
|
|
first = "".join(cells[0]["source"]) if cells else ""
|
|
last = "".join(cells[-1]["source"]) if cells else ""
|
|
r.check("its first cell is what came in", True, "what came in" in first)
|
|
r.check("its last cell is what came out", True, "what came out" in last)
|
|
# Compiling is not enough — see selftest.py. `json.dumps` writes
|
|
# `false`/`true`/`null`, which are valid Python *identifiers*, so a
|
|
# generated body full of them compiles and then raises NameError.
|
|
ran, failure = _run_cells(cells, book.dir)
|
|
r.check(f"its {ran} code cells run, not merely compile", None, failure)
|
|
|
|
r.note("the web output exists and shows both ends")
|
|
index = book.dir / "site" / "index.html"
|
|
if not index.exists():
|
|
r.skip("site", "no site/index.html in this book")
|
|
else:
|
|
html = index.read_text()
|
|
# The site is the last step precisely because it is the artifact somebody
|
|
# definitely opens. A page that shows the result without showing what
|
|
# went in is the thing this whole change corrects.
|
|
r.check("the page says what came in", True, "what came in" in html)
|
|
r.check("the page says what came out", True, "what came out" in html)
|
|
if larder.get("failed"):
|
|
r.check("the page admits the book is incomplete", True,
|
|
"This book is incomplete" in html)
|
|
|
|
r.note("the book is reproducible")
|
|
# A timestamp would make two builds of an unchanged larder differ, which
|
|
# destroys the only useful property a ledger has: that a diff means a real
|
|
# change. Same rule as the IR's `generated_at`.
|
|
r.check("no timestamp in the ledger", True, "generated_at" not in json.dumps(book.data))
|
|
|
|
|
|
def custom(book: Loaded, r: Report) -> None:
|
|
"""This book's own assertions, if it has any."""
|
|
path = book.dir / "checks.py"
|
|
if not path.exists():
|
|
r.note("this book's own checks")
|
|
r.skip("custom checks", f"no {path.name} — write one to assert what this project cares about")
|
|
return
|
|
|
|
namespace: dict = {"__file__": str(path), "__name__": "book_checks"}
|
|
try:
|
|
exec(compile(path.read_text(), str(path), "exec"), namespace)
|
|
except Exception as e: # noqa: BLE001 - report, do not traceback
|
|
r.note("this book's own checks")
|
|
r.check(f"{path.name} loads", None, f"{type(e).__name__}: {e}")
|
|
return
|
|
|
|
fn = namespace.get("checks")
|
|
if not callable(fn):
|
|
r.note("this book's own checks")
|
|
r.check(f"{path.name} defines checks(book, check, note, skip)", True, False)
|
|
return
|
|
|
|
try:
|
|
fn(book, r.check, r.note, r.skip)
|
|
except Exception as e: # noqa: BLE001
|
|
r.check(f"{path.name} ran to completion", None, f"{type(e).__name__}: {e}")
|
|
|
|
|
|
def _run_cells(cells: list, cwd: Path) -> tuple[int, str | None]:
|
|
"""Execute the notebook's code cells from the book directory."""
|
|
import contextlib
|
|
import io
|
|
import os
|
|
|
|
ns, ran, failure = {}, 0, None
|
|
previous = os.getcwd()
|
|
try:
|
|
os.chdir(cwd)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
for c in cells:
|
|
if c.get("cell_type") != "code":
|
|
continue
|
|
src = "".join(c["source"])
|
|
if "urlopen" in src or ("call(" in src and "def call" not in src):
|
|
# Anything that would reach the network is compiled, not run.
|
|
try:
|
|
compile(src, c["id"], "exec")
|
|
ran += 1
|
|
except SyntaxError as e:
|
|
failure = f"{c['id']}: {e}"
|
|
break
|
|
continue
|
|
try:
|
|
exec(compile(src, c["id"], "exec"), ns)
|
|
ran += 1
|
|
except Exception as e: # noqa: BLE001 - any failure is the finding
|
|
failure = f"{c['id']}: {type(e).__name__}: {e}"
|
|
break
|
|
finally:
|
|
os.chdir(previous)
|
|
return ran, failure
|