From a29e0708e84854c1ae5225ff1ac2fe93a3641e81 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Mon, 14 Sep 2026 06:13:22 -0300 Subject: [PATCH] new conventions --- soleprint/atlas2/docgen/Makefile | 71 +- soleprint/atlas2/docgen/book/__init__.py | 315 +++++ soleprint/atlas2/docgen/book/__main__.py | 80 ++ soleprint/atlas2/docgen/book/build.py | 350 ++++++ soleprint/atlas2/docgen/book/checks.py | 305 +++++ soleprint/atlas2/docgen/book/larder.py | 195 ++++ .../atlas2/docgen/docs/img/architecture.svg | 1016 ++++++++++------- soleprint/atlas2/docgen/docs/img/minimap.svg | 536 +++++---- soleprint/atlas2/docgen/docs/index.html | 348 +++++- soleprint/atlas2/docgen/emitters/site.py | 94 +- soleprint/atlas2/docgen/extractors/code.py | 23 +- soleprint/atlas2/docgen/extractors/db.py | 22 +- soleprint/atlas2/docgen/extractors/openapi.py | 109 +- .../docgen/extractors/python/__init__.py | 13 +- .../docgen/extractors/python/resolve.py | 25 +- soleprint/atlas2/docgen/extractors/usage.py | 25 +- soleprint/atlas2/docgen/fixtures/orders.yaml | 154 +++ soleprint/atlas2/docgen/ir/model.py | 20 +- soleprint/atlas2/docgen/ir/schema.json | 27 + soleprint/atlas2/docgen/ir/validate.py | 76 ++ soleprint/atlas2/docgen/reference.py | 107 ++ soleprint/atlas2/docgen/selftest.py | 695 ++++++++++- 22 files changed, 3924 insertions(+), 682 deletions(-) create mode 100644 soleprint/atlas2/docgen/book/__init__.py create mode 100644 soleprint/atlas2/docgen/book/__main__.py create mode 100644 soleprint/atlas2/docgen/book/build.py create mode 100644 soleprint/atlas2/docgen/book/checks.py create mode 100644 soleprint/atlas2/docgen/book/larder.py create mode 100644 soleprint/atlas2/docgen/fixtures/orders.yaml create mode 100644 soleprint/atlas2/docgen/reference.py diff --git a/soleprint/atlas2/docgen/Makefile b/soleprint/atlas2/docgen/Makefile index acb3c21..038b330 100644 --- a/soleprint/atlas2/docgen/Makefile +++ b/soleprint/atlas2/docgen/Makefile @@ -3,13 +3,18 @@ # Derived from where this file sits, so the folder can be copied anywhere and # renamed and still work. The logic lives in the Python, never here. # -# make check prove it, on a tree it builds itself -# make ir SRC=../station extract -> out/ir.json +# make book SRC=../station the whole operation, measured at both ends +# make check prove docgen, on a tree it builds itself +# make check BOOK=out/book/x prove one book — its own level +# make ir SRC=../station extract -> out/ir.json (one step, on its own) # make graph out/ir.json -> out/graph.svg # make index out/ir.json -> out/index.md -# make self run the whole pipeline over soleprint itself +# make self docgen's book of itself, then check it # make doctor what this machine has # +# Every target below is one step of a book and still works alone — that is the +# property the book spine exists to preserve, not to replace. +# # The pipeline is three commands and they compose, which is the point: # # python3 -m docgen.extractors.python --root SRC -o ir.json @@ -25,14 +30,24 @@ RUN := PYTHONPATH=$(PARENT) $(PY) -m OUT ?= $(HERE)/out SRC ?= SCHEMA ?= +OPENAPI ?= +HAR ?= STYLE ?= lucid THEME ?= DEPTH ?= 2 SCALE ?= 0.55 +BOOK ?= +SLUG ?= +# NOT `LANG`: that is the shell's locale variable, so `?=` inherits +# en_US.UTF-8 from the environment and --lang rejects it. +READER ?= python +OVERLAY ?= THEME_ARG := $(if $(THEME),--theme $(THEME)) +SLUG_ARG := $(if $(SLUG),--slug $(SLUG)) +OVER_ARG := $(if $(OVERLAY),--overlay $(OVERLAY)) -.PHONY: help check ir db code graph index site minimap explore docs view self doctor clean +.PHONY: help book check ir db code graph index site minimap explore docs view self doctor clean help: ## List every target @echo "docgen — static analysis of a tree, and the artifacts that fall out of it" @@ -42,10 +57,34 @@ help: ## List every target @echo @echo " SRC=/path/to/tree what to read OUT=/path where output goes" @echo " SCHEMA=schema.json a database instead STYLE=lucid THEME=dark|lucid" + @echo " OPENAPI=spec.yaml an API document HAR=session.har a recording" + @echo " BOOK=/path where a book goes, and which book to check" + @echo " READER=python|code ast, or tree-sitter SLUG=name what to call the book" + @echo " OVERLAY=overlay.json hand-written notebook additions, re-applied every build" @echo " DEPTH=2 how deep to draw" - -check: ## Prove the pipeline, offline, needing nothing installed - @$(PY) $(HERE)/selftest.py + @echo + @echo " Three levels of test, by what they assert about:" + @echo " make doctor the machine. Never fails." + @echo " make check docgen. Exits 1." + @echo " make check BOOK= that book. Exits 1." + +check: ## Prove docgen (or one book, with BOOK=) + @if [ -n "$(BOOK)" ]; then \ + $(RUN) $(PKG).book.checks "$(BOOK)"; \ + else \ + $(PY) $(HERE)/selftest.py; \ + fi + +book: ## SRC (or SCHEMA/OPENAPI/HAR) -> one operation, measured at both ends + @test -n "$(SRC)$(SCHEMA)$(OPENAPI)$(HAR)" \ + || { echo "Error: set SRC=/path/to/tree (or SCHEMA=, OPENAPI=, HAR=)" >&2; exit 1; } + @$(RUN) $(PKG).book \ + $(if $(SRC),--root "$(SRC)" --lang $(READER)) \ + $(if $(SCHEMA),--schema "$(SCHEMA)") \ + $(if $(OPENAPI),--openapi "$(OPENAPI)") \ + $(if $(HAR),--har "$(HAR)") \ + -o "$(if $(BOOK),$(BOOK),$(OUT)/book)" \ + --style $(STYLE) $(THEME_ARG) $(SLUG_ARG) $(OVER_ARG) ir: ## Extract SRC into OUT/ir.json @test -n "$(SRC)" || { echo "Error: set SRC=/path/to/tree" >&2; exit 1; } @@ -104,15 +143,14 @@ index: ## OUT/ir.json -> OUT/index.md and OUT/sidebar.json @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/index.md @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/sidebar.json -self: ## Run the whole pipeline over soleprint itself — the honest end-to-end check - @$(MAKE) --no-print-directory ir SRC=$(PARENT)/.. OUT=$(OUT) - @$(MAKE) --no-print-directory index OUT=$(OUT) - @$(MAKE) --no-print-directory graph OUT=$(OUT) - @$(MAKE) --no-print-directory site OUT=$(OUT) - @$(MAKE) --no-print-directory minimap OUT=$(OUT) - @$(MAKE) --no-print-directory explore OUT=$(OUT) +self: ## docgen's book of the widest tree it can see, then check it + @$(eval SELF_SRC := $(shell PYTHONPATH=$(PARENT) $(PY) -c "from $(PKG) import reference; \ +r = reference.root(); print(r if r else '$(HERE)')")) + @echo " self-hosting on $(SELF_SRC)" + @$(MAKE) --no-print-directory book SRC=$(SELF_SRC) SLUG=self \ + BOOK=$(OUT)/book/self OUT=$(OUT) @echo - @echo " Read $(OUT)/index.md, or open $(OUT)/site/index.html" + @$(MAKE) --no-print-directory check BOOK=$(OUT)/book/self doctor: ## Report whether this machine can run it @printf 'python : '; $(PY) --version 2>&1 || echo MISSING @@ -120,6 +158,9 @@ doctor: ## Report whether this machine can run it @printf 'tree-sit : '; $(PY) -c 'import tree_sitter, tree_sitter_c_sharp, tree_sitter_typescript; print("ok — C# and TypeScript available")' 2>/dev/null || echo 'absent — Python only. pip install tree_sitter tree_sitter_c_sharp tree_sitter_typescript' @printf 'networkx : '; $(PY) -c 'import networkx; print(networkx.__version__ + " — for lab/ experiments")' 2>/dev/null || echo 'absent — only used in lab/' @printf 'package : %s (from %s)\n' '$(PKG)' '$(PARENT)' + @printf 'yaml : '; $(PY) -c 'import yaml; print("ok — needed only to read OpenAPI")' 2>/dev/null || echo 'absent — only used by the OpenAPI reader' + @printf 'reference: '; PYTHONPATH=$(PARENT) $(PY) -c "from $(PKG) import reference; print(reference.describe())" + @printf 'styles : '; $(RUN) $(PKG).style 2>/dev/null \ || $(RUN) $(PKG) 2>/dev/null \ || PYTHONPATH=$(PARENT) $(PY) -c "from $(PKG).style import Style; print(', '.join(Style.available()))" diff --git a/soleprint/atlas2/docgen/book/__init__.py b/soleprint/atlas2/docgen/book/__init__.py new file mode 100644 index 0000000..f5fea7e --- /dev/null +++ b/soleprint/atlas2/docgen/book/__init__.py @@ -0,0 +1,315 @@ +""" +A **book** — one docgen operation, measured at both ends. + + larder ──► step ──► step ──► step ──► book + what each one usable what + came in by itself came out + +The first step says what went into the operation. The last step says what came +out, and renders it for the web. Everything between is an ordinary artifact that +stands on its own — `ir.json` is still an `ir.json`, and `make ir` still works +without knowing this file exists. + +## Why both ends, and why they are steps rather than a wrapper + +Because the two numbers are only worth having **together**. "102 nodes" is not a +fact about anything; "49 files in, 102 nodes out, nothing lost" is. A book that +read 45 of 47 files and drew a clean diagram is lying by omission, and before +this there was no place for the other 2 to be mentioned. + +They are steps, not a wrapper, because a wrapper is something you can forget to +apply. A step is in the sequence, and the sequence is the thing the notebook +emits — so the measure is in the document whether or not anyone remembered. + +## The two things it is not + +**Not a gate.** Running one step alone is still a book, just a short one. An +operation that cannot measure something says what it could not measure and +carries on. Gating would break the property that makes the intermediate +artifacts useful, and that property is the whole reason the sequence is worth +having. + +**Not a new pipeline.** Everything here composes functions that already existed +— `ops.overview`, `emitters.dot`, `emitters.site`, `notebook.spec`. The spine +adds a ledger and two measurements. If it ever starts doing the work itself, +something has gone wrong. + +## The web end is deliberately the loose one + +It is last, so nothing depends on it, so it can be replaced wholesale without +touching a single thing upstream. That is what lets it rule what gets generated +without being a stable contract: the book measure is the promise, and the page +that displays it is free to change drastically and often. +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path + +from .larder import Larder + +# The two spine steps. Named here because the notebook, the site and the checks +# all have to agree on what they are called, and three string literals in three +# files is how they stop agreeing. +FIRST = "larder" +LAST = "book" + + +@dataclass +class Step: + """One thing the operation did, and what it left behind.""" + + id: str + label: str + artifact: str | None = None # relative to the book directory + bytes: int = 0 + note: str = "" + skipped: str = "" # why, when it did not run + + def to_dict(self) -> dict: + out = {"id": self.id, "label": self.label} + if self.artifact: + out["artifact"] = self.artifact + out["bytes"] = self.bytes + if self.note: + out["note"] = self.note + if self.skipped: + out["skipped"] = self.skipped + return out + + +# How a larder's unit shows up on the output side. Per-kind because the relation +# genuinely differs, and because getting it wrong produces a check that passes +# for the wrong reason — which is what the first version of this did. +# +# relation "exact" one unit in, one node out. Fewer means input was dropped. +# "collapse" many units in, fewer nodes out, by design. +# noun what to call the output-side thing, in the claim +# represents whether a unit that FAILED still appears as a node. Where it does, +# that is checkable and is the whole-input form of the rule that an +# unresolved name becomes an `external` node rather than vanishing. +RELATION = { + "python": ("exact", "module", True), + "code": ("exact", "module", True), + "db": ("exact", "table", False), + "openapi": ("exact", "endpoint", False), + # A HAR entry with no URL has nothing to represent, so there is no node to + # look for. Its absence is the correct outcome and is recorded in `failed`. + "usage": ("collapse", "call", False), +} + + +def _unit_counts(ir: dict, larder: Larder) -> tuple[int, int]: + """(nodes from units that were read, nodes from units that failed). + + Split because a failed file still gets a module node — carrying + `attrs.error`, so the gap is visible in the graph rather than only in a log. + Counting them together made "2 files read produced 4 modules" pass a check + that was supposed to prove nothing had been dropped. + """ + nodes = ir.get("nodes") or [] + + if larder.kind in ("python", "code"): + mods = [n for n in nodes + if n["kind"] == "module" and (n.get("attrs") or {}).get("file")] + broken = {n["attrs"]["file"] for n in mods if (n.get("attrs") or {}).get("error")} + whole = {n["attrs"]["file"] for n in mods} - broken + return len(whole), len(broken) + + if larder.kind == "db": + return sum(1 for n in nodes if n["kind"] == "table"), 0 + + if larder.kind == "openapi": + return len({ + (n.get("attrs") or {}).get("path") for n in nodes + if n["kind"] == "endpoint" and (n.get("attrs") or {}).get("path") + }), 0 + + if larder.kind == "usage": + return sum(1 for n in nodes if n["kind"] in ("endpoint", "operation")), 0 + + return 0, 0 + + +class Book: + """A named operation, its ledger, and the two measures that bracket it.""" + + def __init__(self, slug: str, larder: Larder, out): + self.slug = slug + self.larder = larder + self.out = Path(out) + self.steps: list[Step] = [] + self.ir: dict | None = None + self.notebook: str | None = None + + # The first step, recorded before any work happens. Doing it here rather + # than at the end is the difference between a measure and a summary: it + # says what the operation *set out* to read, so a crash halfway leaves a + # book that still says what went in. + self.steps.append(Step( + id=FIRST, + label="what came in", + note=larder.line(), + )) + + # -- the ledger ------------------------------------------------------- + + def step(self, id: str, label: str, path=None, note: str = "", + skipped: str = "") -> Step: + """Record a step. `path` is written already; this measures it.""" + artifact, size = None, 0 + if path is not None: + path = Path(path) + if path.exists(): + artifact = str(path.relative_to(self.out)) if self.out in path.parents \ + or path.parent == self.out else str(path) + size = path.stat().st_size if path.is_file() else _tree_bytes(path) + s = Step(id=id, label=label, artifact=artifact, bytes=size, + note=note, skipped=skipped) + self.steps.append(s) + return s + + # -- the last step ---------------------------------------------------- + + def measure(self) -> dict: + """What came out. Counted off the final IR and the ledger.""" + ir = self.ir or {"nodes": [], "edges": []} + by_kind: dict[str, int] = {} + for n in ir.get("nodes") or []: + by_kind[n["kind"]] = by_kind.get(n["kind"], 0) + 1 + edge_kinds: dict[str, int] = {} + for e in ir.get("edges") or []: + edge_kinds[e["kind"]] = edge_kinds.get(e["kind"], 0) + 1 + + artifacts = [s for s in self.steps if s.artifact] + return { + "nodes": sum(by_kind.values()), + "by_kind": {k: by_kind[k] for k in sorted(by_kind)}, + "edges": sum(edge_kinds.values()), + "edges_by_kind": {k: edge_kinds[k] for k in sorted(edge_kinds)}, + "external": by_kind.get("external", 0), + "steps": len(self.steps), + "artifacts": [{"path": s.artifact, "bytes": s.bytes} for s in artifacts], + "bytes": sum(s.bytes for s in artifacts), + } + + def compare(self) -> list[dict]: + """Reconcile the two ends. This is what having both is *for*. + + Returns observations, each with a verdict, rather than raising: the book + is already built by the time anyone can compare, and refusing to write it + would destroy the evidence. `book/checks.py` turns these into pass/fail + at the book test level, and the CLI exits 1 when one of them is not ok. + """ + out = [] + ir = self.ir or {} + produced, represented = _unit_counts(ir, self.larder) + relation, noun, represents = RELATION.get( + self.larder.kind, ("collapse", "node", False)) + read, unit = self.larder.read, self.larder.unit + n_failed = len(self.larder.failed) + + if relation == "exact": + ok = produced >= read + out.append({ + "id": "units-accounted-for", + "ok": ok, + "claim": f"{read} {unit}(s) read produced {produced} {noun}(s)", + "why": ( + "one unit in, one node out" if ok else + f"{read - produced} {unit}(s) were read and produced nothing — " + "the input was dropped between the extractor and the document, " + "which is the failure this measure exists to catch" + ), + }) + else: + out.append({ + "id": "collapse-is-intended", + "ok": produced >= 1 or read == 0, + "claim": f"{read} {unit}(s) collapsed to {produced} {noun}(s)", + "why": "many recorded requests describe few endpoints — that is the point", + }) + + if n_failed: + out.append({ + "id": "failures-surfaced", + "ok": True, + "claim": f"{n_failed} {unit}(s) could not be read", + "why": "named in the larder measure and on the landing page, not only in a log", + }) + + # The whole-input form of "an unresolved name becomes an `external` node". + # A file that failed to parse must still be in the graph, or the diagram + # shows a tree that is smaller than the tree on disk and says nothing. + if represents and n_failed: + out.append({ + "id": "failures-still-in-the-graph", + "ok": represented >= n_failed, + "claim": f"{n_failed} unreadable {unit}(s) appear as {represented} " + f"marked {noun}(s)", + "why": ( + "a gap that is drawn can be seen" if represented >= n_failed else + f"{n_failed - represented} unreadable {unit}(s) are missing from the " + "graph entirely — the picture is smaller than the source and does " + "not say so" + ), + }) + return out + + # -- writing ---------------------------------------------------------- + + def close(self, *, site=None) -> Step: + """Append the last step. Call once, after the web output is written. + + Must be called BEFORE the notebook is built, because the notebook quotes + the book measure and the measure is not complete until this step exists. + + The site is this step's *artifact*. The notebook is not: it is the whole + sequence's rendering rather than an item in it, and it is set on + `self.notebook` afterwards, by name only — a notebook cannot report its + own byte count without changing it. + """ + m = self.measure() + summary = " · ".join(f"{v} {k}" for k, v in + sorted(m["by_kind"].items(), key=lambda kv: -kv[1])) + artifact, size = None, 0 + if site is not None: + site = Path(site) + if site.exists(): + artifact, size = str(site.relative_to(self.out)), _tree_bytes(site) + self.steps.append(Step( + id=LAST, + label="what came out", + artifact=artifact, + bytes=size, + note=f"{summary} · {m['edges']} edges" if summary else "nothing was produced", + )) + return self.steps[-1] + + def to_dict(self) -> dict: + """The book, as the ledger someone else can read. + + `generated_at` is absent for the same reason the IR's is: an unchanged + larder must serialise to the same bytes, or nothing downstream can tell + a real change from a rebuild. + """ + out = { + "slug": self.slug, + "larder": self.larder.to_dict(), + "steps": [s.to_dict() for s in self.steps], + "book": self.measure(), + "reconciled": self.compare(), + } + if self.notebook: + out["notebook"] = self.notebook + return out + + def write(self) -> Path: + self.out.mkdir(parents=True, exist_ok=True) + path = self.out / "book.json" + path.write_text(json.dumps(self.to_dict(), indent=2, sort_keys=False) + "\n") + return path + + +def _tree_bytes(root: Path) -> int: + return sum(p.stat().st_size for p in root.rglob("*") if p.is_file()) diff --git a/soleprint/atlas2/docgen/book/__main__.py b/soleprint/atlas2/docgen/book/__main__.py new file mode 100644 index 0000000..159d94f --- /dev/null +++ b/soleprint/atlas2/docgen/book/__main__.py @@ -0,0 +1,80 @@ +""" python3 -m docgen.book --root ../station -o out/book/station + +One book: larder measure, the steps, the web output, book measure. The step +artifacts land in `steps/` and are ordinary files — nothing here needs this +command to have been the thing that produced them. +""" + +import argparse +import sys +from pathlib import Path + +KINDS = ("python", "code", "db", "openapi", "usage") + + +def main(argv=None) -> int: + p = argparse.ArgumentParser( + prog="python3 -m docgen.book", + description="Run one docgen operation, measured at both ends.", + ) + src = p.add_mutually_exclusive_group(required=True) + src.add_argument("--root", type=Path, help="A source tree (Python, or --lang code).") + src.add_argument("--schema", type=Path, help="A graphgen-compatible schema.json.") + src.add_argument("--openapi", type=Path, help="An OpenAPI document.") + src.add_argument("--har", type=Path, help="A HAR recording.") + p.add_argument("--lang", choices=("python", "code"), default="python", + help="With --root: the stdlib ast reader, or tree-sitter. Default python.") + p.add_argument("--output", "-o", type=Path, required=True, + help="The book directory. Created if absent.") + p.add_argument("--slug", help="What to call it. Defaults to the source's name.") + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None, help="dark (default) or lucid.") + p.add_argument("--overlay", type=Path, + help="A hand-written overlay, re-applied on every build.") + p.add_argument("--exclude", action="append", default=[], + help="Directory name to skip. Repeatable.") + p.add_argument("--quiet", "-q", action="store_true") + args = p.parse_args(argv) + + if args.root is not None: + kind, source = args.lang, args.root + elif args.schema is not None: + kind, source = "db", args.schema + elif args.openapi is not None: + kind, source = "openapi", args.openapi + else: + kind, source = "usage", args.har + + if not Path(source).exists(): + print(f"Error: {source} does not exist", file=sys.stderr) + return 1 + + overlay = None + if args.overlay: + if not args.overlay.exists(): + # Absent is fine and is the documented default; named-but-missing is + # a typo, and quietly building without it would hide the typo. + print(f"Error: overlay {args.overlay} does not exist", file=sys.stderr) + return 1 + from ..notebook import spec as spec_mod + overlay = spec_mod.load(args.overlay) + + from .build import run + + try: + book = run(kind, source, args.output, slug=args.slug, style=args.style, + theme=args.theme, exclude=tuple(args.exclude), overlay=overlay, + quiet=args.quiet) + except Exception as e: # noqa: BLE001 - the CLI reports, it does not traceback + print(f"Error: {type(e).__name__}: {e}", file=sys.stderr) + return 1 + + # Exit 1 when the two ends do not reconcile. The book is still written — + # the evidence is the point — but a build that lost input should fail a + # pipeline rather than pass quietly. + lost = [r for r in book.compare() if not r.get("ok")] + return 1 if lost else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/book/build.py b/soleprint/atlas2/docgen/book/build.py new file mode 100644 index 0000000..79fc92b --- /dev/null +++ b/soleprint/atlas2/docgen/book/build.py @@ -0,0 +1,350 @@ +""" +Run a book: larder, the steps, the web output, the book measure. + + python3 -m docgen.book --root ../station -o out/book/station + +Composes functions that already exist. Nothing here parses, lays out or styles +anything — `ops.overview`, `emitters.dot`, `emitters.site` and `notebook.spec` +do all of it, and this decides the order and writes the ledger. If this file +ever starts doing the work, the seam has moved to the wrong place. + +## The notebook is the sequence, not an item in it + +So it is not a step. Its **first cell is the larder measure and its last cell is +the book measure**, which is what makes the two ends part of the document rather +than part of the tooling. Between them, one pair of cells per step: what the +step did, and a code cell that loads that step's artifact on its own. + +That last part is the "usable by themselves" property made executable. Each cell +reads one artifact and prints one fact, so a reader can start anywhere in the +sequence, and `selftest.py` runs them. +""" + +import json +from pathlib import Path + +from . import Book +from .larder import Larder + +# Loader for the notebook's step cells. Guarded on purpose: a notebook is +# opened from wherever somebody happens to open it, and a traceback on cell 2 +# is a worse answer than a sentence saying which directory to run it from. +PRELUDE = '''from pathlib import Path +import json + +BOOK = Path.cwd() # the book directory — change if you opened this elsewhere + + +def load(rel): + """One step's artifact, on its own. Returns None when it is not here. + + A step's artifact can be a directory — the explorer is one — so this returns + a file list for those rather than trying to read a directory as text. + """ + p = BOOK / rel + if not p.exists(): + print(f"{rel} is not here — run: make book OUT={BOOK}") + return None + if p.is_dir(): + return sorted(f.relative_to(p).as_posix() for f in p.rglob("*") if f.is_file()) + return json.loads(p.read_text()) if p.suffix == ".json" else p.read_text() +''' + + +# Every extractor a book can run, as data rather than as an if/elif chain. +# +# This is the shape that gives the extension contract teeth. `selftest.py` loops +# over this dict and asserts each entry reports a larder measure — so adding an +# extractor makes the test start asking about it without anyone remembering to +# go and add a case. rig's `CONFIG_OVERRIDABLE` loop is the precedent, and it +# found two real bugs that way. +# +# kind -> (module, function, what the source is) +EXTRACTORS = { + "python": (".extractors.python", "extract", "tree"), + "code": (".extractors.code", "extract", "tree"), + "db": (".extractors.db", "extract", "file"), + "openapi": (".extractors.openapi", "extract", "file"), + "usage": (".extractors.usage", "extract", "file"), +} + +# Extractors that walk a directory take an exclude list; the ones that read a +# single document have nothing to exclude. +TAKES_EXCLUDE = {"python", "code"} + + +def extract(kind: str, source, *, exclude=(), identity=None): + """(IR dict, Larder) for one source type, dispatched through EXTRACTORS.""" + from importlib import import_module + + if kind not in EXTRACTORS: + raise ValueError( + f"no extractor named {kind!r} — have {', '.join(sorted(EXTRACTORS))}" + ) + module_name, fn_name, _ = EXTRACTORS[kind] + # The package name is derived, not written: the Makefile takes PKG from the + # directory name so the folder can be copied anywhere and renamed, and a + # literal "docgen" here would quietly undo that. + root_pkg = __package__.rsplit(".", 1)[0] + fn = getattr(import_module(module_name, package=root_pkg), fn_name) + + kwargs = {"identity": identity or str(source)} + if kind in TAKES_EXCLUDE: + kwargs["exclude"] = exclude + g = fn(source, **kwargs) + + ir = g.to_dict() + measured = ir["meta"].get("larder") + if measured is None: + # Not fatal. An extractor that cannot count its input still produces a + # book; what it does not get to do is pretend it measured one. + return ir, None + return ir, Larder.from_dict(measured) + + +def spec_from(book: Book, ir: dict) -> dict: + """The book's ledger -> a notebook spec, with the two measures at the ends. + + Reuses `notebook.spec`'s step vocabulary rather than inventing one, so + `merge()` keeps working and a hand-written overlay can annotate a spine step + the same way it annotates any other. + """ + from ..notebook import spec as spec_mod + + steps = [ + spec_mod._step( + "larder", "md", title=f"{book.slug} — what came in", + text=( + f"`{book.larder.identity}`\n\n**{book.larder.line()}**\n\n" + "This is the first step of the book and the only measure of the " + "input. Everything below is derived from it, so a number here " + "that looks wrong makes everything below it suspect." + + ( + "\n\nCould not be read:\n\n" + + "\n".join(f"- `{f['name']}` — {f['error']}" + for f in book.larder.failed) + if book.larder.failed else "" + ) + ), + ), + spec_mod._step("prelude", "code", title="Reading a step on its own", + code=PRELUDE), + ] + + for s in book.steps: + if s.id in ("larder", "book") or not s.artifact: + continue + steps.append(spec_mod._step( + f"step-{s.id}", "md", title=s.label, + text=f"`{s.artifact}` — {s.bytes:,} bytes" + (f"\n\n{s.note}" if s.note else ""), + )) + steps.append(spec_mod._step( + f"load-{s.id}", "code", code=_load_cell(s.id, s.artifact), + )) + + # Where the larder is API-shaped, the generated endpoint walkthrough slots in + # as further steps. For a tree of source there are no endpoints and this adds + # nothing, which is the correct amount for it to add. + if any(n["kind"] in ("endpoint", "operation") for n in ir.get("nodes") or []): + generated = spec_mod.from_ir(ir) + steps.extend(s for s in generated["steps"] if s["id"] not in ("intro",)) + + m = book.measure() + # `external` is reported on its own below, so it is dropped here rather than + # appearing twice in one line — which is how it read before. + summary = " · ".join(f"{v} {k}" for k, v in + sorted(m["by_kind"].items(), key=lambda kv: -kv[1]) + if k != "external") + reconciled = "\n".join( + f"- {'✓' if r['ok'] else '✗'} {r['claim']} — {r['why']}" + for r in book.compare() + ) + steps.append(spec_mod._step( + "book", "md", title=f"{book.slug} — what came out", + text=( + f"**{summary} · {m['edges']} edges · {m['external']} external**\n\n" + f"{len(m['artifacts'])} artifact(s), {m['bytes']:,} bytes.\n\n" + f"Reconciled against what came in:\n\n{reconciled}\n\n" + "The web output is `site/index.html`." + ), + )) + + return {"version": spec_mod.SPEC_VERSION, "steps": steps} + + +def _load_cell(step_id: str, artifact: str) -> str: + """A cell that reads one artifact and prints one fact about it. + + The fact has to suit the artifact. An earlier version printed "nodes, edges" + for every JSON file, so `sidebar.json` — which has neither — reported + "0 nodes, 0 edges", which is a true sentence about the wrong thing and worse + than saying nothing. + """ + if not artifact.endswith((".json", ".svg", ".md")): + # A directory, e.g. the explorer. + return ( + f'files = load("{artifact}")\n' + 'if files is not None:\n' + ' print(f"{len(files)} file(s)")\n' + ' print("\\n".join(files[:5]))' + ) + if artifact.endswith(".json"): + return ( + f'data = load("{artifact}")\n' + 'if data:\n' + ' if "nodes" in data:\n' + ' print(f\'{len(data["nodes"])} nodes, {len(data.get("edges", []))} edges\')\n' + ' else:\n' + ' print(", ".join(f"{k}: {len(v) if isinstance(v, (list, dict)) else v}"\n' + ' for k, v in data.items()))' + ) + if artifact.endswith(".svg"): + # No IPython import, guarded or otherwise. A generated notebook is a + # build artifact and has to run wherever it is opened; requiring a + # kernel package in order to *load a file* would make the cell fail on + # the machine that produced it, which is where this was found. + return ( + f'svg = load("{artifact}")\n' + 'if svg:\n' + ' print(f"{len(svg):,} bytes of SVG")\n' + ' # In Jupyter: from IPython.display import SVG; SVG(svg)' + ) + if artifact.endswith(".md"): + return ( + f'text = load("{artifact}")\n' + 'if text:\n' + ' print(text[:400])' + ) + return f'print(load("{artifact}") is not None)' + + +def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid", + theme: str | None = None, exclude=(), overlay=None, quiet: bool = False) -> Book: + """The whole book, in one process. Returns it; `book.json` is written.""" + from ..ir import check + from ..ops import classify, overview + from ..style import Style + + out = Path(out) + slug = slug or Path(str(source)).name or "book" + steps_dir = out / "steps" + steps_dir.mkdir(parents=True, exist_ok=True) + + def say(text): + if not quiet: + print(text) + + ir, larder = extract(kind, source, exclude=exclude) + if larder is None: + larder = Larder(kind=kind, identity=str(source), unit="document", seen=0) + book = Book(slug=slug, larder=larder, out=out) + book.ir = ir + say(f" larder {larder.line()}") + + problems = check(ir) + if problems: + # Recorded as a step rather than raised: a book that cannot be trusted + # should exist and say so, because the alternative is that nobody can + # see what went wrong. + book.step("validate", "the IR did not validate", note="; ".join(problems[:3])) + say(f" WARNING IR has {len(problems)} problem(s)") + + p = steps_dir / "ir.json" + p.write_text(json.dumps(ir, indent=2) + "\n") + book.step("ir", "the graph, extracted", p, note=f"{len(ir['nodes'])} nodes") + + view = overview(ir) + p = steps_dir / "view.json" + p.write_text(json.dumps(view, indent=2) + "\n") + verdict = classify(view) + book.step("view", "the default view for this source", p, + note=f"{verdict['kind']} — {verdict['why']}") + + style_obj = Style.load(style, theme=theme) + + # The drawing, chosen by structure rather than by the caller. `classify` + # already decided; this runs what it named. + graph_rel = None + if verdict["emitter"] == "erd": + from ..emitters.erd import emit as erd_emit + p = steps_dir / "graph.svg" + p.write_text(erd_emit(view, style_obj)) + graph_rel, label = "steps/graph.svg", "drawn as an ERD" + elif verdict["emitter"] == "index": + from ..emitters.index import to_markdown + p = steps_dir / "graph.md" + p.write_text(to_markdown(view)) + graph_rel, label = None, "not a diagram — written as a list" + else: + from ..emitters.dot import emit as dot_emit, render + p = steps_dir / "graph.svg" + try: + p.write_bytes(render(dot_emit(view, style_obj, + rankdir=(verdict.get("options") or {}).get("rankdir")))) + graph_rel, label = "steps/graph.svg", f"drawn as a {verdict['kind']}" + except Exception as e: # noqa: BLE001 - graphviz may not be installed + p = None + label = f"not drawn — {type(e).__name__}" + book.step("graph", label, skipped=str(e)[:200]) + if p is not None: + book.step("graph", label, p, note=verdict["why"]) + + from ..emitters.index import to_markdown, to_sidebar + p = steps_dir / "index.md" + p.write_text(to_markdown(ir)) + book.step("index", "readable without a diagram", p) + p = steps_dir / "sidebar.json" + p.write_text(json.dumps(to_sidebar(ir), indent=2) + "\n") + book.step("sidebar", "navigation, for whatever renders it", p) + + try: + from ..emitters.minimap import emit as mm_emit + p = steps_dir / "minimap.svg" + p.write_text(mm_emit(ir, style_obj)) + book.step("minimap", "what is where, read from the colours", p) + except Exception as e: # noqa: BLE001 + book.step("minimap", "minimap not drawn", skipped=f"{type(e).__name__}: {e}") + + try: + from ..emitters.explore import write as exp_write + exp_write(ir, style_obj, out / "explore") + book.step("explore", "navigate on one side, explore on the other", + out / "explore") + except Exception as e: # noqa: BLE001 + book.step("explore", "explorer not built", skipped=f"{type(e).__name__}: {e}") + + # -- the last step ---------------------------------------------------- + from ..emitters.site import write as site_write + site_dir = out / "site" + site_write(view, style_obj, site_dir, + graph=Path(graph_rel).name if graph_rel else None, + title=slug, book=book.to_dict()) + if graph_rel and (out / graph_rel).exists(): + (site_dir / Path(graph_rel).name).write_bytes((out / graph_rel).read_bytes()) + + # close() BEFORE the notebook spec is built, not after. The spec quotes the + # book measure, and the book measure only includes the site once the last + # step exists — build it the other way round and the notebook says 7 + # artifacts while book.json says 8, which is exactly what it did. + book.close(site=site_dir) + + from ..emitters.notebook import write as nb_write + from ..notebook import spec as spec_mod + spec, spec_problems = spec_mod.merge(spec_from(book, ir), overlay) + nb_write(spec, out / "notebook.ipynb") + # Recorded by name and not by size: a notebook cannot report its own byte + # count without changing it. + book.notebook = "notebook.ipynb" + + for pr in spec_problems: + say(f" overlay {pr}") + path = book.write() + + m = book.measure() + say(f" book {m['nodes']} nodes · {m['edges']} edges · " + f"{m['external']} external · {len(m['artifacts'])} artifacts") + for r in book.compare(): + say(f" {'ok ' if r['ok'] else 'LOST'} {r['claim']}") + say(f" open {site_dir / 'index.html'}") + say(f" {path}") + return book diff --git a/soleprint/atlas2/docgen/book/checks.py b/soleprint/atlas2/docgen/book/checks.py new file mode 100644 index 0000000..28d874e --- /dev/null +++ b/soleprint/atlas2/docgen/book/checks.py @@ -0,0 +1,305 @@ +""" +Does *this* book still hold — the third test level. + + python3 -m docgen.book.checks out/book/station + +docgen has three levels of test, and they differ by what they assert *about*. +The distinction is rig's, from `rig/ctrl/selftest.sh`, and it is worth keeping +because it decides what a failure means: + + make doctor the MACHINE. Never fails; it reports. + make check DOCGEN. Exits 1. "docgen no longer does what it says." + make check BOOK= + THIS BOOK. Exits 1. "this book no longer holds." + +The third is the one that reaches a project docgen has never seen. It is also +where **framework and custom checks live together**, at different levels: + + generated the spine's own assertions, identical for every book. Both + measures present, the reconciliation holding, every artifact + where the ledger says it is, the notebook still executing. + custom hand-written, in the book's own `checks.py`, using the same + check/note/skip helpers — so a project's line and a framework + line read identically and fail identically. + +Same split as the notebook's base and overlay, for the same reason: generation +alone cannot know what *this* project cares about, and hand-authoring alone rots. + +## The discipline these are written in + +Carried from rig verbatim, because it is the point of the whole level: + +> Each check is ONE decision that has already been made, with the reason above +> it — not coverage, and deliberately not an exhaustive sweep of use cases. A +> rule without its reason gets overridden the first time it is inconvenient. +> Failing one should read as **"you are about to undo this"** rather than +> "something broke". + +## Writing a book's own checks + +Put a `checks.py` next to `book.json`: + + def checks(book, check, note, skip): + note("what this project will not give up") + + # Payments moved once already and the move broke three dashboards. + # If it is not here, something renamed it again. + check("payments is still a module", True, + any(n["id"] == "app.payments" for n in book.ir["nodes"])) + +`book` carries `.data` (the parsed `book.json`), `.ir` (the extracted graph), +and `.dir`. `check` takes (name, expected, actual) — expected first, because a +failure report is only useful if it says what was wanted. +""" + +import json +import sys +from pathlib import Path + +from . import FIRST, LAST + + +class Loaded: + """A book read back off disk, for checking rather than building.""" + + def __init__(self, directory): + self.dir = Path(directory) + path = self.dir / "book.json" + if not path.exists(): + raise FileNotFoundError( + f"{path} does not exist — is {self.dir} a book directory?\n" + "Build one with: python3 -m docgen.book --root -o " + ) + self.data = json.loads(path.read_text()) + + ir_path = self.dir / "steps" / "ir.json" + self.ir = json.loads(ir_path.read_text()) if ir_path.exists() else None + + nb_path = self.dir / (self.data.get("notebook") or "notebook.ipynb") + self.notebook = json.loads(nb_path.read_text()) if nb_path.exists() else None + + @property + def larder(self) -> dict: + return self.data.get("larder") or {} + + @property + def measure(self) -> dict: + return self.data.get("book") or {} + + +class Report: + """rig's reporting shape: sections, one line per decision, nothing aborts.""" + + def __init__(self): + self.passed, self.failed, self.skipped = 0, [], 0 + + def note(self, text: str) -> None: + print(f"\n{text}") + + def check(self, name: str, expected, actual) -> bool: + if expected == actual: + print(f" ok {name}") + self.passed += 1 + return True + print(f" FAIL {name}\n expected: {expected!r}\n got: {actual!r}") + self.failed.append(name) + return False + + def skip(self, name: str, why: str) -> None: + print(f" -- {name} ({why})") + self.skipped += 1 + + def total(self) -> int: + print() + # stdout is block-buffered when redirected and stderr is not, so the + # failure summary printed below would otherwise arrive BEFORE the checks + # it summarises — which is how this was found, piping to `tail`. + sys.stdout.flush() + if not self.failed: + print(f"{self.passed} checks passed — this book still holds" + + (f", {self.skipped} skipped" if self.skipped else "")) + return 0 + print(f"FAILED — {len(self.failed)} of {self.passed + len(self.failed)}: " + f"{', '.join(self.failed)}", file=sys.stderr) + print("Read the comment next to the check. A failure here means a decision " + "has drifted, not that a tool is broken.", file=sys.stderr) + return 1 + + +def generated(book: Loaded, r: Report) -> None: + """The spine's own assertions. Identical for every book, by design.""" + + r.note("the book is bracketed — both ends present") + # The one structural promise the whole design makes: an operation is + # measured at both ends. A book missing either end is not a short book, it + # is a book whose numbers cannot be reconciled against anything. + ids = [s["id"] for s in book.data.get("steps") or []] + r.check("the first step is the larder measure", FIRST, ids[0] if ids else None) + r.check("the last step is the book measure", LAST, ids[-1] if ids else None) + r.check("nothing is bracketed twice", [1, 1], + [ids.count(FIRST), ids.count(LAST)]) + + r.note("what came in was measured, not guessed") + larder = book.larder + for key in ("kind", "identity", "unit", "seen", "read", "failed"): + r.check(f"larder records {key}", True, key in larder) + # read is derived, so a stored value that disagrees means somebody wrote it + # by hand. This is the same arithmetic ir/validate.py enforces; asserted + # again here because a book can be assembled without going through the IR. + r.check("read == seen - failed", larder.get("read"), + max(0, larder.get("seen", 0) - len(larder.get("failed") or []))) + # The one field that could carry a secret out of a database URL. + r.check("the identity carries no password", True, + ":***@" in larder.get("identity", "") or "@" not in larder.get("identity", "")) + + r.note("the two ends reconcile") + reconciled = book.data.get("reconciled") or [] + r.check("something was reconciled", True, len(reconciled) > 0) + for item in reconciled: + # Each of these is a claim the book makes about itself. A false one + # means the document below is smaller than the source and does not say + # so, which is the single failure this level exists to catch. + r.check(item["claim"], True, item["ok"]) + + r.note("the ledger describes files that exist") + for step in book.data.get("steps") or []: + artifact = step.get("artifact") + if not artifact: + continue + path = book.dir / artifact + # A ledger naming a file that is not there is worse than no ledger: it + # is a manifest somebody will build tooling against. + r.check(f"{artifact} is where the ledger says", True, path.exists()) + if path.is_file(): + r.check(f"{artifact} is {step['bytes']} bytes", step["bytes"], + path.stat().st_size) + + r.note("the notebook carries the sequence, with the measures at its ends") + if book.notebook is None: + r.skip("notebook", "no notebook.ipynb in this book") + else: + cells = book.notebook.get("cells") or [] + first = "".join(cells[0]["source"]) if cells else "" + last = "".join(cells[-1]["source"]) if cells else "" + r.check("its first cell is what came in", True, "what came in" in first) + r.check("its last cell is what came out", True, "what came out" in last) + # Compiling is not enough — see selftest.py. `json.dumps` writes + # `false`/`true`/`null`, which are valid Python *identifiers*, so a + # generated body full of them compiles and then raises NameError. + ran, failure = _run_cells(cells, book.dir) + r.check(f"its {ran} code cells run, not merely compile", None, failure) + + r.note("the web output exists and shows both ends") + index = book.dir / "site" / "index.html" + if not index.exists(): + r.skip("site", "no site/index.html in this book") + else: + html = index.read_text() + # The site is the last step precisely because it is the artifact somebody + # definitely opens. A page that shows the result without showing what + # went in is the thing this whole change corrects. + r.check("the page says what came in", True, "what came in" in html) + r.check("the page says what came out", True, "what came out" in html) + if larder.get("failed"): + r.check("the page admits the book is incomplete", True, + "This book is incomplete" in html) + + r.note("the book is reproducible") + # A timestamp would make two builds of an unchanged larder differ, which + # destroys the only useful property a ledger has: that a diff means a real + # change. Same rule as the IR's `generated_at`. + r.check("no timestamp in the ledger", True, "generated_at" not in json.dumps(book.data)) + + +def custom(book: Loaded, r: Report) -> None: + """This book's own assertions, if it has any.""" + path = book.dir / "checks.py" + if not path.exists(): + r.note("this book's own checks") + r.skip("custom checks", f"no {path.name} — write one to assert what this project cares about") + return + + namespace: dict = {"__file__": str(path), "__name__": "book_checks"} + try: + exec(compile(path.read_text(), str(path), "exec"), namespace) + except Exception as e: # noqa: BLE001 - report, do not traceback + r.note("this book's own checks") + r.check(f"{path.name} loads", None, f"{type(e).__name__}: {e}") + return + + fn = namespace.get("checks") + if not callable(fn): + r.note("this book's own checks") + r.check(f"{path.name} defines checks(book, check, note, skip)", True, False) + return + + try: + fn(book, r.check, r.note, r.skip) + except Exception as e: # noqa: BLE001 + r.check(f"{path.name} ran to completion", None, f"{type(e).__name__}: {e}") + + +def _run_cells(cells: list, cwd: Path) -> tuple[int, str | None]: + """Execute the notebook's code cells from the book directory.""" + import contextlib + import io + import os + + ns, ran, failure = {}, 0, None + previous = os.getcwd() + try: + os.chdir(cwd) + with contextlib.redirect_stdout(io.StringIO()): + for c in cells: + if c.get("cell_type") != "code": + continue + src = "".join(c["source"]) + if "urlopen" in src or ("call(" in src and "def call" not in src): + # Anything that would reach the network is compiled, not run. + try: + compile(src, c["id"], "exec") + ran += 1 + except SyntaxError as e: + failure = f"{c['id']}: {e}" + break + continue + try: + exec(compile(src, c["id"], "exec"), ns) + ran += 1 + except Exception as e: # noqa: BLE001 - any failure is the finding + failure = f"{c['id']}: {type(e).__name__}: {e}" + break + finally: + os.chdir(previous) + return ran, failure + + +def main(argv=None) -> int: + import argparse + + p = argparse.ArgumentParser( + prog="python3 -m docgen.book.checks", + description="Check one book: the spine's assertions, then its own.", + ) + p.add_argument("book", type=Path, help="A book directory (holding book.json).") + p.add_argument("--only", choices=("generated", "custom"), + help="Run one level rather than both.") + args = p.parse_args(argv) + + try: + book = Loaded(args.book) + except (FileNotFoundError, json.JSONDecodeError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + print(f"{book.dir} — {book.data.get('slug', '?')}") + r = Report() + if args.only != "custom": + generated(book, r) + if args.only != "generated": + custom(book, r) + return r.total() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/book/larder.py b/soleprint/atlas2/docgen/book/larder.py new file mode 100644 index 0000000..e8ceff0 --- /dev/null +++ b/soleprint/atlas2/docgen/book/larder.py @@ -0,0 +1,195 @@ +""" +What went in — the first step of every book. + +A **larder** is the stocked source together with what connects it: a repo, a +live database, an OpenAPI endpoint, a HAR capture. Deliberately not "bucket". +A bucket is somewhere bytes sit; a larder is stocked from outside, has an +inventory, and goes stale. All three of those are properties worth measuring, +and they are the reason this file exists. + +## Why a measure at all + +Every extractor knew what it read and threw the number away. `Meta` carried +`source` and `root` — which extractor ran, and what it was pointed at — and +nothing about what was actually consumed. So a run that read 45 of 47 files +produced exactly the same document as one that read all 47, and the diagram +looked complete either way. + +That is the failure this prevents, and it is the same failure the IR already +guards against one level down: an unresolved name becomes an `external` node +rather than being dropped, because *silently losing a thing is worse than +recording an unresolved one*. A larder measure is that rule applied to the +input as a whole. + +## seen, failed, read + + seen how many units the larder offered + failed the ones that could not be consumed, by name, with the reason + read seen - len(failed), derived and never stored + +Three fields where two numbers would do, on purpose: `read` as a stored value +invites the question "does it include the failures", and every reader answers it +differently. Derived, there is nothing to get wrong. + +## Redaction + +`identity` is the one field in this whole tool that can carry a credential — +a database DSN has the password in it. So it is scrubbed here at construction, +and `ir/validate.py` sweeps for the scrub having worked. That is the same +discipline as `VISUAL_KEYS`: the architectural rule is a check, not a convention. + +Paths are left as the caller gave them rather than resolved. An absolute path is +not a secret but it is machine-specific, and `meta.root` is already a bare name +for exactly that reason. +""" + +import re +from dataclasses import dataclass, field + +# What `seen` counts, per source type. Closed, like the IR's `kind` vocabulary: +# a larder that counts something with no name here is a larder nobody can +# compare against another one. +UNITS = ("file", "table", "path", "entry", "document") + +# Query-string and connection-string keys whose value is a secret. Matched +# case-insensitively, because ODBC writes `Password=` and URLs write `password=`. +SECRET_KEYS = ( + "password", "passwd", "pwd", "secret", "token", "access_token", "refresh_token", + "api_key", "apikey", "key", "sig", "signature", "auth", "credentials", +) + +MASK = "***" + + +def _count(label: str, n: int) -> str: + """Agree a label with its count, both directions. + + Two directions because both arise: the closed `UNITS` are singular and need + pluralising, while `extra` keys are written plural by the extractor that + knows them ("packages", "hosts") and need singularising at one. Naive `+ "s"` + gives "entrys"; naive nothing gives "1 hosts". + """ + if n == 1: + return label[:-3] + "y" if label.endswith("ies") else label.rstrip("s") or label + if label.endswith("y"): + return label[:-1] + "ies" + return label if label.endswith("s") else label + "s" + + +def redact(identity: str) -> str: + """Strip credentials out of a path or connection string. + + Three shapes, which is all of them in practice: + + postgresql://user:hunter2@host:5432/db -> postgresql://user:***@host:5432/db + https://api/x?token=abc123 -> https://api/x?token=*** + Driver=x;Server=y;Password=hunter2; -> Driver=x;Server=y;Password=***; + + The user, host, port and database survive. Those are what someone reading + the measure needs in order to recognise which larder this was, and none of + them is a secret. + """ + if not identity: + return identity + + # scheme://user:secret@host — the password is between the first colon after + # the scheme and the last @ of the authority. + identity = re.sub( + r"(?P[a-zA-Z][a-zA-Z0-9+.\-]*://)(?P[^:/@\s]+):(?P[^@/\s]*)@", + lambda m: f"{m.group('scheme')}{m.group('user')}:{MASK}@", + identity, + ) + + # key=value, in a query string or a semicolon-delimited connection string. + keys = "|".join(re.escape(k) for k in SECRET_KEYS) + identity = re.sub( + rf"(?i)\b(?P{keys})(?P\s*=\s*)(?P[^&;\s]*)", + lambda m: f"{m.group('key')}{m.group('sep')}{MASK}", + identity, + ) + return identity + + +@dataclass +class Larder: + """What one source offered, and how much of it was consumed. + + Lands in `meta.larder`. Never in `nodes` or `attrs` — the IR is structure, + and provenance is meta. That split is the same one that keeps colour out. + """ + + kind: str # which extractor stocked it + identity: str # path or redacted DSN + unit: str # one of UNITS + seen: int = 0 + failed: list[dict] = field(default_factory=list) + extra: dict = field(default_factory=dict) # per-source facts + + def __post_init__(self): + self.identity = redact(self.identity) + if self.unit not in UNITS: + raise ValueError( + f"larder unit {self.unit!r} is not one of {UNITS} — " + "a unit nobody else uses cannot be compared against another larder" + ) + + @property + def read(self) -> int: + """`seen` minus what failed. Derived, so it cannot disagree with itself.""" + return max(0, self.seen - len(self.failed)) + + def fail(self, name: str, error: str) -> None: + """Record a unit that could not be consumed, by name and reason.""" + self.failed.append({"name": name, "error": error}) + + def to_dict(self) -> dict: + """Key order fixed so an unchanged larder serialises byte-identically.""" + out = { + "kind": self.kind, + "identity": self.identity, + "unit": self.unit, + "seen": self.seen, + "read": self.read, + "failed": [{"name": f["name"], "error": f["error"]} for f in self.failed], + } + if self.extra: + out["extra"] = {k: self.extra[k] for k in sorted(self.extra)} + return out + + @classmethod + def from_dict(cls, data: dict) -> "Larder": + """Round-trip. `read` is dropped: it is derived, not carried.""" + return cls( + kind=data["kind"], + identity=data["identity"], + unit=data["unit"], + seen=data.get("seen", 0), + failed=list(data.get("failed") or []), + extra=dict(data.get("extra") or {}), + ) + + def line(self) -> str: + """The one-line human form — the notebook's first cell, and the CLI. + + Reads as a sentence because it is the first thing anyone sees about a + book, and "47 files read, 2 failed" is the fact that decides whether the + rest of the document is worth trusting. Which is also why the grammar + gets attention it would not otherwise deserve: "2 entrys read, 1 hosts" + reads as a machine talking, and a measure nobody reads is not a measure. + """ + parts = [f"{self.read} {_count(self.unit, self.read)} read"] + if self.failed: + parts.append(f"{len(self.failed)} failed") + for key in sorted(self.extra): + value = self.extra[key] + label = key.replace("_", " ") + if isinstance(value, int): + parts.append(f"{value} {_count(label, value)}") + else: + parts.append(f"{value} {label}") + return f"{self.identity} — " + ", ".join(parts) + + +def of(kind: str, identity: str, unit: str, **extra) -> Larder: + """Shorthand for the common case: `of("db", dsn, "table", dialect=...)`.""" + return Larder(kind=kind, identity=identity, unit=unit, extra=extra) diff --git a/soleprint/atlas2/docgen/docs/img/architecture.svg b/soleprint/atlas2/docgen/docs/img/architecture.svg index e4139be..efe5476 100644 --- a/soleprint/atlas2/docgen/docs/img/architecture.svg +++ b/soleprint/atlas2/docgen/docs/img/architecture.svg @@ -4,57 +4,304 @@ - + ir - + cluster_docgen - + docgen + +cluster_docgen_book + +book + cluster_docgen_emitters - -emitters + +emitters cluster_docgen_extractors - -extractors + +extractors cluster_docgen_extractors_python - -python + +python cluster_docgen_ir - -ir + +ir cluster_docgen_lab - -lab + +lab cluster_docgen_notebook - -notebook + +notebook cluster_docgen_ops - -ops + +ops + + +cluster_docgen_style + +style + + + +docgen.book.__main__ + + +__main__ + + + + + +docgen.book.build + + +build + + + + + +docgen.book.__main__->docgen.book.build + + + + + +docgen.book.larder + + +larder + + + + + +docgen.book.__main__->docgen.book.larder + + + + + +docgen.notebook.spec + + +spec + + + + + +docgen.book.__main__->docgen.notebook.spec + + + + + +docgen.book.build->docgen.book.__main__ + + + + + +docgen.book.build->docgen.book.larder + + + + + +docgen.emitters.dot + + +dot + + + + + +docgen.book.build->docgen.emitters.dot + + + + + +docgen.emitters.erd + + +erd + + + + + +docgen.book.build->docgen.emitters.erd + + + + + +docgen.emitters.explore + + +explore + + + + + +docgen.book.build->docgen.emitters.explore + + + + + +docgen.emitters.index + + +index + + + + + +docgen.book.build->docgen.emitters.index + + + + + +docgen.emitters.minimap + + +minimap + + + + + +docgen.book.build->docgen.emitters.minimap + + + + + +docgen.emitters.notebook + + +notebook + + + + + +docgen.book.build->docgen.emitters.notebook + + + + + +docgen.emitters.site + + +site + + + + + +docgen.book.build->docgen.emitters.site + + + + + +docgen.ir.__main__ + + +__main__ + + + + + +docgen.book.build->docgen.ir.__main__ + + + + + +docgen.book.build->docgen.notebook.spec + + + + + +docgen.ops.__main__ + + +__main__ + + + + + +docgen.book.build->docgen.ops.__main__ + + + + + +docgen.style.extract + + +extract + + + + + +docgen.book.build->docgen.style.extract + + + + + +docgen.book.checks + + +checks + + + + + +docgen.book.checks->docgen.book.__main__ + + docgen.emitters.__main__ - -__main__ + +__main__ @@ -62,437 +309,338 @@ docgen.emitters.auto - -auto + +auto - + docgen.emitters.__main__->docgen.emitters.auto - - + + docgen.emitters.cli_dot - -cli_dot + +cli_dot - + docgen.emitters.__main__->docgen.emitters.cli_dot - - + + docgen.emitters.cli_erd - -cli_erd + +cli_erd - + docgen.emitters.__main__->docgen.emitters.cli_erd - - + + docgen.emitters.cli_explore - -cli_explore + +cli_explore - + docgen.emitters.__main__->docgen.emitters.cli_explore - - + + docgen.emitters.cli_index - -cli_index + +cli_index - + docgen.emitters.__main__->docgen.emitters.cli_index - - + + docgen.emitters.cli_minimap - -cli_minimap + +cli_minimap - + docgen.emitters.__main__->docgen.emitters.cli_minimap - - + + docgen.emitters.cli_notebook - -cli_notebook + +cli_notebook - + docgen.emitters.__main__->docgen.emitters.cli_notebook - - + + docgen.emitters.cli_site - -cli_site + +cli_site - + docgen.emitters.__main__->docgen.emitters.cli_site - - - - - -docgen.emitters.dot - - -dot - - + + - + docgen.emitters.auto->docgen.emitters.dot - - - - - -docgen.emitters.erd - - -erd - - + + - + docgen.emitters.auto->docgen.emitters.erd - - - - - -docgen.emitters.index - - -index - - + + - + docgen.emitters.auto->docgen.emitters.index - - - - - -docgen.ir.__main__ - - -__main__ - - + + - + docgen.emitters.auto->docgen.ir.__main__ - - - - - -docgen.ops.__main__ - - -__main__ - - + + - + docgen.emitters.auto->docgen.ops.__main__ - - + + - - -docgen.style - - -style - - - - - -docgen.emitters.auto->docgen.style - - + + +docgen.emitters.auto->docgen.style.extract + + - + docgen.emitters.cli_dot->docgen.emitters.dot - - + + - + docgen.emitters.cli_dot->docgen.ir.__main__ - - + + - + docgen.emitters.cli_dot->docgen.ops.__main__ - - + + - - -docgen.emitters.cli_dot->docgen.style - - + + +docgen.emitters.cli_dot->docgen.style.extract + + - + docgen.emitters.cli_erd->docgen.emitters.erd - - + + - + docgen.emitters.cli_erd->docgen.ir.__main__ - - - - - -docgen.emitters.cli_erd->docgen.style - - - - - -docgen.emitters.explore - - -explore - + + + + +docgen.emitters.cli_erd->docgen.style.extract + + - + docgen.emitters.cli_explore->docgen.emitters.explore - - + + - + docgen.emitters.cli_explore->docgen.ir.__main__ - - + + - - -docgen.emitters.cli_explore->docgen.style - - + + +docgen.emitters.cli_explore->docgen.style.extract + + - + docgen.emitters.cli_index->docgen.emitters.index - - + + - + docgen.emitters.cli_index->docgen.ir.__main__ - - - - - -docgen.emitters.minimap - - -minimap - - + + - + docgen.emitters.cli_minimap->docgen.emitters.minimap - - + + - + docgen.emitters.cli_minimap->docgen.ir.__main__ - - - - - -docgen.emitters.cli_minimap->docgen.style - - - - - -docgen.emitters.notebook - - -notebook - + + + + +docgen.emitters.cli_minimap->docgen.style.extract + + - + docgen.emitters.cli_notebook->docgen.emitters.notebook - - + + - + docgen.emitters.cli_notebook->docgen.ir.__main__ - - - - - -docgen.notebook.spec - - -spec - - + + - + docgen.emitters.cli_notebook->docgen.notebook.spec - - + + - + docgen.emitters.cli_site->docgen.emitters.dot - - + + - + docgen.emitters.cli_site->docgen.emitters.erd - - - - - -docgen.emitters.site - - -site - - + + - + docgen.emitters.cli_site->docgen.emitters.site - - + + - + docgen.emitters.cli_site->docgen.ir.__main__ - - + + - + docgen.emitters.cli_site->docgen.ops.__main__ - - + + - - -docgen.emitters.cli_site->docgen.style - - + + +docgen.emitters.cli_site->docgen.style.extract + + - + docgen.emitters.explore->docgen.emitters.dot - - + + - + docgen.emitters.explore->docgen.emitters.erd - - + + - + docgen.emitters.explore->docgen.emitters.minimap - - + + - + docgen.emitters.explore->docgen.ops.__main__ - - + + - + docgen.emitters.site->docgen.emitters.index - - + + docgen.extractors.__main__ - -__main__ + +__main__ @@ -500,155 +648,191 @@ docgen.extractors.code_main - -code_main + +code_main - + docgen.extractors.__main__->docgen.extractors.code_main - - + + docgen.extractors.db_main - -db_main + +db_main - + docgen.extractors.__main__->docgen.extractors.db_main - - + + docgen.extractors.openapi_main - -openapi_main + +openapi_main - + docgen.extractors.__main__->docgen.extractors.openapi_main - - + + docgen.extractors.usage_main - -usage_main + +usage_main - + docgen.extractors.__main__->docgen.extractors.usage_main - - + + docgen.extractors.code - -code + +code + + +docgen.extractors.code->docgen.book.larder + + + - + docgen.extractors.code->docgen.ir.__main__ - - + + tree_sitter - -tree_sitter + +tree_sitter - + docgen.extractors.code->tree_sitter - - + + - + docgen.extractors.code_main->docgen.extractors.code - - + + docgen.extractors.db - -db + +db + + +docgen.extractors.db->docgen.book.larder + + + - + docgen.extractors.db->docgen.ir.__main__ - - + + - + docgen.extractors.db_main->docgen.extractors.db - - + + docgen.extractors.openapi - -openapi + +openapi + + +docgen.extractors.openapi->docgen.book.__main__ + + + + + +docgen.extractors.openapi->docgen.book.larder + + + - + docgen.extractors.openapi->docgen.ir.__main__ - - + + modelgen.loader.extract.openapi - -openapi + +openapi - + docgen.extractors.openapi->modelgen.loader.extract.openapi - - + + + + + +yaml + +yaml + + + +docgen.extractors.openapi->yaml + + - + docgen.extractors.openapi_main->docgen.extractors.openapi - - + + docgen.extractors.python.__main__ - -__main__ + +__main__ @@ -656,145 +840,205 @@ docgen.extractors.python.collect - -collect + +collect - + docgen.extractors.python.__main__->docgen.extractors.python.collect - - + + docgen.extractors.python.resolve - -resolve + +resolve - + docgen.extractors.python.__main__->docgen.extractors.python.resolve - - + + + + + +docgen.extractors.python.resolve->docgen.book.larder + + - + docgen.extractors.python.resolve->docgen.extractors.python.collect - - + + - + docgen.extractors.python.resolve->docgen.ir.__main__ - - + + docgen.extractors.usage - -usage + +usage + + +docgen.extractors.usage->docgen.book.larder + + + - + docgen.extractors.usage->docgen.ir.__main__ - - + + - + docgen.extractors.usage_main->docgen.extractors.usage - - + + docgen.ir.model - -model + +model - + docgen.ir.__main__->docgen.ir.model - - + + docgen.ir.validate - -validate + +validate - + docgen.ir.__main__->docgen.ir.validate - - + + - + docgen.ir.__main__->docgen.ir.validate - - + + - + docgen.ir.validate->docgen.ir.model - - + + docgen.lab.pg_probe - -pg_probe + +pg_probe - + docgen.ops.__main__->docgen.ir.__main__ - - + + docgen.ops.filter - -filter + +filter - + docgen.ops.__main__->docgen.ops.filter - - + + + + + +docgen.reference + + +reference + + docgen.selftest - -selftest + +selftest + + + + + +lxml + +lxml + + + +docgen.selftest->lxml + + + + + +docgen.style.tokens + + +tokens + + +docgen.style.extract->docgen.style.tokens + + + + + +docgen.style.extract->lxml + + + + + +docgen.style.tokens->docgen.style.extract + + + diff --git a/soleprint/atlas2/docgen/docs/img/minimap.svg b/soleprint/atlas2/docgen/docs/img/minimap.svg index 78880f0..d68a4a0 100644 --- a/soleprint/atlas2/docgen/docs/img/minimap.svg +++ b/soleprint/atlas2/docgen/docs/img/minimap.svg @@ -1,13 +1,14 @@ - - -docgen.emitters.site — 408 lines -_slots — function, 13 lines -_fill — function, 4 lines -_sidebar — function, 17 lines -_sections — function, 15 lines -emit — function, 86 lines -write — function, 9 lines + + +docgen.emitters.site — 494 lines +_ledger — function, 54 lines +_slots — function, 13 lines +_fill — function, 4 lines +_sidebar — function, 17 lines +_sections — function, 15 lines +emit — function, 96 lines +write — function, 10 lines docgen.emitters.explore — 316 lines _is_schema — function, 4 lines _neighbourhood_svgs — function, 48 lines @@ -70,178 +71,257 @@ main — function, 56 lines docgen.emitters.cli_notebook — 74 lines main — function, 57 lines -docgen.emitters.cli_site — 74 lines -main — function, 57 lines -docgen.emitters.cli_erd — 51 lines -main — function, 38 lines -docgen.emitters.cli_minimap — 51 lines -main — function, 37 lines -docgen.emitters.cli_explore — 48 lines -main — function, 35 lines -docgen.emitters.cli_index — 44 lines -main — function, 32 lines -docgen.emitters.__main__ — 37 lines -main — function, 27 lines -docgen.extractors.code — 261 lines -MissingParser — class, 2 lines -_parser — function, 21 lines -_name — function, 10 lines -_walk — function, 34 lines -extract_file — function, 27 lines -_count_errors — function, 5 lines -extract — function, 54 lines -docgen.extractors.usage — 248 lines -_template — function, 27 lines -_body — function, 10 lines -_shape — function, 15 lines -_graphql — function, 11 lines -extract — function, 112 lines -docgen.extractors.db — 145 lines -from_schema_dict — function, 67 lines -_relation — function, 10 lines -_plain_type — function, 6 lines -_dedupe — function, 9 lines -extract — function, 5 lines -docgen.extractors.openapi — 124 lines -_modelgen — function, 21 lines -_type_name — function, 6 lines -extract — function, 60 lines -docgen.extractors.code_main — 40 lines -main — function, 31 lines -docgen.extractors.usage_main — 33 lines -main — function, 24 lines -docgen.extractors.db_main — 32 lines -main — function, 21 lines -docgen.extractors.openapi_main — 30 lines -main — function, 21 lines -docgen.extractors.python — 30 lines -extract — function, 7 lines -docgen.extractors.__main__ — 26 lines -main — function, 16 lines -docgen.selftest — 1333 lines -check — function, 5 lines -_err — function, 7 lines -skip — function, 3 lines -build_tree — function, 5 lines -_entry — function, 7 lines -docgen.style — 180 lines -StyleError — class, 2 lines -Style — class, 133 lines -__init__ — function, 17 lines -load — function, 12 lines -available — function, 2 lines -themes — function, 2 lines -validate — function, 32 lines -_resolve — function, 12 lines -_lookup — function, 3 lines -node — function, 2 lines -group — function, 2 lines -edge — function, 2 lines -graph — function, 2 lines -geom — function, 2 lines -slot — function, 2 lines -domain_slot — function, 13 lines -limits — function, 3 lines -docgen.ops — 28 lines -docgen.notebook — 15 lines -docgen.emitters — 12 lines -docgen.lab — 11 lines -docgen.ir — 7 lines -docgen.extractors — 2 lines -docgen.extractors.python.collect — 236 lines -Definition — class, 10 lines -Module — class, 12 lines -_Collector — class, 67 lines -__init__ — function, 3 lines -_define — function, 17 lines -visit_ClassDef — function, 5 lines -visit_FunctionDef — function, 5 lines -visit_Import — function, 8 lines -visit_ImportFrom — function, 14 lines -_first_line — function, 5 lines -_name_of — function, 15 lines -_resolve_relative — function, 16 lines -module_name — function, 26 lines -collect_file — function, 21 lines -collect — function, 13 lines -docgen.extractors.python.resolve — 163 lines -_id_for — function, 2 lines -_resolve — function, 32 lines -to_ir — function, 96 lines -_point_at — function, 9 lines -docgen.extractors.python.__main__ — 38 lines -main — function, 23 lines -docgen.ir.validate — 237 lines -IRError — class, 2 lines -_schema — function, 2 lines -_props — function, 5 lines -_fields — function, 4 lines -check — function, 106 lines -validate — function, 6 lines -check_model_matches_schema — function, 23 lines -main — function, 32 lines -docgen.ir.model — 147 lines -Meta — class, 21 lines -to_dict — function, 7 lines -Node — class, 21 lines -__post_init__ — function, 3 lines -to_dict — function, 8 lines -Edge — class, 15 lines -to_dict — function, 7 lines -Graph — class, 48 lines -node — function, 4 lines -edge — function, 4 lines -has — function, 2 lines -to_dict — function, 16 lines -from_dict — function, 6 lines -docgen.ir.__main__ — 9 lines -docgen.ops.filter — 485 lines -_rebuild — function, 57 lines -surviving_parent — function, 5 lines -lift — function, 6 lines -drop_kinds — function, 14 lines -only_kinds — function, 14 lines -drop_stdlib — function, 13 lines -drop_external — function, 3 lines -subtree — function, 13 lines -neighbourhood — function, 45 lines -collapse_to_depth — function, 18 lines -level — function, 6 lines -drop_builtins — function, 14 lines -overview — function, 35 lines -shape — function, 72 lines -rank_of — function, 9 lines -split — function, 24 lines -classify — function, 102 lines -docgen.ops.__main__ — 102 lines -main — function, 84 lines -docgen — 2 lines -docgen.lab.pg_probe — 150 lines -probe — function, 30 lines -_simplify — function, 3 lines -main — function, 25 lines -docgen.notebook.spec — 264 lines -_step — function, 4 lines -from_ir — function, 108 lines -_order — function, 5 lines -scaffold — function, 20 lines -merge — function, 58 lines -load — function, 2 lines -dump — function, 5 lines +docgen.emitters.cli_site — 74 lines +main — function, 57 lines +docgen.emitters.cli_erd — 51 lines +main — function, 38 lines +docgen.emitters.cli_minimap — 51 lines +main — function, 37 lines +docgen.emitters.cli_explore — 48 lines +main — function, 35 lines +docgen.emitters.cli_index — 44 lines +main — function, 32 lines +docgen.emitters.__main__ — 37 lines +main — function, 27 lines +docgen.selftest — 2014 lines +check — function, 5 lines +_err — function, 7 lines +skip — function, 3 lines +build_tree — function, 5 lines +_harvesting — function, 194 lines +_entry — function, 7 lines +_books — function, 279 lines +_standalone — function, 153 lines +docgen.book — 316 lines +Step — class, 20 lines +to_dict — function, 10 lines +_unit_counts — function, 30 lines +Book — class, 178 lines +__init__ — function, 17 lines +step — function, 14 lines +measure — function, 21 lines +compare — function, 62 lines +close — function, 27 lines +to_dict — function, 17 lines +write — function, 5 lines +_tree_bytes — function, 2 lines +docgen.style — 194 lines +StyleError — class, 2 lines +Style — class, 133 lines +__init__ — function, 17 lines +load — function, 12 lines +available — function, 2 lines +themes — function, 2 lines +validate — function, 32 lines +_resolve — function, 12 lines +_lookup — function, 3 lines +node — function, 2 lines +group — function, 2 lines +edge — function, 2 lines +graph — function, 2 lines +geom — function, 2 lines +slot — function, 2 lines +domain_slot — function, 13 lines +limits — function, 3 lines +harvest — function, 9 lines +docgen.reference — 108 lines +_candidates — function, 7 lines +root — function, 9 lines +station_tools — function, 4 lines +describe — function, 10 lines +on_path — function, 13 lines +missing — function, 8 lines +docgen.ops — 28 lines +docgen.notebook — 15 lines +docgen.emitters — 12 lines +docgen.lab — 11 lines +docgen.ir — 7 lines +docgen.extractors — 2 lines +docgen.extractors.code — 282 lines +MissingParser — class, 2 lines +_parser — function, 21 lines +_name — function, 10 lines +_walk — function, 34 lines +extract_file — function, 27 lines +_count_errors — function, 5 lines +extract — function, 75 lines +docgen.extractors.usage — 269 lines +_template — function, 27 lines +_body — function, 10 lines +_shape — function, 15 lines +_graphql — function, 11 lines +extract — function, 133 lines +docgen.extractors.openapi — 191 lines +_modelgen — function, 28 lines +_type_name — function, 6 lines +_refs — function, 43 lines +extract — function, 76 lines +docgen.extractors.db — 161 lines +from_schema_dict — function, 82 lines +_relation — function, 10 lines +_plain_type — function, 6 lines +_dedupe — function, 9 lines +extract — function, 6 lines +docgen.extractors.code_main — 40 lines +main — function, 31 lines +docgen.extractors.python — 37 lines +extract — function, 14 lines +docgen.extractors.usage_main — 33 lines +main — function, 24 lines +docgen.extractors.db_main — 32 lines +main — function, 21 lines +docgen.extractors.openapi_main — 30 lines +main — function, 21 lines +docgen.extractors.__main__ — 26 lines +main — function, 16 lines +docgen.book.build — 351 lines +extract — function, 27 lines +spec_from — function, 68 lines +_load_cell — function, 44 lines +run — function, 130 lines +say — function, 3 lines +docgen.book.checks — 306 lines +Loaded — class, 26 lines +__init__ — function, 15 lines +larder — function, 2 lines +measure — function, 2 lines +Report — class, 37 lines +__init__ — function, 2 lines +note — function, 2 lines +check — function, 8 lines +skip — function, 3 lines +total — function, 15 lines +generated — function, 83 lines +custom — function, 26 lines +_run_cells — function, 33 lines +main — function, 25 lines +docgen.book.larder — 196 lines +_count — function, 13 lines +redact — function, 32 lines +Larder — class, 77 lines +__post_init__ — function, 7 lines +read — function, 3 lines +fail — function, 3 lines +to_dict — function, 13 lines +from_dict — function, 10 lines +line — function, 20 lines +of — function, 3 lines +docgen.book.__main__ — 81 lines +main — function, 62 lines +docgen.extractors.python.collect — 236 lines +Definition — class, 10 lines +Module — class, 12 lines +_Collector — class, 67 lines +__init__ — function, 3 lines +_define — function, 17 lines +visit_ClassDef — function, 5 lines +visit_FunctionDef — function, 5 lines +visit_Import — function, 8 lines +visit_ImportFrom — function, 14 lines +_first_line — function, 5 lines +_name_of — function, 15 lines +_resolve_relative — function, 16 lines +module_name — function, 26 lines +collect_file — function, 21 lines +collect — function, 13 lines +docgen.extractors.python.resolve — 184 lines +_id_for — function, 2 lines +_resolve — function, 32 lines +larder_of — function, 17 lines +to_ir — function, 98 lines +_point_at — function, 9 lines +docgen.extractors.python.__main__ — 38 lines +main — function, 23 lines +docgen.ir.validate — 313 lines +IRError — class, 2 lines +_schema — function, 2 lines +_props — function, 5 lines +_fields — function, 4 lines +check — function, 108 lines +_check_larder — function, 58 lines +validate — function, 6 lines +check_model_matches_schema — function, 23 lines +main — function, 32 lines +docgen.ir.model — 165 lines +Meta — class, 38 lines +to_dict — function, 18 lines +Node — class, 21 lines +__post_init__ — function, 3 lines +to_dict — function, 8 lines +Edge — class, 15 lines +to_dict — function, 7 lines +Graph — class, 48 lines +node — function, 4 lines +edge — function, 4 lines +has — function, 2 lines +to_dict — function, 16 lines +from_dict — function, 6 lines +docgen.ir.__main__ — 9 lines +docgen.ops.filter — 485 lines +_rebuild — function, 57 lines +surviving_parent — function, 5 lines +lift — function, 6 lines +drop_kinds — function, 14 lines +only_kinds — function, 14 lines +drop_stdlib — function, 13 lines +drop_external — function, 3 lines +subtree — function, 13 lines +neighbourhood — function, 45 lines +collapse_to_depth — function, 18 lines +level — function, 6 lines +drop_builtins — function, 14 lines +overview — function, 35 lines +shape — function, 72 lines +rank_of — function, 9 lines +split — function, 24 lines +classify — function, 102 lines +docgen.ops.__main__ — 102 lines +main — function, 84 lines +docgen.style.extract — 222 lines +_values_from — function, 17 lines +_normalise — function, 45 lines +_svg_files — function, 25 lines +harvest — function, 43 lines +write — function, 7 lines +summarise — function, 13 lines +docgen.style.tokens — 211 lines +_top — function, 2 lines +_mode — function, 3 lines +_luminance — function, 6 lines +derive — function, 101 lines +accent — function, 2 lines +geometry — function, 14 lines +write — function, 7 lines +from_folder — function, 9 lines +docgen — 2 lines +docgen.lab.pg_probe — 150 lines +probe — function, 30 lines +_simplify — function, 3 lines +main — function, 25 lines +docgen.notebook.spec — 264 lines +_step — function, 4 lines +from_ir — function, 108 lines +_order — function, 5 lines +scaffold — function, 20 lines +merge — function, 58 lines +load — function, 2 lines +dump — function, 5 lines docgen.emitters -docgen.emitters -docgen.extractors -docgen.extractors -docgen -docgen -docgen.extractors.python -docgen.ir -docgen.ir -docgen.ops -(root) -docgen.lab -docgen.notebook -site +docgen.emitters +docgen +docgen +docgen.extractors +docgen.extractors +docgen.book +docgen.extractors.python +docgen.ir +docgen.ops +docgen.style +docgen.style +(root) +docgen.lab +docgen.notebook +site explore dot notebook @@ -251,48 +331,56 @@ cli_dot auto cli_noteboo -cli_site -cli_erd -cli_minimap -cli_explore -cli_index -__main__ -code -usage -db -openapi -code_main -usage_main -db_main -openapi_mai -python -__main__ -selftest -style -ops -notebook -emitters -lab -ir -extractors -collect -resolve -__main__ -validate -model -__main__ -filter -__main__ -docgen -pg_probe -spec - -module - -class - -interface - -function -45 files · 6,933 lines · 1px ≈ 2.0 lines +cli_site +cli_erd +cli_minimap +cli_explore +cli_index +__main__ +selftest +book +style +reference +ops +notebook +emitters +lab +ir +extractors +code +usage +openapi +db +code_main +python +usage_main +db_main +openapi_mai +__main__ +build +checks +larder +__main__ +collect +resolve +__main__ +validate +model +__main__ +filter +__main__ +extract +tokens +docgen +pg_probe +spec + +module + +class + +interface + +function +53 files · 9,752 lines · 1px ≈ 2.0 lines diff --git a/soleprint/atlas2/docgen/docs/index.html b/soleprint/atlas2/docgen/docs/index.html index 40b01c2..7555706 100644 --- a/soleprint/atlas2/docgen/docs/index.html +++ b/soleprint/atlas2/docgen/docs/index.html @@ -22,6 +22,7 @@
Architecture
The three concerns + The book The IR Shape decides the drawing @@ -39,6 +40,7 @@ Style & colour
Reference
+ Standalone Commands Dependencies Testing @@ -107,7 +109,23 @@

Five minutes

-

Three commands, and they compose. That is the whole interface.

+

+ One command, if you want the whole thing — a book: what + went in, every step, what came out, and a page to open. +

+ +
make book SRC=/path/to/repo BOOK=out/book/mine
+make check BOOK=out/book/mine
+ +
  larder   /path/to/repo — 45 files read, 2 failed, 12 packages
+  book     312 nodes · 244 edges · 26 external · 8 artifacts
+  ok       45 file(s) read produced 45 module(s)
+  open     out/book/mine/site/index.html
+ +

+ Underneath it is three commands that compose, and each still works on its own. + That is the interface, and the book does not replace it: +

# 1. read something
 python3 -m docgen.extractors.python --root ../station/tools/histgen -o ir.json
@@ -123,7 +141,7 @@ python3 -m docgen.emitters auto view.json -o out/
make ir SRC=/path/to/repo OUT=out   # extract
 make explore OUT=out                # the two-pane navigator
 make site OUT=out                   # a docs site with a sidebar
-make self                           # run the whole thing over soleprint
+make self # docgen's book of soleprint, then check it

Everything is offline and self-contained. No server, no CDN, no build step — @@ -166,6 +184,151 @@ make self # run the whole thing over s + +

The book

+ +

+ A book is one docgen operation, and it has a fixed shape: it begins by + saying what went in and ends by saying what came out. Everything in between is + an ordinary file that stands on its own. +

+ +
larder ──► step ──► step ──► step ──► book
+  what        each one usable          what
+  came in     by itself                came out
+ +

+ The word comes from Atlas 1.0, where a book is a larder composed with a + pattern into something published and served. docgen had already built all three + parts under other names, so this is less an adoption than a renaming back. +

+ +

Why both ends, rather than just the result

+ +

+ Because the two numbers are only worth having together. "1,505 nodes" + is not a fact about anything. "225 files in, 1,505 nodes out, nothing + lost" is. Before this, an extractor that read 45 of 47 files produced + exactly the same document as one that read all 47, and the diagram looked + complete either way — there was nowhere for the other two to be mentioned. +

+ +
+ A clean diagram over an incomplete read is a lie by omission. It is also + the failure mode the IR already guards against one level down: an unresolved + name becomes an external node rather than being dropped, because + silently losing a thing is worse than recording an unresolved one. The larder + measure is that same rule applied to the input as a whole. +
+ +

The larder — what came in

+ +

+ Deliberately not called a bucket. A bucket is somewhere bytes sit; a larder is + stocked from outside, has an inventory, and goes stale. All three are + worth measuring, and they are what the measure records: +

+ +
"larder": {
+  "kind":     "python",           // which extractor stocked it
+  "identity": "../../station",    // path, or a DSN with the password masked
+  "unit":     "file",             // file | table | path | entry | document
+  "seen":     47,                 // what the larder offered
+  "read":     45,                 // seen - len(failed), derived
+  "failed":   [{"name": "a.py", "error": "syntax: line 3"}],
+  "extra":    {"packages": 12}
+}
+ +

+ read is derived and never stored. Stored, it invites the question + "does that include the failures?" and every reader answers it + differently; derived, there is nothing to get wrong — and + ir/validate.py fails a document whose arithmetic disagrees with + itself. +

+

+ Failures are recorded by name, not counted. A count tells you a book is + incomplete; a name tells you which part of it to distrust. +

+ +
+ identity is the one field in docgen that could carry a secret + — a database DSN has the password in it. It is masked at construction, and + validate.py then sweeps for the mask having worked, using its own + independent key list. A scrubber graded by its own word is not graded. +
+ +

The book measure — what came out, and reconciled

+ +

+ Counts by kind, edges by kind, externals, and every artifact with its byte + count. On its own that is just a summary. What makes it a measure is that it is + reconciled against the larder: +

+ +
  larder   ../../station — 45 files read, 2 failed, 12 packages
+  book     312 nodes · 244 edges · 26 external · 8 artifacts
+  ok       45 file(s) read produced 45 module(s)
+  ok       2 file(s) could not be read
+  ok       2 unreadable file(s) appear as 2 marked module(s)
+ +

+ The relation differs by source and is declared per extractor, because getting + it wrong gives a check that passes for the wrong reason. One file becomes one + module node — fewer means input was dropped. Four hundred HAR entries becoming + twelve endpoints is not a loss, it is the point of the capture. And a + file that failed to parse must still appear in the graph, carrying its error, + or the picture is smaller than the source and says nothing about it. +

+

+ python3 -m docgen.book exits 1 when a reconciliation fails. The + book is still written — the evidence is the point — but a build that lost input + should fail a pipeline rather than pass quietly. +

+ +

The notebook is the sequence, the web is the last step

+ +

+ These two rule what gets generated, and each for its own reason. +

+

+ The notebook is not one artifact; it is the sequence. Its first cell is + the larder measure and its last cell is the book measure, which is what puts + the two ends in the document rather than only in the tooling. Between + them, one pair of cells per step: what the step did, and a cell that loads that + step's artifact and prints one fact about it. That is the "usable by + themselves" property made executable, and the test suite runs those cells. +

+

+ The web output is last, so nothing depends on it, so it can be replaced + wholesale without touching anything upstream. That is exactly what lets it rule + the output without being a stable contract — the book measure is the promise, + and the page displaying it is free to change drastically and often. It is also + the artifact somebody definitely opens, which is why "2 of 47 files could + not be read" has to appear there, above the diagram rather than below it. +

+ +
+ The spine is scaffolding, not a gate. Running one step alone is still a + book, just a short one — make ir works exactly as it did. An + operation that cannot measure something says what it could not measure and + carries on. Gating would destroy the property that makes the intermediate + artifacts useful, which is the whole reason the sequence is worth having. +
+ +

What a book looks like on disk

+ +
book/<slug>/
+├── book.json        both measures, the steps, artifacts with byte counts
+├── steps/           every intermediate — ir.json, view.json, graph.svg, …
+├── notebook.ipynb   the sequence; first and last cells are the measures
+├── overlay.json     hand-written, optional, re-applied every build
+├── checks.py        this book's own assertions — optional
+└── site/            the web output, both measures at the top
+ +
make book SRC=../station BOOK=out/book/station
+make check BOOK=out/book/station
+

The IR

@@ -627,13 +790,96 @@ python3 -m docgen.emitters notebook ir.json --overlay overlay.json -o walkthroug between a representative leaf and clip the line at the cluster border.

+ +

Standalone

+ +

+ Copy the docgen/ folder anywhere and it works. The Makefile + derives its own package name from where it sits, so it can be renamed too, and + everything else resolves inside the directory. +

+ +
cp -r docgen /somewhere/else
+cd /somewhere/else/docgen
+make doctor          # what this machine has
+make check           # the suite, from the copy
+make book SRC=/path/to/any/repo BOOK=out/book/theirs
+ +

One seam, and it is optional

+ +

+ Exactly one capability needs more than the folder: reading an OpenAPI + document goes through station/tools/modelgen, which parses the + spec and resolves $ref. It is deliberately not reimplemented here + — a second OpenAPI reader in one repo is two things to keep correct. +

+

+ So if you are using docgen standalone but keeping the repo alongside as + reference, point at it: +

+ +
export DOCGEN_REFERENCE=/path/to/repo
+make doctor
+# reference: /path/to/repo (from $DOCGEN_REFERENCE)
+ +

+ Resolution is $DOCGEN_REFERENCE first, then walking up from the + package — so in place it needs no configuration, and an explicit path wins + when set. Without it, the four other extractors and every emitter work + unchanged; the OpenAPI reader reports what to set, and the suite + skips rather than fails. +

+ +
+ An env var rather than a config file, because there is one setting and + it is a path. A config file for one path is a file to find, parse, document and + validate, and the first question anyone asks of it is "where does it live" — + which is the same question again. +
+ +

It is asserted, not asserted-in-prose

+ +

+ A standalone claim decays the moment somebody adds a convenient import, and it + decays silently, because the suite still passes inside the repo. So + the suite reads its own source: +

+
    +
  • Imports are the stdlib, docgen itself, and a short allow-list — + tree-sitter, lxml, yaml, networkx, modelgen, each optional and each with a + reason. A new name is a new dependency in a folder meant to be copied.
    • +
  • Only extractors/openapi.py imports modelgen. One seam + is a seam; two is a dependency.
    • +
  • Only reference.py knows the repo layout, so pointing + docgen elsewhere is one change rather than a search.
    • +
  • A book writes only inside its own output directory — nothing is + left in the tree being read.
    • +
+ +

+ The folder was also literally copied to /tmp and run, which is + how the one real bug here was found: a check asserting the reference repo is + reachable, correct in place and wrong the moment there was nothing above. It + now reports which case applies instead of assuming one. +

+ + + + + + + +
contextchecksskipped
in the repo250tree-sitter (259 with it)
copied out242tree-sitter, OpenAPI, the in-place case
copied out, DOCGEN_REFERENCE set249tree-sitter, the in-place case
+

Commands

Make

- + + + @@ -644,18 +890,30 @@ python3 -m docgen.emitters notebook ir.json --overlay overlay.json -o walkthroug - +
targetdoes
make checkthe whole test suite, offline, nothing installed
make book SRC=…one whole operation, measured at both ends
make checkdocgen's own suite, offline, nothing installed
make check BOOK=…one book's own level — generated and custom
make doctorwhat this machine has and what it is missing
make ir SRC=…extract Python into OUT/ir.json
make code SRC=…extract C#/TypeScript tree-sitter
make minimapwhat is where, read from the colours
make explorethe two-pane navigator
make sitea self-contained docs site
make selfthe whole pipeline over soleprint itself
make selfdocgen's book of soleprint, then check it

Variables: SRC, OUT, SCHEMA, - STYLE, THEME, SCALE, DEPTH, - PY. The Makefile derives its own package name from where it sits, - so the folder can be copied anywhere and renamed and still work. + OPENAPI, HAR, BOOK, SLUG, + READER, OVERLAY, STYLE, + THEME, SCALE, DEPTH, PY. + The Makefile derives its own package name from where it sits, so the folder can + be copied anywhere and renamed and still work. +

+

+ READER rather than LANG because LANG is + the shell's locale variable, so ?= inherits + en_US.UTF-8 from the environment and the argument is rejected. + Every target above is one step of a book and still works alone — that is the + property the spine exists to preserve, not to replace.

Modules

-
python3 -m docgen.extractors.python --root SRC -o ir.json
+
python3 -m docgen.book --root SRC -o out/book/slug   # the whole operation
+python3 -m docgen.book.checks out/book/slug          # that book's level
+
+python3 -m docgen.extractors.python --root SRC -o ir.json
 python3 -m docgen.extractors code    --root SRC -o ir.json
 python3 -m docgen.extractors db      --schema schema.json -o ir.json
 python3 -m docgen.extractors openapi --spec spec.yaml -o ir.json
@@ -676,6 +934,11 @@ python3 -m docgen.emitters explore  ir.json -o explore/

 
 
Dependencies

 
+

+  Everything below is optional. The stdlib covers the whole structural path — see standalone
+  for the one seam out of the folder.
+

+
 

   The core is standard library only. Everything else is optional
   and reported by make doctor; when something is missing you lose
@@ -707,11 +970,68 @@ python3 -m docgen.emitters explore  ir.json -o explore/

Testing

-
make check      # 191 checks, offline, no network
+

+ Three levels, and they differ by what they assert about. The + distinction decides what a failure means, which is why it is worth keeping: +

+ + + + + + + + + +
commandasks aboutfails?
make doctorthe machine — what is installednever; it reports
make checkdocgen — 259 checksexit 1
make check BOOK=<dir>that one bookexit 1
+ +

+ All 259 with both optional dependencies installed, 244 with neither — the suite + skips rather than fails when tree-sitter, lxml or the OpenAPI reader is absent. + See standalone for the counts outside the repo. +

+ +

The book level, where custom checks live

+ +

+ The third level is the one that reaches a project docgen has never seen, and it + is where framework and hand-written checks live together. The generated + half is the spine's own assertions, identical for every book: both measures + present, the reconciliation holding, every artifact where the ledger says it + is, the notebook still executing. The custom half goes in the book's own + checks.py and uses the same helpers, so a project's line and a + framework line read identically and fail identically. +

+ +
# out/book/station/checks.py
+def checks(book, check, note, skip):
+    note("what this project will not give up")
+
+    # Payments moved once already and the move broke three dashboards.
+    # If it is not here, something renamed it again.
+    check("payments is still a module", True,
+          any(n["id"] == "app.payments" for n in book.ir["nodes"]))
+ +

+ Same split as the notebook's base and overlay, for the same reason: generation + alone cannot know what this project cares about, and hand-authoring + alone rots. +

+ +
+ Each check is one decision that has already been made, with the reason above + it. Not coverage, and deliberately not an exhaustive sweep. A rule without + its reason gets overridden the first time it is inconvenient, so failing a + check should read as "you are about to undo this" rather than + "something broke". The idiom is carried from rig's + ctrl/selftest.sh, which is where the three-level split comes from. +
+ +

The four that are the design

- Four of those are the design rather than regressions, and they are the - ones to keep if anything is ever cut: + Of docgen's own checks, four assert the architecture rather than guard + a regression, and they are the ones to keep if anything is ever cut:

  • No visual field reaches the IR — extractors cannot decide appearance.
    • @@ -736,8 +1056,10 @@ python3 -m docgen.emitters explore ir.json -o explore/

    Self-hosting is the honest end-to-end check, and it is where the real bugs came from — two name-resolution faults and a duplicate-id crash that no fixture had - reached. make self runs the whole pipeline over soleprint; if the - index does not read like the system, something is wrong. + reached. make self builds docgen's book of soleprint and + then runs that book's own level against it, so the two ends have to reconcile + on 225 real files. If the index does not read like the system, something is + wrong.

    diff --git a/soleprint/atlas2/docgen/emitters/site.py b/soleprint/atlas2/docgen/emitters/site.py index bd4fd3f..7057d9c 100644 --- a/soleprint/atlas2/docgen/emitters/site.py +++ b/soleprint/atlas2/docgen/emitters/site.py @@ -248,9 +248,84 @@ body { .figure a:hover { border-color: var(--accent); } .figure img { display: block; width: 100%; height: auto; } .figure figcaption { color: var(--dim); font-size: 10px; margin-top: .4rem; } + +/* Both ends of the book, side by side and above the diagram. Above, because a + reader who scrolls past the picture has already formed an impression, and + "2 files could not be read" has to arrive before that and not after. */ +.ledger { display: flex; gap: 1px; background: var(--border); border: 1px solid var(--border); + border-radius: 8px; overflow: hidden; margin: 0 0 1.5rem; } +.ledger > div { flex: 1 1 0; background: var(--surface); padding: .7rem .9rem; min-width: 0; } +.ledger dt { color: var(--dim); font-size: 9.5px; text-transform: uppercase; + letter-spacing: .07em; margin-bottom: .3rem; } +.ledger dd { margin: 0; color: var(--text); font-size: 12.5px; } +.ledger dd .sub { display: block; color: var(--muted); font-size: 11px; margin-top: .2rem; + overflow-wrap: anywhere; } +.ledger .lost { color: var(--artery); } +.ledger .kept { color: var(--ok); } +.gap { border: 1px solid var(--artery); border-left-width: 3px; border-radius: 6px; + background: var(--surface); padding: .7rem .9rem; margin: 0 0 1.5rem; font-size: 12px; } +.gap b { color: var(--artery); } +.gap ul { margin: .4rem 0 0 1.1rem; color: var(--muted); } +.gap code { font-size: 11px; } """ +def _ledger(book: dict) -> str: + """The two measures, and the gap between them if there is one. + + This is the whole reason the site is the book's last step rather than just + another emitter: it is the one artifact somebody definitely opens, so it is + where "45 of 47 files" has to appear. A diagram cannot say it, and a log + nobody reads does not count as having said it. + """ + larder = book.get("larder") or {} + measure = book.get("book") or {} + failed = larder.get("failed") or [] + + kinds = measure.get("by_kind") or {} + out_summary = " · ".join(f"{v} {k}" for k, v in + sorted(kinds.items(), key=lambda kv: -kv[1])[:4]) + + unit = larder.get("unit", "unit") + read, seen = larder.get("read", 0), larder.get("seen", 0) + plural = unit if read == 1 else (unit[:-1] + "ies" if unit.endswith("y") else unit + "s") + in_line = f"{read} {plural} read" + if failed: + in_line += f' · {len(failed)} of {seen} could not be' + + panel = ( + '
    ' + f'
    what came in
    {in_line}' + f'{escape(larder.get("identity", "?"))}
    ' + f'
    what came out
    {escape(out_summary) or "nothing"}' + f'{measure.get("edges", 0)} edges · ' + f'{measure.get("external", 0)} external · ' + # "step artifacts", not "artifacts": this page is written before the + # book's last step closes, so it cannot count itself. Saying `step` + # makes the number true rather than one short of book.json's. + f'{len(measure.get("artifacts") or [])} step artifacts
    ' + "
    " + ) + + # A reconciliation that failed is not a footnote. It means the document + # below is incomplete in a way the document below cannot show. + lost = [r for r in (book.get("reconciled") or []) if not r.get("ok")] + if lost or failed: + items = "".join(f"
  • {escape(r['claim'])} — {escape(r['why'])}
    • " for r in lost) + items += "".join( + f"
  • {escape(f['name'])} — {escape(f['error'])}
    • " + for f in failed[:12] + ) + if len(failed) > 12: + items += f"
  • and {len(failed) - 12} more
    • " + panel += ( + '
    This book is incomplete. ' + "What is drawn below is everything that could be read, which is not " + f"everything there is.
      {items}
    " + ) + return panel + + def _slots(style) -> dict: s = style.slot return { @@ -308,8 +383,15 @@ def _sections(items: list, depth: int = 0) -> str: return "".join(out) -def emit(ir: dict, style, *, graph: str | None = None, title: str = "") -> dict: - """IR + Style -> {filename: text}. Write them next to each other.""" +def emit(ir: dict, style, *, graph: str | None = None, title: str = "", + book: dict | None = None) -> dict: + """IR + Style -> {filename: text}. Write them next to each other. + + `book` is a book ledger (`book/__init__.py`). When present the page opens + with both measures — what went in, what came out — because a page that + shows only the result is the thing the measure exists to correct. + Optional, so the site emitter still works on a bare IR. + """ from .index import to_sidebar meta = ir.get("meta", {}) @@ -322,6 +404,8 @@ def emit(ir: dict, style, *, graph: str | None = None, title: str = "") -> dict: counts[n["kind"]] = counts.get(n["kind"], 0) + 1 summary = " · ".join(f"{v} {k}" for k, v in sorted(counts.items(), key=lambda kv: -kv[1])) + ledger = _ledger(book) if book else "" + figure = "" if graph: figure = ( @@ -360,6 +444,7 @@ def emit(ir: dict, style, *, graph: str | None = None, title: str = "") -> dict:

    {escape(name)}

    Generated from {escape(meta.get("source", "?"))} · {escape(summary)}. Regenerated, not edited.

    + {ledger} {figure} {_sections(side["items"])} {ext_html} @@ -396,11 +481,12 @@ document.querySelectorAll('h2[id], h3[id]').forEach(function (h) {{ obs.observe( } -def write(ir: dict, style, out_dir, *, graph: str | None = None, title: str = "") -> list[Path]: +def write(ir: dict, style, out_dir, *, graph: str | None = None, title: str = "", + book: dict | None = None) -> list[Path]: out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) written = [] - for name, text in emit(ir, style, graph=graph, title=title).items(): + for name, text in emit(ir, style, graph=graph, title=title, book=book).items(): path = out_dir / name path.write_text(text) written.append(path) diff --git a/soleprint/atlas2/docgen/extractors/code.py b/soleprint/atlas2/docgen/extractors/code.py index 809812e..2251112 100644 --- a/soleprint/atlas2/docgen/extractors/code.py +++ b/soleprint/atlas2/docgen/extractors/code.py @@ -204,7 +204,8 @@ def _count_errors(node) -> int: return n -def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: +def extract(root, suffixes=None, exclude=(), source: str = "code", + identity: str | None = None) -> Graph: """Walk a tree and map its structure. No edges — see the module docstring.""" root = Path(root).resolve() if not root.is_dir(): @@ -214,6 +215,8 @@ def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: skip = {"__pycache__", ".git", ".venv", "venv", "node_modules", "dist", "build", "bin", "obj", *exclude} + from ..book.larder import Larder + g = Graph(Meta(source=source, root=root.name)) packages: set[str] = set() files = [ @@ -222,6 +225,14 @@ def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: and not any(part in skip for part in p.relative_to(root).parts) ] + # The measure is built alongside the walk rather than recomputed after it. + # tree-sitter never raises on bad syntax — it produces ERROR nodes — so + # "failed" here means a file that could not be handled at all, and the + # partially-parsed ones are counted separately under `unparsed_regions`. + larder = Larder(kind="code", identity=identity or root.name, unit="file", + seen=len(files)) + unparsed = 0 + for path in files: try: module, decls = extract_file(path, root) @@ -229,6 +240,7 @@ def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: raise except Exception as e: # noqa: BLE001 - one bad file must not cost the run rel = path.relative_to(root) + larder.fail(rel.as_posix(), f"{type(e).__name__}: {e}") g.node(".".join([*rel.parts[:-1], rel.stem]), "module", rel.stem, attrs={"file": rel.as_posix(), "error": f"{type(e).__name__}: {e}"}) continue @@ -239,6 +251,7 @@ def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: attrs = {"file": module["file"], "lines": module["lines"]} if module["errors"]: attrs["error"] = f"{module['errors']} unparsed region(s)" + unparsed += 1 g.node(module["id"], "module", module["label"], parent=parent, attrs=attrs) for d in decls: @@ -257,4 +270,12 @@ def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: parent=".".join(parts[: i - 1]) or None, attrs={"lines": 0, "directory": True}) known.add(pid) + + larder.extra["languages"] = len({p.suffix for p in files}) + if unparsed: + # Worth its own number rather than folding into `failed`: a file with an + # unparsed region still contributed structure, so calling it a failure + # would understate what the book contains. + larder.extra["unparsed_regions"] = unparsed + g.meta.larder = larder.to_dict() return g diff --git a/soleprint/atlas2/docgen/extractors/db.py b/soleprint/atlas2/docgen/extractors/db.py index e22aa81..8a28f68 100644 --- a/soleprint/atlas2/docgen/extractors/db.py +++ b/soleprint/atlas2/docgen/extractors/db.py @@ -37,7 +37,8 @@ from pathlib import Path from ..ir import Graph, Meta -def from_schema_dict(data: dict, root: str = "schema", source: str = "db") -> Graph: +def from_schema_dict(data: dict, root: str = "schema", source: str = "db", + identity: str | None = None) -> Graph: """A graphgen-compatible schema dict -> IR. Accepts both spellings of the contract: the on-disk `schema.json` form, @@ -45,6 +46,8 @@ def from_schema_dict(data: dict, root: str = "schema", source: str = "db") -> Gr `graphgen.schema.load_graph_schema` returns, where it is a list. They are the same data and callers have both. """ + from ..book.larder import Larder + g = Graph(Meta(source=source, root=root)) models = data.get("models", {}) listed = models.values() if isinstance(models, dict) else models @@ -96,6 +99,18 @@ def from_schema_dict(data: dict, root: str = "schema", source: str = "db") -> Gr kind = "references" if rel.get("type") == "M2M" else "foreign_key" g.edge(src, dst, kind, attrs={"label": rel.get("from_field", "")}) + # A schema is a published contract, so there is nothing here that can fail + # to be read — every table it declares is a table. `seen == read` always, + # and saying so is more useful than omitting the measure: it distinguishes + # "no failures" from "not measured". + larder = Larder(kind="db", identity=identity or root, unit="table", + seen=len(names)) + rels = data.get("relationships") or [] + larder.extra["relationships"] = len(rels) + if data.get("source"): + larder.extra["dialect"] = str(data["source"]) + g.meta.larder = larder.to_dict() + _dedupe(g) for missing in sorted( {e.target for e in g.edges} - {n.id for n in g.nodes} @@ -137,8 +152,9 @@ def _dedupe(g: Graph) -> None: g.edges = keep -def extract(schema_path, source: str = "db") -> Graph: +def extract(schema_path, source: str = "db", identity: str | None = None) -> Graph: """Read a schema.json from disk.""" path = Path(schema_path) data = json.loads(path.read_text()) - return from_schema_dict(data, root=path.parent.name or path.stem, source=source) + return from_schema_dict(data, root=path.parent.name or path.stem, source=source, + identity=identity or str(schema_path)) diff --git a/soleprint/atlas2/docgen/extractors/openapi.py b/soleprint/atlas2/docgen/extractors/openapi.py index 3e4f2f6..1dea76f 100644 --- a/soleprint/atlas2/docgen/extractors/openapi.py +++ b/soleprint/atlas2/docgen/extractors/openapi.py @@ -24,33 +24,39 @@ are a separate `kind`, so a view can ask for one or the other. only_kinds(ir, {"endpoint"}) -> the surface, as a notebook """ -import sys from pathlib import Path from ..ir import Graph, Meta def _modelgen(): - """modelgen, from wherever this instance keeps station tools. + """modelgen's OpenAPI reader, from wherever the reference repo is. - Imported lazily and by path rather than as a hard dependency: docgen belongs - to atlas and may depend on a station tool, but it should not fail to import - because one is missing. + Imported lazily and by path rather than as a hard dependency. This is the + **only** seam between docgen and the wider repo — see `reference.py`, which + resolves it from `$DOCGEN_REFERENCE` or by walking up. Every other extractor + and every emitter works with nothing above `docgen/`. + + Not reimplemented here on purpose: modelgen already parses the spec and + resolves `$ref`, and a second OpenAPI reader in the same repo is two things + to keep correct. """ - here = Path(__file__).resolve() - for parent in here.parents: - tools = parent / "station" / "tools" - if (tools / "modelgen").is_dir(): - if str(tools) not in sys.path: - sys.path.insert(0, str(tools)) - from modelgen.loader.extract.openapi import OpenAPIExtractor - - return OpenAPIExtractor - raise ImportError( - "modelgen not found — docgen reads OpenAPI through " - "station/tools/modelgen/loader/extract/openapi.py, which parses the spec " - "and resolves $ref. It is not reimplemented here." - ) + from .. import reference + + if reference.on_path() is None: + raise reference.missing( + "modelgen", + "OpenAPI is read through station/tools/modelgen/loader/extract/" + "openapi.py, which parses the spec and resolves $ref", + ) + try: + from modelgen.loader.extract.openapi import OpenAPIExtractor + except ImportError as e: + raise reference.missing( + "modelgen.loader.extract.openapi", + f"the reference repo was found but the module did not import ({e})", + ) from None + return OpenAPIExtractor def _type_name(hint) -> str: @@ -61,8 +67,55 @@ def _type_name(hint) -> str: return getattr(hint, "__name__", str(hint)) -def extract(spec_path, source: str = "openapi") -> Graph: +def _refs(path: Path) -> dict[str, dict[str, str]]: + """{schema: {field: referenced schema}} — the relationships, recovered. + + modelgen resolves an inter-schema `$ref` to the literal string `dict`, so by + the time its fields reach us the *target* is gone. Without this, an API's + data model draws as disconnected cards: three tables, no foreign keys, and + nothing saying the relationships were lost. Which is exactly the failure + this tool is otherwise built to prevent. + + So one key is read directly, and one only: `$ref` under a schema's + `properties`. That is not parsing OpenAPI — no paths, no bodies, no + responses, no `$ref` resolution, no composition keywords. modelgen still + does all of the reading that matters, and this recovers the single fact its + type mapping cannot carry. + + Returns `{}` on anything unexpected. A missing relationship is a worse + diagram; a raised exception here would be no diagram at all. + """ + try: + import yaml # available: modelgen just used it + except ImportError: + return {} + try: + doc = yaml.safe_load(path.read_text()) or {} + schemas = ((doc.get("components") or {}).get("schemas")) or {} + except Exception: # noqa: BLE001 - see the docstring + return {} + + out: dict[str, dict[str, str]] = {} + for name, schema in schemas.items(): + if not isinstance(schema, dict): + continue + for field, spec in (schema.get("properties") or {}).items(): + if not isinstance(spec, dict): + continue + # A direct reference, or an array of them — `lines: [OrderLine]` is + # the same relationship as `order: Order`, pointing the other way. + ref = spec.get("$ref") + if not ref and isinstance(spec.get("items"), dict): + ref = spec["items"].get("$ref") + if isinstance(ref, str) and ref.startswith("#/components/schemas/"): + out.setdefault(name, {})[field] = ref.rsplit("/", 1)[-1] + return out + + +def extract(spec_path, source: str = "openapi", identity: str | None = None) -> Graph: """An OpenAPI file -> IR.""" + from ..book.larder import Larder + OpenAPIExtractor = _modelgen() path = Path(spec_path) extractor = OpenAPIExtractor(path) @@ -71,6 +124,7 @@ def extract(spec_path, source: str = "openapi") -> Graph: g = Graph(Meta(source=source, root=path.name)) known = {m.name for m in models} + refs = _refs(path) for model in models: attrs = {} @@ -83,7 +137,9 @@ def extract(spec_path, source: str = "openapi") -> Graph: a["nullable"] = True if field.name in ("id", "uuid"): a["pk"] = True - target = _type_name(field.type_hint) + # The recovered $ref target wins over the mapped type name: modelgen + # says `dict` where the spec said which schema. + target = refs.get(model.name, {}).get(field.name) or _type_name(field.type_hint) if target in known and target != model.name: a["references"] = target g.edge(model.name, target, "foreign_key", attrs={"label": field.name}) @@ -120,4 +176,15 @@ def extract(spec_path, source: str = "openapi") -> Graph: if getattr(e, "response_model", None) in known: g.edge(eid, e.response_model, "returns") + # A spec is the larder here, and `path` is the unit because that is what a + # spec is an inventory of. Schemas and enums are counted separately: a spec + # with 40 schemas and 3 endpoints is a data model, and the measure should + # make that visible before the diagram does. + larder = Larder(kind="openapi", identity=identity or path.name, unit="path", + seen=len({e.path for e in endpoints})) + larder.extra["operations"] = len(endpoints) + larder.extra["schemas"] = len(models) + if enums: + larder.extra["enums"] = len(enums) + g.meta.larder = larder.to_dict() return g diff --git a/soleprint/atlas2/docgen/extractors/python/__init__.py b/soleprint/atlas2/docgen/extractors/python/__init__.py index ddabf3e..c5889cb 100644 --- a/soleprint/atlas2/docgen/extractors/python/__init__.py +++ b/soleprint/atlas2/docgen/extractors/python/__init__.py @@ -17,13 +17,20 @@ from .collect import collect from .resolve import to_ir -def extract(root, exclude=(), source="python"): - """Walk `root`, return an IR Graph. Never raises on a bad file.""" +def extract(root, exclude=(), source="python", identity=None): + """Walk `root`, return an IR Graph. Never raises on a bad file. + + `identity` is what the larder measure calls this source — the path as the + caller wrote it, which is what they will recognise. It defaults to the + directory name rather than the resolved path, for the same reason + `meta.root` does: an absolute path is not a secret but it is machine + specific, and the measure should mean the same thing on two machines. + """ root = Path(root).resolve() if not root.is_dir(): raise NotADirectoryError(f"not a directory: {root}") modules = collect(root, exclude=exclude) - return to_ir(modules, root=root.name, source=source) + return to_ir(modules, root=root.name, source=source, identity=identity) __all__ = ["extract", "collect", "to_ir"] diff --git a/soleprint/atlas2/docgen/extractors/python/resolve.py b/soleprint/atlas2/docgen/extractors/python/resolve.py index 23cb415..f4190ed 100644 --- a/soleprint/atlas2/docgen/extractors/python/resolve.py +++ b/soleprint/atlas2/docgen/extractors/python/resolve.py @@ -64,9 +64,30 @@ def _resolve(name: str, module: Module, known: set[str]) -> str | None: return None -def to_ir(modules: list[Module], root: str, source: str = "python") -> Graph: +def larder_of(modules: list[Module], identity: str) -> "Larder": + """The input measure for a tree of Python files. + + Every fact here was already computed and then discarded: `collect_file` + records `Module.error` for a file it could not read or parse, and nothing + downstream ever reported how many there were. A run that parsed 45 of 47 + files produced the same document as one that parsed all 47. + """ + from ...book.larder import Larder + + larder = Larder(kind="python", identity=identity, unit="file", + seen=len(modules)) + for m in modules: + if m.error: + larder.fail(m.path, m.error) + larder.extra["packages"] = len({m.package for m in modules if m.package}) + return larder + + +def to_ir(modules: list[Module], root: str, source: str = "python", + identity: str | None = None) -> Graph: """Collected modules -> a validated-shaped IR graph.""" - g = Graph(Meta(source=source, root=root)) + g = Graph(Meta(source=source, root=root, + larder=larder_of(modules, identity or root).to_dict())) # -- nodes we own ------------------------------------------------------ known: set[str] = set() diff --git a/soleprint/atlas2/docgen/extractors/usage.py b/soleprint/atlas2/docgen/extractors/usage.py index 0e0bca3..c4ef41f 100644 --- a/soleprint/atlas2/docgen/extractors/usage.py +++ b/soleprint/atlas2/docgen/extractors/usage.py @@ -133,8 +133,10 @@ def _graphql(body: dict | None) -> tuple[str | None, str | None]: return (name or (m.group(2) if m and m.group(2) else "anonymous")), op_type -def extract(har_path, source: str = "usage") -> Graph: +def extract(har_path, source: str = "usage", identity: str | None = None) -> Graph: """A HAR file -> IR of what was actually called.""" + from ..book.larder import Larder + path = Path(har_path) har = json.loads(path.read_text()) entries = (har.get("log") or {}).get("entries") or [] @@ -143,11 +145,26 @@ def extract(har_path, source: str = "usage") -> Graph: calls = [] # (key, kind, facts) in order seen: dict[str, dict] = {} - for entry in entries: + # The unit is the HAR entry — one recorded request — not the distinct call. + # A capture's size is what it is, and the collapse from many entries to few + # endpoints is a fact worth being able to show against this number. + larder = Larder(kind="usage", identity=identity or path.name, unit="entry", + seen=len(entries)) + + for i, entry in enumerate(entries): req = entry.get("request") or {} res = entry.get("response") or {} method = (req.get("method") or "GET").upper() url = req.get("url") or "" + if not url: + # Without a URL there is nothing to template, and `urlsplit("")` + # yields an empty path that `_template` turns into "/". That made a + # malformed entry indistinguishable from a real call to the root. + # Recorded by name and skipped, because a capture is somebody's + # session and quietly inventing a request in it is worse than + # reporting a gap. + larder.fail(f"entries[{i}]", "no request URL") + continue split = urlsplit(url) templated, id_formats = _template(split.path or "/") body = _body(entry) @@ -199,6 +216,10 @@ def extract(har_path, source: str = "usage") -> Graph: g = Graph(Meta(source=source, root=path.name)) + larder.extra["hosts"] = len({r["host"] for r in seen.values() if r.get("host")}) + larder.extra["calls"] = len(seen) + g.meta.larder = larder.to_dict() + order = {} for i, key in enumerate(calls): order.setdefault(key, i) diff --git a/soleprint/atlas2/docgen/fixtures/orders.yaml b/soleprint/atlas2/docgen/fixtures/orders.yaml new file mode 100644 index 0000000..af2b340 --- /dev/null +++ b/soleprint/atlas2/docgen/fixtures/orders.yaml @@ -0,0 +1,154 @@ +# docgen's own OpenAPI fixture. +# +# Small, and shipped here rather than borrowed from station/tools/shuntgen so the +# framework test does not need the repo to be next door. A test's fixtures belong +# to the test — the suite still skips without modelgen, because reading OpenAPI +# genuinely needs it, but it no longer skips for want of a file. +# +# What it exercises, deliberately: +# - schemas that reference each other, so `foreign_key` edges have somewhere +# to come from and the ERD emitter has something to draw +# - an enum, which is a separate path in modelgen's reader +# - more than three paths, and one non-collection path with a parameter +# - a create body whose `id` is server-assigned, which is what the notebook +# emitter's "primary key is left out of a create body" check is about +openapi: 3.0.3 +info: + title: Orders + version: "1.0.0" +paths: + /customers: + get: + operationId: listCustomers + responses: + "200": + description: Every customer. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Customer" + post: + operationId: createCustomer + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + responses: + "201": + description: Created. + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + /customers/{customerId}: + get: + operationId: getCustomer + parameters: + - name: customerId + in: path + required: true + schema: + type: integer + responses: + "200": + description: One customer. + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + /orders: + get: + operationId: listOrders + parameters: + - name: status + in: query + required: false + schema: + $ref: "#/components/schemas/OrderStatus" + responses: + "200": + description: Orders, newest first. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Order" + post: + operationId: createOrder + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Order" + responses: + "201": + description: Created. + content: + application/json: + schema: + $ref: "#/components/schemas/Order" + /orders/{orderId}/lines: + get: + operationId: listOrderLines + parameters: + - name: orderId + in: path + required: true + schema: + type: integer + responses: + "200": + description: The lines on one order. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/OrderLine" +components: + schemas: + OrderStatus: + type: string + enum: [pending, paid, shipped, cancelled] + Customer: + type: object + required: [id, email] + properties: + id: + type: integer + description: Server-assigned. + email: + type: string + name: + type: string + Order: + type: object + required: [id, customer] + properties: + id: + type: integer + description: Server-assigned. + customer: + $ref: "#/components/schemas/Customer" + status: + $ref: "#/components/schemas/OrderStatus" + total: + type: number + OrderLine: + type: object + required: [id, order, sku] + properties: + id: + type: integer + order: + $ref: "#/components/schemas/Order" + sku: + type: string + quantity: + type: integer diff --git a/soleprint/atlas2/docgen/ir/model.py b/soleprint/atlas2/docgen/ir/model.py index 74a1bbd..bc5873b 100644 --- a/soleprint/atlas2/docgen/ir/model.py +++ b/soleprint/atlas2/docgen/ir/model.py @@ -23,6 +23,7 @@ in a theme that has no cylinders. The previous iteration of this model carried `cls`, `shape` and `style`, and that is exactly the mistake this replaces. """ +from copy import deepcopy from dataclasses import dataclass, field from typing import Any @@ -37,20 +38,37 @@ class Meta: extractions of an unchanged tree differ, which destroys the diff emitter's only useful property. Set it when the run time is the fact being recorded, not by habit. + + `larder` is what was actually read — see `book/larder.py`. It is optional so + that an IR written by hand, or by an extractor that cannot count its input, + is still valid; what it must never be is *wrong*. Absent means "not + measured", which a reader can act on. A zero would be a claim. """ source: str root: str schema_version: str = SCHEMA_VERSION generated_at: str | None = None + larder: dict | None = None def to_dict(self) -> dict: - return { + out = { "source": self.source, "root": self.root, "schema_version": self.schema_version, "generated_at": self.generated_at, } + # Omitted rather than null when unmeasured: `meta.larder` present with a + # null inside is two ways to say the same thing, and consumers end up + # checking for both. + # + # Deep-copied because the rest of this model copies too — `Node.to_dict` + # does `dict(self.attrs)`. A shallow copy is not enough here: `failed` is + # a list of dicts, so aliasing it would let a caller editing a serialised + # document reach back into the live measure. + if self.larder is not None: + out["larder"] = deepcopy(self.larder) + return out @dataclass diff --git a/soleprint/atlas2/docgen/ir/schema.json b/soleprint/atlas2/docgen/ir/schema.json index f1fbe9e..fe38989 100644 --- a/soleprint/atlas2/docgen/ir/schema.json +++ b/soleprint/atlas2/docgen/ir/schema.json @@ -28,6 +28,33 @@ "generated_at": { "type": ["string", "null"], "description": "ISO-8601, or null. Null is the default and the honest one: a timestamp makes two extractions of the same tree differ, which breaks diffing. Set it only when the run time is itself the fact being recorded." + }, + "larder": { + "type": ["object", "null"], + "description": "What was actually read: the first step of a book, and the only measure of the input this document carries. Absent means not measured, which a reader can act on; a zero would be a claim. Provenance lives here and never in nodes or attrs, the same split that keeps colour out of the IR.", + "required": ["kind", "identity", "unit", "seen", "read", "failed"], + "additionalProperties": false, + "properties": { + "kind": { "type": "string", "description": "Which extractor stocked it: python, code, db, openapi, usage." }, + "identity": { "type": "string", "description": "The path, or a connection string with its credentials masked. Scrubbed at construction in book/larder.py and swept for in validate.py — the one field in the IR that could carry a secret." }, + "unit": { "type": "string", "description": "What `seen` counts: file, table, path, entry, document. Closed vocabulary, so two larders can be compared." }, + "seen": { "type": "integer", "description": "How many units the larder offered." }, + "read": { "type": "integer", "description": "How many were consumed. Always seen - len(failed); derived on the dataclass so the two cannot disagree." }, + "failed": { + "type": "array", + "description": "What could not be consumed, by name and reason. Named rather than counted: a count says a book is incomplete, a name says which part of it to distrust.", + "items": { + "type": "object", + "required": ["name", "error"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "error": { "type": "string" } + } + } + }, + "extra": { "type": "object", "description": "Per-source facts that do not generalise: packages for a tree, dialect for a database, hosts for a capture." } + } } } }, diff --git a/soleprint/atlas2/docgen/ir/validate.py b/soleprint/atlas2/docgen/ir/validate.py index 49d4832..9515d5c 100644 --- a/soleprint/atlas2/docgen/ir/validate.py +++ b/soleprint/atlas2/docgen/ir/validate.py @@ -22,6 +22,7 @@ that produce a broken diagram. """ import json +import re import sys from pathlib import Path @@ -90,6 +91,8 @@ def check(data: dict, *, strict_visual: bool = True) -> list[str]: if version is not None and not isinstance(version, str): problems.append(f"meta.schema_version must be a string, got {type(version).__name__}") + problems.extend(_check_larder(data["meta"].get("larder"), meta_schema)) + # -- nodes ------------------------------------------------------------ node_required, node_allowed = _fields(schema, "nodes") seen: set[str] = set() @@ -169,6 +172,79 @@ def check(data: dict, *, strict_visual: bool = True) -> list[str]: return problems +# A credential that survived redaction, detected independently of the code that +# does the redacting. `book/larder.py` has its own key list; this has a second +# one on purpose. If the two ever disagree the check fires, which is the point — +# a scrubber that is graded by its own word is not graded. Same reason +# VISUAL_KEYS lives here and not in schema.json. +_DSN_PASSWORD = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^:/@\s]+:(?P[^@/\s]+)@") +_SECRET_PARAM = re.compile( + r"(?i)\b(?:password|passwd|pwd|secret|token|access_token|refresh_token" + r"|api_key|apikey|sig|signature|credentials)\s*=\s*(?P[^&;\s]+)" +) +_MASKED = {"***", "xxx", "redacted", "[redacted]", "masked"} + + +def _check_larder(larder, meta_schema: dict) -> list[str]: + """The input measure, checked — shape, arithmetic, and no leaked secret. + + Absent is fine and means "not measured". Present and wrong is not, because + the whole value of the measure is that the number can be trusted against the + result; a larder claiming 47 reads when it read 45 is worse than no larder. + """ + if larder is None: + return [] + if not isinstance(larder, dict): + return [f"meta.larder must be an object, got {type(larder).__name__}"] + + problems = [] + spec = meta_schema["properties"]["larder"] + for key in spec["required"]: + if key not in larder: + problems.append(f"meta.larder is missing {key!r}") + for key in larder: + if key not in spec["properties"]: + problems.append(f"meta.larder has unknown key {key!r}") + + failed = larder.get("failed") + if failed is not None and not isinstance(failed, list): + problems.append("meta.larder.failed must be an array") + failed = None + elif failed: + for i, f in enumerate(failed): + if not isinstance(f, dict) or "name" not in f or "error" not in f: + problems.append(f"meta.larder.failed[{i}] needs both 'name' and 'error'") + + # read is derived. Stored, it can disagree with itself, and this is where + # that shows up rather than in a book measure nobody can reconcile. + seen, read = larder.get("seen"), larder.get("read") + if isinstance(seen, int) and isinstance(read, int) and isinstance(failed, list): + if read != max(0, seen - len(failed)): + problems.append( + f"meta.larder.read is {read} but seen={seen} with {len(failed)} failed " + f"implies {max(0, seen - len(failed))} — the measure disagrees with itself" + ) + if isinstance(read, int) and isinstance(seen, int) and read > seen: + problems.append(f"meta.larder.read ({read}) exceeds seen ({seen})") + + unit = larder.get("unit") + if unit is not None and not isinstance(unit, str): + problems.append(f"meta.larder.unit must be a string, got {type(unit).__name__}") + + identity = larder.get("identity") + if isinstance(identity, str): + for pattern, what in ((_DSN_PASSWORD, "a DSN password"), + (_SECRET_PARAM, "a secret parameter")): + m = pattern.search(identity) + if m and m.group("secret").lower() not in _MASKED: + problems.append( + f"meta.larder.identity still contains {what} — " + "redact() in book/larder.py did not mask it, and this document " + "must not be written anywhere" + ) + return problems + + def validate(data: dict, **kw) -> dict: """check(), but raises. For use at a boundary where carrying on is wrong.""" problems = check(data, **kw) diff --git a/soleprint/atlas2/docgen/reference.py b/soleprint/atlas2/docgen/reference.py new file mode 100644 index 0000000..62d21c7 --- /dev/null +++ b/soleprint/atlas2/docgen/reference.py @@ -0,0 +1,107 @@ +""" +Where the rest of spr is, when docgen is not inside it. + +docgen is meant to be usable **standalone** — copied out of the repo, pointed at +someone else's codebase, with nothing above it. One capability genuinely needs +more than that: reading an OpenAPI document goes through +`station/tools/modelgen`, which parses the spec and resolves `$ref`, and is +deliberately not reimplemented here. + +So there is exactly one seam to the wider repo, and this is it. Everything else +in docgen resolves inside its own directory. + +## How it resolves, in order + + 1. $DOCGEN_REFERENCE an explicit path to the repo root + 2. walking up from this file when docgen still sits inside the repo + +(1) is the case that matters when docgen is used standalone with the repo kept +alongside as reference. (2) is what happens in place, and needs no configuration. + +## Why an env var and not a config file + +Because there is one setting and it is a path. A config file for one path is a +file to find, parse, document and validate, and the first question anyone asks +of it is "where does it live" — which is the same question again. The Makefile +passes it through, `make doctor` reports it, and the error message when it is +missing says the variable name. + +## Absent is a normal state, not an error + +Nothing here raises on a failed lookup. The OpenAPI extractor reports what to +set and every other part of docgen carries on, because for four of the five +extractors this file is irrelevant. +""" + +import os +import sys +from pathlib import Path + +ENV_VAR = "DOCGEN_REFERENCE" + +# What proves a directory is the repo root rather than some other directory: +# the station tools tree, which is the only thing docgen ever reaches for. +MARKER = Path("station") / "tools" + + +def _candidates(): + explicit = os.environ.get(ENV_VAR) + if explicit: + yield Path(explicit).expanduser() + # Walking up covers the in-place case and costs nothing when it fails. + for parent in Path(__file__).resolve().parents: + yield parent + + +def root() -> Path | None: + """The repo root holding `station/tools`, or None if it is not reachable.""" + for candidate in _candidates(): + try: + if (candidate / MARKER).is_dir(): + return candidate + except OSError: + continue + return None + + +def station_tools() -> Path | None: + """The `station/tools` directory itself, or None.""" + found = root() + return found / MARKER if found else None + + +def describe() -> str: + """One line for `make doctor`. Says how it resolved, not just whether.""" + explicit = os.environ.get(ENV_VAR) + found = root() + if found is None: + return (f"absent — set {ENV_VAR}=/path/to/repo to read OpenAPI " + "(everything else works without it)") + how = f"from ${ENV_VAR}" if explicit and Path(explicit).expanduser() == found \ + else "found by walking up" + return f"{found} ({how})" + + +def on_path() -> Path | None: + """Put `station/tools` on `sys.path` and return it, or None. + + Prepending rather than appending: a `modelgen` earlier on the path is + somebody else's, and silently reading the wrong one is worse than not + finding it at all. + """ + tools = station_tools() + if tools is None: + return None + if str(tools) not in sys.path: + sys.path.insert(0, str(tools)) + return tools + + +def missing(what: str, why: str) -> ImportError: + """The error to raise when a reference-dependent capability is used.""" + return ImportError( + f"{what} is not reachable — {why}\n" + f"docgen is standalone except for this one seam. Point at the repo with:\n" + f" export {ENV_VAR}=/path/to/repo\n" + f"and check it with: make doctor" + ) diff --git a/soleprint/atlas2/docgen/selftest.py b/soleprint/atlas2/docgen/selftest.py index 32f7b95..38cc3bd 100644 --- a/soleprint/atlas2/docgen/selftest.py +++ b/soleprint/atlas2/docgen/selftest.py @@ -52,6 +52,8 @@ exp_mod = __import__(f"{PKG}.emitters.explore", fromlist=["*"]) spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"]) db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"]) usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"]) +tokens_mod = __import__(f"{PKG}.style.tokens", fromlist=["*"]) +extract_mod = __import__(f"{PKG}.style.extract", fromlist=["*"]) check_ir = ir_mod.check Style, StyleError = style_mod.Style, style_mod.StyleError @@ -361,6 +363,208 @@ check( ) +# -------------------------------------------------------------------------- +print("\n3b. harvesting — a binding recovered from diagrams that cannot leave") + + +def _harvesting(): + """Its own scope on purpose: `tmp` out here is the whole run's temp + directory, and a `with TemporaryDirectory() as tmp` would delete it.""" + + # The shape a Graphviz harvest really has, and the reason frequency is the wrong + # signal for the canvas: every label carries a `fill`, so the ink (87) outnumbers + # the background (43) and "most common is the background" inverts the theme. + HARVEST = { + "source": "/tmp/diagrams", + "files_read": 3, + "files_failed": [], + "tokens": { + "fill": [ + {"value": "#1f2933", "count": 87}, + {"value": "#ffffff", "count": 43}, + {"value": "#e8effd", "count": 20}, + {"value": "#2b5fd9", "count": 12}, + {"value": "#1a7f45", "count": 9}, + {"value": "#c0392b", "count": 4}, + ], + "stroke": [{"value": "#9aa5b1", "count": 60}, {"value": "#616e7c", "count": 18}], + "stroke-width": [{"value": "1", "count": 90}, {"value": "4", "count": 3}], + "font-family": [{"value": "Helvetica", "count": 70}], + "font-size": [{"value": "11", "count": 70}, {"value": "14", "count": 6}], + "rx": [{"value": "4", "count": 12}], + }, + } + + harvested = tokens_mod.derive(HARVEST, "harvested") + check( + "a harvest produces a theme, not a style file", + set(harvested) == {"harvested"} and set(harvested["harvested"]) == {"note", "slots"}, + f"got {sorted(harvested['harvested'])} — what counting recovers is the binding, " + "not what a kind should look like", + ) + + slots = harvested["harvested"]["slots"] + check( + "the canvas and the ink are the lightness extremes", + (slots["surface-0"], slots["text"]) == ("#ffffff", "#1f2933"), + f"surface-0={slots['surface-0']} text={slots['text']} — by frequency the ink " + "wins the background and the theme's own text is invisible against it", + ) + check( + "the accents are the colours that are neither canvas nor ink", + {slots["accent"], slots["artery"], slots["atlas"]} <= {"#e8effd", "#2b5fd9", "#1a7f45", "#c0392b"}, + f"accent={slots['accent']} artery={slots['artery']} atlas={slots['atlas']}", + ) + + missing = [s for s in tokens_mod.REQUIRED_SLOTS if s not in slots] + check( + "every slot a style file needs is bound, so a partial harvest still loads", + not missing, + f"unbound: {missing}", + ) + check( + "a readable harvest says nothing", + "_warning" not in slots, + "a warning on a theme that is fine is a warning nobody reads", + ) + + flipped = tokens_mod.derive(HARVEST, "harvested", dark=True)["harvested"]["slots"] + check( + "polarity flips whole — canvas and ink together, never one of them", + (flipped["surface-0"], flipped["text"]) == (slots["text"], slots["surface-0"]), + ) + + # Polarity is the guess most likely to be wrong, so being wrong has to be loud. + DARK_ONLY = json.loads(json.dumps(HARVEST)) + DARK_ONLY["tokens"]["fill"] = [ + {"value": "#1f2933", "count": 87}, + {"value": "#2b3440", "count": 43}, + ] + warned = tokens_mod.derive(DARK_ONLY, "harvested")["harvested"]["slots"] + check( + "a theme nobody could read says so", + "_warning" in warned and "unreadable" in warned["_warning"], + f"got {warned.get('_warning', 'no warning')!r}", + ) + + geom = tokens_mod.geometry(HARVEST) + check( + "widths take the mode, not the mean", + geom["hairline"] == "1", + f"got {geom['hairline']} — the mean of 1 and 4 is 2.5, a width no diagram uses", + ) + check( + "the type scale is derived from the base size, not harvested three times", + (geom["font-size-sm"], geom["font-size-base"], geom["font-size-header"]) + == ("10", "11", "13"), + f"got {geom['font-size-sm']}/{geom['font-size-base']}/{geom['font-size-header']}", + ) + check( + "rounding survives as the binary DOT can express", + geom["rounded"] is True, + "the radius is dropped on purpose — `rounded` has no scalar", + ) + + # The claim the whole split rests on: one set of rules, several bindings. A theme + # recovered from someone else's diagrams has to render through docgen's own rules + # with nothing rewritten. + merged = json.loads((HERE / "style" / "lucid.json").read_text()) + merged["themes"].update(harvested) + try: + harvested_style = Style(merged, theme="harvested", name="harvested") + rendered, why = slots["surface-0"] in dot_mod.emit(ir, harvested_style), "" + except StyleError as e: + rendered, why = False, str(e) + check( + "a harvested theme drops into the shipped rules and renders", + rendered, + why or "if this fails the rules and the binding are not actually separable", + ) + + # -- the two promises extract.py makes to a confidential folder --------------- + # Both are asserted rather than trusted, because both are the reason it is safe + # to point this at diagrams that may not be sent anywhere. + extract_src = ast.parse((HERE / "style" / "extract.py").read_text()) + imported = set() + for node in ast.walk(extract_src): + if isinstance(node, ast.Import): + imported.update(a.name.split(".")[0] for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + NETWORK = {"socket", "ssl", "urllib", "http", "requests", "httpx", "ftplib", + "smtplib", "asyncio", "xmlrpc", "webbrowser"} + check( + "the harvester imports nothing that can open a socket", + not (imported & NETWORK), + f"{sorted(imported & NETWORK)} — a confidential diagram stays off every network path", + ) + check( + "it never reads .text or .tail", + not [n for n in ast.walk(extract_src) + if isinstance(n, ast.Attribute) and n.attr in ("text", "tail")], + "a palette does not need to know what the diagram says", + ) + + try: + import lxml # noqa: F401 + HAS_LXML = True + except ImportError: + HAS_LXML = False + + if not HAS_LXML: + skip("harvest reads real SVG", "lxml is not installed — pip install lxml") + else: + FIXTURE_SVG = ( + '' + '' + '' + '' + '' + f'{MARKER}' + "" + ) + with tempfile.TemporaryDirectory(prefix="docgen-harvest-") as tmp: + folder = Path(tmp) + (folder / "one.svg").write_text(FIXTURE_SVG) + data = extract_mod.harvest(folder) + + blob = json.dumps(data) + check( + "no text from the diagram reaches the output", + MARKER not in blob, + "the fixture's label is in the harvest — rule 1 of extract.py is broken", + ) + + found = {p: {e["value"] for e in data["tokens"][p]} for p in data["tokens"]} + check( + "one spelling per value — #FFF and #ffffff are not two entries", + found["fill"] >= {"#ffffff", "#1a7f45"} and "#fff" not in found["fill"], + f"fills: {sorted(found['fill'])}", + ) + check( + "an inline style beats the presentation attribute, as the spec says", + "#2b5fd9" in found["fill"], + f"fills: {sorted(found['fill'])} — the rect declares both", + ) + check( + "a gradient reference is not a colour", + not any(v.startswith("url(") for v in found["fill"]), + f"fills: {sorted(found['fill'])}", + ) + check( + "a font stack collapses to the font that will actually render", + found["font-family"] == {"Helvetica"}, + f"got {sorted(found['font-family'])}", + ) + check( + "1.00 and 11.00 are tidied to 1 and 11", + found["stroke-width"] == {"1"} and found["font-size"] == {"11"}, + f"widths {sorted(found['stroke-width'])}, sizes {sorted(found['font-size'])}", + ) + + +_harvesting() + # -------------------------------------------------------------------------- print("\n4. emitters") @@ -794,15 +998,20 @@ check( "the server assigns it", ) -# The chain, if modelgen is next door: spec -> IR -> notebook, nothing by hand. +# The chain: spec -> IR -> notebook, nothing by hand. +# +# The fixture is docgen's own, shipped in `fixtures/`, so this no longer skips +# for want of a file when docgen is used standalone. It still skips without +# modelgen, because reading OpenAPI genuinely needs it — that is the one seam +# out of docgen, and `reference.py` is where it resolves. try: oa = __import__(f"{PKG}.extractors.openapi", fromlist=["*"]) - spec = HERE.parent.parent / "station/tools/shuntgen/fixtures/petstore.yaml" + spec = HERE / "fixtures" / "orders.yaml" if not spec.exists(): - raise ImportError("no petstore fixture") + raise ImportError(f"fixture missing: {spec}") real = oa.extract(spec).to_dict() -except (ImportError, Exception) as e: # noqa: BLE001 - skip("openapi -> IR", str(e).splitlines()[0][:60]) +except Exception as e: # noqa: BLE001 - absent modelgen is a skip, not a failure + skip("openapi -> IR", str(e).splitlines()[0][:70]) else: check("a real spec extracts", check_ir(real) == [], str(check_ir(real)[:2])) eps = [n for n in real["nodes"] if n["kind"] == "endpoint"] @@ -812,9 +1021,29 @@ else: {"table", "column"} <= {n["kind"] for n in real["nodes"]}, "so the ER emitter draws an API's data model without knowing it is one", ) + # modelgen maps an inter-schema `$ref` to the literal string `dict`, so the + # target is gone by the time its fields reach docgen. Without recovering it + # an API's data model draws as disconnected cards — three tables, no keys, + # and nothing saying the relationships were lost. Found by writing a fixture + # that has refs; the borrowed one was never checked for this. + fks = {(e["source"], e["target"]) for e in real["edges"] if e["kind"] == "foreign_key"} check( - "and the ER emitter does draw it", - erd_mod.emit(ops_mod.only_kinds(real, {"table"}), lucid).startswith("= len(fks), + f"{svg.count(' {larder_mod.redact(src)}" for src, want in cases + if larder_mod.redact(src) != want] + check("credentials are masked and nothing else is", not wrong, "; ".join(wrong)) + check( + "the secret is gone, the recognisable part is not", + "hunter2" not in larder_mod.redact(cases[0][0]) and "db:5432/shop" in larder_mod.redact(cases[0][0]), + ) + + # The scrubber is graded by a second, independent key list in validate.py. + leaked = dict(ir["meta"], larder={ + "kind": "db", "identity": "postgresql://app:hunter2@db/shop", + "unit": "table", "seen": 1, "read": 1, "failed": [], + }) + problems = ir_validate.check({**ir, "meta": leaked}) + check( + "a secret that survived redaction fails validation", + any("identity still contains" in p for p in problems), + f"got {problems[:2]} — a scrubber graded by its own word is not graded", + ) + # And the arithmetic, because a book can be assembled without the IR. + lying = dict(ir["meta"], larder={ + "kind": "python", "identity": "src", "unit": "file", + "seen": 10, "read": 10, "failed": [{"name": "a.py", "error": "x"}], + }) + check( + "a measure that disagrees with itself fails validation", + any("disagrees with itself" in p for p in ir_validate.check({**ir, "meta": lying})), + ) + + # -- every extractor reports one. The extension contract, with teeth. -- + # Looped over the registry rather than a hand-written list, so adding an + # extractor makes this start asking about it. + check( + "every registered extractor is in the relation table", + set(build_mod.EXTRACTORS) == set(book_mod.RELATION), + f"registry {sorted(build_mod.EXTRACTORS)} vs relations {sorted(book_mod.RELATION)}" + " — a new extractor must say how its units show up in the output", + ) + unmeasured = [] + for kind in sorted(build_mod.EXTRACTORS): + module_name, fn_name, _ = build_mod.EXTRACTORS[kind] + try: + mod = __import__(f"{PKG}{module_name}", fromlist=["*"]) + except ImportError as exc: + unmeasured.append(f"{kind}: not importable ({exc})") + continue + fn = getattr(mod, fn_name, None) + if fn is None: + unmeasured.append(f"{kind}: no {fn_name}()") + elif "identity" not in fn.__code__.co_varnames[:fn.__code__.co_argcount]: + unmeasured.append(f"{kind}: {fn_name}() takes no `identity`") + check( + "every registered extractor can name its larder", + not unmeasured, + "; ".join(unmeasured) + " <- an extractor is done when it measures its input", + ) + + # -- the reconciliation, which is what having both ends is for --------- + b = book_mod.Book("pretend", Larder(kind="python", identity="p", unit="file", seen=10), + "/nonexistent") + b.ir = {"nodes": [{"id": f"m{i}", "kind": "module", "label": "m", + "attrs": {"file": f"m{i}.py"}} for i in range(3)], "edges": []} + lost = [r for r in b.compare() if not r["ok"]] + check( + "input read but not produced is reported as lost", + len(lost) == 1 and lost[0]["id"] == "units-accounted-for", + f"got {[r['id'] for r in b.compare()]}", + ) + check("and it says how much", "7 file(s)" in lost[0]["why"], lost[0]["why"][:70]) + + # A file that failed to parse still gets a node carrying attrs.error, so the + # two populations must be counted apart. Counting them together made + # "2 files read produced 4 modules" pass a check meant to prove nothing was + # dropped — which is how it was written first. + b2 = book_mod.Book("split", Larder(kind="python", identity="p", unit="file", seen=4), + "/nonexistent") + b2.larder.fail("c.py", "syntax"); b2.larder.fail("d.py", "syntax") + b2.ir = {"edges": [], "nodes": [ + {"id": "a", "kind": "module", "label": "a", "attrs": {"file": "a.py"}}, + {"id": "b", "kind": "module", "label": "b", "attrs": {"file": "b.py"}}, + {"id": "c", "kind": "module", "label": "c", "attrs": {"file": "c.py", "error": "syntax"}}, + {"id": "d", "kind": "module", "label": "d", "attrs": {"file": "d.py", "error": "syntax"}}, + ]} + ids = {r["id"]: r for r in b2.compare()} + check( + "read and failed units are reconciled separately", + ids["units-accounted-for"]["claim"] == "2 file(s) read produced 2 module(s)", + f"got {ids['units-accounted-for']['claim']!r}", + ) + check( + "an unreadable unit must still appear in the graph", + ids["failures-still-in-the-graph"]["ok"], + "the whole-input form of `an unresolved name becomes an external node`", + ) + b2.ir["nodes"] = b2.ir["nodes"][:2] # drop the marked ones + check( + "...and its absence is caught", + not {r["id"]: r for r in b2.compare()}["failures-still-in-the-graph"]["ok"], + "a picture smaller than its source, with nothing saying so", + ) + + # -- end to end, on a tree with a file that cannot be parsed ---------- + fixture = Path(tmp.name) / "book-src" + (fixture / "app").mkdir(parents=True, exist_ok=True) + (fixture / "app" / "__init__.py").write_text('"""An app."""\n') + (fixture / "app" / "models.py").write_text('"""Models."""\n\n\nclass User:\n pass\n') + (fixture / "app" / "broken.py").write_text("def oops(:\n") + + out = Path(tmp.name) / "book-out" + built = build_mod.run("python", fixture, out, slug="fixture", quiet=True) + + check("a book writes its ledger", (out / "book.json").exists()) + ledger = json.loads((out / "book.json").read_text()) + step_ids = [s["id"] for s in ledger["steps"]] + check( + "the first and last steps are the two measures", + (step_ids[0], step_ids[-1]) == (book_mod.FIRST, book_mod.LAST), + f"got {step_ids}", + ) + check( + "the larder measured the broken file as failed, by name", + [f["name"] for f in ledger["larder"]["failed"]] == ["app/broken.py"], + f"got {ledger['larder']['failed']}", + ) + check( + "the book measure counts what came out", + ledger["book"]["nodes"] == len(json.loads((out / "steps" / "ir.json").read_text())["nodes"]), + ) + check( + "every intermediate step is still a file on its own", + all((out / s["artifact"]).exists() for s in ledger["steps"] if s.get("artifact")), + "that property is the reason the notebook is a sequence and not one artifact", + ) + + # The notebook is the sequence, so the measures are its first and last cells + # rather than something the tooling knows and the document does not. + nb = json.loads((out / "notebook.ipynb").read_text()) + check( + "the notebook opens with what came in and closes with what came out", + "what came in" in "".join(nb["cells"][0]["source"]) + and "what came out" in "".join(nb["cells"][-1]["source"]), + ) + check( + "it names the file it could not read", + "app/broken.py" in "".join(nb["cells"][0]["source"]), + ) + # A generated notebook is a build artifact and must run where it was built. + # An earlier version imported IPython to *load* an SVG and failed here. + ran, failure = checks_mod._run_cells(nb["cells"], out) + check(f"its {ran} code cells run from the book directory", failure is None, failure or "") + + html = (out / "site" / "index.html").read_text() + check( + "the web output shows both ends", + "what came in" in html and "what came out" in html, + "the site is the last step because it is the artifact somebody opens", + ) + check( + "and admits when the book is incomplete", + "This book is incomplete" in html and "app/broken.py" in html, + "a clean diagram over 2 of 3 files is a lie by omission", + ) + + # Reproducible in the strict sense, for the same reason the IR is: a diff + # has to mean a real change. + again = Path(tmp.name) / "book-out-2" + build_mod.run("python", fixture, again, slug="fixture", quiet=True) + differs = [ + name for name in ("book.json", "notebook.ipynb", "steps/ir.json", "site/index.html") + if (out / name).read_bytes() != (again / name).read_bytes() + ] + check("the same larder builds the same bytes", not differs, f"differ: {differs}") + + # -- the book test level ---------------------------------------------- + loaded = checks_mod.Loaded(out) + report = checks_mod.Report() + import io as _io2 + import contextlib as _ctx2 + with _ctx2.redirect_stdout(_io2.StringIO()): + checks_mod.generated(loaded, report) + check( + f"the generated book checks pass on a real book ({report.passed})", + not report.failed, + f"failed: {report.failed}", + ) + + # Framework and custom living together is the whole point of the level. + (out / "checks.py").write_text( + "def checks(book, check, note, skip):\n" + " note('this project')\n" + " check('models survived', True,\n" + " any(n['id'] == 'app.models' for n in book.ir['nodes']))\n" + " check('wrong on purpose', 9, 0)\n" + ) + report2 = checks_mod.Report() + with _ctx2.redirect_stdout(_io2.StringIO()): + checks_mod.custom(checks_mod.Loaded(out), report2) + check( + "a book's own checks run beside the generated ones", + report2.passed == 1 and report2.failed == ["wrong on purpose"], + f"passed={report2.passed} failed={report2.failed}", + ) + + # -- the architectural rules, still holding --------------------------- + blob = json.dumps(ledger) + # Swept where a visual field could actually land — `larder.extra` is the one + # open bag in the ledger. NOT over the whole document: `by_kind` is keyed by + # IR node kind, and `class` is both a legitimate kind and a member of + # VISUAL_KEYS, so a text sweep reports `"class": 15` as a style leak. The + # first version of this check did exactly that. + leaked_keys = sorted(set(ledger["larder"].get("extra") or {}) & ir_validate.VISUAL_KEYS) + check( + "no visual field reached the ledger's open bag", + not leaked_keys, + f"larder.extra has {leaked_keys} — the ledger records what was read, not how it looks", + ) + node_kinds = {n["kind"] for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]} + check( + "the book measure is keyed by IR kind, nothing else", + set(ledger["book"]["by_kind"]) <= node_kinds, + f"{sorted(set(ledger['book']['by_kind']) - node_kinds)} is in by_kind but is not a kind", + ) + check( + "no timestamp reached the ledger", + "generated_at" not in blob, + "a timestamp makes two builds of an unchanged larder differ", + ) + check( + "provenance is in meta, never in nodes", + not any("larder" in (n.get("attrs") or {}) + for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]), + "the IR is structure; provenance is meta, the same split that keeps colour out", + ) + + +_books() + + + +# -------------------------------------------------------------------------- +print("\n16. standalone — docgen works with nothing above it") + + +def _standalone(): + """docgen is meant to be copied out of the repo and used on its own. + + That claim decays the moment somebody adds a convenient import, and it + decays silently: the suite still passes *inside* the repo. So it is asserted + from the source rather than by copying the folder in CI, which is the same + move as rig's `no host-project references` grep. + """ + ref_mod = __import__(f"{PKG}.reference", fromlist=["*"]) + + # Third-party modules docgen is allowed to reach for, each optional and each + # with a documented reason. Anything else is a new dependency, and a new + # dependency in a folder meant to be copied around is worth noticing. + ALLOWED = { + "tree_sitter", "tree_sitter_c_sharp", "tree_sitter_typescript", + "tree_sitter_python", # extractors/code.py, optional + "lxml", # style/extract.py, optional + "yaml", # extractors/openapi.py, optional + "networkx", # lab/ only + "modelgen", # the one seam — see reference.py + } + # The one module allowed to import `modelgen`, and the one allowed to go + # looking above the package. Two names, so a third is a decision. + SEAM_IMPORTER = "extractors/openapi.py" + SEAM_RESOLVER = "reference.py" + + stdlib = set(sys.stdlib_module_names) + own = {PKG} + files = sorted(p for p in HERE.rglob("*.py") if "__pycache__" not in p.parts) + + foreign, seam_leaks = [], [] + for path in files: + rel = path.relative_to(HERE).as_posix() + try: + tree = ast.parse(path.read_text()) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [a.name.split(".")[0] for a in node.names] + elif isinstance(node, ast.ImportFrom): + if node.level: # relative — inside the package by definition + continue + names = [node.module.split(".")[0]] if node.module else [] + else: + continue + for name in names: + if name in stdlib or name in own: + continue + if name not in ALLOWED: + foreign.append(f"{rel}: {name}") + elif name == "modelgen" and rel != SEAM_IMPORTER: + seam_leaks.append(f"{rel}: {name}") + + check( + "docgen imports the stdlib, itself, and a short allow-list", + not foreign, + "; ".join(foreign) + " <- a new dependency in a folder meant to be copied", + ) + check( + f"only {SEAM_IMPORTER} imports modelgen", + not seam_leaks, + "; ".join(seam_leaks) + " <- one seam is a seam; two is a dependency", + ) + + # The repo path is named in exactly one place, so pointing docgen at a + # reference repo is one change rather than a search. + hardcoded = [] + for path in files: + rel = path.relative_to(HERE).as_posix() + if rel in (SEAM_RESOLVER, "selftest.py"): + continue + text = path.read_text() + for line in text.splitlines(): + if "station/tools" in line and not line.lstrip().startswith("#") \ + and '"""' not in line and "station/tools" not in line.split("#")[-1]: + hardcoded.append(f"{rel}: {line.strip()[:60]}") + check( + f"the repo layout is known only to {SEAM_RESOLVER}", + not hardcoded, + "; ".join(hardcoded), + ) + + # Resolution order is the thing a standalone user depends on, so it is + # asserted rather than described: an explicit path must win over the walk, + # or setting the variable would appear to do nothing inside the repo. + import os as _os + saved = _os.environ.get(ref_mod.ENV_VAR) + try: + fake = Path(tmp.name) / "fake-repo" + (fake / "station" / "tools" / "modelgen").mkdir(parents=True, exist_ok=True) + _os.environ[ref_mod.ENV_VAR] = str(fake) + check( + "an explicit path wins over walking up", + ref_mod.root() == fake, + f"got {ref_mod.root()} — setting the variable inside the repo would do nothing", + ) + check( + "and it says how it resolved, not just whether", + ref_mod.ENV_VAR in ref_mod.describe(), + ref_mod.describe(), + ) + _os.environ.pop(ref_mod.ENV_VAR) + # Both outcomes are correct and which one applies is the fact worth + # printing: inside the repo, no configuration should be needed; copied + # out, there is nothing to find and saying so is the point. Asserting + # the in-repo answer unconditionally failed the moment the folder was + # actually copied out, which is how this was found. + walked = ref_mod.root() + if walked is None: + skip("the in-place case", "standalone — nothing above this folder, as intended") + else: + # What root() promises is a directory holding station/tools — not a + # particular depth above docgen, which is an assumption about where + # somebody chose to put it. + check( + "inside a repo it needs no configuration", + (walked / "station" / "tools").is_dir(), + f"walked to {walked}, which has no station/tools", + ) + finally: + if saved is None: + _os.environ.pop(ref_mod.ENV_VAR, None) + else: + _os.environ[ref_mod.ENV_VAR] = saved + + # Absent is a normal state. Four of the five extractors never touch it. + check( + "a missing reference is reported, not raised at import time", + ref_mod.missing("x", "y").__class__ is ImportError + and ref_mod.ENV_VAR in str(ref_mod.missing("x", "y")), + "the error has to name the variable to set", + ) + + # Everything a book writes has to stay inside the book directory, or a + # standalone user cannot tell what docgen created. + fixture = Path(tmp.name) / "sa-src" + fixture.mkdir(exist_ok=True) + (fixture / "one.py").write_text('"""One."""\n\n\nclass A:\n pass\n') + out = Path(tmp.name) / "sa-book" + build_mod = __import__(f"{PKG}.book.build", fromlist=["*"]) + build_mod.run("python", fixture, out, slug="sa", quiet=True) + strays = [p.name for p in fixture.iterdir() if p.name != "one.py"] + check( + "a book writes only inside its own output directory", + not strays, + f"left {strays} in the source tree", + ) + check( + "the shipped OpenAPI fixture travels with docgen", + (HERE / "fixtures" / "orders.yaml").exists(), + "a test's fixtures belong to the test, not to a sibling project", + ) + + +_standalone() + + print(f"{len(PASS)} passed, {len(FAIL)} failed, {len(SKIP)} skipped") if FAIL: print("\nfailed:")