""" Prove the pipeline, offline, on a tree it builds itself. python3 selftest.py # or: make check No network, nothing installed. The render steps skip with a message when graphviz is absent rather than failing — a machine without `dot` can still check extraction, the IR and the style layer, and saying so is more useful than a red mark about the host. **A check that only ever proves things work is not worth running.** So the negative cases carry equal weight, and the four checks that are the *design* rather than a regression are marked below. If anything ever gets cut, those are the ones to keep: the IR carries no visual field extractors cannot decide appearance an emitter never reads a source file the layering, from the other side style names slots, not colours one colour language, not three ids are stable across runs without it, diffing is noise ## Golden tests go on the IR, never on the SVG Graphviz measures label text with the host's fonts to size nodes, so identical input produces different geometry on a machine with different fontconfig. The IR is deterministic; the SVG is not. Pinning the wrong one gives a suite that fails on someone else's laptop for no reason anyone can act on. """ import ast import json import re import sys import tempfile from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE.parent)) PKG = HERE.name ir_mod = __import__(f"{PKG}.ir", fromlist=["*"]) ir_validate = __import__(f"{PKG}.ir.validate", fromlist=["*"]) py_ex = __import__(f"{PKG}.extractors.python", fromlist=["*"]) style_mod = __import__(f"{PKG}.style", fromlist=["*"]) dot_mod = __import__(f"{PKG}.emitters.dot", fromlist=["*"]) index_mod = __import__(f"{PKG}.emitters.index", fromlist=["*"]) ops_mod = __import__(f"{PKG}.ops", fromlist=["*"]) erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"]) nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"]) site_mod = __import__(f"{PKG}.emitters.site", fromlist=["*"]) mm_mod = __import__(f"{PKG}.emitters.minimap", fromlist=["*"]) exp_mod = __import__(f"{PKG}.emitters.explore", fromlist=["*"]) spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"]) db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"]) usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"]) tokens_mod = __import__(f"{PKG}.style.tokens", fromlist=["*"]) extract_mod = __import__(f"{PKG}.style.extract", fromlist=["*"]) check_ir = ir_mod.check Style, StyleError = style_mod.Style, style_mod.StyleError PASS, FAIL, SKIP = [], [], [] # A docstring that could not plausibly be a style value or a module name, so # finding it somewhere it should not be means something really read it. MARKER = "Quarterly Revenue Ledger" FIXTURE = { "app/__init__.py": '"""The app package."""\nfrom .db import Base\n', "app/db.py": '"""Database plumbing."""\n\n\nclass Base:\n """Declarative base."""\n', "app/models.py": ( '"""Domain models."""\n' "from .db import Base\n" "import json\n" "from third_party.orm import Mixin\n" "\n\n" "class User(Base):\n" f' """{MARKER}."""\n' "\n" " def save(self):\n" " pass\n" "\n\n" "class Admin(User, Mixin):\n" " pass\n" ), "app/sub/deep.py": "from ..db import Base\n\n\nclass Deep(Base):\n class Inner:\n pass\n", "app/broken.py": "def oops(:\n", # must not cost the run } # docgen's own Python — what a source scan should read. Not `HERE.rglob`: once # `make sync` has run, `.venv/` under this folder holds thousands of third-party # modules, and the standalone check reported lxml's own imports as docgen's. NOT_OURS = {"__pycache__", ".venv", "venv", "node_modules", "out"} def own_py_files(): return sorted(p for p in HERE.rglob("*.py") if not NOT_OURS & set(p.relative_to(HERE).parts)) def check(name, condition, detail=""): (PASS if condition else FAIL).append(name) print(f" {'ok ' if condition else 'FAIL'} {name}" + (f"\n {detail}" if detail and not condition else "")) return condition def _err(fn): """The message from a call that is expected to fail.""" try: fn() except Exception as e: # noqa: BLE001 - the message is the assertion return str(e) return "" def skip(name, why): SKIP.append(name) print(f" -- {name} ({why})") def build_tree(root: Path): for rel, text in FIXTURE.items(): p = root / rel p.parent.mkdir(parents=True, exist_ok=True) p.write_text(text) tmp = tempfile.TemporaryDirectory(prefix="docgen-selftest-") ROOT = Path(tmp.name) / "fx" build_tree(ROOT) ir = py_ex.extract(ROOT).to_dict() ids = {n["id"] for n in ir["nodes"]} edges = {(e["source"], e["target"], e["kind"]) for e in ir["edges"]} # -------------------------------------------------------------------------- print("\n1. extraction — is it analysis, or is it guessing") check("the IR validates", check_ir(ir) == [], str(check_ir(ir)[:3])) # The claim the whole thing rests on: `ast` hands over the string "Base", and # two-pass resolution turns it into the id of the thing it actually names. check( "a relative import resolves to the real id", ("app.models.User", "app.db.Base", "inherits") in edges, f"got {sorted(e for e in edges if e[2] == 'inherits')}", ) check( "a relative import two levels up resolves", ("app.sub.deep.Deep", "app.db.Base", "inherits") in edges, ) check( "a base defined in the same module resolves", ("app.models.Admin", "app.models.User", "inherits") in edges, ) check( "a package's own __init__ resolves against itself", ("app", "app.db", "imports") in edges, "from .db inside app/__init__.py must mean app.db, not db", ) check( "no raw source name leaks in as an id", "Base" not in ids and "User" not in ids, f"unresolved names present as bare ids: {sorted(ids & {'Base', 'User'})}", ) # Dropping an edge you cannot resolve is the worse failure: the diagram looks # complete and is quietly missing a dependency. externals = {n["id"] for n in ir["nodes"] if n["kind"] == "external"} check( "an unresolvable name survives as `external`", "third_party.orm.Mixin" in externals, f"got {sorted(externals)}", ) check( "and it keeps its edge", ("app.models.Admin", "third_party.orm.Mixin", "inherits") in edges, ) check( "nested definitions are reached", "app.sub.deep.Deep.Inner" in ids and "app.models.User.save" in ids, "a missing generic_visit() silently loses everything below a definition", ) check( "containment is recorded", next(n for n in ir["nodes"] if n["id"] == "app.models.User.save")["parent"] == "app.models.User", ) # Spans, not just a start line. "This class is 400 lines" is structure, and a # density map sizes a block by it — `ast` carries end_lineno, so recording only # where something begins throws away half the fact for nothing. spans = [n for n in ir["nodes"] if n["kind"] in ("class", "function")] check( "a construct records how many lines it spans", all((n.get("attrs") or {}).get("lines", 0) >= 1 for n in spans), f"missing on {[n['id'] for n in spans if not (n.get('attrs') or {}).get('lines')][:3]}", ) check( "a multi-line class spans more than one", any((n.get("attrs") or {}).get("lines", 0) > 1 for n in spans), ) check( "a module records its length", all((n.get("attrs") or {}).get("lines", 0) >= 1 for n in ir["nodes"] if n["kind"] == "module" and not (n.get("attrs") or {}).get("error")), ) check( "a span is not a visual field", check_ir(ir) == [], "`lines` is a fact about the code, not about how it is drawn", ) check( "source anchors are carried", all( (n.get("attrs") or {}).get("file") for n in ir["nodes"] if n["kind"] in ("class", "function") ), ) broken = [n for n in ir["nodes"] if (n.get("attrs") or {}).get("error")] check( "an unparseable file is recorded, not fatal", len(broken) == 1 and "broken.py" in broken[0]["attrs"]["file"], f"got {[(n['id'], n['attrs'].get('error')) for n in broken]}", ) check("...and the rest of the tree still extracted", len(ids) > 10) # -------------------------------------------------------------------------- print("\n2. the design rules") # (design) Without this the diff emitter reports noise and nobody trusts it. again = py_ex.extract(ROOT).to_dict() check( "ids are stable — the same tree twice is the same bytes", json.dumps(ir, sort_keys=True) == json.dumps(again, sort_keys=True), ) # (design) An extractor that sets a colour decides how the graph looks forever. visual = set() for n in ir["nodes"]: visual |= set(n.get("attrs") or {}) & ir_validate.VISUAL_KEYS for e in ir["edges"]: visual |= set(e.get("attrs") or {}) & ir_validate.VISUAL_KEYS check("no visual field reaches the IR", not visual, f"found {sorted(visual)}") check( "the IR is readable without this package", isinstance(json.loads(json.dumps(ir)), dict), "it must be plain JSON, not something that needs a library to open", ) # (design) The layering, checked from the emitter side. Parsed, not grepped: # a docstring mentioning `ast` is prose, an import is a dependency. offenders = [] for src in sorted((HERE / "emitters").glob("*.py")): tree = ast.parse(src.read_text(), str(src)) for node in ast.walk(tree): names = [] if isinstance(node, ast.Import): names = [a.name for a in node.names] elif isinstance(node, ast.ImportFrom): names = [node.module or ""] for name in names: if name.split(".")[0] in {"ast", "sqlalchemy", "griffe"}: offenders.append(f"{src.name}:{node.lineno} imports {name}") check( "no emitter reads source — it has never heard of Python or SQL", not offenders, "; ".join(offenders), ) extractor_offenders = [] for src in sorted((HERE / "extractors").rglob("*.py")): tree = ast.parse(src.read_text(), str(src)) for node in ast.walk(tree): names = [] if isinstance(node, ast.Import): names = [a.name for a in node.names] elif isinstance(node, ast.ImportFrom): names = [node.module or ""] if any(n and ("emitters" in n or "style" in n) for n in names): extractor_offenders.append(f"{src.name}:{node.lineno}") check( "no extractor knows about style or emitters", not extractor_offenders, "; ".join(extractor_offenders), ) # -------------------------------------------------------------------------- print("\n3. style — one colour language") styles = Style.available() check("a style ships", "lucid" in styles, f"found {styles}") lucid = Style.load("lucid") # the default: dark print_theme = Style.load("lucid", theme="lucid") # the print palette check("it carries more than one theme", set(lucid.themes()) >= {"lucid", "dark"}) check( "an unknown kind falls back rather than crashing", lucid.node("no-such-kind-at-all") == lucid.node("default") != {}, ) # (design) A rule naming a hex is a rule outside the shared vocabulary. raw = json.loads((HERE / "style" / "lucid.json").read_text()) leaked = [] for section in ("nodes", "groups", "edges", "graph"): for kind, rule in (raw.get(section) or {}).items(): if not isinstance(rule, dict): continue for key, value in rule.items(): if isinstance(value, str) and re.fullmatch(r"#[0-9A-Fa-f]{3,8}", value): leaked.append(f"{section}.{kind}.{key}={value}") check( "style rules name slots, never colours", not leaked, "; ".join(leaked) + " <- the colour language has stopped being one language", ) # A theme that defines only some of the slots renders half a diagram in the # right colours and the rest in whatever DOT does with an empty string. half_bound = [] for theme in lucid.themes(): try: Style.load("lucid", theme=theme) except StyleError as e: half_bound.append(f"{theme}: {e}") check("every slot resolves in every theme", not half_bound, "; ".join(half_bound)) # The point of binding to spr's models: a diagram and the page around it must # not drift apart. docs/graphs/README.md states the rule. SYSTEM_ACCENTS = { "artery": ("artery/index.html", "#b91c1c"), "atlas": ("atlas/index.html", "#15803d"), "station": ("station/index.html", "#1d4ed8"), } spr_root = HERE.parent.parent mismatched = [] for slot, (page, expected) in SYSTEM_ACCENTS.items(): got = lucid.slot(slot).lower() if got != expected: mismatched.append(f"{slot}: style has {got}, {page} sets {expected}") src = spr_root / page if src.exists() and expected not in src.read_text(): mismatched.append(f"{slot}: {page} no longer sets {expected}") check( "the dark theme's slots are spr's own --system-accent values", not mismatched, "; ".join(mismatched), ) check( "domain -> slot is a mapping, not a colour choice", lucid.domain_slot("atlas") == "atlas" and lucid.domain_slot("station") == "station", ) check( "an undomained group still gets a deterministic slot", [lucid.domain_slot(None, i) for i in range(4)] == [lucid.domain_slot(None, i) for i in range(4)], ) check( "the style records where DOT runs out", {"header-bar", "dasharray", "sequence-badge"} <= set(lucid.limits()), f"got {sorted(lucid.limits())}", ) # -------------------------------------------------------------------------- print("\n3b. harvesting — a binding recovered from diagrams that cannot leave") def _harvesting(): """Its own scope on purpose: `tmp` out here is the whole run's temp directory, and a `with TemporaryDirectory() as tmp` would delete it.""" # The shape a Graphviz harvest really has, and the reason frequency is the wrong # signal for the canvas: every label carries a `fill`, so the ink (87) outnumbers # the background (43) and "most common is the background" inverts the theme. HARVEST = { "source": "/tmp/diagrams", "files_read": 3, "files_failed": [], "tokens": { "fill": [ {"value": "#1f2933", "count": 87}, {"value": "#ffffff", "count": 43}, {"value": "#e8effd", "count": 20}, {"value": "#2b5fd9", "count": 12}, {"value": "#1a7f45", "count": 9}, {"value": "#c0392b", "count": 4}, ], "stroke": [{"value": "#9aa5b1", "count": 60}, {"value": "#616e7c", "count": 18}], "stroke-width": [{"value": "1", "count": 90}, {"value": "4", "count": 3}], "font-family": [{"value": "Helvetica", "count": 70}], "font-size": [{"value": "11", "count": 70}, {"value": "14", "count": 6}], "rx": [{"value": "4", "count": 12}], }, } harvested = tokens_mod.derive(HARVEST, "harvested") check( "a harvest produces a theme, not a style file", set(harvested) == {"harvested"} and set(harvested["harvested"]) == {"note", "slots"}, f"got {sorted(harvested['harvested'])} — what counting recovers is the binding, " "not what a kind should look like", ) slots = harvested["harvested"]["slots"] check( "the canvas and the ink are the lightness extremes", (slots["surface-0"], slots["text"]) == ("#ffffff", "#1f2933"), f"surface-0={slots['surface-0']} text={slots['text']} — by frequency the ink " "wins the background and the theme's own text is invisible against it", ) check( "the accents are the colours that are neither canvas nor ink", {slots["accent"], slots["artery"], slots["atlas"]} <= {"#e8effd", "#2b5fd9", "#1a7f45", "#c0392b"}, f"accent={slots['accent']} artery={slots['artery']} atlas={slots['atlas']}", ) missing = [s for s in tokens_mod.REQUIRED_SLOTS if s not in slots] check( "every slot a style file needs is bound, so a partial harvest still loads", not missing, f"unbound: {missing}", ) check( "a readable harvest says nothing", "_warning" not in slots, "a warning on a theme that is fine is a warning nobody reads", ) flipped = tokens_mod.derive(HARVEST, "harvested", dark=True)["harvested"]["slots"] check( "polarity flips whole — canvas and ink together, never one of them", (flipped["surface-0"], flipped["text"]) == (slots["text"], slots["surface-0"]), ) # Polarity is the guess most likely to be wrong, so being wrong has to be loud. DARK_ONLY = json.loads(json.dumps(HARVEST)) DARK_ONLY["tokens"]["fill"] = [ {"value": "#1f2933", "count": 87}, {"value": "#2b3440", "count": 43}, ] warned = tokens_mod.derive(DARK_ONLY, "harvested")["harvested"]["slots"] check( "a theme nobody could read says so", "_warning" in warned and "unreadable" in warned["_warning"], f"got {warned.get('_warning', 'no warning')!r}", ) geom = tokens_mod.geometry(HARVEST) check( "widths take the mode, not the mean", geom["hairline"] == "1", f"got {geom['hairline']} — the mean of 1 and 4 is 2.5, a width no diagram uses", ) check( "the type scale is derived from the base size, not harvested three times", (geom["font-size-sm"], geom["font-size-base"], geom["font-size-header"]) == ("10", "11", "13"), f"got {geom['font-size-sm']}/{geom['font-size-base']}/{geom['font-size-header']}", ) check( "rounding survives as the binary DOT can express", geom["rounded"] is True, "the radius is dropped on purpose — `rounded` has no scalar", ) # The claim the whole split rests on: one set of rules, several bindings. A theme # recovered from someone else's diagrams has to render through docgen's own rules # with nothing rewritten. merged = json.loads((HERE / "style" / "lucid.json").read_text()) merged["themes"].update(harvested) try: harvested_style = Style(merged, theme="harvested", name="harvested") rendered, why = slots["surface-0"] in dot_mod.emit(ir, harvested_style), "" except StyleError as e: rendered, why = False, str(e) check( "a harvested theme drops into the shipped rules and renders", rendered, why or "if this fails the rules and the binding are not actually separable", ) # -- the two promises extract.py makes to a confidential folder --------------- # Both are asserted rather than trusted, because both are the reason it is safe # to point this at diagrams that may not be sent anywhere. extract_src = ast.parse((HERE / "style" / "extract.py").read_text()) imported = set() for node in ast.walk(extract_src): if isinstance(node, ast.Import): imported.update(a.name.split(".")[0] for a in node.names) elif isinstance(node, ast.ImportFrom) and node.module: imported.add(node.module.split(".")[0]) NETWORK = {"socket", "ssl", "urllib", "http", "requests", "httpx", "ftplib", "smtplib", "asyncio", "xmlrpc", "webbrowser"} check( "the harvester imports nothing that can open a socket", not (imported & NETWORK), f"{sorted(imported & NETWORK)} — a confidential diagram stays off every network path", ) check( "it never reads .text or .tail", not [n for n in ast.walk(extract_src) if isinstance(n, ast.Attribute) and n.attr in ("text", "tail")], "a palette does not need to know what the diagram says", ) try: import lxml # noqa: F401 HAS_LXML = True except ImportError: HAS_LXML = False if not HAS_LXML: skip("harvest reads real SVG", "lxml is not installed — pip install lxml") else: FIXTURE_SVG = ( '' '' '' '' '' f'{MARKER}' "" ) with tempfile.TemporaryDirectory(prefix="docgen-harvest-") as tmp: folder = Path(tmp) (folder / "one.svg").write_text(FIXTURE_SVG) data = extract_mod.harvest(folder) blob = json.dumps(data) check( "no text from the diagram reaches the output", MARKER not in blob, "the fixture's label is in the harvest — rule 1 of extract.py is broken", ) found = {p: {e["value"] for e in data["tokens"][p]} for p in data["tokens"]} check( "one spelling per value — #FFF and #ffffff are not two entries", found["fill"] >= {"#ffffff", "#1a7f45"} and "#fff" not in found["fill"], f"fills: {sorted(found['fill'])}", ) check( "an inline style beats the presentation attribute, as the spec says", "#2b5fd9" in found["fill"], f"fills: {sorted(found['fill'])} — the rect declares both", ) check( "a gradient reference is not a colour", not any(v.startswith("url(") for v in found["fill"]), f"fills: {sorted(found['fill'])}", ) check( "a font stack collapses to the font that will actually render", found["font-family"] == {"Helvetica"}, f"got {sorted(found['font-family'])}", ) check( "1.00 and 11.00 are tidied to 1 and 11", found["stroke-width"] == {"1"} and found["font-size"] == {"11"}, f"widths {sorted(found['stroke-width'])}, sizes {sorted(found['font-size'])}", ) _harvesting() # -------------------------------------------------------------------------- print("\n4. emitters") dot_text = dot_mod.emit(ir, lucid) check("DOT is produced", dot_text.startswith("digraph ir {")) check( "every IR edge reaches the DOT", dot_text.count("->") == len(ir["edges"]), f"{dot_text.count('->')} of {len(ir['edges'])} — cluster endpoints need lhead/ltail", ) check("the profile's colours are applied", lucid.node("class")["border"] in dot_text) check( "ids and kinds reach the output for a front end to bind to", 'id="app.models.User"' in dot_text and 'class="class"' in dot_text, ) check("source anchors become links", "href=" in dot_text) print_text = dot_mod.emit(ir, print_theme) check( "one IR, two looks, no re-extraction", print_text != dot_text and print_theme.slot("surface-0") in print_text, ) md = index_mod.to_markdown(ir) check("the index names what exists", "app.models" in md or "models" in md) check("the index carries the docstrings", MARKER in md) check( "the index reports the dependency surface", "third_party.orm.Mixin" in md, "externals are the thing a reader most often wants and a diagram scatters", ) check("the index reports what could not be parsed", "Not parsed" in md) sidebar = index_mod.to_sidebar(ir) check("the sidebar is nested, not flat", any(i.get("children") for i in sidebar["items"])) # -------------------------------------------------------------------------- print("\n5. views") no_std = ops_mod.drop_stdlib(ir) check( "dropping the stdlib removes json and keeps third-party", "json" not in {n["id"] for n in no_std["nodes"]} and "third_party.orm.Mixin" in {n["id"] for n in no_std["nodes"]}, ) check("a view still validates", check_ir(no_std) == [], str(check_ir(no_std)[:3])) classes = ops_mod.only_kinds(ir, {"class"}) check( "keeping one kind keeps its ancestors, so containment survives", check_ir(classes) == [] and "app.models" in {n["id"] for n in classes["nodes"]}, ) around = ops_mod.neighbourhood(ir, "app.db.Base", hops=1) check( "a neighbourhood keeps what points at it", "app.models.User" in {n["id"] for n in around["nodes"]} and check_ir(around) == [], ) shallow = ops_mod.collapse_to_depth(ir, 1) check( "collapsing to a depth drops what is inside", "app.models.User.save" not in {n["id"] for n in shallow["nodes"]} and check_ir(shallow) == [], ) # The difference between a simpler picture and a wrong one. `app.models.User` # inherits from `app.db.Base`; look at the module level and that is a dependency # of app.models on app.db. Dropping the edge says they are unrelated. mods = ops_mod.only_kinds(ir, {"module"}) mod_edges = {(e["source"], e["target"], e["kind"]) for e in mods["edges"]} check( "an edge is lifted to the surviving node, not dropped", ("app.models", "app.db", "inherits") in mod_edges, f"got {sorted(mod_edges)}", ) check( "a lifted edge says how many it stands for", any((e.get("attrs") or {}).get("weight") for e in mods["edges"]), ) check( "lifting never leaves a self-loop", not [e for e in mods["edges"] if e["source"] == e["target"]], "both ends inside one node is a fact about that node, not a line", ) check( "a label that no longer applies is dropped with the lift", not any( (e.get("attrs") or {}).get("label") for e in mods["edges"] if (e.get("attrs") or {}).get("weight") ), ) # The default view: what you get without knowing which filters to ask for. ov = ops_mod.overview(ir) ov_kinds = {n["kind"] for n in ov["nodes"]} check( "the python overview is modules and what is outside", ov_kinds <= {"module", "external"} and check_ir(ov) == [], f"got {sorted(ov_kinds)}", ) check( "it drops the stdlib but keeps third-party", "json" not in {n["id"] for n in ov["nodes"]} and "third_party.orm.Mixin" in {n["id"] for n in ov["nodes"]}, ) # Depth is the tempting knob and the wrong one: a directory without # `__init__.py` is not a package, so its modules have no parent and sit at # depth 0. soleprint has 173 such roots. fragmented = ops_mod.overview( {"meta": {"source": "python", "root": "x", "schema_version": "1"}, "nodes": [{"id": "loose", "kind": "module", "label": "loose", "parent": None, "attrs": {}}, {"id": "loose.C", "kind": "class", "label": "C", "parent": "loose", "attrs": {}}], "edges": []} ) check( "a parentless module still reduces to its module", [n["id"] for n in fragmented["nodes"]] == ["loose"], "depth-based collapsing would have kept the class; kind-based does not care", ) # -------------------------------------------------------------------------- print("\n5b. shape — knowing the picture before rendering it") # Aspect ratio is a property of the graph, not of the renderer. A layered # engine puts one dependency level in one row, so the widest level is the # width. Measured on soleprint: 7 levels x 109 nodes rendered 14:1, and no # layout flag helped — LR merely rotated it (1818x11183), ratio=compress # squashed it to an unreadable 1008x75. wide = { "meta": {"source": "python", "root": "x", "schema_version": "1"}, "nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}} for i in range(40)] + [{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}], "edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}} for i in range(40)], } sh = ops_mod.shape(wide) check("shape counts the widest level", sh["widest_level"] == 40, str(sh)) check("shape counts levels", sh["levels"] == 2, str(sh)) check("shape finds isolated nodes", ops_mod.shape(ir)["isolated"] >= 0) check( "shape needs no renderer", isinstance(sh["nodes"], int) and "width" not in sh, "it is computed from the IR, so it can warn before writing a 375 KB image", ) parts = ops_mod.split(ir) check("split produces one document per subsystem", len(parts) >= 1, str(list(parts))) check("each part still validates", all(check_ir(p) == [] for p in parts.values())) check( "a crossing edge is kept on the side that reaches out", any(p["edges"] for p in parts.values()), "dropping it would understate the coupling", ) # -------------------------------------------------------------------------- print("\n6. render") if not dot_mod.have_graphviz(): skip("render", "graphviz not installed — sudo apt install graphviz") else: svg = dot_mod.render(dot_text) check("SVG comes out", svg.startswith(b"" in svg) check("the SVG is addressable", b'id="app.models.User"' in svg) check( "kinds survive as classes for CSS to reach", b'class="node class"' in svg and b'class="edge inherits"' in svg, ) print_svg = dot_mod.render(print_text) check("the two themes really differ", print_svg != svg) # `fontname="Arial Bold"` passes through as a family no browser has, so it # renders neither Arial nor bold. Only the PostScript spelling produces a # real weight — and a correctly measured box to hold it. check( "labels are actually bold, not nominally bold", b'font-weight="bold"' in svg, "no font-weight in the SVG — the face name did not map to a weight", ) check( "the font resolves to a real stack", b'font-family="Helvetica,sans-Serif"' in svg, ) # -------------------------------------------------------------------------- print("\n7. a second domain — the schema checkpoint") # The brief makes this a checkpoint: if tables and foreign keys need the schema # contorted, the schema is wrong and must be fixed before more extractors. SCHEMA = { "models": { "Customer": {"doc": "A buyer.", "fields": {"id": {"type": "int", "pk": True}, "email": {"type": "str"}}}, "Invoice": {"fields": {"id": {"type": "int", "pk": True}, "customer_id": {"type": "FK:Customer"}, "due_at": {"type": "datetime", "nullable": True}}}, "Tag": {"fields": {"id": {"type": "int", "pk": True}, "invoices": {"type": "M2M:Invoice"}}}, "Orphan": {"fields": {"ref": {"type": "FK:NotDefinedHere"}}}, } } db_ir = db_ex.from_schema_dict(SCHEMA, root="fixture").to_dict() db_ids = {n["id"] for n in db_ir["nodes"]} db_edges = {(e["source"], e["target"], e["kind"]) for e in db_ir["edges"]} db_kinds = {n["kind"] for n in db_ir["nodes"]} check("the db IR validates", check_ir(db_ir) == [], str(check_ir(db_ir)[:3])) check( "it needed no new top-level field", set(db_ir) == set(ir), f"{sorted(set(db_ir) ^ set(ir))} — the schema did not survive the second domain", ) check("tables and columns are kinds, not fields", db_kinds >= {"table", "column"}) check( "a column is contained by its table", next(n for n in db_ir["nodes"] if n["id"] == "Invoice.customer_id")["parent"] == "Invoice", ) check("a foreign key is an edge", ("Invoice", "Customer", "foreign_key") in db_edges) check("a many-to-many is a different edge kind", ("Tag", "Invoice", "references") in db_edges) check( "domain detail lives in attrs, not in new columns", next(n for n in db_ir["nodes"] if n["id"] == "Customer.id")["attrs"].get("pk") is True, ) check( "a key naming a table the schema does not define survives as external", "NotDefinedHere" in db_ids and next(n for n in db_ir["nodes"] if n["id"] == "NotDefinedHere")["kind"] == "external", ) # "Done means: adding a source type costs one adapter and every existing emitter # works on it unchanged." This is that claim, executed. db_md = index_mod.to_markdown(db_ir) check("the index emitter needed no change for a new domain", "Customer" in db_md) db_dot = dot_mod.emit(db_ir, lucid) check("the dot emitter needed no change either", db_dot.startswith("digraph ir {")) check( "an unstyled kind still renders, via `default`", lucid.node("table") == lucid.node("default"), "no `table` rule exists yet, and the diagram is still legible", ) check( "the same view filters both domains", check_ir(ops_mod.only_kinds(db_ir, {"table"})) == [], ) # -------------------------------------------------------------------------- print("\n8. the structure picks the drawing") # The lesson this encodes: a diagram that fights its layout engine is usually # the wrong kind of diagram. The same 24-table schema rendered 32034x136 through # `dot` (235:1) and 1740x1860 through `erd` (0.9:1) — not because one engine is # better, but because a schema is peer entities with references, and drawing it # in dependency ranks was never its shape. verdict_db = ops_mod.classify(db_ir) check("a schema is classified as entities", verdict_db["kind"] == "erd", str(verdict_db)) check("...and routed to the card emitter", verdict_db["emitter"] == "erd") verdict_code = ops_mod.classify(ops_mod.overview(ir)) check( "a small module graph still goes to ranks", verdict_code["emitter"] == "dot", str(verdict_code), ) sheet = { "meta": {"source": "python", "root": "x", "schema_version": "1"}, "nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}} for i in range(40)] + [{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}], "edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}} for i in range(40)], } verdict_sheet = ops_mod.classify(sheet) check( "a 40-wide level is called a sheet, not a diagram", verdict_sheet["kind"] == "sheet" and verdict_sheet["emitter"] == "index", str(verdict_sheet), ) check("every verdict explains itself", all( ops_mod.classify(g)["why"] for g in (db_ir, sheet, ir) ), "advice without a reason gets overridden") # The property that makes the card layout work, and the one dot cannot offer. svg = erd_mod.emit(db_ir, lucid) import re as _re m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg) w, h = int(m.group(1)), int(m.group(2)) check( f"the ER layout stays near-square ({w}x{h})", 0.25 < w / h < 4, "sqrt(n) columns should bound the aspect ratio whatever the table count", ) wide_schema = { "meta": {"source": "db", "root": "x", "schema_version": "1"}, "nodes": [{"id": f"t{i}", "kind": "table", "label": f"t{i}", "parent": None, "attrs": {}} for i in range(60)] + [{"id": f"t{i}.c", "kind": "column", "label": "c", "parent": f"t{i}", "attrs": {"type": "int"}} for i in range(60)], "edges": [], } svg60 = erd_mod.emit(wide_schema, lucid) m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg60) w60, h60 = int(m.group(1)), int(m.group(2)) check( f"...and still does at 60 tables ({w60}x{h60})", 0.25 < w60 / h60 < 4, "this is the whole reason it is not a rank-based layout", ) check( "the ER geometry is deterministic", erd_mod.emit(db_ir, lucid) == svg, "nothing here measures a font, so it renders identically on any machine", ) check( "it draws no tables only when there are none", "no tables" in ( _err(lambda: erd_mod.emit(ops_mod.overview(ir), lucid)) ), "pointing it at a code graph should say so, not emit an empty canvas", ) # -------------------------------------------------------------------------- print("\n9. the notebook is a build artifact, not a source file") # The whole argument: a hand-confected notebook drifts from whatever it # documents and there is no way to tell by looking. Generated from the spec the # server is built from, "is this current" becomes a question about the build. SPEC_IR = { "meta": {"source": "openapi", "root": "petstore.yaml", "schema_version": "1"}, "nodes": [ {"id": "Pet", "kind": "table", "label": "Pet", "parent": None, "attrs": {}}, {"id": "Pet.id", "kind": "column", "label": "id", "parent": "Pet", "attrs": {"type": "int", "pk": True}}, {"id": "Pet.name", "kind": "column", "label": "name", "parent": "Pet", "attrs": {"type": "str"}}, {"id": "Pet.neutered", "kind": "column", "label": "neutered", "parent": "Pet", "attrs": {"type": "bool"}}, {"id": "GET /pets", "kind": "endpoint", "label": "GET /pets", "parent": None, "attrs": {"method": "GET", "path": "/pets", "summary": "List pets", "status": 200, "response_model": "Pet", "returns_list": True}}, {"id": "POST /pets", "kind": "endpoint", "label": "POST /pets", "parent": None, "attrs": {"method": "POST", "path": "/pets", "summary": "Create a pet", "status": 201, "request_model": "Pet", "response_model": "Pet"}}, ], "edges": [{"source": "POST /pets", "target": "Pet", "kind": "accepts", "attrs": {}}], } check("the spec IR validates", check_ir(SPEC_IR) == [], str(check_ir(SPEC_IR)[:2])) base = spec_mod.from_ir(SPEC_IR) book_spec, _ = spec_mod.merge(base, None) book = nb_mod.build(book_spec) text = nb_mod.emit(book_spec) check("it is nbformat 4", book["nbformat"] == 4 and book["nbformat_minor"] == 5) check( "regenerating gives byte-identical bytes", nb_mod.emit(book_spec) == text, "a notebook that changes every build cannot be reviewed", ) check("nothing has run in it", all( c["outputs"] == [] and c["execution_count"] is None for c in book["cells"] if c["cell_type"] == "code" )) intro = "".join(book["cells"][0]["source"]).lower() check( "it says not to edit it, and where to put changes instead", "not be edited" in intro and "overlay" in intro, f"the frame only holds if the artifact states it — got: {intro[:120]!r}", ) body = "".join("".join(c["source"]) for c in book["cells"]) check("every endpoint in the spec has a section", "GET `/pets`" in body and "POST `/pets`" in body) check("the facts come from the spec", "returns **Pet**" in body and "`201`" in body) check( "no credential is written into it", "Bearer sk-" not in body and "password" not in body.lower(), ) check( "it needs nothing installed", "pip install" not in body and "import requests" not in body, ) # Compiling is not enough. `json.dumps` writes `false`/`true`/`null`, which are # valid *identifiers* in Python — a generated body full of them compiles and # then raises NameError on the first run. This is the check that caught it. import io as _io import contextlib as _ctx ns, ran, failure = {}, 0, None with _ctx.redirect_stdout(_io.StringIO()): for c in book["cells"]: if c["cell_type"] != "code": continue src = "".join(c["source"]) if "urlopen" in src or "call(" in src and "def call" not in src: # Anything that would reach the network is compiled, not run. try: compile(src, c["id"], "exec") ran += 1 except SyntaxError as e: failure = f"{c['id']}: {e}" break continue try: exec(compile(src, c["id"], "exec"), ns) ran += 1 except Exception as e: # noqa: BLE001 - any failure is the finding failure = f"{c['id']}: {type(e).__name__}: {e}" break check(f"its cells run, not merely compile ({ran})", failure is None, failure or "") literal = next( ("".join(c["source"]) for c in book["cells"] if c["cell_type"] == "code" and "BODY" in "".join(c["source"])), "" ) check( "a generated body is a Python literal, not JSON", "False" in literal and "false" not in literal, f"got: {literal[:120]}", ) check( "the primary key is left out of a create body", "'id'" not in literal, "the server assigns it", ) # The chain: spec -> IR -> notebook, nothing by hand. # # The fixture is docgen's own, shipped in `fixtures/`, so this no longer skips # for want of a file when docgen is used standalone. It still skips without # modelgen, because reading OpenAPI genuinely needs it — that is the one seam # out of docgen, and `reference.py` is where it resolves. try: oa = __import__(f"{PKG}.extractors.openapi", fromlist=["*"]) spec = HERE / "fixtures" / "orders.yaml" if not spec.exists(): raise ImportError(f"fixture missing: {spec}") real = oa.extract(spec).to_dict() except Exception as e: # noqa: BLE001 - absent modelgen is a skip, not a failure skip("openapi -> IR", str(e).splitlines()[0][:70]) else: check("a real spec extracts", check_ir(real) == [], str(check_ir(real)[:2])) eps = [n for n in real["nodes"] if n["kind"] == "endpoint"] check(f"its endpoints become nodes ({len(eps)})", len(eps) >= 3) check( "its schemas reuse the db vocabulary", {"table", "column"} <= {n["kind"] for n in real["nodes"]}, "so the ER emitter draws an API's data model without knowing it is one", ) # modelgen maps an inter-schema `$ref` to the literal string `dict`, so the # target is gone by the time its fields reach docgen. Without recovering it # an API's data model draws as disconnected cards — three tables, no keys, # and nothing saying the relationships were lost. Found by writing a fixture # that has refs; the borrowed one was never checked for this. fks = {(e["source"], e["target"]) for e in real["edges"] if e["kind"] == "foreign_key"} check( "a $ref between schemas survives as a foreign key", fks == {("Order", "Customer"), ("OrderLine", "Order")}, f"got {sorted(fks)} — an ERD of an API with no edges is the failure this tool " "exists to prevent, one level up", ) check( "and the referencing column says what it points at", [n["attrs"]["references"] for n in real["nodes"] if n["id"] == "Order.customer"] == ["Customer"], ) svg = erd_mod.emit(real, lucid) check("and the ER emitter does draw it", svg.startswith("= len(fks), f"{svg.count(' 1) bare, drift3 = spec_mod.merge(spec_mod.from_ir(SPEC_IR), None) check( "extraction works with the overlay absent", drift3 == [] and len(bare["steps"]) > 1, "the overlay is an addition, never a dependency", ) blank = spec_mod.scaffold(spec_mod.from_ir(SPEC_IR)) check( "a scaffold hands over every step id", set(blank["steps"]) == {st["id"] for st in spec_mod.from_ir(SPEC_IR)["steps"]}, "listing the ids is the difference between an overlay being written and meant to be", ) check( "a blank scaffold changes nothing", nb_mod.emit(spec_mod.merge(spec_mod.from_ir(SPEC_IR), blank)[0]) == nb_mod.emit(bare), ) # -------------------------------------------------------------------------- print("\n10. defaults") check( "dark is the default theme", Style.load("lucid").theme == "dark", "a diagram lands in a dark docs page far more often than in print", ) check("print is one flag away", Style.load("lucid", theme="lucid").theme == "lucid") pipeline = { "meta": {"source": "airflow", "root": "etl", "schema_version": "1"}, "nodes": [{"id": x, "kind": "task", "label": x, "parent": None, "attrs": {}} for x in ("extract", "transform", "load", "notify")], "edges": [{"source": a, "target": b, "kind": "depends", "attrs": {}} for a, b in (("extract", "transform"), ("transform", "load"), ("load", "notify"))], } verdict = ops_mod.classify(pipeline) check( "a DAG is recognised as a pipeline", verdict["kind"] == "pipeline", str(verdict), ) check( "...and is drawn left to right", verdict.get("options", {}).get("rankdir") == "LR", "a sequence reads across, which is how every scheduler's own UI draws it", ) check( "a task has its own style, not a module's", lucid.node("task") != lucid.node("module"), "a schedule should not render as if it were code structure", ) check( "the emitter honours the rankdir it is given", "rankdir=LR" in dot_mod.emit(pipeline, lucid, rankdir="LR"), ) check("every verdict carries options", "options" in ops_mod.classify(ir)) # -------------------------------------------------------------------------- print("\n11. the docs site") # Not invented here: five demos under semester/ carry the same viewer, differing # only in comments and one background colour. This generates that arrangement # rather than a sixth copy — and adds the 1:1 toggle none of them has. site_dir = ROOT.parent / "site" files = site_mod.write(ops_mod.overview(ir), lucid, site_dir, graph="graph.svg", title="fx") names = {f.name for f in files} check("it writes the three files", names == {"index.html", "viewer.html", "site.css"}, str(names)) index_html = (site_dir / "index.html").read_text() css = (site_dir / "site.css").read_text() viewer = (site_dir / "viewer.html").read_text() check( "the graph links to the viewer, the way both demos already do it", 'href="viewer.html?src=graph.svg"' in index_html and "" in index_html) check( "every sidebar link is styled", ".sidebar a {" in css, "selecting `li > a` misses links inside a , which is most of them", ) check( "colours are baked from the theme, not hardcoded", lucid.slot("surface-0") in css and lucid.slot("accent") in css, ) light = site_mod.emit(ops_mod.overview(ir), print_theme, graph="graph.svg") check( "a light theme gives a light page", print_theme.slot("surface-0") in light["site.css"] and lucid.slot("surface-0") not in light["site.css"], "the page and the diagram on it move together", ) check( "it is self-contained and offline", "http://" not in index_html and "https://" not in index_html and "cdn" not in index_html.lower(), ) check("the viewer carries a 1:1 toggle", "1:1" in viewer and "fitScale" in viewer) # The viewer is JavaScript, so it is tested as JavaScript. import shutil as _shutil import subprocess as _sub if not _shutil.which("node"): skip("viewer behaviour", "node not installed") else: harness = HERE / "viewer_test.js" if not harness.exists(): skip("viewer behaviour", "viewer_test.js missing") else: proc = _sub.run(["node", str(harness), str(site_dir / "viewer.html")], capture_output=True, text=True) for line in proc.stdout.strip().splitlines(): name = line.strip()[5:] (PASS if line.strip().startswith("ok") else FAIL).append(f"viewer: {name}") print(f" {line.strip()[:4]} viewer: {name}") if proc.returncode and not proc.stdout.strip(): check("the viewer harness runs", False, proc.stderr.strip()[:200]) _shutil.rmtree(site_dir, ignore_errors=True) # -------------------------------------------------------------------------- print("\n12. the minimap — what is where, read from the colours") mm = mm_mod.emit(ir, lucid, scale=1.0) check("it produces SVG", mm.startswith("" in mm) check( "a block is sized by the lines it occupies", mm_mod._files(ir)[0]["total"] > 1 and all(f["total"] >= 1 for f in mm_mod._files(ir)), ) check( "no text is rendered inside a block", " mm.count(" CountAsync(); }\n' '\n' ' public class Repo : IRepo\n' ' {\n' ' public async Task CountAsync()\n' ' {\n' ' var sql = "SELECT 1 WHERE x = {0}";\n' ' return 1;\n' ' }\n' '\n' ' private enum Mode { A, B }\n' ' }\n' '}\n' ) got = code_ex.extract(cs).to_dict() kinds = {n["id"].rsplit(".", 1)[-1]: n["kind"] for n in got["nodes"]} check("the C# IR validates", check_ir(got) == [], str(check_ir(got)[:2])) check("a class is a class", kinds.get("Repo") == "class", str(kinds)) check( "an interface is kept apart from a class", kinds.get("IRepo") == "interface", "in C# and TypeScript that distinction is most of what a file tells you", ) check("a method is a function", kinds.get("CountAsync") == "function") iface_method = next(n for n in got["nodes"] if n["id"].endswith(".IRepo.CountAsync")) check( "an interface's one-line method is one line", iface_method["attrs"]["lines"] == 1, "and is a different node from the class's implementation of it", ) check("a nested enum is found", kinds.get("Mode") == "class") # `.endswith("Repo.CountAsync")` also matches `IRepo.CountAsync`, whose # one-line declaration is correct — so name the class's method exactly. method = next(n for n in got["nodes"] if n["id"].endswith(".Repo.CountAsync")) check( "a brace inside a string does not end a block", method["attrs"]["lines"] >= 4, f'got {method["attrs"]["lines"]} lines — a scanner would stop at the brace in the string', ) check( "structure only — no edges are invented", got["edges"] == [], "resolving a `using` is a different job, and half a graph looks whole", ) check("the minimap draws it", mm_mod.emit(got, lucid).startswith("= 1) check( "a table's neighbourhood keeps its columns", any("column" not in g.read_text() or "PK" in g.read_text() for g in graphs), "a table without its columns is not a table", ) # A module's neighbourhood must NOT drag its contents in — that is the sheet. code_ir = ops_mod.overview(ir) mod_view = ops_mod.neighbourhood(code_ir, next( n["id"] for n in code_ir["nodes"] if n["kind"] == "module"), hops=1) check( "a module's neighbourhood stays small", len(mod_view["nodes"]) <= 24, "its contents are the hundred functions that made the sheet unreadable", ) if not _shutil.which("node"): skip("explore behaviour", "node not installed") else: harness = HERE / "explore_test.js" if not harness.exists(): skip("explore behaviour", "explore_test.js missing") else: proc = _sub.run(["node", str(harness), str(page)], capture_output=True, text=True) for line in proc.stdout.strip().splitlines(): name = line.strip()[5:] (PASS if line.strip().startswith("ok") else FAIL).append(f"explore: {name}") print(f" {line.strip()[:4]} explore: {name}") if proc.returncode and not proc.stdout.strip(): check("the explore harness runs", False, proc.stderr.strip()[:200]) _shutil.rmtree(exp_dir, ignore_errors=True) # -------------------------------------------------------------------------- tmp.cleanup() print() # -------------------------------------------------------------------------- print("\n15. the book — every operation measured at both ends") def _books(): """Its own scope, so nothing here can rebind the run's `tmp` (see 3b).""" larder_mod = __import__(f"{PKG}.book.larder", fromlist=["*"]) book_mod = __import__(f"{PKG}.book", fromlist=["*"]) build_mod = __import__(f"{PKG}.book.build", fromlist=["*"]) checks_mod = __import__(f"{PKG}.book.checks", fromlist=["*"]) Larder = larder_mod.Larder # -- the input measure, which nothing had before ---------------------- check( "the unit vocabulary is closed", larder_mod.UNITS == ("file", "table", "path", "entry", "document"), f"got {larder_mod.UNITS} — a unit nobody else uses cannot be compared", ) try: Larder(kind="x", identity="y", unit="thingy") refused = False except ValueError: refused = True check("a unit outside it is refused, not recorded", refused) # `read` derived rather than stored is the one design decision here, and it # is the one that stops the measure disagreeing with itself. l = Larder(kind="python", identity="src", unit="file", seen=47) l.fail("a.py", "syntax"); l.fail("b.py", "unreadable") check("read is seen minus failed, derived", (l.seen, l.read, len(l.failed)) == (47, 45, 2), f"seen={l.seen} read={l.read} failed={len(l.failed)}") check( "the measure reads as a sentence", l.line() == "src — 45 files read, 2 failed", f"got {l.line()!r}", ) # "2 entrys read, 1 hosts" is how this read before. A measure nobody reads # is not a measure, so the grammar gets attention it would not otherwise. e = Larder(kind="usage", identity="s.har", unit="entry", seen=2) e.extra["hosts"] = 1 check("counts and labels agree", e.line() == "s.har — 2 entries read, 1 host", f"got {e.line()!r}") # -- redaction, the one field that can carry a secret ----------------- cases = [ ("postgresql://app:hunter2@db:5432/shop", "postgresql://app:***@db:5432/shop"), ("https://api/v1?token=abc123&page=2", "https://api/v1?token=***&page=2"), ("Driver=x;Server=y;Password=hunter2;Uid=app", "Driver=x;Server=y;Password=***;Uid=app"), ("../../station", "../../station"), ] wrong = [f"{src} -> {larder_mod.redact(src)}" for src, want in cases if larder_mod.redact(src) != want] check("credentials are masked and nothing else is", not wrong, "; ".join(wrong)) check( "the secret is gone, the recognisable part is not", "hunter2" not in larder_mod.redact(cases[0][0]) and "db:5432/shop" in larder_mod.redact(cases[0][0]), ) # The scrubber is graded by a second, independent key list in validate.py. leaked = dict(ir["meta"], larder={ "kind": "db", "identity": "postgresql://app:hunter2@db/shop", "unit": "table", "seen": 1, "read": 1, "failed": [], }) problems = ir_validate.check({**ir, "meta": leaked}) check( "a secret that survived redaction fails validation", any("identity still contains" in p for p in problems), f"got {problems[:2]} — a scrubber graded by its own word is not graded", ) # And the arithmetic, because a book can be assembled without the IR. lying = dict(ir["meta"], larder={ "kind": "python", "identity": "src", "unit": "file", "seen": 10, "read": 10, "failed": [{"name": "a.py", "error": "x"}], }) check( "a measure that disagrees with itself fails validation", any("disagrees with itself" in p for p in ir_validate.check({**ir, "meta": lying})), ) # -- every extractor reports one. The extension contract, with teeth. -- # Looped over the registry rather than a hand-written list, so adding an # extractor makes this start asking about it. check( "every registered extractor is in the relation table", set(build_mod.EXTRACTORS) == set(book_mod.RELATION), f"registry {sorted(build_mod.EXTRACTORS)} vs relations {sorted(book_mod.RELATION)}" " — a new extractor must say how its units show up in the output", ) unmeasured = [] for kind in sorted(build_mod.EXTRACTORS): module_name, fn_name, _ = build_mod.EXTRACTORS[kind] try: mod = __import__(f"{PKG}{module_name}", fromlist=["*"]) except ImportError as exc: unmeasured.append(f"{kind}: not importable ({exc})") continue fn = getattr(mod, fn_name, None) if fn is None: unmeasured.append(f"{kind}: no {fn_name}()") elif "identity" not in fn.__code__.co_varnames[:fn.__code__.co_argcount]: unmeasured.append(f"{kind}: {fn_name}() takes no `identity`") check( "every registered extractor can name its larder", not unmeasured, "; ".join(unmeasured) + " <- an extractor is done when it measures its input", ) # -- the reconciliation, which is what having both ends is for --------- b = book_mod.Book("pretend", Larder(kind="python", identity="p", unit="file", seen=10), "/nonexistent") b.ir = {"nodes": [{"id": f"m{i}", "kind": "module", "label": "m", "attrs": {"file": f"m{i}.py"}} for i in range(3)], "edges": []} lost = [r for r in b.compare() if not r["ok"]] check( "input read but not produced is reported as lost", len(lost) == 1 and lost[0]["id"] == "units-accounted-for", f"got {[r['id'] for r in b.compare()]}", ) check("and it says how much", "7 file(s)" in lost[0]["why"], lost[0]["why"][:70]) # A file that failed to parse still gets a node carrying attrs.error, so the # two populations must be counted apart. Counting them together made # "2 files read produced 4 modules" pass a check meant to prove nothing was # dropped — which is how it was written first. b2 = book_mod.Book("split", Larder(kind="python", identity="p", unit="file", seen=4), "/nonexistent") b2.larder.fail("c.py", "syntax"); b2.larder.fail("d.py", "syntax") b2.ir = {"edges": [], "nodes": [ {"id": "a", "kind": "module", "label": "a", "attrs": {"file": "a.py"}}, {"id": "b", "kind": "module", "label": "b", "attrs": {"file": "b.py"}}, {"id": "c", "kind": "module", "label": "c", "attrs": {"file": "c.py", "error": "syntax"}}, {"id": "d", "kind": "module", "label": "d", "attrs": {"file": "d.py", "error": "syntax"}}, ]} ids = {r["id"]: r for r in b2.compare()} check( "read and failed units are reconciled separately", ids["units-accounted-for"]["claim"] == "2 file(s) read produced 2 module(s)", f"got {ids['units-accounted-for']['claim']!r}", ) check( "an unreadable unit must still appear in the graph", ids["failures-still-in-the-graph"]["ok"], "the whole-input form of `an unresolved name becomes an external node`", ) b2.ir["nodes"] = b2.ir["nodes"][:2] # drop the marked ones check( "...and its absence is caught", not {r["id"]: r for r in b2.compare()}["failures-still-in-the-graph"]["ok"], "a picture smaller than its source, with nothing saying so", ) # -- end to end, on a tree with a file that cannot be parsed ---------- fixture = Path(tmp.name) / "book-src" (fixture / "app").mkdir(parents=True, exist_ok=True) (fixture / "app" / "__init__.py").write_text('"""An app."""\n') (fixture / "app" / "models.py").write_text('"""Models."""\n\n\nclass User:\n pass\n') (fixture / "app" / "broken.py").write_text("def oops(:\n") out = Path(tmp.name) / "book-out" built = build_mod.run("python", fixture, out, slug="fixture", quiet=True) check("a book writes its ledger", (out / "book.json").exists()) ledger = json.loads((out / "book.json").read_text()) step_ids = [s["id"] for s in ledger["steps"]] check( "the first and last steps are the two measures", (step_ids[0], step_ids[-1]) == (book_mod.FIRST, book_mod.LAST), f"got {step_ids}", ) check( "the larder measured the broken file as failed, by name", [f["name"] for f in ledger["larder"]["failed"]] == ["app/broken.py"], f"got {ledger['larder']['failed']}", ) check( "the book measure counts what came out", ledger["book"]["nodes"] == len(json.loads((out / "steps" / "ir.json").read_text())["nodes"]), ) check( "every intermediate step is still a file on its own", all((out / s["artifact"]).exists() for s in ledger["steps"] if s.get("artifact")), "that property is the reason the notebook is a sequence and not one artifact", ) # The notebook is the sequence, so the measures are its first and last cells # rather than something the tooling knows and the document does not. nb = json.loads((out / "notebook.ipynb").read_text()) check( "the notebook opens with what came in and closes with what came out", "what came in" in "".join(nb["cells"][0]["source"]) and "what came out" in "".join(nb["cells"][-1]["source"]), ) check( "it names the file it could not read", "app/broken.py" in "".join(nb["cells"][0]["source"]), ) # A generated notebook is a build artifact and must run where it was built. # An earlier version imported IPython to *load* an SVG and failed here. ran, failure = checks_mod._run_cells(nb["cells"], out) check(f"its {ran} code cells run from the book directory", failure is None, failure or "") html = (out / "site" / "index.html").read_text() check( "the web output shows both ends", "what came in" in html and "what came out" in html, "the site is the last step because it is the artifact somebody opens", ) check( "and admits when the book is incomplete", "This book is incomplete" in html and "app/broken.py" in html, "a clean diagram over 2 of 3 files is a lie by omission", ) # Reproducible in the strict sense, for the same reason the IR is: a diff # has to mean a real change. again = Path(tmp.name) / "book-out-2" build_mod.run("python", fixture, again, slug="fixture", quiet=True) differs = [ name for name in ("book.json", "notebook.ipynb", "steps/ir.json", "site/index.html") if (out / name).read_bytes() != (again / name).read_bytes() ] check("the same larder builds the same bytes", not differs, f"differ: {differs}") # -- the book test level ---------------------------------------------- loaded = checks_mod.Loaded(out) report = checks_mod.Report() import io as _io2 import contextlib as _ctx2 with _ctx2.redirect_stdout(_io2.StringIO()): checks_mod.generated(loaded, report) check( f"the generated book checks pass on a real book ({report.passed})", not report.failed, f"failed: {report.failed}", ) # Framework and custom living together is the whole point of the level. (out / "checks.py").write_text( "def checks(book, check, note, skip):\n" " note('this project')\n" " check('models survived', True,\n" " any(n['id'] == 'app.models' for n in book.ir['nodes']))\n" " check('wrong on purpose', 9, 0)\n" ) report2 = checks_mod.Report() with _ctx2.redirect_stdout(_io2.StringIO()): checks_mod.custom(checks_mod.Loaded(out), report2) check( "a book's own checks run beside the generated ones", report2.passed == 1 and report2.failed == ["wrong on purpose"], f"passed={report2.passed} failed={report2.failed}", ) # -- the architectural rules, still holding --------------------------- blob = json.dumps(ledger) # Swept where a visual field could actually land — `larder.extra` is the one # open bag in the ledger. NOT over the whole document: `by_kind` is keyed by # IR node kind, and `class` is both a legitimate kind and a member of # VISUAL_KEYS, so a text sweep reports `"class": 15` as a style leak. The # first version of this check did exactly that. leaked_keys = sorted(set(ledger["larder"].get("extra") or {}) & ir_validate.VISUAL_KEYS) check( "no visual field reached the ledger's open bag", not leaked_keys, f"larder.extra has {leaked_keys} — the ledger records what was read, not how it looks", ) node_kinds = {n["kind"] for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]} check( "the book measure is keyed by IR kind, nothing else", set(ledger["book"]["by_kind"]) <= node_kinds, f"{sorted(set(ledger['book']['by_kind']) - node_kinds)} is in by_kind but is not a kind", ) check( "no timestamp reached the ledger", "generated_at" not in blob, "a timestamp makes two builds of an unchanged larder differ", ) check( "provenance is in meta, never in nodes", not any("larder" in (n.get("attrs") or {}) for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]), "the IR is structure; provenance is meta, the same split that keeps colour out", ) _books() # -------------------------------------------------------------------------- print("\n16. standalone — docgen works with nothing above it") def _standalone(): """docgen is meant to be copied out of the repo and used on its own. That claim decays the moment somebody adds a convenient import, and it decays silently: the suite still passes *inside* the repo. So it is asserted from the source rather than by copying the folder in CI, which is the same move as rig's `no host-project references` grep. """ ref_mod = __import__(f"{PKG}.reference", fromlist=["*"]) # Third-party modules docgen is allowed to reach for, each optional and each # with a documented reason. Anything else is a new dependency, and a new # dependency in a folder meant to be copied around is worth noticing. ALLOWED = { "tree_sitter", "tree_sitter_c_sharp", "tree_sitter_typescript", "tree_sitter_python", # extractors/code.py, optional "lxml", # style/extract.py, optional "yaml", # extractors/openapi.py, optional "networkx", # lab/ only "modelgen", # the one seam — see reference.py } # The one module allowed to import `modelgen`, and the one allowed to go # looking above the package. Two names, so a third is a decision. SEAM_IMPORTER = "extractors/openapi.py" SEAM_RESOLVER = "reference.py" stdlib = set(sys.stdlib_module_names) own = {PKG} files = own_py_files() foreign, seam_leaks = [], [] for path in files: rel = path.relative_to(HERE).as_posix() try: tree = ast.parse(path.read_text()) except SyntaxError: continue for node in ast.walk(tree): if isinstance(node, ast.Import): names = [a.name.split(".")[0] for a in node.names] elif isinstance(node, ast.ImportFrom): if node.level: # relative — inside the package by definition continue names = [node.module.split(".")[0]] if node.module else [] else: continue for name in names: if name in stdlib or name in own: continue if name not in ALLOWED: foreign.append(f"{rel}: {name}") elif name == "modelgen" and rel != SEAM_IMPORTER: seam_leaks.append(f"{rel}: {name}") check( "docgen imports the stdlib, itself, and a short allow-list", not foreign, "; ".join(foreign) + " <- a new dependency in a folder meant to be copied", ) check( f"only {SEAM_IMPORTER} imports modelgen", not seam_leaks, "; ".join(seam_leaks) + " <- one seam is a seam; two is a dependency", ) # The repo path is named in exactly one place, so pointing docgen at a # reference repo is one change rather than a search. hardcoded = [] for path in files: rel = path.relative_to(HERE).as_posix() if rel in (SEAM_RESOLVER, "selftest.py"): continue text = path.read_text() for line in text.splitlines(): if "station/tools" in line and not line.lstrip().startswith("#") \ and '"""' not in line and "station/tools" not in line.split("#")[-1]: hardcoded.append(f"{rel}: {line.strip()[:60]}") check( f"the repo layout is known only to {SEAM_RESOLVER}", not hardcoded, "; ".join(hardcoded), ) # Resolution order is the thing a standalone user depends on, so it is # asserted rather than described: an explicit path must win over the walk, # or setting the variable would appear to do nothing inside the repo. import os as _os saved = _os.environ.get(ref_mod.ENV_VAR) try: fake = Path(tmp.name) / "fake-repo" (fake / "station" / "tools" / "modelgen").mkdir(parents=True, exist_ok=True) _os.environ[ref_mod.ENV_VAR] = str(fake) check( "an explicit path wins over walking up", ref_mod.root() == fake, f"got {ref_mod.root()} — setting the variable inside the repo would do nothing", ) check( "and it says how it resolved, not just whether", ref_mod.ENV_VAR in ref_mod.describe(), ref_mod.describe(), ) _os.environ.pop(ref_mod.ENV_VAR) # Both outcomes are correct and which one applies is the fact worth # printing: inside the repo, no configuration should be needed; copied # out, there is nothing to find and saying so is the point. Asserting # the in-repo answer unconditionally failed the moment the folder was # actually copied out, which is how this was found. walked = ref_mod.root() if walked is None: skip("the in-place case", "standalone — nothing above this folder, as intended") else: # What root() promises is a directory holding station/tools — not a # particular depth above docgen, which is an assumption about where # somebody chose to put it. check( "inside a repo it needs no configuration", (walked / "station" / "tools").is_dir(), f"walked to {walked}, which has no station/tools", ) finally: if saved is None: _os.environ.pop(ref_mod.ENV_VAR, None) else: _os.environ[ref_mod.ENV_VAR] = saved # Absent is a normal state. Four of the five extractors never touch it. check( "a missing reference is reported, not raised at import time", ref_mod.missing("x", "y").__class__ is ImportError and ref_mod.ENV_VAR in str(ref_mod.missing("x", "y")), "the error has to name the variable to set", ) # Everything a book writes has to stay inside the book directory, or a # standalone user cannot tell what docgen created. fixture = Path(tmp.name) / "sa-src" fixture.mkdir(exist_ok=True) (fixture / "one.py").write_text('"""One."""\n\n\nclass A:\n pass\n') out = Path(tmp.name) / "sa-book" build_mod = __import__(f"{PKG}.book.build", fromlist=["*"]) build_mod.run("python", fixture, out, slug="sa", quiet=True) strays = [p.name for p in fixture.iterdir() if p.name != "one.py"] check( "a book writes only inside its own output directory", not strays, f"left {strays} in the source tree", ) check( "the shipped OpenAPI fixture travels with docgen", (HERE / "fixtures" / "orders.yaml").exists(), "a test's fixtures belong to the test, not to a sibling project", ) _standalone() # -------------------------------------------------------------------------- print("\n17. one command line, and library code that has none") def _layout(): """Library packages hold library code; the command line lives in cli/. The rule exists because two kinds of file in one folder — `cli_dot.py` next to `dot.py` — make every folder answer two questions, and the command-line half is the half that grows copies. It decays by accretion: one convenient `argparse` in a library module and the split is gone. So it is read from the source rather than trusted. """ cli_mod = __import__(f"{PKG}.cli", fromlist=["*"]) # Where command-line code may live. lab/ is sanctioned experiments, and the # suite is itself a script. allowed = {"cli", "lab"} offenders = [] for path in own_py_files(): rel = path.relative_to(HERE) if rel.parts[0] in allowed or rel.as_posix() in ("__main__.py", "selftest.py"): continue tree = ast.parse(path.read_text()) for node in ast.walk(tree): names = [] if isinstance(node, ast.Import): names = [a.name for a in node.names] elif isinstance(node, ast.ImportFrom) and not node.level: names = [node.module or ""] if "argparse" in names: offenders.append(rel.as_posix()) check( "nothing outside cli/ parses a command line", not offenders, f"{sorted(set(offenders))} import argparse — command-line code belongs in cli/", ) stray = sorted( p.relative_to(HERE).as_posix() for p in own_py_files() if (p.name == "__main__.py" and p.parent != HERE) or p.name.startswith("cli_") or p.stem.endswith("_main") ) check( "one entry point — no __main__.py, cli_*.py or *_main.py in the packages", not stray, f"found {stray}", ) missing = [] for name, (module, _) in cli_mod.COMMANDS.items(): try: mod = __import__(f"{PKG}.cli.{module}", fromlist=["*"]) except ImportError as e: missing.append(f"{name}: {e}") continue if not callable(getattr(mod, "main", None)): missing.append(f"{name}: no main()") check("every listed command has a module with main()", not missing, "; ".join(missing)) # How a failure reaches the user is part of the interface: one line on # stderr, exit 1, never a traceback. Run as a real process, the way a shell # or a Makefile sees it. env = dict(__import__("os").environ, PYTHONPATH=str(HERE.parent)) run = lambda *a: _sub.run([sys.executable, "-m", PKG, *a], capture_output=True, text=True, env=env) bare = run() check("with no command it prints the commands and exits 2", bare.returncode == 2 and "emit" in bare.stdout and "run" in bare.stdout, f"exit {bare.returncode}") bad = run("validate", str(Path(tmp.name) / "no-such.json")) check( "an expected failure is one line on stderr, exit 1, no traceback", bad.returncode == 1 and bad.stderr.startswith("Error:") and "Traceback" not in bad.stderr and bad.stderr.count("\n") == 1, f"exit {bad.returncode}: {bad.stderr[:120]!r}", ) # The structure-picks-the-drawing branch used to be written three times — # the `auto` command, the `site` command, the book. Now it is `draw()`, and # the book reaching past it to the concrete emitters would be the first copy. # Parsed, not grepped, for the reason section 2 gives: the docstring naming # `emitters.dot` is prose, and the first version of this check failed on it. imported = set() for node in ast.walk(ast.parse((HERE / "book" / "build.py").read_text())): if isinstance(node, ast.ImportFrom) and node.module: imported |= {f"{node.module}.{a.name}" for a in node.names} check( "the book chooses its drawing through draw(), not by hand", "emitters.auto.draw" in imported and not any(m.startswith(("emitters.erd", "emitters.dot")) for m in imported), f"imports {sorted(m for m in imported if m.startswith('emitters'))}", ) _layout() # -------------------------------------------------------------------------- print("\n18. run files — every book, rebuilt with one command") def _runfiles(): cfg = __import__(f"{PKG}.book.config", fromlist=["*"]) build_mod = __import__(f"{PKG}.book.build", fromlist=["*"]) ref_mod = __import__(f"{PKG}.reference", fromlist=["*"]) import os as _os base = Path(tmp.name) / "runfiles" (base / "src" / "app").mkdir(parents=True, exist_ok=True) (base / "src" / "app" / "__init__.py").write_text('"""App."""\n') (base / "src" / "app" / "models.py").write_text('"""Models."""\n\n\nclass User:\n pass\n') (base / "shop.json").write_text((HERE / "fixtures" / "shop.json").read_text()) def write(text, name="docgen.toml"): path = base / name path.write_text(text) return path def problems(text): try: cfg.load(write(text, "bad.toml")) except cfg.ConfigError as e: return e.problems return [] example = cfg.load(HERE / "docgen.example.toml") check( "the shipped example loads", [b.name for b in example.books] == ["extractors", "schema", "api"], f"got {[b.name for b in example.books]}", ) good = write( '[defaults]\nout = "books"\nexclude = ["vendor"]\n\n' '[[book]]\nname = "app"\nroot = "src"\n\n' '[[book]]\nname = "shop"\nschema = "shop.json"\n\n' '[[book]]\nname = "narrow"\nroot = "src/app"\nout = "elsewhere/narrow"\nexclude = []\n' ) # Relative to the FILE. Loaded from another working directory, a run file # that resolved against cwd would build different books from the same text. previous = _os.getcwd() try: _os.chdir(Path(tmp.name)) rf = cfg.load(good) finally: _os.chdir(previous) by = {b.name: b for b in rf.books} check( "paths resolve against the run file, not the working directory", by["app"].source == base / "src" and by["app"].out == base / "books" / "app", f"source {by['app'].source}, out {by['app'].out}", ) check("a book's own out overrides /", by["narrow"].out == base / "elsewhere" / "narrow") check( "a book's value replaces the default — exclude included, no merging", by["app"].exclude == ("vendor",) and by["narrow"].exclude == (), f"app {by['app'].exclude}, narrow {by['narrow'].exclude}", ) check( "an inherited exclude does not apply to a non-tree source, and is not an error", by["shop"].exclude == () and by["shop"].kind == "db", ) # Refused, not ignored — and every problem at once, not the first. found = problems( 'colour = "red"\n[defaults]\nstyel = "x"\n\n' '[[book]]\nname = "a"\nroot = "src"\nexlude = ["x"]\n' ) check( "unknown keys are refused at every level, all reported together", len(found) == 3 and all("unknown" in f for f in found), f"got {found}", ) check( "a book names exactly one source", any("exactly one source" in f for f in problems( '[[book]]\nname = "a"\nroot = "src"\nschema = "shop.json"\n')), ) check( "an explicit exclude on a non-tree source is refused", any("only applies to a root" in f for f in problems( '[[book]]\nname = "a"\nschema = "shop.json"\nexclude = ["x"]\n')), ) # The next run would read the last run's output: a rebuild that changes on # every rebuild. check( "a book written inside the tree it reads is refused", any("inside the tree it reads" in f for f in problems( '[[book]]\nname = "a"\nroot = "src"\nout = "src/book"\n')), ) check( "two books writing to one directory are refused", any("both write to" in f for f in problems( '[[book]]\nname = "a"\nroot = "src"\nout = "same"\n\n' '[[book]]\nname = "b"\nschema = "shop.json"\nout = "same"\n')), "the second would clear the first on every run, silently", ) try: rf.select(["app", "nope"]) selected = False except cfg.ConfigError: selected = True check("--only with an unknown name is an error, not an empty run", selected) # The caller's env beats the file — rig's precedence rule for every setting. saved = _os.environ.pop(ref_mod.ENV_VAR, None) try: with_ref = cfg.load(write( f'reference = "{base}"\n[[book]]\nname = "a"\nroot = "src"\n', "ref.toml")) check("a run file can name the reference repo", cfg.apply_reference(with_ref) == str(base) and _os.environ.get(ref_mod.ENV_VAR) == str(base)) _os.environ[ref_mod.ENV_VAR] = "/from/the/environment" check("but the environment wins over the file", cfg.apply_reference(with_ref) is None and _os.environ[ref_mod.ENV_VAR] == "/from/the/environment") finally: _os.environ.pop(ref_mod.ENV_VAR, None) if saved is not None: _os.environ[ref_mod.ENV_VAR] = saved # -- running it ---------------------------------------------------------- broken = write( '[[book]]\nname = "app"\nroot = "src"\nout = "run/app"\n\n' '[[book]]\nname = "bad"\nhar = "not-a-har.har"\nout = "run/bad"\n\n' '[[book]]\nname = "shop"\nschema = "shop.json"\nout = "run/shop"\n', "run.toml", ) (base / "not-a-har.har").write_text("this is not json") results = {r["name"]: r for r in cfg.run(cfg.load(broken), check=True)} check( "one broken book does not stop the others", results["app"]["ok"] and results["shop"]["ok"] and not results["bad"]["ok"] and results["bad"]["error"], {k: (v["ok"], v["error"]) for k, v in results.items()}, ) check("each built book passed its own level", not results["app"]["checks_failed"]) # Rerunning is the point of a run file, so a rerun must not leave the last # run's artifacts beside the new ledger, and must not touch what a person wrote. app = base / "run" / "app" (app / "steps" / "stale.svg").write_text("") (app / "checks.py").write_text("def checks(book, check, note, skip):\n pass\n") again = {r["name"]: r for r in cfg.run(cfg.load(broken), ["app"])} check( "a rebuild removes the previous build's outputs", again["app"]["ok"] and not (app / "steps" / "stale.svg").exists(), "a stale graph.svg would go on showing in the site, unlisted in the ledger", ) check("...and keeps the hand-written checks.py", (app / "checks.py").exists()) foreign = base / "not-a-book" foreign.mkdir(exist_ok=True) (foreign / "steps").mkdir(exist_ok=True) (foreign / "site").mkdir(exist_ok=True) check( "clear() never touches a directory it did not write", build_mod.clear(foreign) == [] and (foreign / "steps").exists(), "a mistyped `out` must not delete somebody's `site/`", ) # -- the environment the lock pins --------------------------------------- import tomllib as _toml project = _toml.loads((HERE / "pyproject.toml").read_text()) check( "pyproject declares no required dependencies", project["project"]["dependencies"] == [], "the structural path is the stdlib; everything else is an optional group", ) check( "it requires the Python that reads run files", project["project"]["requires-python"] == ">=3.11", "tomllib is 3.11+", ) # A module the suite allows docgen to import must be a dependency somebody # can install from the lock, or the allow-list and pyproject have drifted. declared = {d.split(">")[0].split("=")[0].strip().lower().replace("-", "_") for group in project["dependency-groups"].values() for d in group} import_to_dist = {"tree_sitter": "tree_sitter", "tree_sitter_c_sharp": "tree_sitter_c_sharp", "tree_sitter_typescript": "tree_sitter_typescript", "tree_sitter_python": "tree_sitter_python", "lxml": "lxml", "yaml": "pyyaml", "networkx": "networkx"} undeclared = sorted(m for m, dist in import_to_dist.items() if dist not in declared) check("every optional import is a declared dependency group", not undeclared, f"{undeclared} are allowed but not in pyproject") lock = HERE / "uv.lock" locked = set() if lock.exists(): locked = {pkg["name"].replace("-", "_") for pkg in _toml.loads(lock.read_text()).get("package", [])} check( "uv.lock pins every declared dependency", lock.exists() and declared <= locked, f"not in the lock: {sorted(declared - locked)} — run `make lock`", ) _runfiles() print(f"{len(PASS)} passed, {len(FAIL)} failed, {len(SKIP)} skipped") if FAIL: print("\nfailed:") for name in FAIL: print(f" {name}") sys.exit(1 if FAIL else 0)