2328 lines
94 KiB
Python
2328 lines
94 KiB
Python
"""
|
|
Prove the pipeline, offline, on a tree it builds itself.
|
|
|
|
python3 selftest.py # or: make check
|
|
|
|
No network, nothing installed. The render steps skip with a message when
|
|
graphviz is absent rather than failing — a machine without `dot` can still check
|
|
extraction, the IR and the style layer, and saying so is more useful than a red
|
|
mark about the host.
|
|
|
|
**A check that only ever proves things work is not worth running.** So the
|
|
negative cases carry equal weight, and the four checks that are the *design*
|
|
rather than a regression are marked below. If anything ever gets cut, those are
|
|
the ones to keep:
|
|
|
|
the IR carries no visual field extractors cannot decide appearance
|
|
an emitter never reads a source file the layering, from the other side
|
|
style names slots, not colours one colour language, not three
|
|
ids are stable across runs without it, diffing is noise
|
|
|
|
## Golden tests go on the IR, never on the SVG
|
|
|
|
Graphviz measures label text with the host's fonts to size nodes, so identical
|
|
input produces different geometry on a machine with different fontconfig. The IR
|
|
is deterministic; the SVG is not. Pinning the wrong one gives a suite that fails
|
|
on someone else's laptop for no reason anyone can act on.
|
|
"""
|
|
|
|
import ast
|
|
import json
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE.parent))
|
|
|
|
PKG = HERE.name
|
|
ir_mod = __import__(f"{PKG}.ir", fromlist=["*"])
|
|
ir_validate = __import__(f"{PKG}.ir.validate", fromlist=["*"])
|
|
py_ex = __import__(f"{PKG}.extractors.python", fromlist=["*"])
|
|
style_mod = __import__(f"{PKG}.style", fromlist=["*"])
|
|
dot_mod = __import__(f"{PKG}.emitters.dot", fromlist=["*"])
|
|
index_mod = __import__(f"{PKG}.emitters.index", fromlist=["*"])
|
|
ops_mod = __import__(f"{PKG}.ops", fromlist=["*"])
|
|
erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"])
|
|
nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"])
|
|
site_mod = __import__(f"{PKG}.emitters.site", fromlist=["*"])
|
|
mm_mod = __import__(f"{PKG}.emitters.minimap", fromlist=["*"])
|
|
exp_mod = __import__(f"{PKG}.emitters.explore", fromlist=["*"])
|
|
spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"])
|
|
db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"])
|
|
usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"])
|
|
tokens_mod = __import__(f"{PKG}.style.tokens", fromlist=["*"])
|
|
extract_mod = __import__(f"{PKG}.style.extract", fromlist=["*"])
|
|
|
|
check_ir = ir_mod.check
|
|
Style, StyleError = style_mod.Style, style_mod.StyleError
|
|
|
|
PASS, FAIL, SKIP = [], [], []
|
|
|
|
# A docstring that could not plausibly be a style value or a module name, so
|
|
# finding it somewhere it should not be means something really read it.
|
|
MARKER = "Quarterly Revenue Ledger"
|
|
|
|
FIXTURE = {
|
|
"app/__init__.py": '"""The app package."""\nfrom .db import Base\n',
|
|
"app/db.py": '"""Database plumbing."""\n\n\nclass Base:\n """Declarative base."""\n',
|
|
"app/models.py": (
|
|
'"""Domain models."""\n'
|
|
"from .db import Base\n"
|
|
"import json\n"
|
|
"from third_party.orm import Mixin\n"
|
|
"\n\n"
|
|
"class User(Base):\n"
|
|
f' """{MARKER}."""\n'
|
|
"\n"
|
|
" def save(self):\n"
|
|
" pass\n"
|
|
"\n\n"
|
|
"class Admin(User, Mixin):\n"
|
|
" pass\n"
|
|
),
|
|
"app/sub/deep.py": "from ..db import Base\n\n\nclass Deep(Base):\n class Inner:\n pass\n",
|
|
"app/broken.py": "def oops(:\n", # must not cost the run
|
|
}
|
|
|
|
|
|
|
|
# docgen's own Python — what a source scan should read. Not `HERE.rglob`: once
|
|
# `make sync` has run, `.venv/` under this folder holds thousands of third-party
|
|
# modules, and the standalone check reported lxml's own imports as docgen's.
|
|
NOT_OURS = {"__pycache__", ".venv", "venv", "node_modules", "out"}
|
|
|
|
|
|
def own_py_files():
|
|
return sorted(p for p in HERE.rglob("*.py")
|
|
if not NOT_OURS & set(p.relative_to(HERE).parts))
|
|
|
|
def check(name, condition, detail=""):
|
|
(PASS if condition else FAIL).append(name)
|
|
print(f" {'ok ' if condition else 'FAIL'} {name}"
|
|
+ (f"\n {detail}" if detail and not condition else ""))
|
|
return condition
|
|
|
|
|
|
def _err(fn):
|
|
"""The message from a call that is expected to fail."""
|
|
try:
|
|
fn()
|
|
except Exception as e: # noqa: BLE001 - the message is the assertion
|
|
return str(e)
|
|
return ""
|
|
|
|
|
|
def skip(name, why):
|
|
SKIP.append(name)
|
|
print(f" -- {name} ({why})")
|
|
|
|
|
|
def build_tree(root: Path):
|
|
for rel, text in FIXTURE.items():
|
|
p = root / rel
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(text)
|
|
|
|
|
|
tmp = tempfile.TemporaryDirectory(prefix="docgen-selftest-")
|
|
ROOT = Path(tmp.name) / "fx"
|
|
build_tree(ROOT)
|
|
|
|
ir = py_ex.extract(ROOT).to_dict()
|
|
ids = {n["id"] for n in ir["nodes"]}
|
|
edges = {(e["source"], e["target"], e["kind"]) for e in ir["edges"]}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n1. extraction — is it analysis, or is it guessing")
|
|
|
|
check("the IR validates", check_ir(ir) == [], str(check_ir(ir)[:3]))
|
|
|
|
# The claim the whole thing rests on: `ast` hands over the string "Base", and
|
|
# two-pass resolution turns it into the id of the thing it actually names.
|
|
check(
|
|
"a relative import resolves to the real id",
|
|
("app.models.User", "app.db.Base", "inherits") in edges,
|
|
f"got {sorted(e for e in edges if e[2] == 'inherits')}",
|
|
)
|
|
check(
|
|
"a relative import two levels up resolves",
|
|
("app.sub.deep.Deep", "app.db.Base", "inherits") in edges,
|
|
)
|
|
check(
|
|
"a base defined in the same module resolves",
|
|
("app.models.Admin", "app.models.User", "inherits") in edges,
|
|
)
|
|
check(
|
|
"a package's own __init__ resolves against itself",
|
|
("app", "app.db", "imports") in edges,
|
|
"from .db inside app/__init__.py must mean app.db, not db",
|
|
)
|
|
check(
|
|
"no raw source name leaks in as an id",
|
|
"Base" not in ids and "User" not in ids,
|
|
f"unresolved names present as bare ids: {sorted(ids & {'Base', 'User'})}",
|
|
)
|
|
|
|
# Dropping an edge you cannot resolve is the worse failure: the diagram looks
|
|
# complete and is quietly missing a dependency.
|
|
externals = {n["id"] for n in ir["nodes"] if n["kind"] == "external"}
|
|
check(
|
|
"an unresolvable name survives as `external`",
|
|
"third_party.orm.Mixin" in externals,
|
|
f"got {sorted(externals)}",
|
|
)
|
|
check(
|
|
"and it keeps its edge",
|
|
("app.models.Admin", "third_party.orm.Mixin", "inherits") in edges,
|
|
)
|
|
|
|
check(
|
|
"nested definitions are reached",
|
|
"app.sub.deep.Deep.Inner" in ids and "app.models.User.save" in ids,
|
|
"a missing generic_visit() silently loses everything below a definition",
|
|
)
|
|
check(
|
|
"containment is recorded",
|
|
next(n for n in ir["nodes"] if n["id"] == "app.models.User.save")["parent"]
|
|
== "app.models.User",
|
|
)
|
|
# Spans, not just a start line. "This class is 400 lines" is structure, and a
|
|
# density map sizes a block by it — `ast` carries end_lineno, so recording only
|
|
# where something begins throws away half the fact for nothing.
|
|
spans = [n for n in ir["nodes"] if n["kind"] in ("class", "function")]
|
|
check(
|
|
"a construct records how many lines it spans",
|
|
all((n.get("attrs") or {}).get("lines", 0) >= 1 for n in spans),
|
|
f"missing on {[n['id'] for n in spans if not (n.get('attrs') or {}).get('lines')][:3]}",
|
|
)
|
|
check(
|
|
"a multi-line class spans more than one",
|
|
any((n.get("attrs") or {}).get("lines", 0) > 1 for n in spans),
|
|
)
|
|
check(
|
|
"a module records its length",
|
|
all((n.get("attrs") or {}).get("lines", 0) >= 1
|
|
for n in ir["nodes"] if n["kind"] == "module"
|
|
and not (n.get("attrs") or {}).get("error")),
|
|
)
|
|
check(
|
|
"a span is not a visual field",
|
|
check_ir(ir) == [],
|
|
"`lines` is a fact about the code, not about how it is drawn",
|
|
)
|
|
|
|
check(
|
|
"source anchors are carried",
|
|
all(
|
|
(n.get("attrs") or {}).get("file")
|
|
for n in ir["nodes"]
|
|
if n["kind"] in ("class", "function")
|
|
),
|
|
)
|
|
|
|
broken = [n for n in ir["nodes"] if (n.get("attrs") or {}).get("error")]
|
|
check(
|
|
"an unparseable file is recorded, not fatal",
|
|
len(broken) == 1 and "broken.py" in broken[0]["attrs"]["file"],
|
|
f"got {[(n['id'], n['attrs'].get('error')) for n in broken]}",
|
|
)
|
|
check("...and the rest of the tree still extracted", len(ids) > 10)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n2. the design rules")
|
|
|
|
# (design) Without this the diff emitter reports noise and nobody trusts it.
|
|
again = py_ex.extract(ROOT).to_dict()
|
|
check(
|
|
"ids are stable — the same tree twice is the same bytes",
|
|
json.dumps(ir, sort_keys=True) == json.dumps(again, sort_keys=True),
|
|
)
|
|
|
|
# (design) An extractor that sets a colour decides how the graph looks forever.
|
|
visual = set()
|
|
for n in ir["nodes"]:
|
|
visual |= set(n.get("attrs") or {}) & ir_validate.VISUAL_KEYS
|
|
for e in ir["edges"]:
|
|
visual |= set(e.get("attrs") or {}) & ir_validate.VISUAL_KEYS
|
|
check("no visual field reaches the IR", not visual, f"found {sorted(visual)}")
|
|
|
|
check(
|
|
"the IR is readable without this package",
|
|
isinstance(json.loads(json.dumps(ir)), dict),
|
|
"it must be plain JSON, not something that needs a library to open",
|
|
)
|
|
|
|
# (design) The layering, checked from the emitter side. Parsed, not grepped:
|
|
# a docstring mentioning `ast` is prose, an import is a dependency.
|
|
offenders = []
|
|
for src in sorted((HERE / "emitters").glob("*.py")):
|
|
tree = ast.parse(src.read_text(), str(src))
|
|
for node in ast.walk(tree):
|
|
names = []
|
|
if isinstance(node, ast.Import):
|
|
names = [a.name for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
names = [node.module or ""]
|
|
for name in names:
|
|
if name.split(".")[0] in {"ast", "sqlalchemy", "griffe"}:
|
|
offenders.append(f"{src.name}:{node.lineno} imports {name}")
|
|
check(
|
|
"no emitter reads source — it has never heard of Python or SQL",
|
|
not offenders,
|
|
"; ".join(offenders),
|
|
)
|
|
|
|
extractor_offenders = []
|
|
for src in sorted((HERE / "extractors").rglob("*.py")):
|
|
tree = ast.parse(src.read_text(), str(src))
|
|
for node in ast.walk(tree):
|
|
names = []
|
|
if isinstance(node, ast.Import):
|
|
names = [a.name for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
names = [node.module or ""]
|
|
if any(n and ("emitters" in n or "style" in n) for n in names):
|
|
extractor_offenders.append(f"{src.name}:{node.lineno}")
|
|
check(
|
|
"no extractor knows about style or emitters",
|
|
not extractor_offenders,
|
|
"; ".join(extractor_offenders),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n3. style — one colour language")
|
|
|
|
styles = Style.available()
|
|
check("a style ships", "lucid" in styles, f"found {styles}")
|
|
|
|
lucid = Style.load("lucid") # the default: dark
|
|
print_theme = Style.load("lucid", theme="lucid") # the print palette
|
|
|
|
check("it carries more than one theme", set(lucid.themes()) >= {"lucid", "dark"})
|
|
check(
|
|
"an unknown kind falls back rather than crashing",
|
|
lucid.node("no-such-kind-at-all") == lucid.node("default") != {},
|
|
)
|
|
|
|
# (design) A rule naming a hex is a rule outside the shared vocabulary.
|
|
raw = json.loads((HERE / "style" / "lucid.json").read_text())
|
|
leaked = []
|
|
for section in ("nodes", "groups", "edges", "graph"):
|
|
for kind, rule in (raw.get(section) or {}).items():
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
for key, value in rule.items():
|
|
if isinstance(value, str) and re.fullmatch(r"#[0-9A-Fa-f]{3,8}", value):
|
|
leaked.append(f"{section}.{kind}.{key}={value}")
|
|
check(
|
|
"style rules name slots, never colours",
|
|
not leaked,
|
|
"; ".join(leaked) + " <- the colour language has stopped being one language",
|
|
)
|
|
|
|
# A theme that defines only some of the slots renders half a diagram in the
|
|
# right colours and the rest in whatever DOT does with an empty string.
|
|
half_bound = []
|
|
for theme in lucid.themes():
|
|
try:
|
|
Style.load("lucid", theme=theme)
|
|
except StyleError as e:
|
|
half_bound.append(f"{theme}: {e}")
|
|
check("every slot resolves in every theme", not half_bound, "; ".join(half_bound))
|
|
|
|
# The point of binding to spr's models: a diagram and the page around it must
|
|
# not drift apart. docs/graphs/README.md states the rule.
|
|
SYSTEM_ACCENTS = {
|
|
"artery": ("artery/index.html", "#b91c1c"),
|
|
"atlas": ("atlas/index.html", "#15803d"),
|
|
"station": ("station/index.html", "#1d4ed8"),
|
|
}
|
|
spr_root = HERE.parent.parent
|
|
mismatched = []
|
|
for slot, (page, expected) in SYSTEM_ACCENTS.items():
|
|
got = lucid.slot(slot).lower()
|
|
if got != expected:
|
|
mismatched.append(f"{slot}: style has {got}, {page} sets {expected}")
|
|
src = spr_root / page
|
|
if src.exists() and expected not in src.read_text():
|
|
mismatched.append(f"{slot}: {page} no longer sets {expected}")
|
|
check(
|
|
"the dark theme's slots are spr's own --system-accent values",
|
|
not mismatched,
|
|
"; ".join(mismatched),
|
|
)
|
|
|
|
check(
|
|
"domain -> slot is a mapping, not a colour choice",
|
|
lucid.domain_slot("atlas") == "atlas" and lucid.domain_slot("station") == "station",
|
|
)
|
|
check(
|
|
"an undomained group still gets a deterministic slot",
|
|
[lucid.domain_slot(None, i) for i in range(4)]
|
|
== [lucid.domain_slot(None, i) for i in range(4)],
|
|
)
|
|
|
|
check(
|
|
"the style records where DOT runs out",
|
|
{"header-bar", "dasharray", "sequence-badge"} <= set(lucid.limits()),
|
|
f"got {sorted(lucid.limits())}",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n3b. harvesting — a binding recovered from diagrams that cannot leave")
|
|
|
|
|
|
def _harvesting():
|
|
"""Its own scope on purpose: `tmp` out here is the whole run's temp
|
|
directory, and a `with TemporaryDirectory() as tmp` would delete it."""
|
|
|
|
# The shape a Graphviz harvest really has, and the reason frequency is the wrong
|
|
# signal for the canvas: every label carries a `fill`, so the ink (87) outnumbers
|
|
# the background (43) and "most common is the background" inverts the theme.
|
|
HARVEST = {
|
|
"source": "/tmp/diagrams",
|
|
"files_read": 3,
|
|
"files_failed": [],
|
|
"tokens": {
|
|
"fill": [
|
|
{"value": "#1f2933", "count": 87},
|
|
{"value": "#ffffff", "count": 43},
|
|
{"value": "#e8effd", "count": 20},
|
|
{"value": "#2b5fd9", "count": 12},
|
|
{"value": "#1a7f45", "count": 9},
|
|
{"value": "#c0392b", "count": 4},
|
|
],
|
|
"stroke": [{"value": "#9aa5b1", "count": 60}, {"value": "#616e7c", "count": 18}],
|
|
"stroke-width": [{"value": "1", "count": 90}, {"value": "4", "count": 3}],
|
|
"font-family": [{"value": "Helvetica", "count": 70}],
|
|
"font-size": [{"value": "11", "count": 70}, {"value": "14", "count": 6}],
|
|
"rx": [{"value": "4", "count": 12}],
|
|
},
|
|
}
|
|
|
|
harvested = tokens_mod.derive(HARVEST, "harvested")
|
|
check(
|
|
"a harvest produces a theme, not a style file",
|
|
set(harvested) == {"harvested"} and set(harvested["harvested"]) == {"note", "slots"},
|
|
f"got {sorted(harvested['harvested'])} — what counting recovers is the binding, "
|
|
"not what a kind should look like",
|
|
)
|
|
|
|
slots = harvested["harvested"]["slots"]
|
|
check(
|
|
"the canvas and the ink are the lightness extremes",
|
|
(slots["surface-0"], slots["text"]) == ("#ffffff", "#1f2933"),
|
|
f"surface-0={slots['surface-0']} text={slots['text']} — by frequency the ink "
|
|
"wins the background and the theme's own text is invisible against it",
|
|
)
|
|
check(
|
|
"the accents are the colours that are neither canvas nor ink",
|
|
{slots["accent"], slots["artery"], slots["atlas"]} <= {"#e8effd", "#2b5fd9", "#1a7f45", "#c0392b"},
|
|
f"accent={slots['accent']} artery={slots['artery']} atlas={slots['atlas']}",
|
|
)
|
|
|
|
missing = [s for s in tokens_mod.REQUIRED_SLOTS if s not in slots]
|
|
check(
|
|
"every slot a style file needs is bound, so a partial harvest still loads",
|
|
not missing,
|
|
f"unbound: {missing}",
|
|
)
|
|
check(
|
|
"a readable harvest says nothing",
|
|
"_warning" not in slots,
|
|
"a warning on a theme that is fine is a warning nobody reads",
|
|
)
|
|
|
|
flipped = tokens_mod.derive(HARVEST, "harvested", dark=True)["harvested"]["slots"]
|
|
check(
|
|
"polarity flips whole — canvas and ink together, never one of them",
|
|
(flipped["surface-0"], flipped["text"]) == (slots["text"], slots["surface-0"]),
|
|
)
|
|
|
|
# Polarity is the guess most likely to be wrong, so being wrong has to be loud.
|
|
DARK_ONLY = json.loads(json.dumps(HARVEST))
|
|
DARK_ONLY["tokens"]["fill"] = [
|
|
{"value": "#1f2933", "count": 87},
|
|
{"value": "#2b3440", "count": 43},
|
|
]
|
|
warned = tokens_mod.derive(DARK_ONLY, "harvested")["harvested"]["slots"]
|
|
check(
|
|
"a theme nobody could read says so",
|
|
"_warning" in warned and "unreadable" in warned["_warning"],
|
|
f"got {warned.get('_warning', 'no warning')!r}",
|
|
)
|
|
|
|
geom = tokens_mod.geometry(HARVEST)
|
|
check(
|
|
"widths take the mode, not the mean",
|
|
geom["hairline"] == "1",
|
|
f"got {geom['hairline']} — the mean of 1 and 4 is 2.5, a width no diagram uses",
|
|
)
|
|
check(
|
|
"the type scale is derived from the base size, not harvested three times",
|
|
(geom["font-size-sm"], geom["font-size-base"], geom["font-size-header"])
|
|
== ("10", "11", "13"),
|
|
f"got {geom['font-size-sm']}/{geom['font-size-base']}/{geom['font-size-header']}",
|
|
)
|
|
check(
|
|
"rounding survives as the binary DOT can express",
|
|
geom["rounded"] is True,
|
|
"the radius is dropped on purpose — `rounded` has no scalar",
|
|
)
|
|
|
|
# The claim the whole split rests on: one set of rules, several bindings. A theme
|
|
# recovered from someone else's diagrams has to render through docgen's own rules
|
|
# with nothing rewritten.
|
|
merged = json.loads((HERE / "style" / "lucid.json").read_text())
|
|
merged["themes"].update(harvested)
|
|
try:
|
|
harvested_style = Style(merged, theme="harvested", name="harvested")
|
|
rendered, why = slots["surface-0"] in dot_mod.emit(ir, harvested_style), ""
|
|
except StyleError as e:
|
|
rendered, why = False, str(e)
|
|
check(
|
|
"a harvested theme drops into the shipped rules and renders",
|
|
rendered,
|
|
why or "if this fails the rules and the binding are not actually separable",
|
|
)
|
|
|
|
# -- the two promises extract.py makes to a confidential folder ---------------
|
|
# Both are asserted rather than trusted, because both are the reason it is safe
|
|
# to point this at diagrams that may not be sent anywhere.
|
|
extract_src = ast.parse((HERE / "style" / "extract.py").read_text())
|
|
imported = set()
|
|
for node in ast.walk(extract_src):
|
|
if isinstance(node, ast.Import):
|
|
imported.update(a.name.split(".")[0] for a in node.names)
|
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
imported.add(node.module.split(".")[0])
|
|
NETWORK = {"socket", "ssl", "urllib", "http", "requests", "httpx", "ftplib",
|
|
"smtplib", "asyncio", "xmlrpc", "webbrowser"}
|
|
check(
|
|
"the harvester imports nothing that can open a socket",
|
|
not (imported & NETWORK),
|
|
f"{sorted(imported & NETWORK)} — a confidential diagram stays off every network path",
|
|
)
|
|
check(
|
|
"it never reads .text or .tail",
|
|
not [n for n in ast.walk(extract_src)
|
|
if isinstance(n, ast.Attribute) and n.attr in ("text", "tail")],
|
|
"a palette does not need to know what the diagram says",
|
|
)
|
|
|
|
try:
|
|
import lxml # noqa: F401
|
|
HAS_LXML = True
|
|
except ImportError:
|
|
HAS_LXML = False
|
|
|
|
if not HAS_LXML:
|
|
skip("harvest reads real SVG", "lxml is not installed — pip install lxml")
|
|
else:
|
|
FIXTURE_SVG = (
|
|
'<svg xmlns="http://www.w3.org/2000/svg">'
|
|
'<g><rect fill="#FFF" stroke="#9AA5B1" stroke-width="1.00" rx="4"/>'
|
|
'<rect style="fill:#2b5fd9;stroke-width:1" fill="#ffffff"/>'
|
|
'<circle fill="rgb(26,127,69)"/>'
|
|
'<path fill="url(#gradient-3)"/>'
|
|
f'<text font-family="Helvetica, Arial" font-size="11.00" fill="#1f2933">{MARKER}</text>'
|
|
"</g></svg>"
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="docgen-harvest-") as tmp:
|
|
folder = Path(tmp)
|
|
(folder / "one.svg").write_text(FIXTURE_SVG)
|
|
data = extract_mod.harvest(folder)
|
|
|
|
blob = json.dumps(data)
|
|
check(
|
|
"no text from the diagram reaches the output",
|
|
MARKER not in blob,
|
|
"the fixture's label is in the harvest — rule 1 of extract.py is broken",
|
|
)
|
|
|
|
found = {p: {e["value"] for e in data["tokens"][p]} for p in data["tokens"]}
|
|
check(
|
|
"one spelling per value — #FFF and #ffffff are not two entries",
|
|
found["fill"] >= {"#ffffff", "#1a7f45"} and "#fff" not in found["fill"],
|
|
f"fills: {sorted(found['fill'])}",
|
|
)
|
|
check(
|
|
"an inline style beats the presentation attribute, as the spec says",
|
|
"#2b5fd9" in found["fill"],
|
|
f"fills: {sorted(found['fill'])} — the rect declares both",
|
|
)
|
|
check(
|
|
"a gradient reference is not a colour",
|
|
not any(v.startswith("url(") for v in found["fill"]),
|
|
f"fills: {sorted(found['fill'])}",
|
|
)
|
|
check(
|
|
"a font stack collapses to the font that will actually render",
|
|
found["font-family"] == {"Helvetica"},
|
|
f"got {sorted(found['font-family'])}",
|
|
)
|
|
check(
|
|
"1.00 and 11.00 are tidied to 1 and 11",
|
|
found["stroke-width"] == {"1"} and found["font-size"] == {"11"},
|
|
f"widths {sorted(found['stroke-width'])}, sizes {sorted(found['font-size'])}",
|
|
)
|
|
|
|
|
|
_harvesting()
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n4. emitters")
|
|
|
|
dot_text = dot_mod.emit(ir, lucid)
|
|
check("DOT is produced", dot_text.startswith("digraph ir {"))
|
|
check(
|
|
"every IR edge reaches the DOT",
|
|
dot_text.count("->") == len(ir["edges"]),
|
|
f"{dot_text.count('->')} of {len(ir['edges'])} — cluster endpoints need lhead/ltail",
|
|
)
|
|
check("the profile's colours are applied", lucid.node("class")["border"] in dot_text)
|
|
check(
|
|
"ids and kinds reach the output for a front end to bind to",
|
|
'id="app.models.User"' in dot_text and 'class="class"' in dot_text,
|
|
)
|
|
check("source anchors become links", "href=" in dot_text)
|
|
|
|
print_text = dot_mod.emit(ir, print_theme)
|
|
check(
|
|
"one IR, two looks, no re-extraction",
|
|
print_text != dot_text and print_theme.slot("surface-0") in print_text,
|
|
)
|
|
|
|
md = index_mod.to_markdown(ir)
|
|
check("the index names what exists", "app.models" in md or "models" in md)
|
|
check("the index carries the docstrings", MARKER in md)
|
|
check(
|
|
"the index reports the dependency surface",
|
|
"third_party.orm.Mixin" in md,
|
|
"externals are the thing a reader most often wants and a diagram scatters",
|
|
)
|
|
check("the index reports what could not be parsed", "Not parsed" in md)
|
|
|
|
sidebar = index_mod.to_sidebar(ir)
|
|
check("the sidebar is nested, not flat", any(i.get("children") for i in sidebar["items"]))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n5. views")
|
|
|
|
no_std = ops_mod.drop_stdlib(ir)
|
|
check(
|
|
"dropping the stdlib removes json and keeps third-party",
|
|
"json" not in {n["id"] for n in no_std["nodes"]}
|
|
and "third_party.orm.Mixin" in {n["id"] for n in no_std["nodes"]},
|
|
)
|
|
check("a view still validates", check_ir(no_std) == [], str(check_ir(no_std)[:3]))
|
|
|
|
classes = ops_mod.only_kinds(ir, {"class"})
|
|
check(
|
|
"keeping one kind keeps its ancestors, so containment survives",
|
|
check_ir(classes) == [] and "app.models" in {n["id"] for n in classes["nodes"]},
|
|
)
|
|
|
|
around = ops_mod.neighbourhood(ir, "app.db.Base", hops=1)
|
|
check(
|
|
"a neighbourhood keeps what points at it",
|
|
"app.models.User" in {n["id"] for n in around["nodes"]} and check_ir(around) == [],
|
|
)
|
|
|
|
shallow = ops_mod.collapse_to_depth(ir, 1)
|
|
check(
|
|
"collapsing to a depth drops what is inside",
|
|
"app.models.User.save" not in {n["id"] for n in shallow["nodes"]}
|
|
and check_ir(shallow) == [],
|
|
)
|
|
|
|
# The difference between a simpler picture and a wrong one. `app.models.User`
|
|
# inherits from `app.db.Base`; look at the module level and that is a dependency
|
|
# of app.models on app.db. Dropping the edge says they are unrelated.
|
|
mods = ops_mod.only_kinds(ir, {"module"})
|
|
mod_edges = {(e["source"], e["target"], e["kind"]) for e in mods["edges"]}
|
|
check(
|
|
"an edge is lifted to the surviving node, not dropped",
|
|
("app.models", "app.db", "inherits") in mod_edges,
|
|
f"got {sorted(mod_edges)}",
|
|
)
|
|
check(
|
|
"a lifted edge says how many it stands for",
|
|
any((e.get("attrs") or {}).get("weight") for e in mods["edges"]),
|
|
)
|
|
check(
|
|
"lifting never leaves a self-loop",
|
|
not [e for e in mods["edges"] if e["source"] == e["target"]],
|
|
"both ends inside one node is a fact about that node, not a line",
|
|
)
|
|
check(
|
|
"a label that no longer applies is dropped with the lift",
|
|
not any(
|
|
(e.get("attrs") or {}).get("label")
|
|
for e in mods["edges"]
|
|
if (e.get("attrs") or {}).get("weight")
|
|
),
|
|
)
|
|
|
|
# The default view: what you get without knowing which filters to ask for.
|
|
ov = ops_mod.overview(ir)
|
|
ov_kinds = {n["kind"] for n in ov["nodes"]}
|
|
check(
|
|
"the python overview is modules and what is outside",
|
|
ov_kinds <= {"module", "external"} and check_ir(ov) == [],
|
|
f"got {sorted(ov_kinds)}",
|
|
)
|
|
check(
|
|
"it drops the stdlib but keeps third-party",
|
|
"json" not in {n["id"] for n in ov["nodes"]}
|
|
and "third_party.orm.Mixin" in {n["id"] for n in ov["nodes"]},
|
|
)
|
|
# Depth is the tempting knob and the wrong one: a directory without
|
|
# `__init__.py` is not a package, so its modules have no parent and sit at
|
|
# depth 0. soleprint has 173 such roots.
|
|
fragmented = ops_mod.overview(
|
|
{"meta": {"source": "python", "root": "x", "schema_version": "1"},
|
|
"nodes": [{"id": "loose", "kind": "module", "label": "loose", "parent": None, "attrs": {}},
|
|
{"id": "loose.C", "kind": "class", "label": "C", "parent": "loose", "attrs": {}}],
|
|
"edges": []}
|
|
)
|
|
check(
|
|
"a parentless module still reduces to its module",
|
|
[n["id"] for n in fragmented["nodes"]] == ["loose"],
|
|
"depth-based collapsing would have kept the class; kind-based does not care",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n5b. shape — knowing the picture before rendering it")
|
|
|
|
# Aspect ratio is a property of the graph, not of the renderer. A layered
|
|
# engine puts one dependency level in one row, so the widest level is the
|
|
# width. Measured on soleprint: 7 levels x 109 nodes rendered 14:1, and no
|
|
# layout flag helped — LR merely rotated it (1818x11183), ratio=compress
|
|
# squashed it to an unreadable 1008x75.
|
|
wide = {
|
|
"meta": {"source": "python", "root": "x", "schema_version": "1"},
|
|
"nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}}
|
|
for i in range(40)] +
|
|
[{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}],
|
|
"edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}}
|
|
for i in range(40)],
|
|
}
|
|
sh = ops_mod.shape(wide)
|
|
check("shape counts the widest level", sh["widest_level"] == 40, str(sh))
|
|
check("shape counts levels", sh["levels"] == 2, str(sh))
|
|
check("shape finds isolated nodes", ops_mod.shape(ir)["isolated"] >= 0)
|
|
check(
|
|
"shape needs no renderer",
|
|
isinstance(sh["nodes"], int) and "width" not in sh,
|
|
"it is computed from the IR, so it can warn before writing a 375 KB image",
|
|
)
|
|
|
|
parts = ops_mod.split(ir)
|
|
check("split produces one document per subsystem", len(parts) >= 1, str(list(parts)))
|
|
check("each part still validates", all(check_ir(p) == [] for p in parts.values()))
|
|
check(
|
|
"a crossing edge is kept on the side that reaches out",
|
|
any(p["edges"] for p in parts.values()),
|
|
"dropping it would understate the coupling",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n6. render")
|
|
|
|
if not dot_mod.have_graphviz():
|
|
skip("render", "graphviz not installed — sudo apt install graphviz")
|
|
else:
|
|
svg = dot_mod.render(dot_text)
|
|
check("SVG comes out", svg.startswith(b"<?xml") and b"</svg>" in svg)
|
|
check("the SVG is addressable", b'id="app.models.User"' in svg)
|
|
check(
|
|
"kinds survive as classes for CSS to reach",
|
|
b'class="node class"' in svg and b'class="edge inherits"' in svg,
|
|
)
|
|
print_svg = dot_mod.render(print_text)
|
|
check("the two themes really differ", print_svg != svg)
|
|
|
|
# `fontname="Arial Bold"` passes through as a family no browser has, so it
|
|
# renders neither Arial nor bold. Only the PostScript spelling produces a
|
|
# real weight — and a correctly measured box to hold it.
|
|
check(
|
|
"labels are actually bold, not nominally bold",
|
|
b'font-weight="bold"' in svg,
|
|
"no font-weight in the SVG — the face name did not map to a weight",
|
|
)
|
|
check(
|
|
"the font resolves to a real stack",
|
|
b'font-family="Helvetica,sans-Serif"' in svg,
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n7. a second domain — the schema checkpoint")
|
|
|
|
# The brief makes this a checkpoint: if tables and foreign keys need the schema
|
|
# contorted, the schema is wrong and must be fixed before more extractors.
|
|
SCHEMA = {
|
|
"models": {
|
|
"Customer": {"doc": "A buyer.", "fields": {"id": {"type": "int", "pk": True},
|
|
"email": {"type": "str"}}},
|
|
"Invoice": {"fields": {"id": {"type": "int", "pk": True},
|
|
"customer_id": {"type": "FK:Customer"},
|
|
"due_at": {"type": "datetime", "nullable": True}}},
|
|
"Tag": {"fields": {"id": {"type": "int", "pk": True},
|
|
"invoices": {"type": "M2M:Invoice"}}},
|
|
"Orphan": {"fields": {"ref": {"type": "FK:NotDefinedHere"}}},
|
|
}
|
|
}
|
|
db_ir = db_ex.from_schema_dict(SCHEMA, root="fixture").to_dict()
|
|
db_ids = {n["id"] for n in db_ir["nodes"]}
|
|
db_edges = {(e["source"], e["target"], e["kind"]) for e in db_ir["edges"]}
|
|
db_kinds = {n["kind"] for n in db_ir["nodes"]}
|
|
|
|
check("the db IR validates", check_ir(db_ir) == [], str(check_ir(db_ir)[:3]))
|
|
check(
|
|
"it needed no new top-level field",
|
|
set(db_ir) == set(ir),
|
|
f"{sorted(set(db_ir) ^ set(ir))} — the schema did not survive the second domain",
|
|
)
|
|
check("tables and columns are kinds, not fields", db_kinds >= {"table", "column"})
|
|
check(
|
|
"a column is contained by its table",
|
|
next(n for n in db_ir["nodes"] if n["id"] == "Invoice.customer_id")["parent"] == "Invoice",
|
|
)
|
|
check("a foreign key is an edge", ("Invoice", "Customer", "foreign_key") in db_edges)
|
|
check("a many-to-many is a different edge kind", ("Tag", "Invoice", "references") in db_edges)
|
|
check(
|
|
"domain detail lives in attrs, not in new columns",
|
|
next(n for n in db_ir["nodes"] if n["id"] == "Customer.id")["attrs"].get("pk") is True,
|
|
)
|
|
check(
|
|
"a key naming a table the schema does not define survives as external",
|
|
"NotDefinedHere" in db_ids
|
|
and next(n for n in db_ir["nodes"] if n["id"] == "NotDefinedHere")["kind"] == "external",
|
|
)
|
|
|
|
# "Done means: adding a source type costs one adapter and every existing emitter
|
|
# works on it unchanged." This is that claim, executed.
|
|
db_md = index_mod.to_markdown(db_ir)
|
|
check("the index emitter needed no change for a new domain", "Customer" in db_md)
|
|
db_dot = dot_mod.emit(db_ir, lucid)
|
|
check("the dot emitter needed no change either", db_dot.startswith("digraph ir {"))
|
|
check(
|
|
"an unstyled kind still renders, via `default`",
|
|
lucid.node("table") == lucid.node("default"),
|
|
"no `table` rule exists yet, and the diagram is still legible",
|
|
)
|
|
check(
|
|
"the same view filters both domains",
|
|
check_ir(ops_mod.only_kinds(db_ir, {"table"})) == [],
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n8. the structure picks the drawing")
|
|
|
|
# The lesson this encodes: a diagram that fights its layout engine is usually
|
|
# the wrong kind of diagram. The same 24-table schema rendered 32034x136 through
|
|
# `dot` (235:1) and 1740x1860 through `erd` (0.9:1) — not because one engine is
|
|
# better, but because a schema is peer entities with references, and drawing it
|
|
# in dependency ranks was never its shape.
|
|
verdict_db = ops_mod.classify(db_ir)
|
|
check("a schema is classified as entities", verdict_db["kind"] == "erd", str(verdict_db))
|
|
check("...and routed to the card emitter", verdict_db["emitter"] == "erd")
|
|
|
|
verdict_code = ops_mod.classify(ops_mod.overview(ir))
|
|
check(
|
|
"a small module graph still goes to ranks",
|
|
verdict_code["emitter"] == "dot",
|
|
str(verdict_code),
|
|
)
|
|
|
|
sheet = {
|
|
"meta": {"source": "python", "root": "x", "schema_version": "1"},
|
|
"nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}}
|
|
for i in range(40)] +
|
|
[{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}],
|
|
"edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}}
|
|
for i in range(40)],
|
|
}
|
|
verdict_sheet = ops_mod.classify(sheet)
|
|
check(
|
|
"a 40-wide level is called a sheet, not a diagram",
|
|
verdict_sheet["kind"] == "sheet" and verdict_sheet["emitter"] == "index",
|
|
str(verdict_sheet),
|
|
)
|
|
check("every verdict explains itself", all(
|
|
ops_mod.classify(g)["why"] for g in (db_ir, sheet, ir)
|
|
), "advice without a reason gets overridden")
|
|
|
|
# The property that makes the card layout work, and the one dot cannot offer.
|
|
svg = erd_mod.emit(db_ir, lucid)
|
|
import re as _re
|
|
m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg)
|
|
w, h = int(m.group(1)), int(m.group(2))
|
|
check(
|
|
f"the ER layout stays near-square ({w}x{h})",
|
|
0.25 < w / h < 4,
|
|
"sqrt(n) columns should bound the aspect ratio whatever the table count",
|
|
)
|
|
|
|
wide_schema = {
|
|
"meta": {"source": "db", "root": "x", "schema_version": "1"},
|
|
"nodes": [{"id": f"t{i}", "kind": "table", "label": f"t{i}", "parent": None, "attrs": {}}
|
|
for i in range(60)] +
|
|
[{"id": f"t{i}.c", "kind": "column", "label": "c", "parent": f"t{i}",
|
|
"attrs": {"type": "int"}} for i in range(60)],
|
|
"edges": [],
|
|
}
|
|
svg60 = erd_mod.emit(wide_schema, lucid)
|
|
m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg60)
|
|
w60, h60 = int(m.group(1)), int(m.group(2))
|
|
check(
|
|
f"...and still does at 60 tables ({w60}x{h60})",
|
|
0.25 < w60 / h60 < 4,
|
|
"this is the whole reason it is not a rank-based layout",
|
|
)
|
|
check(
|
|
"the ER geometry is deterministic",
|
|
erd_mod.emit(db_ir, lucid) == svg,
|
|
"nothing here measures a font, so it renders identically on any machine",
|
|
)
|
|
check(
|
|
"it draws no tables only when there are none",
|
|
"no tables" in (
|
|
_err(lambda: erd_mod.emit(ops_mod.overview(ir), lucid))
|
|
),
|
|
"pointing it at a code graph should say so, not emit an empty canvas",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n9. the notebook is a build artifact, not a source file")
|
|
|
|
# The whole argument: a hand-confected notebook drifts from whatever it
|
|
# documents and there is no way to tell by looking. Generated from the spec the
|
|
# server is built from, "is this current" becomes a question about the build.
|
|
SPEC_IR = {
|
|
"meta": {"source": "openapi", "root": "petstore.yaml", "schema_version": "1"},
|
|
"nodes": [
|
|
{"id": "Pet", "kind": "table", "label": "Pet", "parent": None, "attrs": {}},
|
|
{"id": "Pet.id", "kind": "column", "label": "id", "parent": "Pet",
|
|
"attrs": {"type": "int", "pk": True}},
|
|
{"id": "Pet.name", "kind": "column", "label": "name", "parent": "Pet",
|
|
"attrs": {"type": "str"}},
|
|
{"id": "Pet.neutered", "kind": "column", "label": "neutered", "parent": "Pet",
|
|
"attrs": {"type": "bool"}},
|
|
{"id": "GET /pets", "kind": "endpoint", "label": "GET /pets", "parent": None,
|
|
"attrs": {"method": "GET", "path": "/pets", "summary": "List pets",
|
|
"status": 200, "response_model": "Pet", "returns_list": True}},
|
|
{"id": "POST /pets", "kind": "endpoint", "label": "POST /pets", "parent": None,
|
|
"attrs": {"method": "POST", "path": "/pets", "summary": "Create a pet",
|
|
"status": 201, "request_model": "Pet", "response_model": "Pet"}},
|
|
],
|
|
"edges": [{"source": "POST /pets", "target": "Pet", "kind": "accepts", "attrs": {}}],
|
|
}
|
|
check("the spec IR validates", check_ir(SPEC_IR) == [], str(check_ir(SPEC_IR)[:2]))
|
|
|
|
base = spec_mod.from_ir(SPEC_IR)
|
|
book_spec, _ = spec_mod.merge(base, None)
|
|
book = nb_mod.build(book_spec)
|
|
text = nb_mod.emit(book_spec)
|
|
check("it is nbformat 4", book["nbformat"] == 4 and book["nbformat_minor"] == 5)
|
|
check(
|
|
"regenerating gives byte-identical bytes",
|
|
nb_mod.emit(book_spec) == text,
|
|
"a notebook that changes every build cannot be reviewed",
|
|
)
|
|
check("nothing has run in it", all(
|
|
c["outputs"] == [] and c["execution_count"] is None
|
|
for c in book["cells"] if c["cell_type"] == "code"
|
|
))
|
|
intro = "".join(book["cells"][0]["source"]).lower()
|
|
check(
|
|
"it says not to edit it, and where to put changes instead",
|
|
"not be edited" in intro and "overlay" in intro,
|
|
f"the frame only holds if the artifact states it — got: {intro[:120]!r}",
|
|
)
|
|
body = "".join("".join(c["source"]) for c in book["cells"])
|
|
check("every endpoint in the spec has a section", "GET `/pets`" in body and "POST `/pets`" in body)
|
|
check("the facts come from the spec", "returns **Pet**" in body and "`201`" in body)
|
|
check(
|
|
"no credential is written into it",
|
|
"Bearer sk-" not in body and "password" not in body.lower(),
|
|
)
|
|
check(
|
|
"it needs nothing installed",
|
|
"pip install" not in body and "import requests" not in body,
|
|
)
|
|
|
|
# Compiling is not enough. `json.dumps` writes `false`/`true`/`null`, which are
|
|
# valid *identifiers* in Python — a generated body full of them compiles and
|
|
# then raises NameError on the first run. This is the check that caught it.
|
|
import io as _io
|
|
import contextlib as _ctx
|
|
|
|
ns, ran, failure = {}, 0, None
|
|
with _ctx.redirect_stdout(_io.StringIO()):
|
|
for c in book["cells"]:
|
|
if c["cell_type"] != "code":
|
|
continue
|
|
src = "".join(c["source"])
|
|
if "urlopen" in src or "call(" in src and "def call" not in src:
|
|
# Anything that would reach the network is compiled, not run.
|
|
try:
|
|
compile(src, c["id"], "exec")
|
|
ran += 1
|
|
except SyntaxError as e:
|
|
failure = f"{c['id']}: {e}"
|
|
break
|
|
continue
|
|
try:
|
|
exec(compile(src, c["id"], "exec"), ns)
|
|
ran += 1
|
|
except Exception as e: # noqa: BLE001 - any failure is the finding
|
|
failure = f"{c['id']}: {type(e).__name__}: {e}"
|
|
break
|
|
check(f"its cells run, not merely compile ({ran})", failure is None, failure or "")
|
|
|
|
literal = next(
|
|
("".join(c["source"]) for c in book["cells"]
|
|
if c["cell_type"] == "code" and "BODY" in "".join(c["source"])), ""
|
|
)
|
|
check(
|
|
"a generated body is a Python literal, not JSON",
|
|
"False" in literal and "false" not in literal,
|
|
f"got: {literal[:120]}",
|
|
)
|
|
check(
|
|
"the primary key is left out of a create body",
|
|
"'id'" not in literal,
|
|
"the server assigns it",
|
|
)
|
|
|
|
# The chain: spec -> IR -> notebook, nothing by hand.
|
|
#
|
|
# The fixture is docgen's own, shipped in `fixtures/`, so this no longer skips
|
|
# for want of a file when docgen is used standalone. It still skips without
|
|
# modelgen, because reading OpenAPI genuinely needs it — that is the one seam
|
|
# out of docgen, and `reference.py` is where it resolves.
|
|
try:
|
|
oa = __import__(f"{PKG}.extractors.openapi", fromlist=["*"])
|
|
spec = HERE / "fixtures" / "orders.yaml"
|
|
if not spec.exists():
|
|
raise ImportError(f"fixture missing: {spec}")
|
|
real = oa.extract(spec).to_dict()
|
|
except Exception as e: # noqa: BLE001 - absent modelgen is a skip, not a failure
|
|
skip("openapi -> IR", str(e).splitlines()[0][:70])
|
|
else:
|
|
check("a real spec extracts", check_ir(real) == [], str(check_ir(real)[:2]))
|
|
eps = [n for n in real["nodes"] if n["kind"] == "endpoint"]
|
|
check(f"its endpoints become nodes ({len(eps)})", len(eps) >= 3)
|
|
check(
|
|
"its schemas reuse the db vocabulary",
|
|
{"table", "column"} <= {n["kind"] for n in real["nodes"]},
|
|
"so the ER emitter draws an API's data model without knowing it is one",
|
|
)
|
|
# modelgen maps an inter-schema `$ref` to the literal string `dict`, so the
|
|
# target is gone by the time its fields reach docgen. Without recovering it
|
|
# an API's data model draws as disconnected cards — three tables, no keys,
|
|
# and nothing saying the relationships were lost. Found by writing a fixture
|
|
# that has refs; the borrowed one was never checked for this.
|
|
fks = {(e["source"], e["target"]) for e in real["edges"] if e["kind"] == "foreign_key"}
|
|
check(
|
|
"a $ref between schemas survives as a foreign key",
|
|
fks == {("Order", "Customer"), ("OrderLine", "Order")},
|
|
f"got {sorted(fks)} — an ERD of an API with no edges is the failure this tool "
|
|
"exists to prevent, one level up",
|
|
)
|
|
check(
|
|
"and the referencing column says what it points at",
|
|
[n["attrs"]["references"] for n in real["nodes"]
|
|
if n["id"] == "Order.customer"] == ["Customer"],
|
|
)
|
|
svg = erd_mod.emit(real, lucid)
|
|
check("and the ER emitter does draw it", svg.startswith("<?xml"))
|
|
check(
|
|
"...with the relationships in the drawing",
|
|
svg.count("<path") >= len(fks),
|
|
f"{svg.count('<path')} paths for {len(fks)} foreign keys",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n8b. usage — what a spec cannot say")
|
|
|
|
# A spec is a set; usage is a sequence. Everything asserted here is something no
|
|
# amount of reading the OpenAPI document harder would recover.
|
|
def _entry(t, method, url, status, body=None):
|
|
req = {"method": method, "url": url,
|
|
"headers": [{"name": "Authorization", "value": "Bearer LEAK-ME-IF-BROKEN"}]}
|
|
if body is not None:
|
|
req["postData"] = {"mimeType": "application/json", "text": json.dumps(body)}
|
|
return {"startedDateTime": f"2026-01-01T00:00:{t:02d}Z",
|
|
"request": req, "response": {"status": status, "content": {}}}
|
|
|
|
|
|
HAR = {"log": {"version": "1.2", "entries": [
|
|
_entry(1, "POST", "https://x.test/auth", 200, {"user": "a", "secret": "b"}),
|
|
_entry(2, "GET", "https://x.test/pets?status=available&limit=20", 200),
|
|
_entry(3, "GET", "https://x.test/pets/1042", 200),
|
|
_entry(4, "GET", "https://x.test/pets/7f3e9b2a-1c4d-4a5b-9e8f-0a1b2c3d4e5f", 200),
|
|
_entry(5, "POST", "https://x.test/graphql", 200,
|
|
{"operationName": "PetWithTags", "query": "query PetWithTags($id: ID!) { pet }",
|
|
"variables": {"id": "1042"}}),
|
|
_entry(6, "POST", "https://x.test/pets", 422, {"name": "Rex"}),
|
|
_entry(7, "POST", "https://x.test/pets", 201, {"name": "Rex", "status": "available"}),
|
|
_entry(8, "GET", "https://x.test/pets?status=available&limit=20", 200),
|
|
]}}
|
|
har_path = ROOT.parent / "session.har"
|
|
har_path.write_text(json.dumps(HAR))
|
|
use = usage_ex.extract(har_path).to_dict()
|
|
by = {n["id"]: n for n in use["nodes"]}
|
|
raw_json = json.dumps(use)
|
|
|
|
check("the usage IR validates", check_ir(use) == [], str(check_ir(use)[:2]))
|
|
check(
|
|
"**no credential reaches the IR**",
|
|
"LEAK-ME-IF-BROKEN" not in raw_json and "Authorization" not in raw_json,
|
|
"a HAR is full of live tokens and a document gets committed",
|
|
)
|
|
check(
|
|
"paths are templated into one route",
|
|
"GET /pets/{id}" in by and "GET /pets/1042" not in by,
|
|
)
|
|
check(
|
|
"...and both id formats stay on that one route",
|
|
by["GET /pets/{id}"]["attrs"]["id_formats"] == ["numeric", "uuid"]
|
|
and by["GET /pets/{id}"]["attrs"]["calls"] == 2,
|
|
"splitting by format invents an endpoint that does not exist",
|
|
)
|
|
check(
|
|
"what is always sent is distinguished from what is sometimes",
|
|
by["GET /pets"]["attrs"]["params_always"] == ["limit", "status"],
|
|
"a spec calls these optional; traffic says otherwise",
|
|
)
|
|
check(
|
|
"the statuses that really happen are recorded",
|
|
by["POST /pets"]["attrs"]["statuses"] == [201, 422],
|
|
"the 422 everyone hits is not in the spec",
|
|
)
|
|
check(
|
|
"a body field seen once out of twice is not marked always",
|
|
[f["name"] for f in by["POST /pets"]["attrs"]["body_fields"] if f["always"]] == ["name"],
|
|
)
|
|
check(
|
|
"a GraphQL operation is found and named",
|
|
"query PetWithTags" in by and by["query PetWithTags"]["kind"] == "operation",
|
|
"one endpoint carries many operations; the path alone says nothing",
|
|
)
|
|
check(
|
|
"only the shape of a payload is kept, never a value",
|
|
'"a"' not in raw_json and '"b"' not in raw_json,
|
|
)
|
|
seq = {(e["source"], e["target"]) for e in use["edges"]}
|
|
check(
|
|
"the sequence is recorded as edges",
|
|
("POST /auth", "GET /pets") in seq and all(e["kind"] == "follows" for e in use["edges"]),
|
|
"this is the half a spec structurally cannot contain",
|
|
)
|
|
check(
|
|
"a repeated call is not a step in the sequence",
|
|
not [e for e in use["edges"] if e["source"] == e["target"]],
|
|
)
|
|
check(
|
|
"usage is a pipeline, and drawn as one",
|
|
ops_mod.classify(use)["kind"] == "pipeline",
|
|
)
|
|
|
|
# The payoff: a walkthrough in the order people actually call things.
|
|
u_spec, _ = spec_mod.merge(spec_mod.from_ir(use), None)
|
|
ids = [st["id"] for st in u_spec["steps"] if st["kind"] == "call"]
|
|
check(
|
|
"the notebook keeps the observed order",
|
|
ids[0] == "POST /auth" and ids.index("GET /pets") < ids.index("POST /pets"),
|
|
f"got {ids}",
|
|
)
|
|
u_body = "".join("".join(c["source"]) for c in nb_mod.build(u_spec)["cells"])
|
|
check(
|
|
"it sends the parameters that are always sent",
|
|
"PARAMS = " in u_body and "'status'" in u_body,
|
|
)
|
|
check(
|
|
"a templated path becomes a variable, not a literal",
|
|
'ID = "<id>"' in u_body and '"/pets/{id}"' not in u_body,
|
|
"a literal {id} in the URL would 404",
|
|
)
|
|
check("the GraphQL operation gets a query cell", "QUERY = " in u_body)
|
|
har_path.unlink()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n9b. generated base + hand-written overlay")
|
|
|
|
# The arrangement that gets both halves: generation alone says nothing a parser
|
|
# could not work out; hand-authoring alone rots. The base is never edited, the
|
|
# overlay is the only file anyone touches, and it is re-applied every build.
|
|
OVERLAY = {
|
|
"steps": {
|
|
"GET /pets": {
|
|
"note": "always called with status=available first",
|
|
"code": 'show(call("GET", "/pets", params={"status": "available"}))',
|
|
},
|
|
"setup": {"_kind": "params", "_title": ""}, # scaffold hints, not edits
|
|
},
|
|
"insert": [
|
|
{"id": "graphql", "after": "client", "kind": "md",
|
|
"title": "The GraphQL endpoint",
|
|
"text": "Not in the OpenAPI document at all."},
|
|
],
|
|
"drop": ["shapes"],
|
|
}
|
|
|
|
merged, drift = spec_mod.merge(spec_mod.from_ir(SPEC_IR), OVERLAY)
|
|
ids = [st["id"] for st in merged["steps"]]
|
|
body_o = "".join("".join(c["source"]) for c in nb_mod.build(merged)["cells"])
|
|
|
|
check("no drift against a matching base", drift == [], str(drift))
|
|
check("an inserted step lands where it was asked to", ids.index("graphql") == ids.index("client") + 1)
|
|
check("a dropped step is gone", "shapes" not in ids)
|
|
check(
|
|
"replaced code wins over the generated call",
|
|
'params={"status": "available"}' in body_o,
|
|
"this is how real usage gets in, and a spec cannot supply it",
|
|
)
|
|
check("an annotation reaches the prose", "always called with status=available" in body_o)
|
|
check(
|
|
"scaffold hints are not mistaken for edits",
|
|
"_kind" not in body_o and "_title" not in body_o,
|
|
)
|
|
|
|
# The two properties the whole arrangement rests on.
|
|
again, _ = spec_mod.merge(spec_mod.from_ir(SPEC_IR), OVERLAY)
|
|
check(
|
|
"regenerating re-applies the overlay, byte for byte",
|
|
nb_mod.emit(again) == nb_mod.emit(merged),
|
|
"a regeneration that loses someone's work will not be run twice",
|
|
)
|
|
|
|
moved = dict(SPEC_IR, nodes=[n for n in SPEC_IR["nodes"] if n["id"] != "GET /pets"])
|
|
shrunk, drift2 = spec_mod.merge(spec_mod.from_ir(moved), OVERLAY)
|
|
check(
|
|
"when the base moves, the overlay says so",
|
|
any("GET /pets" in d for d in drift2),
|
|
"silently dropping it is how an overlay goes stale without anyone noticing",
|
|
)
|
|
check("...and a document is still produced", len(shrunk["steps"]) > 1)
|
|
|
|
bare, drift3 = spec_mod.merge(spec_mod.from_ir(SPEC_IR), None)
|
|
check(
|
|
"extraction works with the overlay absent",
|
|
drift3 == [] and len(bare["steps"]) > 1,
|
|
"the overlay is an addition, never a dependency",
|
|
)
|
|
|
|
blank = spec_mod.scaffold(spec_mod.from_ir(SPEC_IR))
|
|
check(
|
|
"a scaffold hands over every step id",
|
|
set(blank["steps"]) == {st["id"] for st in spec_mod.from_ir(SPEC_IR)["steps"]},
|
|
"listing the ids is the difference between an overlay being written and meant to be",
|
|
)
|
|
check(
|
|
"a blank scaffold changes nothing",
|
|
nb_mod.emit(spec_mod.merge(spec_mod.from_ir(SPEC_IR), blank)[0]) == nb_mod.emit(bare),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n10. defaults")
|
|
|
|
check(
|
|
"dark is the default theme",
|
|
Style.load("lucid").theme == "dark",
|
|
"a diagram lands in a dark docs page far more often than in print",
|
|
)
|
|
check("print is one flag away", Style.load("lucid", theme="lucid").theme == "lucid")
|
|
|
|
pipeline = {
|
|
"meta": {"source": "airflow", "root": "etl", "schema_version": "1"},
|
|
"nodes": [{"id": x, "kind": "task", "label": x, "parent": None, "attrs": {}}
|
|
for x in ("extract", "transform", "load", "notify")],
|
|
"edges": [{"source": a, "target": b, "kind": "depends", "attrs": {}}
|
|
for a, b in (("extract", "transform"), ("transform", "load"),
|
|
("load", "notify"))],
|
|
}
|
|
verdict = ops_mod.classify(pipeline)
|
|
check(
|
|
"a DAG is recognised as a pipeline",
|
|
verdict["kind"] == "pipeline",
|
|
str(verdict),
|
|
)
|
|
check(
|
|
"...and is drawn left to right",
|
|
verdict.get("options", {}).get("rankdir") == "LR",
|
|
"a sequence reads across, which is how every scheduler's own UI draws it",
|
|
)
|
|
check(
|
|
"a task has its own style, not a module's",
|
|
lucid.node("task") != lucid.node("module"),
|
|
"a schedule should not render as if it were code structure",
|
|
)
|
|
check(
|
|
"the emitter honours the rankdir it is given",
|
|
"rankdir=LR" in dot_mod.emit(pipeline, lucid, rankdir="LR"),
|
|
)
|
|
check("every verdict carries options", "options" in ops_mod.classify(ir))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n11. the docs site")
|
|
|
|
# Not invented here: five demos under semester/ carry the same viewer, differing
|
|
# only in comments and one background colour. This generates that arrangement
|
|
# rather than a sixth copy — and adds the 1:1 toggle none of them has.
|
|
site_dir = ROOT.parent / "site"
|
|
files = site_mod.write(ops_mod.overview(ir), lucid, site_dir, graph="graph.svg", title="fx")
|
|
names = {f.name for f in files}
|
|
check("it writes the three files", names == {"index.html", "viewer.html", "site.css"}, str(names))
|
|
|
|
index_html = (site_dir / "index.html").read_text()
|
|
css = (site_dir / "site.css").read_text()
|
|
viewer = (site_dir / "viewer.html").read_text()
|
|
|
|
check(
|
|
"the graph links to the viewer, the way both demos already do it",
|
|
'href="viewer.html?src=graph.svg"' in index_html and "<img src=" in index_html,
|
|
"spr/docs/docs.js:229 and sms/docs/index.html:311 arrived at this independently",
|
|
)
|
|
check("the sidebar nests", "<details" in index_html and "<summary>" in index_html)
|
|
check(
|
|
"every sidebar link is styled",
|
|
".sidebar a {" in css,
|
|
"selecting `li > a` misses links inside a <summary>, which is most of them",
|
|
)
|
|
check(
|
|
"colours are baked from the theme, not hardcoded",
|
|
lucid.slot("surface-0") in css and lucid.slot("accent") in css,
|
|
)
|
|
light = site_mod.emit(ops_mod.overview(ir), print_theme, graph="graph.svg")
|
|
check(
|
|
"a light theme gives a light page",
|
|
print_theme.slot("surface-0") in light["site.css"]
|
|
and lucid.slot("surface-0") not in light["site.css"],
|
|
"the page and the diagram on it move together",
|
|
)
|
|
check(
|
|
"it is self-contained and offline",
|
|
"http://" not in index_html and "https://" not in index_html
|
|
and "cdn" not in index_html.lower(),
|
|
)
|
|
check("the viewer carries a 1:1 toggle", "1:1" in viewer and "fitScale" in viewer)
|
|
|
|
# The viewer is JavaScript, so it is tested as JavaScript.
|
|
import shutil as _shutil
|
|
import subprocess as _sub
|
|
|
|
if not _shutil.which("node"):
|
|
skip("viewer behaviour", "node not installed")
|
|
else:
|
|
harness = HERE / "viewer_test.js"
|
|
if not harness.exists():
|
|
skip("viewer behaviour", "viewer_test.js missing")
|
|
else:
|
|
proc = _sub.run(["node", str(harness), str(site_dir / "viewer.html")],
|
|
capture_output=True, text=True)
|
|
for line in proc.stdout.strip().splitlines():
|
|
name = line.strip()[5:]
|
|
(PASS if line.strip().startswith("ok") else FAIL).append(f"viewer: {name}")
|
|
print(f" {line.strip()[:4]} viewer: {name}")
|
|
if proc.returncode and not proc.stdout.strip():
|
|
check("the viewer harness runs", False, proc.stderr.strip()[:200])
|
|
|
|
_shutil.rmtree(site_dir, ignore_errors=True)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n12. the minimap — what is where, read from the colours")
|
|
|
|
mm = mm_mod.emit(ir, lucid, scale=1.0)
|
|
check("it produces SVG", mm.startswith("<?xml") and "</svg>" in mm)
|
|
check(
|
|
"a block is sized by the lines it occupies",
|
|
mm_mod._files(ir)[0]["total"] > 1
|
|
and all(f["total"] >= 1 for f in mm_mod._files(ir)),
|
|
)
|
|
check(
|
|
"no text is rendered inside a block",
|
|
"<text" in mm and mm.count("<rect") > mm.count("<text"),
|
|
"the shape is the message; labels are for the margins",
|
|
)
|
|
|
|
# The claim: kind is the only thing that varies, so the pattern is in the colour.
|
|
slots = {mm_mod.KIND_SLOT[k] for k in ("class", "interface", "function")}
|
|
check(
|
|
"class, interface and function are three different colours",
|
|
len({lucid.slot(x) for x in slots}) == 3,
|
|
f"got {[lucid.slot(x) for x in slots]}",
|
|
)
|
|
for kind in ("class", "function"):
|
|
check(f"a {kind} is drawn in its slot",
|
|
lucid.slot(mm_mod.KIND_SLOT[kind]) in mm)
|
|
|
|
# A namespace and a file both arrive as `module`; drawing both puts a file's
|
|
# contents on the canvas twice and insets everything for nothing.
|
|
wrapped = {
|
|
"meta": {"source": "code", "root": "x", "schema_version": "1"},
|
|
"nodes": [
|
|
{"id": "F", "kind": "module", "label": "F", "parent": None,
|
|
"attrs": {"file": "F.cs", "lines": 40}},
|
|
{"id": "F.Ns", "kind": "module", "label": "Ns", "parent": "F",
|
|
"attrs": {"file": "F.cs", "line": 2, "lines": 38}},
|
|
{"id": "F.Ns.C", "kind": "class", "label": "C", "parent": "F.Ns",
|
|
"attrs": {"file": "F.cs", "line": 4, "lines": 30}},
|
|
],
|
|
"edges": [],
|
|
}
|
|
mapped = mm_mod._files(wrapped)
|
|
check(
|
|
"a namespace wrapper is dissolved, not drawn as a second file",
|
|
len(mapped) == 1 and mapped[0]["id"] == "F",
|
|
f"got {[f['id'] for f in mapped]}",
|
|
)
|
|
check(
|
|
"...and what was inside it survives, re-parented",
|
|
[c["label"] for c in mapped[0]["children"]] == ["C"],
|
|
f"got {[c['label'] for c in mapped[0]['children']]}",
|
|
)
|
|
|
|
# Layout is chosen, not emergent — the same lesson as the aspect-ratio work.
|
|
# A band per package gave 1196x10165 on a tree of 234 files in 70 packages.
|
|
many = {
|
|
"meta": {"source": "python", "root": "x", "schema_version": "1"},
|
|
"nodes": [
|
|
{"id": f"p{i}.m", "kind": "module", "label": "m", "parent": None,
|
|
"attrs": {"file": f"p{i}/m.py", "lines": 60}}
|
|
for i in range(60)
|
|
],
|
|
"edges": [],
|
|
}
|
|
wide = mm_mod.emit(many, lucid, scale=0.5, target_width=1000)
|
|
import re as _re2
|
|
m2 = _re2.search(r'width="(\d+)pt" height="(\d+)pt"', wide)
|
|
w2, h2 = int(m2.group(1)), int(m2.group(2))
|
|
check(
|
|
f"60 files in 60 packages still pack ({w2}x{h2})",
|
|
h2 < w2 * 3,
|
|
"a row per package turns a wide map into a ribbon",
|
|
)
|
|
|
|
check(
|
|
"an IR with no line spans says so rather than drawing nothing",
|
|
"lines" in _err(lambda: mm_mod.emit(
|
|
{"meta": {"source": "x", "root": "x", "schema_version": "1"},
|
|
"nodes": [], "edges": []}, lucid)),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n13. tree-sitter — C# and TypeScript, when it is installed")
|
|
|
|
try:
|
|
code_ex = __import__(f"{PKG}.extractors.code", fromlist=["*"])
|
|
code_ex._parser(".cs")
|
|
except Exception as e: # noqa: BLE001
|
|
skip("tree-sitter", str(e).splitlines()[0][:64])
|
|
else:
|
|
cs = ROOT.parent / "cs"
|
|
(cs / "Core").mkdir(parents=True, exist_ok=True)
|
|
(cs / "Core" / "Shop.cs").write_text(
|
|
'using System;\n'
|
|
'\n'
|
|
'namespace Shop.Core\n'
|
|
'{\n'
|
|
' public interface IRepo { Task<int> CountAsync(); }\n'
|
|
'\n'
|
|
' public class Repo : IRepo\n'
|
|
' {\n'
|
|
' public async Task<int> CountAsync()\n'
|
|
' {\n'
|
|
' var sql = "SELECT 1 WHERE x = {0}";\n'
|
|
' return 1;\n'
|
|
' }\n'
|
|
'\n'
|
|
' private enum Mode { A, B }\n'
|
|
' }\n'
|
|
'}\n'
|
|
)
|
|
got = code_ex.extract(cs).to_dict()
|
|
kinds = {n["id"].rsplit(".", 1)[-1]: n["kind"] for n in got["nodes"]}
|
|
check("the C# IR validates", check_ir(got) == [], str(check_ir(got)[:2]))
|
|
check("a class is a class", kinds.get("Repo") == "class", str(kinds))
|
|
check(
|
|
"an interface is kept apart from a class",
|
|
kinds.get("IRepo") == "interface",
|
|
"in C# and TypeScript that distinction is most of what a file tells you",
|
|
)
|
|
check("a method is a function", kinds.get("CountAsync") == "function")
|
|
iface_method = next(n for n in got["nodes"] if n["id"].endswith(".IRepo.CountAsync"))
|
|
check(
|
|
"an interface's one-line method is one line",
|
|
iface_method["attrs"]["lines"] == 1,
|
|
"and is a different node from the class's implementation of it",
|
|
)
|
|
check("a nested enum is found", kinds.get("Mode") == "class")
|
|
# `.endswith("Repo.CountAsync")` also matches `IRepo.CountAsync`, whose
|
|
# one-line declaration is correct — so name the class's method exactly.
|
|
method = next(n for n in got["nodes"] if n["id"].endswith(".Repo.CountAsync"))
|
|
check(
|
|
"a brace inside a string does not end a block",
|
|
method["attrs"]["lines"] >= 4,
|
|
f'got {method["attrs"]["lines"]} lines — a scanner would stop at the brace in the string',
|
|
)
|
|
check(
|
|
"structure only — no edges are invented",
|
|
got["edges"] == [],
|
|
"resolving a `using` is a different job, and half a graph looks whole",
|
|
)
|
|
check("the minimap draws it", mm_mod.emit(got, lucid).startswith("<?xml"))
|
|
_shutil.rmtree(cs, ignore_errors=True)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n14. explore — navigate on one side, explore on the other")
|
|
|
|
# The minimap alone showed shape and no meaning. It is not the artifact, it is
|
|
# the selector — and that is also what retires the 14:1 sheet: the whole graph
|
|
# is never drawn, only the neighbourhood of a selection.
|
|
exp_dir = ROOT.parent / "exp"
|
|
page = exp_mod.write(db_ir, lucid, exp_dir, title="schema")
|
|
html = page.read_text()
|
|
|
|
check("it writes one self-contained page", page.name == "explore.html")
|
|
check(
|
|
"a schema navigates on table headers, not line spans",
|
|
"erd" not in html and 'data-kind="table"' in html,
|
|
"the minimap's geometry is lines-of-source, which means nothing for a table",
|
|
)
|
|
check(
|
|
"the overview carries no columns",
|
|
html.count('data-kind="table"') == sum(
|
|
1 for n in db_ir["nodes"] if n["kind"] == "table"
|
|
),
|
|
)
|
|
check("the minimap is inline, so a block can be clicked", "<svg" in html and "<img src=" in html)
|
|
check("facts travel with the page", '"kind"' in html and '"out"' in html)
|
|
# `xmlns="http://www.w3.org/2000/svg"` is an XML namespace — an identifier, not
|
|
# something anything fetches. What must not appear is a *reference*.
|
|
fetched = re.findall(r'(?:src|href)\s*=\s*"(https?://[^"]+)"', html)
|
|
check("nothing is fetched from the network", not fetched, str(fetched[:3]))
|
|
|
|
graphs = sorted((exp_dir / "graphs").glob("*.svg"))
|
|
check(f"neighbourhoods are pre-rendered ({len(graphs)})", len(graphs) >= 1)
|
|
check(
|
|
"a table's neighbourhood keeps its columns",
|
|
any("column" not in g.read_text() or "PK" in g.read_text() for g in graphs),
|
|
"a table without its columns is not a table",
|
|
)
|
|
|
|
# A module's neighbourhood must NOT drag its contents in — that is the sheet.
|
|
code_ir = ops_mod.overview(ir)
|
|
mod_view = ops_mod.neighbourhood(code_ir, next(
|
|
n["id"] for n in code_ir["nodes"] if n["kind"] == "module"), hops=1)
|
|
check(
|
|
"a module's neighbourhood stays small",
|
|
len(mod_view["nodes"]) <= 24,
|
|
"its contents are the hundred functions that made the sheet unreadable",
|
|
)
|
|
|
|
if not _shutil.which("node"):
|
|
skip("explore behaviour", "node not installed")
|
|
else:
|
|
harness = HERE / "explore_test.js"
|
|
if not harness.exists():
|
|
skip("explore behaviour", "explore_test.js missing")
|
|
else:
|
|
proc = _sub.run(["node", str(harness), str(page)], capture_output=True, text=True)
|
|
for line in proc.stdout.strip().splitlines():
|
|
name = line.strip()[5:]
|
|
(PASS if line.strip().startswith("ok") else FAIL).append(f"explore: {name}")
|
|
print(f" {line.strip()[:4]} explore: {name}")
|
|
if proc.returncode and not proc.stdout.strip():
|
|
check("the explore harness runs", False, proc.stderr.strip()[:200])
|
|
|
|
_shutil.rmtree(exp_dir, ignore_errors=True)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
tmp.cleanup()
|
|
print()
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n15. the book — every operation measured at both ends")
|
|
|
|
|
|
def _books():
|
|
"""Its own scope, so nothing here can rebind the run's `tmp` (see 3b)."""
|
|
larder_mod = __import__(f"{PKG}.book.larder", fromlist=["*"])
|
|
book_mod = __import__(f"{PKG}.book", fromlist=["*"])
|
|
build_mod = __import__(f"{PKG}.book.build", fromlist=["*"])
|
|
checks_mod = __import__(f"{PKG}.book.checks", fromlist=["*"])
|
|
Larder = larder_mod.Larder
|
|
|
|
# -- the input measure, which nothing had before ----------------------
|
|
check(
|
|
"the unit vocabulary is closed",
|
|
larder_mod.UNITS == ("file", "table", "path", "entry", "document"),
|
|
f"got {larder_mod.UNITS} — a unit nobody else uses cannot be compared",
|
|
)
|
|
try:
|
|
Larder(kind="x", identity="y", unit="thingy")
|
|
refused = False
|
|
except ValueError:
|
|
refused = True
|
|
check("a unit outside it is refused, not recorded", refused)
|
|
|
|
# `read` derived rather than stored is the one design decision here, and it
|
|
# is the one that stops the measure disagreeing with itself.
|
|
l = Larder(kind="python", identity="src", unit="file", seen=47)
|
|
l.fail("a.py", "syntax"); l.fail("b.py", "unreadable")
|
|
check("read is seen minus failed, derived", (l.seen, l.read, len(l.failed)) == (47, 45, 2),
|
|
f"seen={l.seen} read={l.read} failed={len(l.failed)}")
|
|
check(
|
|
"the measure reads as a sentence",
|
|
l.line() == "src — 45 files read, 2 failed",
|
|
f"got {l.line()!r}",
|
|
)
|
|
# "2 entrys read, 1 hosts" is how this read before. A measure nobody reads
|
|
# is not a measure, so the grammar gets attention it would not otherwise.
|
|
e = Larder(kind="usage", identity="s.har", unit="entry", seen=2)
|
|
e.extra["hosts"] = 1
|
|
check("counts and labels agree", e.line() == "s.har — 2 entries read, 1 host",
|
|
f"got {e.line()!r}")
|
|
|
|
# -- redaction, the one field that can carry a secret -----------------
|
|
cases = [
|
|
("postgresql://app:hunter2@db:5432/shop", "postgresql://app:***@db:5432/shop"),
|
|
("https://api/v1?token=abc123&page=2", "https://api/v1?token=***&page=2"),
|
|
("Driver=x;Server=y;Password=hunter2;Uid=app", "Driver=x;Server=y;Password=***;Uid=app"),
|
|
("../../station", "../../station"),
|
|
]
|
|
wrong = [f"{src} -> {larder_mod.redact(src)}" for src, want in cases
|
|
if larder_mod.redact(src) != want]
|
|
check("credentials are masked and nothing else is", not wrong, "; ".join(wrong))
|
|
check(
|
|
"the secret is gone, the recognisable part is not",
|
|
"hunter2" not in larder_mod.redact(cases[0][0]) and "db:5432/shop" in larder_mod.redact(cases[0][0]),
|
|
)
|
|
|
|
# The scrubber is graded by a second, independent key list in validate.py.
|
|
leaked = dict(ir["meta"], larder={
|
|
"kind": "db", "identity": "postgresql://app:hunter2@db/shop",
|
|
"unit": "table", "seen": 1, "read": 1, "failed": [],
|
|
})
|
|
problems = ir_validate.check({**ir, "meta": leaked})
|
|
check(
|
|
"a secret that survived redaction fails validation",
|
|
any("identity still contains" in p for p in problems),
|
|
f"got {problems[:2]} — a scrubber graded by its own word is not graded",
|
|
)
|
|
# And the arithmetic, because a book can be assembled without the IR.
|
|
lying = dict(ir["meta"], larder={
|
|
"kind": "python", "identity": "src", "unit": "file",
|
|
"seen": 10, "read": 10, "failed": [{"name": "a.py", "error": "x"}],
|
|
})
|
|
check(
|
|
"a measure that disagrees with itself fails validation",
|
|
any("disagrees with itself" in p for p in ir_validate.check({**ir, "meta": lying})),
|
|
)
|
|
|
|
# -- every extractor reports one. The extension contract, with teeth. --
|
|
# Looped over the registry rather than a hand-written list, so adding an
|
|
# extractor makes this start asking about it.
|
|
check(
|
|
"every registered extractor is in the relation table",
|
|
set(build_mod.EXTRACTORS) == set(book_mod.RELATION),
|
|
f"registry {sorted(build_mod.EXTRACTORS)} vs relations {sorted(book_mod.RELATION)}"
|
|
" — a new extractor must say how its units show up in the output",
|
|
)
|
|
unmeasured = []
|
|
for kind in sorted(build_mod.EXTRACTORS):
|
|
module_name, fn_name, _ = build_mod.EXTRACTORS[kind]
|
|
try:
|
|
mod = __import__(f"{PKG}{module_name}", fromlist=["*"])
|
|
except ImportError as exc:
|
|
unmeasured.append(f"{kind}: not importable ({exc})")
|
|
continue
|
|
fn = getattr(mod, fn_name, None)
|
|
if fn is None:
|
|
unmeasured.append(f"{kind}: no {fn_name}()")
|
|
elif "identity" not in fn.__code__.co_varnames[:fn.__code__.co_argcount]:
|
|
unmeasured.append(f"{kind}: {fn_name}() takes no `identity`")
|
|
check(
|
|
"every registered extractor can name its larder",
|
|
not unmeasured,
|
|
"; ".join(unmeasured) + " <- an extractor is done when it measures its input",
|
|
)
|
|
|
|
# -- the reconciliation, which is what having both ends is for ---------
|
|
b = book_mod.Book("pretend", Larder(kind="python", identity="p", unit="file", seen=10),
|
|
"/nonexistent")
|
|
b.ir = {"nodes": [{"id": f"m{i}", "kind": "module", "label": "m",
|
|
"attrs": {"file": f"m{i}.py"}} for i in range(3)], "edges": []}
|
|
lost = [r for r in b.compare() if not r["ok"]]
|
|
check(
|
|
"input read but not produced is reported as lost",
|
|
len(lost) == 1 and lost[0]["id"] == "units-accounted-for",
|
|
f"got {[r['id'] for r in b.compare()]}",
|
|
)
|
|
check("and it says how much", "7 file(s)" in lost[0]["why"], lost[0]["why"][:70])
|
|
|
|
# A file that failed to parse still gets a node carrying attrs.error, so the
|
|
# two populations must be counted apart. Counting them together made
|
|
# "2 files read produced 4 modules" pass a check meant to prove nothing was
|
|
# dropped — which is how it was written first.
|
|
b2 = book_mod.Book("split", Larder(kind="python", identity="p", unit="file", seen=4),
|
|
"/nonexistent")
|
|
b2.larder.fail("c.py", "syntax"); b2.larder.fail("d.py", "syntax")
|
|
b2.ir = {"edges": [], "nodes": [
|
|
{"id": "a", "kind": "module", "label": "a", "attrs": {"file": "a.py"}},
|
|
{"id": "b", "kind": "module", "label": "b", "attrs": {"file": "b.py"}},
|
|
{"id": "c", "kind": "module", "label": "c", "attrs": {"file": "c.py", "error": "syntax"}},
|
|
{"id": "d", "kind": "module", "label": "d", "attrs": {"file": "d.py", "error": "syntax"}},
|
|
]}
|
|
ids = {r["id"]: r for r in b2.compare()}
|
|
check(
|
|
"read and failed units are reconciled separately",
|
|
ids["units-accounted-for"]["claim"] == "2 file(s) read produced 2 module(s)",
|
|
f"got {ids['units-accounted-for']['claim']!r}",
|
|
)
|
|
check(
|
|
"an unreadable unit must still appear in the graph",
|
|
ids["failures-still-in-the-graph"]["ok"],
|
|
"the whole-input form of `an unresolved name becomes an external node`",
|
|
)
|
|
b2.ir["nodes"] = b2.ir["nodes"][:2] # drop the marked ones
|
|
check(
|
|
"...and its absence is caught",
|
|
not {r["id"]: r for r in b2.compare()}["failures-still-in-the-graph"]["ok"],
|
|
"a picture smaller than its source, with nothing saying so",
|
|
)
|
|
|
|
# -- end to end, on a tree with a file that cannot be parsed ----------
|
|
fixture = Path(tmp.name) / "book-src"
|
|
(fixture / "app").mkdir(parents=True, exist_ok=True)
|
|
(fixture / "app" / "__init__.py").write_text('"""An app."""\n')
|
|
(fixture / "app" / "models.py").write_text('"""Models."""\n\n\nclass User:\n pass\n')
|
|
(fixture / "app" / "broken.py").write_text("def oops(:\n")
|
|
|
|
out = Path(tmp.name) / "book-out"
|
|
built = build_mod.run("python", fixture, out, slug="fixture", quiet=True)
|
|
|
|
check("a book writes its ledger", (out / "book.json").exists())
|
|
ledger = json.loads((out / "book.json").read_text())
|
|
step_ids = [s["id"] for s in ledger["steps"]]
|
|
check(
|
|
"the first and last steps are the two measures",
|
|
(step_ids[0], step_ids[-1]) == (book_mod.FIRST, book_mod.LAST),
|
|
f"got {step_ids}",
|
|
)
|
|
check(
|
|
"the larder measured the broken file as failed, by name",
|
|
[f["name"] for f in ledger["larder"]["failed"]] == ["app/broken.py"],
|
|
f"got {ledger['larder']['failed']}",
|
|
)
|
|
check(
|
|
"the book measure counts what came out",
|
|
ledger["book"]["nodes"] == len(json.loads((out / "steps" / "ir.json").read_text())["nodes"]),
|
|
)
|
|
check(
|
|
"every intermediate step is still a file on its own",
|
|
all((out / s["artifact"]).exists() for s in ledger["steps"] if s.get("artifact")),
|
|
"that property is the reason the notebook is a sequence and not one artifact",
|
|
)
|
|
|
|
# The notebook is the sequence, so the measures are its first and last cells
|
|
# rather than something the tooling knows and the document does not.
|
|
nb = json.loads((out / "notebook.ipynb").read_text())
|
|
check(
|
|
"the notebook opens with what came in and closes with what came out",
|
|
"what came in" in "".join(nb["cells"][0]["source"])
|
|
and "what came out" in "".join(nb["cells"][-1]["source"]),
|
|
)
|
|
check(
|
|
"it names the file it could not read",
|
|
"app/broken.py" in "".join(nb["cells"][0]["source"]),
|
|
)
|
|
# A generated notebook is a build artifact and must run where it was built.
|
|
# An earlier version imported IPython to *load* an SVG and failed here.
|
|
ran, failure = checks_mod._run_cells(nb["cells"], out)
|
|
check(f"its {ran} code cells run from the book directory", failure is None, failure or "")
|
|
|
|
html = (out / "site" / "index.html").read_text()
|
|
check(
|
|
"the web output shows both ends",
|
|
"what came in" in html and "what came out" in html,
|
|
"the site is the last step because it is the artifact somebody opens",
|
|
)
|
|
check(
|
|
"and admits when the book is incomplete",
|
|
"This book is incomplete" in html and "app/broken.py" in html,
|
|
"a clean diagram over 2 of 3 files is a lie by omission",
|
|
)
|
|
|
|
# Reproducible in the strict sense, for the same reason the IR is: a diff
|
|
# has to mean a real change.
|
|
again = Path(tmp.name) / "book-out-2"
|
|
build_mod.run("python", fixture, again, slug="fixture", quiet=True)
|
|
differs = [
|
|
name for name in ("book.json", "notebook.ipynb", "steps/ir.json", "site/index.html")
|
|
if (out / name).read_bytes() != (again / name).read_bytes()
|
|
]
|
|
check("the same larder builds the same bytes", not differs, f"differ: {differs}")
|
|
|
|
# -- the book test level ----------------------------------------------
|
|
loaded = checks_mod.Loaded(out)
|
|
report = checks_mod.Report()
|
|
import io as _io2
|
|
import contextlib as _ctx2
|
|
with _ctx2.redirect_stdout(_io2.StringIO()):
|
|
checks_mod.generated(loaded, report)
|
|
check(
|
|
f"the generated book checks pass on a real book ({report.passed})",
|
|
not report.failed,
|
|
f"failed: {report.failed}",
|
|
)
|
|
|
|
# Framework and custom living together is the whole point of the level.
|
|
(out / "checks.py").write_text(
|
|
"def checks(book, check, note, skip):\n"
|
|
" note('this project')\n"
|
|
" check('models survived', True,\n"
|
|
" any(n['id'] == 'app.models' for n in book.ir['nodes']))\n"
|
|
" check('wrong on purpose', 9, 0)\n"
|
|
)
|
|
report2 = checks_mod.Report()
|
|
with _ctx2.redirect_stdout(_io2.StringIO()):
|
|
checks_mod.custom(checks_mod.Loaded(out), report2)
|
|
check(
|
|
"a book's own checks run beside the generated ones",
|
|
report2.passed == 1 and report2.failed == ["wrong on purpose"],
|
|
f"passed={report2.passed} failed={report2.failed}",
|
|
)
|
|
|
|
# -- the architectural rules, still holding ---------------------------
|
|
blob = json.dumps(ledger)
|
|
# Swept where a visual field could actually land — `larder.extra` is the one
|
|
# open bag in the ledger. NOT over the whole document: `by_kind` is keyed by
|
|
# IR node kind, and `class` is both a legitimate kind and a member of
|
|
# VISUAL_KEYS, so a text sweep reports `"class": 15` as a style leak. The
|
|
# first version of this check did exactly that.
|
|
leaked_keys = sorted(set(ledger["larder"].get("extra") or {}) & ir_validate.VISUAL_KEYS)
|
|
check(
|
|
"no visual field reached the ledger's open bag",
|
|
not leaked_keys,
|
|
f"larder.extra has {leaked_keys} — the ledger records what was read, not how it looks",
|
|
)
|
|
node_kinds = {n["kind"] for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]}
|
|
check(
|
|
"the book measure is keyed by IR kind, nothing else",
|
|
set(ledger["book"]["by_kind"]) <= node_kinds,
|
|
f"{sorted(set(ledger['book']['by_kind']) - node_kinds)} is in by_kind but is not a kind",
|
|
)
|
|
check(
|
|
"no timestamp reached the ledger",
|
|
"generated_at" not in blob,
|
|
"a timestamp makes two builds of an unchanged larder differ",
|
|
)
|
|
check(
|
|
"provenance is in meta, never in nodes",
|
|
not any("larder" in (n.get("attrs") or {})
|
|
for n in json.loads((out / "steps" / "ir.json").read_text())["nodes"]),
|
|
"the IR is structure; provenance is meta, the same split that keeps colour out",
|
|
)
|
|
|
|
|
|
_books()
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n16. standalone — docgen works with nothing above it")
|
|
|
|
|
|
def _standalone():
|
|
"""docgen is meant to be copied out of the repo and used on its own.
|
|
|
|
That claim decays the moment somebody adds a convenient import, and it
|
|
decays silently: the suite still passes *inside* the repo. So it is asserted
|
|
from the source rather than by copying the folder in CI, which is the same
|
|
move as rig's `no host-project references` grep.
|
|
"""
|
|
ref_mod = __import__(f"{PKG}.reference", fromlist=["*"])
|
|
|
|
# Third-party modules docgen is allowed to reach for, each optional and each
|
|
# with a documented reason. Anything else is a new dependency, and a new
|
|
# dependency in a folder meant to be copied around is worth noticing.
|
|
ALLOWED = {
|
|
"tree_sitter", "tree_sitter_c_sharp", "tree_sitter_typescript",
|
|
"tree_sitter_python", # extractors/code.py, optional
|
|
"lxml", # style/extract.py, optional
|
|
"yaml", # extractors/openapi.py, optional
|
|
"networkx", # lab/ only
|
|
"modelgen", # the one seam — see reference.py
|
|
}
|
|
# The one module allowed to import `modelgen`, and the one allowed to go
|
|
# looking above the package. Two names, so a third is a decision.
|
|
SEAM_IMPORTER = "extractors/openapi.py"
|
|
SEAM_RESOLVER = "reference.py"
|
|
|
|
stdlib = set(sys.stdlib_module_names)
|
|
own = {PKG}
|
|
files = own_py_files()
|
|
|
|
foreign, seam_leaks = [], []
|
|
for path in files:
|
|
rel = path.relative_to(HERE).as_posix()
|
|
try:
|
|
tree = ast.parse(path.read_text())
|
|
except SyntaxError:
|
|
continue
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
names = [a.name.split(".")[0] for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
if node.level: # relative — inside the package by definition
|
|
continue
|
|
names = [node.module.split(".")[0]] if node.module else []
|
|
else:
|
|
continue
|
|
for name in names:
|
|
if name in stdlib or name in own:
|
|
continue
|
|
if name not in ALLOWED:
|
|
foreign.append(f"{rel}: {name}")
|
|
elif name == "modelgen" and rel != SEAM_IMPORTER:
|
|
seam_leaks.append(f"{rel}: {name}")
|
|
|
|
check(
|
|
"docgen imports the stdlib, itself, and a short allow-list",
|
|
not foreign,
|
|
"; ".join(foreign) + " <- a new dependency in a folder meant to be copied",
|
|
)
|
|
check(
|
|
f"only {SEAM_IMPORTER} imports modelgen",
|
|
not seam_leaks,
|
|
"; ".join(seam_leaks) + " <- one seam is a seam; two is a dependency",
|
|
)
|
|
|
|
# The repo path is named in exactly one place, so pointing docgen at a
|
|
# reference repo is one change rather than a search.
|
|
hardcoded = []
|
|
for path in files:
|
|
rel = path.relative_to(HERE).as_posix()
|
|
if rel in (SEAM_RESOLVER, "selftest.py"):
|
|
continue
|
|
text = path.read_text()
|
|
for line in text.splitlines():
|
|
if "station/tools" in line and not line.lstrip().startswith("#") \
|
|
and '"""' not in line and "station/tools" not in line.split("#")[-1]:
|
|
hardcoded.append(f"{rel}: {line.strip()[:60]}")
|
|
check(
|
|
f"the repo layout is known only to {SEAM_RESOLVER}",
|
|
not hardcoded,
|
|
"; ".join(hardcoded),
|
|
)
|
|
|
|
# Resolution order is the thing a standalone user depends on, so it is
|
|
# asserted rather than described: an explicit path must win over the walk,
|
|
# or setting the variable would appear to do nothing inside the repo.
|
|
import os as _os
|
|
saved = _os.environ.get(ref_mod.ENV_VAR)
|
|
try:
|
|
fake = Path(tmp.name) / "fake-repo"
|
|
(fake / "station" / "tools" / "modelgen").mkdir(parents=True, exist_ok=True)
|
|
_os.environ[ref_mod.ENV_VAR] = str(fake)
|
|
check(
|
|
"an explicit path wins over walking up",
|
|
ref_mod.root() == fake,
|
|
f"got {ref_mod.root()} — setting the variable inside the repo would do nothing",
|
|
)
|
|
check(
|
|
"and it says how it resolved, not just whether",
|
|
ref_mod.ENV_VAR in ref_mod.describe(),
|
|
ref_mod.describe(),
|
|
)
|
|
_os.environ.pop(ref_mod.ENV_VAR)
|
|
# Both outcomes are correct and which one applies is the fact worth
|
|
# printing: inside the repo, no configuration should be needed; copied
|
|
# out, there is nothing to find and saying so is the point. Asserting
|
|
# the in-repo answer unconditionally failed the moment the folder was
|
|
# actually copied out, which is how this was found.
|
|
walked = ref_mod.root()
|
|
if walked is None:
|
|
skip("the in-place case", "standalone — nothing above this folder, as intended")
|
|
else:
|
|
# What root() promises is a directory holding station/tools — not a
|
|
# particular depth above docgen, which is an assumption about where
|
|
# somebody chose to put it.
|
|
check(
|
|
"inside a repo it needs no configuration",
|
|
(walked / "station" / "tools").is_dir(),
|
|
f"walked to {walked}, which has no station/tools",
|
|
)
|
|
finally:
|
|
if saved is None:
|
|
_os.environ.pop(ref_mod.ENV_VAR, None)
|
|
else:
|
|
_os.environ[ref_mod.ENV_VAR] = saved
|
|
|
|
# Absent is a normal state. Four of the five extractors never touch it.
|
|
check(
|
|
"a missing reference is reported, not raised at import time",
|
|
ref_mod.missing("x", "y").__class__ is ImportError
|
|
and ref_mod.ENV_VAR in str(ref_mod.missing("x", "y")),
|
|
"the error has to name the variable to set",
|
|
)
|
|
|
|
# Everything a book writes has to stay inside the book directory, or a
|
|
# standalone user cannot tell what docgen created.
|
|
fixture = Path(tmp.name) / "sa-src"
|
|
fixture.mkdir(exist_ok=True)
|
|
(fixture / "one.py").write_text('"""One."""\n\n\nclass A:\n pass\n')
|
|
out = Path(tmp.name) / "sa-book"
|
|
build_mod = __import__(f"{PKG}.book.build", fromlist=["*"])
|
|
build_mod.run("python", fixture, out, slug="sa", quiet=True)
|
|
strays = [p.name for p in fixture.iterdir() if p.name != "one.py"]
|
|
check(
|
|
"a book writes only inside its own output directory",
|
|
not strays,
|
|
f"left {strays} in the source tree",
|
|
)
|
|
check(
|
|
"the shipped OpenAPI fixture travels with docgen",
|
|
(HERE / "fixtures" / "orders.yaml").exists(),
|
|
"a test's fixtures belong to the test, not to a sibling project",
|
|
)
|
|
|
|
|
|
_standalone()
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n17. one command line, and library code that has none")
|
|
|
|
|
|
def _layout():
|
|
"""Library packages hold library code; the command line lives in cli/.
|
|
|
|
The rule exists because two kinds of file in one folder — `cli_dot.py` next
|
|
to `dot.py` — make every folder answer two questions, and the command-line
|
|
half is the half that grows copies. It decays by accretion: one convenient
|
|
`argparse` in a library module and the split is gone. So it is read from the
|
|
source rather than trusted.
|
|
"""
|
|
cli_mod = __import__(f"{PKG}.cli", fromlist=["*"])
|
|
|
|
# Where command-line code may live. lab/ is sanctioned experiments, and the
|
|
# suite is itself a script.
|
|
allowed = {"cli", "lab"}
|
|
offenders = []
|
|
for path in own_py_files():
|
|
rel = path.relative_to(HERE)
|
|
if rel.parts[0] in allowed or rel.as_posix() in ("__main__.py", "selftest.py"):
|
|
continue
|
|
tree = ast.parse(path.read_text())
|
|
for node in ast.walk(tree):
|
|
names = []
|
|
if isinstance(node, ast.Import):
|
|
names = [a.name for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom) and not node.level:
|
|
names = [node.module or ""]
|
|
if "argparse" in names:
|
|
offenders.append(rel.as_posix())
|
|
check(
|
|
"nothing outside cli/ parses a command line",
|
|
not offenders,
|
|
f"{sorted(set(offenders))} import argparse — command-line code belongs in cli/",
|
|
)
|
|
|
|
stray = sorted(
|
|
p.relative_to(HERE).as_posix() for p in own_py_files()
|
|
if (p.name == "__main__.py" and p.parent != HERE)
|
|
or p.name.startswith("cli_") or p.stem.endswith("_main")
|
|
)
|
|
check(
|
|
"one entry point — no __main__.py, cli_*.py or *_main.py in the packages",
|
|
not stray,
|
|
f"found {stray}",
|
|
)
|
|
|
|
missing = []
|
|
for name, (module, _) in cli_mod.COMMANDS.items():
|
|
try:
|
|
mod = __import__(f"{PKG}.cli.{module}", fromlist=["*"])
|
|
except ImportError as e:
|
|
missing.append(f"{name}: {e}")
|
|
continue
|
|
if not callable(getattr(mod, "main", None)):
|
|
missing.append(f"{name}: no main()")
|
|
check("every listed command has a module with main()", not missing, "; ".join(missing))
|
|
|
|
# How a failure reaches the user is part of the interface: one line on
|
|
# stderr, exit 1, never a traceback. Run as a real process, the way a shell
|
|
# or a Makefile sees it.
|
|
env = dict(__import__("os").environ, PYTHONPATH=str(HERE.parent))
|
|
run = lambda *a: _sub.run([sys.executable, "-m", PKG, *a], capture_output=True,
|
|
text=True, env=env)
|
|
bare = run()
|
|
check("with no command it prints the commands and exits 2",
|
|
bare.returncode == 2 and "emit" in bare.stdout and "run" in bare.stdout,
|
|
f"exit {bare.returncode}")
|
|
bad = run("validate", str(Path(tmp.name) / "no-such.json"))
|
|
check(
|
|
"an expected failure is one line on stderr, exit 1, no traceback",
|
|
bad.returncode == 1 and bad.stderr.startswith("Error:")
|
|
and "Traceback" not in bad.stderr and bad.stderr.count("\n") == 1,
|
|
f"exit {bad.returncode}: {bad.stderr[:120]!r}",
|
|
)
|
|
|
|
# The structure-picks-the-drawing branch used to be written three times —
|
|
# the `auto` command, the `site` command, the book. Now it is `draw()`, and
|
|
# the book reaching past it to the concrete emitters would be the first copy.
|
|
# Parsed, not grepped, for the reason section 2 gives: the docstring naming
|
|
# `emitters.dot` is prose, and the first version of this check failed on it.
|
|
imported = set()
|
|
for node in ast.walk(ast.parse((HERE / "book" / "build.py").read_text())):
|
|
if isinstance(node, ast.ImportFrom) and node.module:
|
|
imported |= {f"{node.module}.{a.name}" for a in node.names}
|
|
check(
|
|
"the book chooses its drawing through draw(), not by hand",
|
|
"emitters.auto.draw" in imported
|
|
and not any(m.startswith(("emitters.erd", "emitters.dot")) for m in imported),
|
|
f"imports {sorted(m for m in imported if m.startswith('emitters'))}",
|
|
)
|
|
|
|
|
|
_layout()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
print("\n18. run files — every book, rebuilt with one command")
|
|
|
|
|
|
def _runfiles():
|
|
cfg = __import__(f"{PKG}.book.config", fromlist=["*"])
|
|
build_mod = __import__(f"{PKG}.book.build", fromlist=["*"])
|
|
ref_mod = __import__(f"{PKG}.reference", fromlist=["*"])
|
|
import os as _os
|
|
|
|
base = Path(tmp.name) / "runfiles"
|
|
(base / "src" / "app").mkdir(parents=True, exist_ok=True)
|
|
(base / "src" / "app" / "__init__.py").write_text('"""App."""\n')
|
|
(base / "src" / "app" / "models.py").write_text('"""Models."""\n\n\nclass User:\n pass\n')
|
|
(base / "shop.json").write_text((HERE / "fixtures" / "shop.json").read_text())
|
|
|
|
def write(text, name="docgen.toml"):
|
|
path = base / name
|
|
path.write_text(text)
|
|
return path
|
|
|
|
def problems(text):
|
|
try:
|
|
cfg.load(write(text, "bad.toml"))
|
|
except cfg.ConfigError as e:
|
|
return e.problems
|
|
return []
|
|
|
|
example = cfg.load(HERE / "docgen.example.toml")
|
|
check(
|
|
"the shipped example loads",
|
|
[b.name for b in example.books] == ["extractors", "schema", "api"],
|
|
f"got {[b.name for b in example.books]}",
|
|
)
|
|
|
|
good = write(
|
|
'[defaults]\nout = "books"\nexclude = ["vendor"]\n\n'
|
|
'[[book]]\nname = "app"\nroot = "src"\n\n'
|
|
'[[book]]\nname = "shop"\nschema = "shop.json"\n\n'
|
|
'[[book]]\nname = "narrow"\nroot = "src/app"\nout = "elsewhere/narrow"\nexclude = []\n'
|
|
)
|
|
# Relative to the FILE. Loaded from another working directory, a run file
|
|
# that resolved against cwd would build different books from the same text.
|
|
previous = _os.getcwd()
|
|
try:
|
|
_os.chdir(Path(tmp.name))
|
|
rf = cfg.load(good)
|
|
finally:
|
|
_os.chdir(previous)
|
|
by = {b.name: b for b in rf.books}
|
|
check(
|
|
"paths resolve against the run file, not the working directory",
|
|
by["app"].source == base / "src" and by["app"].out == base / "books" / "app",
|
|
f"source {by['app'].source}, out {by['app'].out}",
|
|
)
|
|
check("a book's own out overrides <defaults.out>/<name>",
|
|
by["narrow"].out == base / "elsewhere" / "narrow")
|
|
check(
|
|
"a book's value replaces the default — exclude included, no merging",
|
|
by["app"].exclude == ("vendor",) and by["narrow"].exclude == (),
|
|
f"app {by['app'].exclude}, narrow {by['narrow'].exclude}",
|
|
)
|
|
check(
|
|
"an inherited exclude does not apply to a non-tree source, and is not an error",
|
|
by["shop"].exclude == () and by["shop"].kind == "db",
|
|
)
|
|
|
|
# Refused, not ignored — and every problem at once, not the first.
|
|
found = problems(
|
|
'colour = "red"\n[defaults]\nstyel = "x"\n\n'
|
|
'[[book]]\nname = "a"\nroot = "src"\nexlude = ["x"]\n'
|
|
)
|
|
check(
|
|
"unknown keys are refused at every level, all reported together",
|
|
len(found) == 3 and all("unknown" in f for f in found),
|
|
f"got {found}",
|
|
)
|
|
check(
|
|
"a book names exactly one source",
|
|
any("exactly one source" in f for f in problems(
|
|
'[[book]]\nname = "a"\nroot = "src"\nschema = "shop.json"\n')),
|
|
)
|
|
check(
|
|
"an explicit exclude on a non-tree source is refused",
|
|
any("only applies to a root" in f for f in problems(
|
|
'[[book]]\nname = "a"\nschema = "shop.json"\nexclude = ["x"]\n')),
|
|
)
|
|
# The next run would read the last run's output: a rebuild that changes on
|
|
# every rebuild.
|
|
check(
|
|
"a book written inside the tree it reads is refused",
|
|
any("inside the tree it reads" in f for f in problems(
|
|
'[[book]]\nname = "a"\nroot = "src"\nout = "src/book"\n')),
|
|
)
|
|
check(
|
|
"two books writing to one directory are refused",
|
|
any("both write to" in f for f in problems(
|
|
'[[book]]\nname = "a"\nroot = "src"\nout = "same"\n\n'
|
|
'[[book]]\nname = "b"\nschema = "shop.json"\nout = "same"\n')),
|
|
"the second would clear the first on every run, silently",
|
|
)
|
|
try:
|
|
rf.select(["app", "nope"])
|
|
selected = False
|
|
except cfg.ConfigError:
|
|
selected = True
|
|
check("--only with an unknown name is an error, not an empty run", selected)
|
|
|
|
# The caller's env beats the file — rig's precedence rule for every setting.
|
|
saved = _os.environ.pop(ref_mod.ENV_VAR, None)
|
|
try:
|
|
with_ref = cfg.load(write(
|
|
f'reference = "{base}"\n[[book]]\nname = "a"\nroot = "src"\n', "ref.toml"))
|
|
check("a run file can name the reference repo",
|
|
cfg.apply_reference(with_ref) == str(base)
|
|
and _os.environ.get(ref_mod.ENV_VAR) == str(base))
|
|
_os.environ[ref_mod.ENV_VAR] = "/from/the/environment"
|
|
check("but the environment wins over the file",
|
|
cfg.apply_reference(with_ref) is None
|
|
and _os.environ[ref_mod.ENV_VAR] == "/from/the/environment")
|
|
finally:
|
|
_os.environ.pop(ref_mod.ENV_VAR, None)
|
|
if saved is not None:
|
|
_os.environ[ref_mod.ENV_VAR] = saved
|
|
|
|
# -- running it ----------------------------------------------------------
|
|
broken = write(
|
|
'[[book]]\nname = "app"\nroot = "src"\nout = "run/app"\n\n'
|
|
'[[book]]\nname = "bad"\nhar = "not-a-har.har"\nout = "run/bad"\n\n'
|
|
'[[book]]\nname = "shop"\nschema = "shop.json"\nout = "run/shop"\n',
|
|
"run.toml",
|
|
)
|
|
(base / "not-a-har.har").write_text("this is not json")
|
|
results = {r["name"]: r for r in cfg.run(cfg.load(broken), check=True)}
|
|
check(
|
|
"one broken book does not stop the others",
|
|
results["app"]["ok"] and results["shop"]["ok"]
|
|
and not results["bad"]["ok"] and results["bad"]["error"],
|
|
{k: (v["ok"], v["error"]) for k, v in results.items()},
|
|
)
|
|
check("each built book passed its own level", not results["app"]["checks_failed"])
|
|
|
|
# Rerunning is the point of a run file, so a rerun must not leave the last
|
|
# run's artifacts beside the new ledger, and must not touch what a person wrote.
|
|
app = base / "run" / "app"
|
|
(app / "steps" / "stale.svg").write_text("<svg/>")
|
|
(app / "checks.py").write_text("def checks(book, check, note, skip):\n pass\n")
|
|
again = {r["name"]: r for r in cfg.run(cfg.load(broken), ["app"])}
|
|
check(
|
|
"a rebuild removes the previous build's outputs",
|
|
again["app"]["ok"] and not (app / "steps" / "stale.svg").exists(),
|
|
"a stale graph.svg would go on showing in the site, unlisted in the ledger",
|
|
)
|
|
check("...and keeps the hand-written checks.py", (app / "checks.py").exists())
|
|
|
|
foreign = base / "not-a-book"
|
|
foreign.mkdir(exist_ok=True)
|
|
(foreign / "steps").mkdir(exist_ok=True)
|
|
(foreign / "site").mkdir(exist_ok=True)
|
|
check(
|
|
"clear() never touches a directory it did not write",
|
|
build_mod.clear(foreign) == [] and (foreign / "steps").exists(),
|
|
"a mistyped `out` must not delete somebody's `site/`",
|
|
)
|
|
|
|
# -- the environment the lock pins ---------------------------------------
|
|
import tomllib as _toml
|
|
project = _toml.loads((HERE / "pyproject.toml").read_text())
|
|
check(
|
|
"pyproject declares no required dependencies",
|
|
project["project"]["dependencies"] == [],
|
|
"the structural path is the stdlib; everything else is an optional group",
|
|
)
|
|
check(
|
|
"it requires the Python that reads run files",
|
|
project["project"]["requires-python"] == ">=3.11",
|
|
"tomllib is 3.11+",
|
|
)
|
|
# A module the suite allows docgen to import must be a dependency somebody
|
|
# can install from the lock, or the allow-list and pyproject have drifted.
|
|
declared = {d.split(">")[0].split("=")[0].strip().lower().replace("-", "_")
|
|
for group in project["dependency-groups"].values() for d in group}
|
|
import_to_dist = {"tree_sitter": "tree_sitter", "tree_sitter_c_sharp": "tree_sitter_c_sharp",
|
|
"tree_sitter_typescript": "tree_sitter_typescript",
|
|
"tree_sitter_python": "tree_sitter_python",
|
|
"lxml": "lxml", "yaml": "pyyaml", "networkx": "networkx"}
|
|
undeclared = sorted(m for m, dist in import_to_dist.items() if dist not in declared)
|
|
check("every optional import is a declared dependency group", not undeclared,
|
|
f"{undeclared} are allowed but not in pyproject")
|
|
lock = HERE / "uv.lock"
|
|
locked = set()
|
|
if lock.exists():
|
|
locked = {pkg["name"].replace("-", "_")
|
|
for pkg in _toml.loads(lock.read_text()).get("package", [])}
|
|
check(
|
|
"uv.lock pins every declared dependency",
|
|
lock.exists() and declared <= locked,
|
|
f"not in the lock: {sorted(declared - locked)} — run `make lock`",
|
|
)
|
|
|
|
|
|
_runfiles()
|
|
|
|
|
|
print(f"{len(PASS)} passed, {len(FAIL)} failed, {len(SKIP)} skipped")
|
|
if FAIL:
|
|
print("\nfailed:")
|
|
for name in FAIL:
|
|
print(f" {name}")
|
|
sys.exit(1 if FAIL else 0)
|