From 542d704da4401e3b194d624e90e1ec5c4b593e21 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Sat, 12 Sep 2026 06:42:49 -0300 Subject: [PATCH] docgen iter 2 --- soleprint/atlas2/docgen/.gitignore | 4 + soleprint/atlas2/docgen/Makefile | 90 ++ soleprint/atlas2/docgen/README.md | 225 +++++ soleprint/atlas2/docgen/__init__.py | 1 + soleprint/atlas2/docgen/emitters/__init__.py | 11 + soleprint/atlas2/docgen/emitters/__main__.py | 30 + soleprint/atlas2/docgen/emitters/auto.py | 79 ++ soleprint/atlas2/docgen/emitters/cli_dot.py | 86 ++ soleprint/atlas2/docgen/emitters/cli_erd.py | 50 + soleprint/atlas2/docgen/emitters/cli_index.py | 43 + .../atlas2/docgen/emitters/cli_notebook.py | 73 ++ soleprint/atlas2/docgen/emitters/dot.py | 291 ++++++ soleprint/atlas2/docgen/emitters/erd.py | 247 +++++ soleprint/atlas2/docgen/emitters/index.py | 163 +++ soleprint/atlas2/docgen/emitters/notebook.py | 248 +++++ .../atlas2/docgen/extractors/__init__.py | 1 + .../atlas2/docgen/extractors/__main__.py | 19 + soleprint/atlas2/docgen/extractors/db.py | 144 +++ soleprint/atlas2/docgen/extractors/db_main.py | 31 + soleprint/atlas2/docgen/extractors/openapi.py | 123 +++ .../atlas2/docgen/extractors/openapi_main.py | 29 + .../docgen/extractors/python/__init__.py | 29 + .../docgen/extractors/python/__main__.py | 37 + .../docgen/extractors/python/collect.py | 235 +++++ .../docgen/extractors/python/resolve.py | 154 +++ soleprint/atlas2/docgen/ir/__init__.py | 6 + soleprint/atlas2/docgen/ir/__main__.py | 8 + soleprint/atlas2/docgen/ir/model.py | 146 +++ soleprint/atlas2/docgen/ir/schema.json | 87 ++ soleprint/atlas2/docgen/ir/validate.py | 236 +++++ soleprint/atlas2/docgen/lab/__init__.py | 10 + soleprint/atlas2/docgen/lab/pg_probe.py | 149 +++ soleprint/atlas2/docgen/notebook/__init__.py | 14 + soleprint/atlas2/docgen/notebook/spec.py | 226 +++++ soleprint/atlas2/docgen/ops/__init__.py | 27 + soleprint/atlas2/docgen/ops/__main__.py | 101 ++ soleprint/atlas2/docgen/ops/filter.py | 468 +++++++++ soleprint/atlas2/docgen/selftest.py | 942 ++++++++++++++++++ soleprint/atlas2/docgen/style/__init__.py | 179 ++++ soleprint/atlas2/docgen/style/lucid.json | 336 +++++++ soleprint/station/tools/docgen/.gitignore | 4 + soleprint/station/tools/docgen/Makefile | 88 ++ soleprint/station/tools/docgen/README.md | 256 +++++ soleprint/station/tools/docgen/__init__.py | 60 ++ soleprint/station/tools/docgen/demo.py | 40 + soleprint/station/tools/docgen/dot.py | 232 +++++ .../station/tools/docgen/export/__init__.py | 6 + soleprint/station/tools/docgen/export/doc.py | 71 ++ .../station/tools/docgen/export/notebook.py | 102 ++ .../tools/docgen/export/specs/__init__.py | 1 + .../tools/docgen/export/specs/vanilla.py | 292 ++++++ soleprint/station/tools/docgen/profile.py | 176 ++++ .../tools/docgen/profiles/default.json | 56 ++ .../station/tools/docgen/profiles/lucid.json | 56 ++ soleprint/station/tools/docgen/render.py | 107 ++ soleprint/station/tools/docgen/selftest.py | 585 +++++++++++ soleprint/station/tools/docgen/shape.py | 131 +++ .../station/tools/docgen/style/__init__.py | 7 + .../station/tools/docgen/style/extract.py | 221 ++++ .../station/tools/docgen/style/tokens.py | 161 +++ soleprint/station/tools/graphgen/README.md | 81 ++ soleprint/station/tools/graphgen/__init__.py | 44 +- soleprint/station/tools/graphgen/examples.py | 71 ++ soleprint/station/tools/graphgen/graph.py | 185 ++++ soleprint/station/tools/graphgen/selftest.py | 131 +++ 65 files changed, 8541 insertions(+), 1 deletion(-) create mode 100644 soleprint/atlas2/docgen/.gitignore create mode 100644 soleprint/atlas2/docgen/Makefile create mode 100644 soleprint/atlas2/docgen/README.md create mode 100644 soleprint/atlas2/docgen/__init__.py create mode 100644 soleprint/atlas2/docgen/emitters/__init__.py create mode 100644 soleprint/atlas2/docgen/emitters/__main__.py create mode 100644 soleprint/atlas2/docgen/emitters/auto.py create mode 100644 soleprint/atlas2/docgen/emitters/cli_dot.py create mode 100644 soleprint/atlas2/docgen/emitters/cli_erd.py create mode 100644 soleprint/atlas2/docgen/emitters/cli_index.py create mode 100644 soleprint/atlas2/docgen/emitters/cli_notebook.py create mode 100644 soleprint/atlas2/docgen/emitters/dot.py create mode 100644 soleprint/atlas2/docgen/emitters/erd.py create mode 100644 soleprint/atlas2/docgen/emitters/index.py create mode 100644 soleprint/atlas2/docgen/emitters/notebook.py create mode 100644 soleprint/atlas2/docgen/extractors/__init__.py create mode 100644 soleprint/atlas2/docgen/extractors/__main__.py create mode 100644 soleprint/atlas2/docgen/extractors/db.py create mode 100644 soleprint/atlas2/docgen/extractors/db_main.py create mode 100644 soleprint/atlas2/docgen/extractors/openapi.py create mode 100644 soleprint/atlas2/docgen/extractors/openapi_main.py create mode 100644 soleprint/atlas2/docgen/extractors/python/__init__.py create mode 100644 soleprint/atlas2/docgen/extractors/python/__main__.py create mode 100644 soleprint/atlas2/docgen/extractors/python/collect.py create mode 100644 soleprint/atlas2/docgen/extractors/python/resolve.py create mode 100644 soleprint/atlas2/docgen/ir/__init__.py create mode 100644 soleprint/atlas2/docgen/ir/__main__.py create mode 100644 soleprint/atlas2/docgen/ir/model.py create mode 100644 soleprint/atlas2/docgen/ir/schema.json create mode 100644 soleprint/atlas2/docgen/ir/validate.py create mode 100644 soleprint/atlas2/docgen/lab/__init__.py create mode 100644 soleprint/atlas2/docgen/lab/pg_probe.py create mode 100644 soleprint/atlas2/docgen/notebook/__init__.py create mode 100644 soleprint/atlas2/docgen/notebook/spec.py create mode 100644 soleprint/atlas2/docgen/ops/__init__.py create mode 100644 soleprint/atlas2/docgen/ops/__main__.py create mode 100644 soleprint/atlas2/docgen/ops/filter.py create mode 100644 soleprint/atlas2/docgen/selftest.py create mode 100644 soleprint/atlas2/docgen/style/__init__.py create mode 100644 soleprint/atlas2/docgen/style/lucid.json create mode 100644 soleprint/station/tools/docgen/.gitignore create mode 100644 soleprint/station/tools/docgen/Makefile create mode 100644 soleprint/station/tools/docgen/README.md create mode 100644 soleprint/station/tools/docgen/__init__.py create mode 100644 soleprint/station/tools/docgen/demo.py create mode 100644 soleprint/station/tools/docgen/dot.py create mode 100644 soleprint/station/tools/docgen/export/__init__.py create mode 100644 soleprint/station/tools/docgen/export/doc.py create mode 100644 soleprint/station/tools/docgen/export/notebook.py create mode 100644 soleprint/station/tools/docgen/export/specs/__init__.py create mode 100644 soleprint/station/tools/docgen/export/specs/vanilla.py create mode 100644 soleprint/station/tools/docgen/profile.py create mode 100644 soleprint/station/tools/docgen/profiles/default.json create mode 100644 soleprint/station/tools/docgen/profiles/lucid.json create mode 100644 soleprint/station/tools/docgen/render.py create mode 100644 soleprint/station/tools/docgen/selftest.py create mode 100644 soleprint/station/tools/docgen/shape.py create mode 100644 soleprint/station/tools/docgen/style/__init__.py create mode 100644 soleprint/station/tools/docgen/style/extract.py create mode 100644 soleprint/station/tools/docgen/style/tokens.py create mode 100644 soleprint/station/tools/graphgen/README.md create mode 100644 soleprint/station/tools/graphgen/examples.py create mode 100644 soleprint/station/tools/graphgen/graph.py create mode 100644 soleprint/station/tools/graphgen/selftest.py diff --git a/soleprint/atlas2/docgen/.gitignore b/soleprint/atlas2/docgen/.gitignore new file mode 100644 index 0000000..b455f56 --- /dev/null +++ b/soleprint/atlas2/docgen/.gitignore @@ -0,0 +1,4 @@ +# Everything this makes. +out/ +__pycache__/ +*.pyc diff --git a/soleprint/atlas2/docgen/Makefile b/soleprint/atlas2/docgen/Makefile new file mode 100644 index 0000000..d407ec1 --- /dev/null +++ b/soleprint/atlas2/docgen/Makefile @@ -0,0 +1,90 @@ +# docgen — code to diagram, and to everything else the IR can feed. +# +# Derived from where this file sits, so the folder can be copied anywhere and +# renamed and still work. The logic lives in the Python, never here. +# +# make check prove it, on a tree it builds itself +# make ir SRC=../station extract -> out/ir.json +# make graph out/ir.json -> out/graph.svg +# make index out/ir.json -> out/index.md +# make self run the whole pipeline over soleprint itself +# make doctor what this machine has +# +# The pipeline is three commands and they compose, which is the point: +# +# python3 -m docgen.extractors.python --root SRC -o ir.json +# python3 -m docgen.ops ir.json --overview -o view.json +# python3 -m docgen.emitters dot view.json -o graph.svg --theme dark + +HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +PKG := $(notdir $(HERE)) +PARENT := $(patsubst %/,%,$(dir $(HERE))) +PY ?= python3 +RUN := PYTHONPATH=$(PARENT) $(PY) -m + +OUT ?= $(HERE)/out +SRC ?= +SCHEMA ?= +STYLE ?= lucid +THEME ?= +DEPTH ?= 2 + +THEME_ARG := $(if $(THEME),--theme $(THEME)) + +.PHONY: help check ir db graph index view self doctor clean + +help: ## List every target + @echo "docgen — static analysis of a tree, and the artifacts that fall out of it" + @echo + @grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-10s\033[0m %s\n", $$1, $$2}' + @echo + @echo " SRC=/path/to/tree what to read OUT=/path where output goes" + @echo " SCHEMA=schema.json a database instead STYLE=lucid THEME=dark|lucid" + @echo " DEPTH=2 how deep to draw" + +check: ## Prove the pipeline, offline, needing nothing installed + @$(PY) $(HERE)/selftest.py + +ir: ## Extract SRC into OUT/ir.json + @test -n "$(SRC)" || { echo "Error: set SRC=/path/to/tree" >&2; exit 1; } + @mkdir -p $(OUT) + @$(RUN) $(PKG).extractors.python --root "$(SRC)" -o $(OUT)/ir.json + @$(RUN) $(PKG).ir $(OUT)/ir.json + +db: ## Extract a graphgen-compatible SCHEMA into OUT/ir.json + @test -n "$(SCHEMA)" || { echo "Error: set SCHEMA=/path/to/schema.json" >&2; exit 1; } + @mkdir -p $(OUT) + @$(RUN) $(PKG).extractors --schema "$(SCHEMA)" -o $(OUT)/ir.json + @$(RUN) $(PKG).ir $(OUT)/ir.json + +view: ## OUT/ir.json -> OUT/view.json, the default view for its source type + @$(RUN) $(PKG).ops $(OUT)/ir.json --overview -o $(OUT)/view.json + +graph: view ## OUT/view.json -> whatever its structure asks for + @$(RUN) $(PKG).emitters auto $(OUT)/view.json -o $(OUT) \ + --style $(STYLE) $(THEME_ARG) + +index: ## OUT/ir.json -> OUT/index.md and OUT/sidebar.json + @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/index.md + @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/sidebar.json + +self: ## Run the whole pipeline over soleprint itself — the honest end-to-end check + @$(MAKE) --no-print-directory ir SRC=$(PARENT)/.. OUT=$(OUT) + @$(MAKE) --no-print-directory index OUT=$(OUT) + @$(MAKE) --no-print-directory graph OUT=$(OUT) + @echo + @echo " Read $(OUT)/index.md — it should read like the system." + +doctor: ## Report whether this machine can run it + @printf 'python : '; $(PY) --version 2>&1 || echo MISSING + @printf 'dot : '; (dot -V 2>&1) || echo 'MISSING — sudo apt install graphviz (only to render)' + @printf 'package : %s (from %s)\n' '$(PKG)' '$(PARENT)' + @printf 'styles : '; $(RUN) $(PKG).style 2>/dev/null \ + || $(RUN) $(PKG) 2>/dev/null \ + || PYTHONPATH=$(PARENT) $(PY) -c "from $(PKG).style import Style; print(', '.join(Style.available()))" + @PYTHONPATH=$(PARENT) $(PY) -c "import $(PKG).ir, $(PKG).emitters.dot, $(PKG).ops" >/dev/null 2>&1 \ + && echo 'import : ok' || echo 'import : FAILED — is the folder intact?' + +clean: ## Delete OUT. Nothing else is ever written to + @rm -rf "$(OUT)" && echo "Removed $(OUT)" diff --git a/soleprint/atlas2/docgen/README.md b/soleprint/atlas2/docgen/README.md new file mode 100644 index 0000000..938444d --- /dev/null +++ b/soleprint/atlas2/docgen/README.md @@ -0,0 +1,225 @@ +# docgen + +Static analysis of a tree, and the artifacts that fall out of it. + +The point is not the diagram. The point is the format in the middle — diagrams +are one consumer of it, and not the one that reaches the most people. + +``` +extractors/ → graph IR (JSON) → emitters/ +(per source type) (one schema) (per output target) + ↑ + style/*.json + (consumed by emitters only) +``` + +```bash +make check # prove it, on a tree it builds itself +make self # run the whole thing over soleprint +make ir SRC=../../station/tools/histgen +make index && make graph +make help +``` + +Or as three composable commands, which is what the Makefile is wrapping: + +```bash +python3 -m docgen.extractors.python --root SRC -o ir.json +python3 -m docgen.ops ir.json --drop-stdlib -o view.json +python3 -m docgen.emitters dot view.json -o graph.svg --theme dark +``` + +## DOT collapses three concerns; this separates them + +| concern | question | owner | +|---|---|---| +| **structure** | what the graph *is* | `ir/schema.json` — versioned, golden-tested | +| **meaning** | what things *mean visually* | `style/*.json`, keyed on `kind` | +| **placement** | where things *go* | Graphviz defaults. Phase two | + +An extractor has never heard of SVG, colours or layout. An emitter has never +heard of Python, `ast` or SQL. **The IR carries no visual information** — if a +field would change between light and dark theme, it does not belong in it. +`shape="cylinder"` is not a field; it is `kind="datastore"` plus a style rule, +which is what lets the same IR render in a theme that has no cylinders. + +The selftest asserts all three of those, because they are the design rather than +a nicety and they are exactly what erodes first. + +## The IR + +```json +{ + "meta": { "source": "python", "root": "app/", "schema_version": "1" }, + "nodes": [ { "id": "app.models.User", "kind": "class", "label": "User", + "parent": "app.models", + "attrs": { "file": "app/models.py", "line": 12 } } ], + "edges": [ { "source": "app.models.User", "target": "app.db.Base", + "kind": "inherits", "attrs": {} } ] +} +``` + +- **`id`** is fully qualified and **stable across runs**. That is what makes two + graphs from two commits diffable. +- **`kind`** is the hinge, and the only field style and layout may key on. +- **`parent`** is containment. Relationships are edges. +- **`attrs`** is an open bag; `file`/`line` let a UI link a box to a line. + +Stdlib dataclasses, not Pydantic. A format that needs a library installed to be +opened is not a format, it is an API. `ir/validate.py` is the check at the +boundary, and it reads the field lists out of `schema.json` so the two cannot +drift. + +```bash +python3 -m docgen.ir ir.json +``` + +It catches what a schema cannot: an edge naming a node that does not exist, a +containment cycle, duplicate ids, and a visual field smuggled into `attrs`. + +## Extraction is deterministic + +**No LLM in the structural path.** A diagram from an AST cannot be out of date +with the code; one from a model's reading of the code is wrong the moment the +model has a bad day, which is the problem this exists to fix. + +`ast` resolves nothing on its own — `class User(Base)` yields the literal string +`"Base"`. So there are two passes: one collects each module's definitions and +imports, the other resolves names against those tables. + +``` +from .db import Base ; class User(Base) + → app.models.User --inherits--> app.db.Base not "Base" +``` + +**Unresolved names become `kind: "external"` nodes and keep their edges.** +Dropping them is the worse failure: the diagram looks complete and has quietly +lost a dependency. Gathered by the index emitter, they *are* the project's +dependency surface. + +An unparseable file is recorded as a node with an `error` attr, not a crash — +one bad file must not cost you the other four hundred. + +`calls` edges are deliberately **not** attempted. Resolving `self.foo()` needs +type inference, and a call graph that is quietly 60% right is worse than none +because it reads as authoritative. + +### A second source + +`extractors/db.py` reads the published `{models, relationships, source}` +contract that `modelgen` already emits and `graphgen` already consumes. Tables +become nodes, columns become contained nodes, foreign keys become edges — with +no new top-level field, which was the checkpoint on whether the schema was right. + +Connecting to a live database is not here. `modelgen from-db --url ...` does +that and writes the schema this reads; the two-step also keeps credentials out +of this pipeline entirely. + +## Views are not an emitter concern + +The first real diagram out of this pipeline was a 3000px strip: four modules of +content and sixty `sys`/`json`/`typing` boxes, all peers. The emitter was +correct and the picture was useless. That is a **missing view**, and the fix +belongs to every consumer at once — the index, the diagram and the diff all want +"just this subsystem, two hops out, without the stdlib". + +```bash +python3 -m docgen.ops ir.json --drop-stdlib --around docgen.ir --hops 2 -o view.json +``` + +`drop_stdlib`, `drop_external`, `only_kinds`, `drop_kinds`, `subtree`, +`neighbourhood`, `collapse_to_depth`. All IR→IR, all composable, each producing +a document that still validates. + +Graph *algorithms* are not here. Transitive reduction, cycle detection and +dominators are `networkx`'s, and reimplementing them is the classic way to +acquire a quiet bug. `lab/` is where that dependency gets tried against real IRs +before anything depends on it — the aim being to learn which part of it is +actually attractive, rather than adopting all of it on faith. + +## One colour language + +A style rule names a **slot**, never a colour. `"border": "atlas"` is the rule; +the theme binds `atlas` to `#43A047` in print and `#15803d` on the docs site. + +That indirection is the whole point. `common/theme/tokens.css`, +`docs/graphs/themes/*.gvpr` and `style/lucid.json` use the same slot names, so a +diagram and the page around it match by construction — which is the rule +`docs/graphs/README.md` already states. The `dark` theme's `artery`, `atlas` and +`station` slots are exactly the `--system-accent` values set in +`artery/index.html:30`, `atlas/index.html:25` and `station/index.html:29`, and +the selftest fails if they drift apart. + +An unknown `kind` falls back to `default` rather than crashing, so a new +extractor renders plainly and legibly on day one instead of needing a style file +written first. + +**How a container picks its colour without the IR naming one:** it does not. The +IR says which spr model a group belongs to (`attrs.domain` — semantic), and +`domain_slots` maps that to a slot. Same mechanism as `--system-accent`. With no +domain, the emitter assigns by sorted id, so two runs agree. + +## Use DOT until it hits its limits + +The emitter writes what DOT expresses natively and stops at the boundary rather +than growing machinery. The limits are recorded in `style/lucid.json` under +`limits` and reachable as `Style.limits()`: + +| | | +|---|---| +| header bars | a cluster has a label and a fill, not a 100%-width header rectangle | +| `stroke-dasharray` | not parameterised — `4,4` and `5,5` collapse to one dash | +| corner radius | `rounded` is binary, so 4px and 6px are identical | +| icon above label | needs an HTML-like label table | +| sequence badges | `xlabel` carries the number; the circle does not exist | + +Those mark where a richer emitter would begin. The style file carries the full +spec regardless, so that emitter needs no re-authoring. + +One limit that *was* worth solving: DOT cannot use a cluster as an edge +endpoint, so every module-to-module import silently vanished. The native answer +is `compound=true` with `lhead`/`ltail` — draw between a representative leaf and +clip at the cluster border. + +## The output is addressable + +`id` and `kind` pass through to the SVG as the element's `id` and `class`, and +`attrs.file`/`attrs.line` become an `href`. A front end can bind behaviour to a +box and a box can link to the line it came from, without the emitter knowing +about either. + +## Testing + +```bash +make check # 59 checks, offline, nothing installed +``` + +**Golden tests go on the IR, never on the SVG.** Graphviz measures label text +with the host's fonts to size nodes, so identical input gives different geometry +on a machine with different fontconfig. The IR is deterministic; the SVG is not. + +Self-hosting is the honest end-to-end check, and it is where the real bugs came +from — two name-resolution faults that no fixture had reached: + +```bash +make self # extract soleprint, and read out/index.md +``` + +## Where this sits + +`docgen` belongs to Atlas — documentation is whose concern it is. It is **not** a +station tool and is not under `station/tools/`; it *may depend on* station tools, +which is the permitted direction. + +Atlas 2 is a successor, not a replacement. `soleprint/atlas/` is untouched: it +carries client information and an idea still worth extracting — deriving frontend +and backend tests from one source, which is the same shape as this pointed the +other way. + +## Not here + +No layout system, no positioning, no ELK. No HTML-like labels, no SVG post-pass. +No LLM in the structural path — annotation (summarising a module, naming a +cluster) is a later layer, cached to its own file keyed by node `id`, merged into +`attrs` at emit time, and extraction must work with it absent. No configuration +knobs until two real consumers disagree. diff --git a/soleprint/atlas2/docgen/__init__.py b/soleprint/atlas2/docgen/__init__.py new file mode 100644 index 0000000..7b93c2b --- /dev/null +++ b/soleprint/atlas2/docgen/__init__.py @@ -0,0 +1 @@ +"""Docgen — code to diagram. The IR is the product; diagrams are one consumer.""" diff --git a/soleprint/atlas2/docgen/emitters/__init__.py b/soleprint/atlas2/docgen/emitters/__init__.py new file mode 100644 index 0000000..4e8b77a --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/__init__.py @@ -0,0 +1,11 @@ +""" +Emitters: IR -> an artifact. None of them has heard of Python, `ast` or SQL. + + dot .dot -> Graphviz -> SVG static docs, embedding + index markdown / sidebar JSON no graph literacy required + diff two IRs -> what changed review + notebook .ipynb a runnable document + +The non-visual ones matter most for reach. A sorted, described index of what +exists is readable by people who will never open a diagram. +""" diff --git a/soleprint/atlas2/docgen/emitters/__main__.py b/soleprint/atlas2/docgen/emitters/__main__.py new file mode 100644 index 0000000..a03d2fe --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/__main__.py @@ -0,0 +1,30 @@ +""" python3 -m docgen.emitters [options]""" + +import sys + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + if not argv: + print("usage: python3 -m docgen.emitters [-o OUT] [--style NAME] [--theme NAME]", + file=sys.stderr) + return 2 + name, rest = argv[0], argv[1:] + if name == "dot": + from .cli_dot import main as run + elif name == "index": + from .cli_index import main as run + elif name == "erd": + from .cli_erd import main as run + elif name == "auto": + from .auto import main as run + elif name == "notebook": + from .cli_notebook import main as run + else: + print(f"Error: no emitter {name!r} — have: auto, dot, index, erd, notebook", file=sys.stderr) + return 1 + return run(rest) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/emitters/auto.py b/soleprint/atlas2/docgen/emitters/auto.py new file mode 100644 index 0000000..832d132 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/auto.py @@ -0,0 +1,79 @@ +""" +Draw it the way its structure asks to be drawn. + + python3 -m docgen.emitters auto ir.json -o out/ + +`ops.classify` reads the structure and names an emitter; this runs it. The whole +point is that nobody should have to know that a schema wants cards and a module +graph wants ranks — or discover it from a 235:1 image. + +When the answer is "this is not a diagram", it says so and writes the index, +because that *is* the right artifact for a flat list of peers. +""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from ..ops import classify +from ..style import Style, StyleError + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters auto") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path, help="Directory to write into.") + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None) + p.add_argument("--force", help="Use this emitter regardless of what fits.") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)}):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + verdict = classify(data) + chosen = args.force or verdict["emitter"] + print(f" {verdict['kind']:<8} -> {chosen}") + print(f" {verdict['why']}") + + out_dir = args.output or Path(".") + out_dir.mkdir(parents=True, exist_ok=True) + stem = args.ir.stem + + try: + style = Style.load(args.style, theme=args.theme) + except StyleError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if chosen == "erd": + from .erd import emit as erd_emit + + path = out_dir / f"{stem}.svg" + path.write_text(erd_emit(data, style)) + elif chosen == "index": + from .index import to_markdown + + path = out_dir / f"{stem}.md" + path.write_text(to_markdown(data)) + else: + from .dot import emit as dot_emit, render + + path = out_dir / f"{stem}.svg" + opts = verdict.get("options") or {} + path.write_bytes(render(dot_emit(data, style, rankdir=opts.get("rankdir")))) + + print(f" {path}") + return 0 diff --git a/soleprint/atlas2/docgen/emitters/cli_dot.py b/soleprint/atlas2/docgen/emitters/cli_dot.py new file mode 100644 index 0000000..d346f80 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_dot.py @@ -0,0 +1,86 @@ +""" python3 -m docgen.emitters dot [-o out.svg] [--style lucid] [--theme dark]""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from ..ops import shape +from ..style import Style, StyleError +from .dot import RenderError, emit, render + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters dot") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path, help="Write here. .dot or .svg by suffix.") + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None) + p.add_argument("--max-depth", type=int, default=None) + p.add_argument("--quiet", "-q", action="store_true", help="Do not warn about shape.") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)} problem(s)):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + try: + style = Style.load(args.style, theme=args.theme) + except StyleError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + # Aspect ratio is a property of the graph, not of the renderer: a layered + # engine puts one dependency level in one row, so the widest level is the + # width. Say so before writing the file, because the alternative is finding + # out from a 13671pt image — and the fix is never a layout flag, it is a + # smaller question. + if not args.quiet: + sh = shape(data) + if sh["widest_level"] > 20 or sh["nodes"] > 60: + est = sh["widest_level"] / max(sh["levels"], 1) + print( + f" note: {sh['nodes']} nodes, {sh['levels']} levels, widest level " + f"{sh['widest_level']} — this will render roughly {est:.0f}:1.", + file=sys.stderr, + ) + print( + " Around 20 nodes is where it stops being a diagram. Try " + "`ops --split`,\n `--around --hops 2`, or `--subtree `. " + "Layout flags will not fix it.", + file=sys.stderr, + ) + if sh["isolated"] > sh["nodes"] // 3: + print( + f" {sh['isolated']} of {sh['nodes']} nodes have no edges; they are " + "laid out side by side.", + file=sys.stderr, + ) + + dot_text = emit(data, style, max_depth=args.max_depth) + + if not args.output: + sys.stdout.write(dot_text) + return 0 + + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.output.suffix == ".dot": + args.output.write_text(dot_text) + else: + try: + args.output.write_bytes(render(dot_text, fmt=args.output.suffix.lstrip(".") or "svg")) + except RenderError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + print(f" {args.style}/{style.theme:6} {args.output}") + return 0 diff --git a/soleprint/atlas2/docgen/emitters/cli_erd.py b/soleprint/atlas2/docgen/emitters/cli_erd.py new file mode 100644 index 0000000..aedd46e --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_erd.py @@ -0,0 +1,50 @@ +""" python3 -m docgen.emitters erd [-o out.svg] [--theme dark]""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from ..style import Style, StyleError +from .erd import emit + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters erd") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path) + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None) + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)} problem(s)):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + try: + style = Style.load(args.style, theme=args.theme) + svg = emit(data, style) + except (StyleError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(svg) + import re + m = re.search(r'width="(\d+)pt" height="(\d+)pt"', svg) + size = f"{m.group(1)}x{m.group(2)} {int(m.group(1))/int(m.group(2)):.1f}:1" if m else "" + print(f" erd/{style.theme:6} {args.output} {size}") + else: + sys.stdout.write(svg) + return 0 diff --git a/soleprint/atlas2/docgen/emitters/cli_index.py b/soleprint/atlas2/docgen/emitters/cli_index.py new file mode 100644 index 0000000..0ab5944 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_index.py @@ -0,0 +1,43 @@ +""" python3 -m docgen.emitters index [-o out.md|out.json]""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from .index import to_markdown, to_sidebar + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters index") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path, help=".md for the document, .json for a sidebar.") + p.add_argument("--title", default="") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)} problem(s)):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + if args.output and args.output.suffix == ".json": + text = json.dumps(to_sidebar(data), indent=2) + "\n" + else: + text = to_markdown(data, title=args.title) + + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + print(f" index {args.output}") + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/emitters/cli_notebook.py b/soleprint/atlas2/docgen/emitters/cli_notebook.py new file mode 100644 index 0000000..1ad4905 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_notebook.py @@ -0,0 +1,73 @@ +""" python3 -m docgen.emitters notebook [-o out.ipynb] [--overlay f.json] + + --spec-out FILE write the generated spec (the base), for reading/diffing + --scaffold FILE write a blank overlay listing every step id +""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from ..notebook import dump, from_ir, merge, scaffold +from .notebook import emit + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters notebook") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path) + p.add_argument("--overlay", type=Path, help="Hand-written additions, re-applied.") + p.add_argument("--spec-out", type=Path, help="Write the generated spec too.") + p.add_argument("--scaffold", type=Path, help="Write a blank overlay and stop.") + p.add_argument("--base-url", default="https://api.example.invalid") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)}):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + base = from_ir(data, base_url=args.base_url) + + if args.scaffold: + dump(scaffold(base), args.scaffold) + print(f" overlay {args.scaffold} {len(base['steps'])} step(s), none filled in") + return 0 + + overlay = None + if args.overlay: + if args.overlay.exists(): + overlay = json.loads(args.overlay.read_text()) + else: + print(f" note: no overlay at {args.overlay} — generating the base only", + file=sys.stderr) + + spec, drift = merge(base, overlay) + for d in drift: + # The base moved under the overlay. Worth saying out loud; not a reason + # to refuse to build the document. + print(f" drift: {d}", file=sys.stderr) + + if args.spec_out: + dump(spec, args.spec_out) + print(f" spec {args.spec_out}") + + text = emit(spec) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + n = len(json.loads(text)["cells"]) + extra = f", {len(drift)} drift" if drift else "" + print(f" notebook {args.output} {len(spec['steps'])} steps, {n} cells{extra}") + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/emitters/dot.py b/soleprint/atlas2/docgen/emitters/dot.py new file mode 100644 index 0000000..0e4f2c7 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/dot.py @@ -0,0 +1,291 @@ +""" +IR + style -> DOT -> SVG. + +This module has never heard of Python, `ast` or SQL. It walks nodes, looks up a +rule by `kind`, and writes attributes. That is deliberately the whole algorithm: +if it starts making decisions about what something *is*, the decision belongs in +an extractor, and if it starts making decisions about what something *looks +like*, it belongs in a style file. + + from docgen.emitters.dot import emit, render + svg = render(emit(ir, Style.load("lucid"))) + +## Containment becomes clusters + +A node with children is a `subgraph cluster_*`; a leaf is a node. That is the +only structural interpretation made here, and it follows from `parent` meaning +containment and nothing else. + +## The SVG is addressable + +`id` and `kind` are written through to the SVG as the element's `id` and +`class`, and `attrs.file`/`attrs.line` become an `href`. So a front end can +attach behaviour to a box, and a box can link to the line it came from, without +this emitter knowing anything about either. + +## Known limits + +Recorded in `style/lucid.json` under `limits` and reachable as `Style.limits()`. +DOT is used until it genuinely cannot express a rule, and then it stops rather +than growing machinery — an HTML-like label table for header bars, a post-pass +for badge circles. Those mark where a richer emitter begins. +""" + +import shutil +import subprocess + + +class RenderError(RuntimeError): + """Graphviz is absent, or refused the graph.""" + + +def _esc(text: str) -> str: + return str(text).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +def _attrs(pairs: dict) -> str: + inner = " ".join(f'{k}="{_esc(v)}"' for k, v in pairs.items() if v not in (None, "", [])) + return f" [{inner}]" if inner else "" + + +def _style_words(rule: dict, *, filled: bool = True) -> str: + words = ["filled"] if filled else [] + # `record` and `plaintext` ignore rounding; asking for it warns and changes + # nothing, which is noise in the build output. + if rule.get("rounded") and rule.get("shape") not in ("record", "Mrecord", "plaintext"): + words.append("rounded") + if rule.get("dashed"): + words.append("dashed") + return ",".join(words) + + +def _node_attrs(node, rule: dict, style) -> dict: + a = { + "label": node.get("label") or node["id"], + "shape": rule.get("shape", "box"), + "style": _style_words(rule), + "fillcolor": rule.get("fill"), + "color": rule.get("border"), + "fontcolor": rule.get("text"), + "penwidth": style.geom("hairline"), + "fontname": style.geom("font-bold" if rule.get("bold") else "font"), + "fontsize": rule.get("font-size") or style.geom("font-size-base"), + "margin": style.geom("padding"), + # Addressability: through to the SVG, for whatever reads it later. + "id": node["id"], + "class": node["kind"], + } + attrs = node.get("attrs") or {} + if attrs.get("file"): + line = attrs.get("line") + a["href"] = f"{attrs['file']}#L{line}" if line else attrs["file"] + a["tooltip"] = attrs.get("doc") or node["id"] + elif attrs.get("doc"): + a["tooltip"] = attrs["doc"] + return a + + +def emit(ir: dict, style, *, max_depth: int | None = None, + rankdir: str | None = None) -> str: + """IR document (a dict) + Style -> DOT text.""" + nodes = {n["id"]: n for n in ir["nodes"]} + children: dict[str | None, list[str]] = {} + for n in ir["nodes"]: + children.setdefault(n.get("parent"), []).append(n["id"]) + + g = style.graph() + out = [ + "digraph ir {", + f' bgcolor="{g.get("bgcolor", "transparent")}"', + f' rankdir={rankdir or g.get("rankdir", "TB")}', + f' nodesep="{g.get("nodesep", 0.5)}"', + f' ranksep="{g.get("ranksep", 0.6)}"', + f' pad="{g.get("pad", 0.3)}"', + f' fontname="{style.geom("font")}"', + " compound=true", + "", + ] + + # Deterministic: groups are numbered by sorted id, so the rotation of + # domain colours is the same on every run. + group_index = { + nid: i for i, nid in enumerate(sorted(k for k in children if k is not None)) + } + + def write(node_id: str, depth: int) -> None: + node = nodes[node_id] + kids = sorted(children.get(node_id, [])) + too_deep = max_depth is not None and depth >= max_depth + pad = " " * (depth + 1) + + if not kids or too_deep: + out.append(f"{pad}{_q(node_id)}{_attrs(_node_attrs(node, style.node(node['kind']), style))}") + return + + rule = style.group(node["kind"]) + domain = (node.get("attrs") or {}).get("domain") + border = rule.get("border") or style.slot( + style.domain_slot(domain, group_index.get(node_id, 0)) + ) + out.append(f"{pad}subgraph cluster_{_safe(node_id)} {{") + out.append(f'{pad} label="{_esc(node.get("label") or node_id)}"') + out.append(f'{pad} style="{_style_words(rule)}"') + out.append(f'{pad} color="{border}"') + out.append(f'{pad} fillcolor="{rule.get("fill", "transparent")}"') + out.append(f'{pad} fontcolor="{rule.get("text", "")}"') + out.append(f'{pad} fontname="{style.geom("font-bold" if rule.get("bold") else "font")}"') + out.append(f'{pad} fontsize="{style.geom("font-size-header")}"') + out.append(f'{pad} labeljust=l') + out.append(f'{pad} id="{_esc(node_id)}"') + out.append(f'{pad} class="{node["kind"]}"') + for kid in kids: + write(kid, depth + 1) + out.append(f"{pad}}}") + + for root in sorted(children.get(None, [])): + write(root, 0) + out.append("") + + # DOT cannot use a cluster as an edge endpoint. The native answer is + # `compound=true` plus lhead/ltail: draw between a representative leaf + # inside each cluster and clip the line at the cluster boundary. Without + # this, every module-to-module import silently disappears — which is most of + # the graph a Python extractor produces. + endpoint = _endpoints(nodes, children, max_depth) + for e in ir["edges"]: + src, dst = endpoint.get(e["source"]), endpoint.get(e["target"]) + if not src or not dst or src[0] == dst[0]: + continue + if src[1] and src[1] == dst[1]: + continue # both collapsed into the same cluster + # A package importing its own submodule gives an edge whose head sits + # inside its tail's cluster. Graphviz warns and draws it oddly; clipping + # to the enclosing boundary is meaningless there, so drop that side's + # clip and let the line run to the box. + ltail, lhead = src[1], dst[1] + if ltail and _within(ltail, dst[0], nodes): + ltail = None + if lhead and _within(lhead, src[0], nodes): + lhead = None + rule = style.edge(e["kind"]) + out.append( + f" {_q(src[0])} -> {_q(dst[0])}" + + _attrs( + { + "color": rule.get("color"), + "fontcolor": rule.get("text"), + "penwidth": style.geom("hairline"), + "arrowhead": rule.get("arrowhead", "normal"), + "arrowsize": rule.get("arrowsize", 0.7), + "style": "dashed" if rule.get("dashed") else None, + "fontname": style.geom("font"), + "fontsize": style.geom("font-size-sm"), + "label": (e.get("attrs") or {}).get("label"), + "ltail": ltail, + "lhead": lhead, + "class": e["kind"], + } + ) + ) + + out.append("}") + return "\n".join(out) + "\n" + + +def _within(cluster_name: str, node_id: str, nodes) -> bool: + """Is `node_id` inside the cluster named `cluster_name`?""" + cur = node_id + while cur: + if f"cluster_{_safe(cur)}" == cluster_name: + return True + cur = nodes.get(cur, {}).get("parent") + return False + + +def _endpoints(nodes, children, max_depth): + """id -> (leaf to draw from, cluster to clip to or None). + + A leaf is its own endpoint. A node that became a cluster is represented by + its first leaf descendant in sorted order — deterministic, so the same graph + twice produces the same DOT — with `ltail`/`lhead` naming the cluster so the + line stops at its border instead of burrowing to the inner box. + """ + depth_of: dict[str, int] = {} + + def walk(nid, depth): + depth_of[nid] = depth + for kid in children.get(nid, []): + walk(kid, depth + 1) + + for root in children.get(None, []): + walk(root, 0) + + def collapsed(nid: str) -> bool: + return max_depth is not None and depth_of.get(nid, 0) >= max_depth + + def first_leaf(nid: str) -> str: + while children.get(nid) and not collapsed(nid): + nid = sorted(children[nid])[0] + return nid + + out: dict[str, tuple[str, str | None]] = {} + for nid in nodes: + cur = nid + # Anything past the depth limit is represented by the ancestor that + # survived it. + while max_depth is not None and depth_of.get(cur, 0) > max_depth: + parent = nodes[cur].get("parent") + if not parent: + break + cur = parent + if children.get(cur) and not collapsed(cur): + out[nid] = (first_leaf(cur), f"cluster_{_safe(cur)}") + else: + out[nid] = (cur, None) + return out + + +def _safe(text: str) -> str: + return "".join(c if c.isalnum() else "_" for c in text) + + +def _q(text: str) -> str: + return f'"{_esc(text)}"' + + +# -- render ---------------------------------------------------------------- + + +def have_graphviz(engine: str = "dot") -> bool: + return shutil.which(engine) is not None + + +def render(dot_text: str, fmt: str = "svg", engine: str = "dot") -> bytes: + """DOT -> bytes, via the graphviz binary. + + The binary, not a wrapper library: it is what the render hosts have and what + `docs/graphs/render.sh` already shells out to. + + Note for anything reading geometry back out: Graphviz is y-up in points and + the SVG backend flips it with a wrapper ``, and layout + measures label text with the host's fonts — so the same graph on a machine + with different fontconfig produces different coordinates. Pin golden tests + to the IR, never to the SVG. + """ + if not have_graphviz(engine): + raise RenderError( + f"{engine!r} not found — install with: sudo apt install graphviz\n" + "(already-rendered files keep working; this is only needed to re-render)" + ) + proc = subprocess.run([engine, f"-T{fmt}"], input=dot_text.encode(), capture_output=True) + if proc.returncode != 0: + raise RenderError( + f"{engine} -T{fmt} failed ({proc.returncode}):\n" + + proc.stderr.decode("utf-8", "replace").strip() + ) + if proc.stderr.strip(): + # Graphviz warns and still renders — a missing font, an ignored + # attribute. Worth seeing, not worth failing on. + for line in proc.stderr.decode("utf-8", "replace").strip().splitlines(): + print(f" graphviz: {line}") + return proc.stdout diff --git a/soleprint/atlas2/docgen/emitters/erd.py b/soleprint/atlas2/docgen/emitters/erd.py new file mode 100644 index 0000000..0b96d48 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/erd.py @@ -0,0 +1,247 @@ +""" +A schema, as an entity-relationship diagram. SVG written directly — no Graphviz. + +**This is the answer to the aspect-ratio problem, and it is not a layout engine.** +A layered engine puts every node at one dependency level into one row, so the +widest level is the width; soleprint's 7-by-109 overview rendered 14:1 and no +Graphviz flag helped. A schema is not layered anyway — tables are peers that +reference each other — so laying it out in ranks was the wrong shape from the +start. + +The design is lifted from `station/tools/graphgen/templates/index.html`, the +Supabase-style schema explorer already in this repo. It had solved this: + + const cols = Math.max(2, Math.ceil(Math.sqrt(sorted.length * 1.2))); + +**Columns from the square root of the table count.** The result is near-square +whatever the size — 4 tables or 400 — because the aspect ratio is chosen rather +than emergent. That is the one thing a rank-based engine cannot do. + +Three more things it gets right that a generic node-edge drawing does not: + +- **A table is a card**, not a box: a header and a list of its columns. That is + what a schema *is*, and it is the form every ER tool has converged on. +- **An edge starts at the column that holds the key** and ends on the target's + primary key, rather than joining two box centres. That is what makes a + foreign key readable rather than merely present. +- **The geometry is computed, never measured.** Card width and row height are + constants, so this produces identical bytes on any machine — unlike Graphviz, + which measures label text with the host's fonts and so renders differently + wherever fontconfig differs. + +Colours come from the same style slots as every other emitter, so an ER diagram +and a code diagram are still one visual language. +""" + +from xml.sax.saxutils import escape + +# Geometry, from the explorer. Fixed on purpose: nothing here measures text, so +# the output is deterministic and can be golden-tested. +CARD_W = 210 +HDR_H = 36 +HDR_H_DOC = 50 +FIELD_H = 26 +COL_GAP = 80 +ROW_GAP = 60 +PAD = 40 +BADGE_W = 28 +RADIUS = 8 + +# ~6.2px per character at 11px in a humanist sans. An estimate, and it only +# decides where a long name is cut — never where anything is placed. +CHAR_W = 6.2 + + +def _truncate(text: str, px: float) -> str: + limit = max(3, int(px / CHAR_W)) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _tables(ir: dict) -> list[dict]: + """Tables with their columns, in a stable order.""" + columns: dict[str, list] = {} + for n in ir["nodes"]: + if n["kind"] == "column" and n.get("parent"): + columns.setdefault(n["parent"], []).append(n) + out = [] + for n in ir["nodes"]: + if n["kind"] != "table": + continue + fields = columns.get(n["id"], []) + out.append( + { + "id": n["id"], + "name": n.get("label") or n["id"], + "doc": (n.get("attrs") or {}).get("doc"), + "fields": fields, + } + ) + return sorted(out, key=lambda t: t["id"]) + + +def _card_height(table: dict) -> int: + header = HDR_H_DOC if table["doc"] else HDR_H + return header + len(table["fields"]) * FIELD_H + + +def layout(tables: list[dict], edges: list[dict]) -> dict[str, tuple[int, int]]: + """Place cards in √n columns, referenced tables first. + + Sorting by "is the target of a foreign key" puts the tables everything + points at into the left columns, so the majority of edges run left to right + and stop crossing each other. Same trick the explorer uses. + """ + referenced = {e["target"] for e in edges} + ordered = sorted(tables, key=lambda t: (t["id"] not in referenced, t["id"])) + + cols = max(2, int((len(ordered) * 1.2) ** 0.5 + 0.999)) + col_w = CARD_W + COL_GAP + cursor = [PAD] * cols + + pos: dict[str, tuple[int, int]] = {} + for i, table in enumerate(ordered): + col = i % cols + pos[table["id"]] = (col * col_w + PAD, cursor[col]) + cursor[col] += _card_height(table) + ROW_GAP + return pos + + +def _field_y(table: dict, index: int, top: int) -> float: + header = HDR_H_DOC if table["doc"] else HDR_H + return top + header + (max(index, 0) + 0.5) * FIELD_H + + +def emit(ir: dict, style) -> str: + """IR (a db document) + Style -> SVG text.""" + tables = _tables(ir) + if not tables: + raise ValueError( + "no tables in this IR — erd draws a schema, and this one has none. " + "Was it extracted with the python reader?" + ) + edges = [e for e in ir["edges"] if e["kind"] in ("foreign_key", "references")] + by_id = {t["id"]: t for t in tables} + pos = layout(tables, edges) + + s = style.slot + width = max(x for x, _ in pos.values()) + CARD_W + PAD + height = max(y + _card_height(by_id[t]) for t, (_, y) in pos.items()) + PAD + + out = [ + '', + f'', + f'', + "", + f'' + f'', + "", + ] + + # Edges first, so cards sit on top of them where they meet. + out.append('') + for e in edges: + src, dst = by_id.get(e["source"]), by_id.get(e["target"]) + if not src or not dst: + continue + sx, sy_top = pos[src["id"]] + dx_, dy_top = pos[dst["id"]] + + label = (e.get("attrs") or {}).get("label") + from_idx = next( + (i for i, f in enumerate(src["fields"]) if f.get("label") == label), 0 + ) + to_idx = next( + (i for i, f in enumerate(dst["fields"]) if (f.get("attrs") or {}).get("pk")), 0 + ) + + # Leave from whichever side faces the target, so a line never crosses + # its own card to get out. + leaving_right = dx_ >= sx + x1 = sx + CARD_W if leaving_right else sx + x2 = dx_ if leaving_right else dx_ + CARD_W + y1 = _field_y(src, from_idx, sy_top) + y2 = _field_y(dst, to_idx, dy_top) + + ctrl = min(max(abs(x2 - x1) * 0.5, 50), 180) + c1 = x1 + ctrl if leaving_right else x1 - ctrl + c2 = x2 - ctrl if leaving_right else x2 + ctrl + out.append( + f'' + ) + out.append("") + + # Cards. + for table in tables: + x, y = pos[table["id"]] + header = HDR_H_DOC if table["doc"] else HDR_H + h = _card_height(table) + out.append(f'') + out.append( + f'' + ) + # Header band, clipped to the card's rounded top by drawing a rounded + # rect and squaring its bottom with a second one. + out.append( + f'' + ) + out.append( + f'' + ) + out.append( + f'' + f'{escape(_truncate(table["name"], CARD_W - 24))}' + ) + if table["doc"]: + out.append( + f'' + f'{escape(_truncate(table["doc"], CARD_W - 24))}' + ) + + for i, field in enumerate(table["fields"]): + fy = y + header + i * FIELD_H + attrs = field.get("attrs") or {} + name = field.get("label") or field["id"].rsplit(".", 1)[-1] + if attrs.get("pk"): + badge, badge_fill = "PK", s("accent") + elif attrs.get("references"): + badge, badge_fill = "FK", s("station") + else: + badge, badge_fill = "", s("text-dim") + + if i: + out.append( + f'' + ) + if badge: + out.append( + f'{badge}' + ) + out.append( + f'' + f'{escape(_truncate(name, 96))}' + ) + type_text = attrs.get("references") or attrs.get("type", "") + if type_text: + out.append( + f'{escape(_truncate(str(type_text), 60))}' + ) + out.append("") + + out.append("") + return "\n".join(out) + "\n" diff --git a/soleprint/atlas2/docgen/emitters/index.py b/soleprint/atlas2/docgen/emitters/index.py new file mode 100644 index 0000000..a90344c --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/index.py @@ -0,0 +1,163 @@ +""" +IR -> an index. Markdown for reading, JSON for a sidebar. + +**This is the emitter that matters most for reach.** A sorted, described list of +what exists is readable by people who will never open a diagram — a PM checking +that a feature has a home, QA looking for the surface to test, someone new +trying to find where anything is. A diagram asks for graph literacy and a +screen; this asks for neither. + +It is also the checkpoint on the whole design. If the IR were secretly +diagram-shaped, this emitter would be awkward to write — it would be reaching +for positions, or re-deriving containment from edges. It is not, because +`parent` is containment and `kind` is meaning, and that is all a table of +contents needs. + + python3 -m docgen.emitters index ir.json # markdown to stdout + python3 -m docgen.emitters index ir.json -o x.json # sidebar JSON + +## What it reports that a diagram cannot + +- **what depends on what is outside**, gathered in one place. `external` nodes + are the project's real dependency surface, and in a diagram they are scattered + boxes. +- **what could not be parsed.** A file the extractor choked on is a hole in the + analysis; it is listed rather than quietly absent. +""" + +import json +from collections import Counter, defaultdict + +# Order matters for reading, not for correctness: containers before contents. +KIND_ORDER = ["module", "class", "function", "table", "column", "external"] + + +def _tree(ir: dict): + children = defaultdict(list) + for n in ir["nodes"]: + children[n.get("parent")].append(n) + for kids in children.values(): + kids.sort(key=lambda n: (KIND_ORDER.index(n["kind"]) if n["kind"] in KIND_ORDER else 99, + n["id"])) + return children + + +def _anchor(node: dict) -> str: + attrs = node.get("attrs") or {} + if not attrs.get("file"): + return "" + return f"{attrs['file']}:{attrs['line']}" if attrs.get("line") else attrs["file"] + + +def to_markdown(ir: dict, title: str = "") -> str: + """A document. Headings for containers, a list for their contents.""" + children = _tree(ir) + nodes = {n["id"]: n for n in ir["nodes"]} + meta = ir.get("meta", {}) + counts = Counter(n["kind"] for n in ir["nodes"]) + edge_counts = Counter(e["kind"] for e in ir["edges"]) + + out = [f"# {title or meta.get('root', 'index')}", ""] + out.append( + f"Extracted from `{meta.get('root', '?')}` by the `{meta.get('source', '?')}` " + f"reader. {len(ir['nodes'])} nodes, {len(ir['edges'])} edges." + ) + out.append("") + out.append("| | |") + out.append("|---|---|") + for kind, n in sorted(counts.items(), key=lambda kv: -kv[1]): + out.append(f"| {kind} | {n} |") + for kind, n in sorted(edge_counts.items(), key=lambda kv: -kv[1]): + out.append(f"| *{kind}* (edges) | {n} |") + out.append("") + + # -- the contents ----------------------------------------------------- + def walk(node: dict, depth: int): + kids = [k for k in children.get(node["id"], [])] + doc = (node.get("attrs") or {}).get("doc") + anchor = _anchor(node) + + if depth == 0: + out.append(f"## {node['label']} {node['kind']}") + out.append("") + if doc: + out.append(doc) + out.append("") + if anchor: + out.append(f"`{anchor}`") + out.append("") + else: + bullet = " " * (depth - 1) + "-" + parts = [f"**{node['label']}**", f"*{node['kind']}*"] + if doc: + parts.append(f"— {doc}") + if anchor: + parts.append(f"`{anchor}`") + out.append(f"{bullet} {' '.join(parts)}") + + for kid in kids: + walk(kid, depth + 1) + if depth == 0 and kids: + out.append("") + + roots = [n for n in children.get(None, []) if n["kind"] != "external"] + for root in roots: + walk(root, 0) + + # -- what is outside -------------------------------------------------- + externals = sorted(n["id"] for n in ir["nodes"] if n["kind"] == "external") + if externals: + out.append("## Depends on, outside this tree") + out.append("") + out.append( + "Names that could not be resolved to anything in the source. This is the " + "dependency surface — third-party imports, and anything reached dynamically." + ) + out.append("") + incoming = Counter(e["target"] for e in ir["edges"] if e["target"] in set(externals)) + for eid in sorted(externals, key=lambda e: (-incoming[e], e)): + n = incoming[eid] + out.append(f"- `{eid}`" + (f" — referenced {n}×" if n > 1 else "")) + out.append("") + + # -- holes in the analysis -------------------------------------------- + broken = [n for n in ir["nodes"] if (n.get("attrs") or {}).get("error")] + if broken: + out.append("## Not parsed") + out.append("") + out.append("These files were skipped, so anything they define is missing below.") + out.append("") + for n in sorted(broken, key=lambda n: n["id"]): + out.append(f"- `{(n.get('attrs') or {}).get('file', n['id'])}` — " + f"{(n.get('attrs') or {}).get('error')}") + out.append("") + + return "\n".join(out).rstrip() + "\n" + + +def to_sidebar(ir: dict) -> dict: + """Nested JSON for a navigation pane. + + Shaped for a UI to render directly: `label`, `kind`, `href`, `children`. + """ + children = _tree(ir) + + def build(node: dict) -> dict: + attrs = node.get("attrs") or {} + item = {"id": node["id"], "label": node["label"], "kind": node["kind"]} + if attrs.get("doc"): + item["doc"] = attrs["doc"] + if attrs.get("file"): + item["href"] = ( + f"{attrs['file']}#L{attrs['line']}" if attrs.get("line") else attrs["file"] + ) + kids = [build(k) for k in children.get(node["id"], [])] + if kids: + item["children"] = kids + return item + + return { + "meta": ir.get("meta", {}), + "items": [build(n) for n in children.get(None, []) if n["kind"] != "external"], + "external": sorted(n["id"] for n in ir["nodes"] if n["kind"] == "external"), + } diff --git a/soleprint/atlas2/docgen/emitters/notebook.py b/soleprint/atlas2/docgen/emitters/notebook.py new file mode 100644 index 0000000..75abe4b --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/notebook.py @@ -0,0 +1,248 @@ +""" +IR -> a Jupyter notebook. Generated, never hand-authored. + +## The frame, because it is the whole argument + +A notebook is normally a **source file** that someone confects by hand — prose, +code and stored output braided together, diffing badly, drifting from whatever +it documents the moment either moves, with no way to tell by looking. jupytext +addresses the diffing and leaves the rest: it makes the notebook editable as +text, so you still hand-author it. + +Here a notebook is a **build artifact**. The source is the OpenAPI document — +the same file the server is built from — and the notebook is regenerated from +it. Nobody edits the `.ipynb`, the same way nobody edits a `.o` file. "Is this +document current" stops being a question about somebody's diligence and becomes +a question about whether the build ran. + +That is also why it suits a mixed audience rather than a data-science one. The +endpoints, their methods, their payload shapes and their status codes are facts +taken from the spec, so a PM reading it is reading the API, not somebody's +recollection of it. + +## Reproducible in the strict sense + +Cell ids come from position, `execution_count` is null, `outputs` is empty, and +the JSON is key-sorted. **The same IR produces byte-identical bytes.** A +notebook that changes on every build cannot be reviewed, and one that cannot be +reviewed will not be trusted. + +Written against the nbformat 4 schema directly. `nbformat` is not installed on +the machines this runs on, and the schema has six required keys. +""" + +import json +from pathlib import Path + +METADATA = { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python"}, +} + + +def _cell(index: int, kind: str, text: str) -> dict: + lines = text.split("\n") + source = [ln + "\n" for ln in lines[:-1]] + ([lines[-1]] if lines[-1] else []) + cell = { + "cell_type": "markdown" if kind == "md" else "code", + "id": f"cell-{index:03d}", + "metadata": {}, + "source": source, + } + if kind == "code": + cell["execution_count"] = None + cell["outputs"] = [] + return cell + + +def _example(fields: list[dict]) -> str: + """A request body shaped like the schema, with placeholder values.""" + sample = {} + for f in fields: + if f.get("pk"): + continue # the server assigns it + t = str(f.get("type", "str")).lower() + name = f["name"] + if "int" in t: + sample[name] = 0 + elif "float" in t or "decimal" in t: + sample[name] = 0.0 + elif "bool" in t: + sample[name] = False + elif "date" in t or "time" in t: + sample[name] = "2026-01-01T00:00:00Z" + elif "list" in t: + sample[name] = [] + else: + sample[name] = f"<{name}>" + # A Python literal, not JSON. `json.dumps` writes `false`/`true`/`null`, + # which are valid *identifiers* in Python — so the cell compiles and then + # raises NameError the moment anyone runs it. Compiling is not enough of a + # check; the notebook selftest executes these cells for exactly this reason. + body = ",\n".join(f" {k!r}: {v!r}" for k, v in sorted(sample.items())) + return "{\n" + body + ",\n}" if body else "{}" + + +CLIENT = '''def call(method, path, params=None, body=None): + """One request. Returns (status, parsed_body). + + A non-2xx is returned rather than raised: the error body usually names the + field that was wrong, and an exception throws that away. + """ + url = BASE_URL.rstrip("/") + "/" + path.lstrip("/") + if params: + url += "?" + urllib.parse.urlencode(params) + data = json.dumps(body).encode() if body is not None else None + headers = {"Accept": "application/json", **AUTH} + if data: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + print(f"-> {method} {url}") + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + status, raw = r.status, r.read() + except urllib.error.HTTPError as e: + status, raw = e.code, e.read() + except urllib.error.URLError as e: + print(f"<- unreachable: {e.reason}") + return None, None + try: + parsed = json.loads(raw) if raw else None + except json.JSONDecodeError: + parsed = raw.decode("utf-8", "replace") + print(f"<- {status}") + return status, parsed + + +def show(result, limit=1500): + status, body = result + if status is None: + return + text = body if isinstance(body, str) else json.dumps(body, indent=2) + print(text[:limit] + (f"\\n… {len(text) - limit} more" if len(text) > limit else ""))''' + + +def _params_cell(step: dict) -> str: + env = step.get("env_var", "API_TOKEN") + return f'''import json +import os +import urllib.error +import urllib.parse +import urllib.request + +BASE_URL = os.environ.get("API_BASE_URL", "{step.get("base_url", "")}") +TOKEN = os.environ.get("{env}", "") +TIMEOUT = 30 + +# Read from the environment, never written here: a token pasted into a cell +# travels with every copy of this notebook from then on. +AUTH = {{"Authorization": f"Bearer {{TOKEN}}"}} if TOKEN else {{}} + +print("base ", BASE_URL) +print("token", f"set ({{len(TOKEN)}} chars)" if TOKEN else "NOT SET — export {env}=…")''' + + +def _call_cell(step: dict) -> str: + """The generated call. An overlay's `code` replaces this wholesale.""" + method, path = step.get("method", "GET"), step.get("path", "/") + params = step.get("path_params") or [] + lines = [f'{p.upper()} = "<{p}>" # path parameter' for p in params] + call_path = path + for p in params: + call_path = call_path.replace("{" + p + "}", f'" + str({p.upper()}) + "') + expr = f'"{call_path}"' if params else f'"{path}"' + + if step.get("body_fields"): + lines.append("BODY = " + _example(step["body_fields"])) + lines.append("") + lines.append(f'show(call("{method}", {expr}, body=BODY))') + else: + if lines: + lines.append("") + lines.append(f'show(call("{method}", {expr}))') + return "\n".join(lines) + + +def _call_md(step: dict) -> str: + out = [f'## {step.get("method", "GET")} `{step.get("path", "/")}`'] + if step.get("summary"): + out += ["", step["summary"]] + facts = [] + if step.get("response_model"): + facts.append(f'returns **{step["response_model"]}**' + + (" (a list)" if step.get("returns_list") else "")) + if step.get("request_model"): + facts.append(f'accepts **{step["request_model"]}**') + if step.get("status"): + facts.append(f'expects `{step["status"]}`') + if facts: + out += ["", " · ".join(facts)] + if step.get("note"): + out += ["", step["note"]] + return "\n".join(out) + + +def build(spec: dict) -> dict: + """A merged notebook spec -> the notebook, as a dict.""" + blocks: list[tuple[str, str]] = [] + + for step in spec["steps"]: + kind = step.get("kind", "md") + before = step.get("before") + if before: + blocks.append(("md", before)) + + if kind == "md": + text = [] + if step.get("title") and step["id"] != "intro": + text.append(f'## {step["title"]}') + elif step.get("title"): + text.append(f'# {step["title"]}') + if step.get("text"): + text += ["", step["text"]] + if step.get("table"): + text += ["", "| | fields |", "|---|---|"] + for row in step["table"]: + names = ", ".join(f'`{f}`' for f in row.get("fields", [])) + text.append(f'| **{row["name"]}** | {names or "—"} |') + blocks.append(("md", "\n".join(text))) + + elif kind == "params": + if step.get("title"): + blocks.append(("md", f'## {step["title"]}')) + blocks.append(("code", step.get("code") or _params_cell(step))) + + elif kind == "code": + if step.get("title"): + blocks.append(("md", f'## {step["title"]}')) + code = step.get("code") + if code is None and step.get("builtin") == "client": + code = CLIENT + blocks.append(("code", code or "")) + + elif kind == "call": + blocks.append(("md", _call_md(step))) + blocks.append(("code", step.get("code") or _call_cell(step))) + + after = step.get("after_text") + if after: + blocks.append(("md", after)) + + return { + "cells": [_cell(i, k, t) for i, (k, t) in enumerate(blocks)], + "metadata": METADATA, + "nbformat": 4, + "nbformat_minor": 5, + } + + +def emit(spec: dict) -> str: + """The notebook as text. Key-sorted, so the same spec gives the same bytes.""" + return json.dumps(build(spec), indent=1, sort_keys=True, ensure_ascii=False) + "\n" + + +def write(spec: dict, path) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(emit(spec)) + return path diff --git a/soleprint/atlas2/docgen/extractors/__init__.py b/soleprint/atlas2/docgen/extractors/__init__.py new file mode 100644 index 0000000..cf261ca --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/__init__.py @@ -0,0 +1 @@ +"""Extractors: source artifacts -> IR. None of them has heard of SVG.""" diff --git a/soleprint/atlas2/docgen/extractors/__main__.py b/soleprint/atlas2/docgen/extractors/__main__.py new file mode 100644 index 0000000..4d38704 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/__main__.py @@ -0,0 +1,19 @@ +""" python3 -m docgen.extractors [options]""" + +import sys + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + if argv and argv[0] == "openapi": + from .openapi_main import main as run + return run(argv[1:]) + if argv and argv[0] == "db": + from .db_main import main as run + return run(argv[1:]) + from .db_main import main as run # bare form stays the db one, as before + return run(argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/extractors/db.py b/soleprint/atlas2/docgen/extractors/db.py new file mode 100644 index 0000000..e22aa81 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/db.py @@ -0,0 +1,144 @@ +""" +A database schema -> IR. + +Reads the **published** `{models, relationships, source}` contract rather than +importing modelgen's Python. That contract is already emitted by +`modelgen/generator/jsonschema.py`, consumed by `graphgen/schema.py` and +`datagen`, and asserted by two modelgen tests — so it is the stable surface, and +reading it means this extractor works for every source modelgen supports +(Django, SQLAlchemy, OpenAPI, CSV/ODS, a live database) without knowing about +any of them. + + python3 -m docgen.extractors.db --schema cfg/sample/.../graphgen/schema.json + +Connecting to a live database is **not here**. `modelgen from-db --url ...` +reflects via SQLAlchemy's Inspector across dialects and writes the schema.json +this reads; that is its job and it is already done. The two-step is also the +safer one — the URL, and therefore the credentials, never enters this pipeline. + +## The schema checkpoint + +The brief requires the IR to survive a second domain without new top-level +fields. It does, and the mapping is not a squeeze: + + table -> node, kind "table" column -> node, kind "column" + column -> parent is its table FK -> edge, kind "foreign_key" + M2M -> edge, kind "references" + +`kind` carries the domain vocabulary, `parent` carries containment, `attrs` +carries what only this domain cares about — `pk`, `nullable`, the column type. +Nothing needed a field that `module`/`class`/`function` did not also use, which +is the result the checkpoint was there to confirm. +""" + +import json +from pathlib import Path + +from ..ir import Graph, Meta + + +def from_schema_dict(data: dict, root: str = "schema", source: str = "db") -> Graph: + """A graphgen-compatible schema dict -> IR. + + Accepts both spellings of the contract: the on-disk `schema.json` form, + where `models` is a mapping of name to definition, and the loaded form that + `graphgen.schema.load_graph_schema` returns, where it is a list. They are + the same data and callers have both. + """ + g = Graph(Meta(source=source, root=root)) + models = data.get("models", {}) + listed = models.values() if isinstance(models, dict) else models + names = ( + set(models) + if isinstance(models, dict) + else {m.get("id") or m.get("name") for m in models} + ) + + for name, model in ( + models.items() if isinstance(models, dict) + else ((m.get("id") or m.get("name"), m) for m in models) + ): + attrs = {} + if model.get("doc"): + attrs["doc"] = model["doc"] + g.node(name, "table", name, attrs=attrs) + + fields = model.get("fields", {}) + pairs = fields.items() if isinstance(fields, dict) else ( + (f.get("name"), f) for f in fields + ) + for field_name, field in pairs: + type_str = field.get("type", "str") + target, kind = _relation(type_str, field) + + a = {"type": _plain_type(type_str)} + if field.get("pk"): + a["pk"] = True + if field.get("nullable"): + a["nullable"] = True + if target: + # Kept on the column so the index can say what it points at + # without walking the edge list. + a["references"] = target + g.node(f"{name}.{field_name}", "column", field_name, parent=name, attrs=a) + + if target and kind: + # The edge is table -> table: a diagram of forty columns joined + # column-to-column is unreadable, and the relationship is + # between the tables. Which column carries it is in `attrs`. + g.edge(name, target, kind, attrs={"label": field_name}) + + # Some producers give `relationships` alongside the fields; take them too, + # and let the dedupe below settle it. + for rel in data.get("relationships", []): + src, dst = rel.get("from_model"), rel.get("to_model") + if src in names and dst in names: + kind = "references" if rel.get("type") == "M2M" else "foreign_key" + g.edge(src, dst, kind, attrs={"label": rel.get("from_field", "")}) + + _dedupe(g) + for missing in sorted( + {e.target for e in g.edges} - {n.id for n in g.nodes} + ): + # A foreign key naming a table the schema does not define. Recorded + # rather than dropped, for the same reason an unresolved import is. + g.node(missing, "external", missing, attrs={"unresolved": True}) + return g + + +def _relation(type_str: str, field: dict) -> tuple[str | None, str | None]: + """(target table, edge kind) for a field, or (None, None).""" + if isinstance(type_str, str): + if type_str.startswith("FK:"): + return type_str[3:], "foreign_key" + if type_str.startswith("M2M:"): + return type_str[4:], "references" + if field.get("fk"): + return field["fk"], "references" if field.get("m2m") else "foreign_key" + return None, None + + +def _plain_type(type_str) -> str: + if isinstance(type_str, str): + for prefix in ("FK:", "M2M:"): + if type_str.startswith(prefix): + return prefix.rstrip(":") + return type_str + + +def _dedupe(g: Graph) -> None: + seen, keep = set(), [] + for e in g.edges: + key = (e.source, e.target, e.kind) + if key in seen: + continue + seen.add(key) + keep.append(e) + g.edges = keep + + +def extract(schema_path, source: str = "db") -> Graph: + """Read a schema.json from disk.""" + path = Path(schema_path) + data = json.loads(path.read_text()) + return from_schema_dict(data, root=path.parent.name or path.stem, source=source) diff --git a/soleprint/atlas2/docgen/extractors/db_main.py b/soleprint/atlas2/docgen/extractors/db_main.py new file mode 100644 index 0000000..79c52e4 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/db_main.py @@ -0,0 +1,31 @@ +""" python3 -m docgen.extractors.db --schema path/to/schema.json [-o ir.json]""" + +import argparse +import json +import sys +from pathlib import Path + +from .db import extract + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.extractors.db") + p.add_argument("--schema", "-s", required=True, type=Path, + help="A graphgen-compatible schema.json, as modelgen emits.") + p.add_argument("--output", "-o", type=Path) + args = p.parse_args(argv) + + try: + ir = extract(args.schema) + except (OSError, json.JSONDecodeError, KeyError) as e: + print(f"Error: could not read {args.schema}: {e}", file=sys.stderr) + return 1 + + text = json.dumps(ir.to_dict(), indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + print(f"{len(ir.nodes)} nodes, {len(ir.edges)} edges -> {args.output}") + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/extractors/openapi.py b/soleprint/atlas2/docgen/extractors/openapi.py new file mode 100644 index 0000000..3e4f2f6 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/openapi.py @@ -0,0 +1,123 @@ +""" +An OpenAPI document -> IR: the endpoints, and the shapes they carry. + +Adapts `modelgen/loader/extract/openapi.py`, which already parses OpenAPI 3.x +and Swagger 2.0 and resolves `$ref`. This turns its output into IR nodes; it +does not re-parse anything. + + python3 -m docgen.extractors.openapi --spec petstore.yaml -o ir.json + +## Why this one matters more than it looks + +A hand-written API notebook is the thing nobody can keep current: the spec moves +and the document does not, and there is no way to tell by looking. Extracting +the endpoints means the document is **generated from the same file the server is +built from**, so "is this current" becomes a question about a build rather than +about somebody's diligence. + +It emits schemas as `table`/`column`, the same vocabulary the database extractor +uses. That is deliberate: an API's data model and a database's are the same kind +of thing, so the ER emitter draws either without knowing which it got. Endpoints +are a separate `kind`, so a view can ask for one or the other. + + only_kinds(ir, {"table", "column"}) -> the data model, as an ER diagram + only_kinds(ir, {"endpoint"}) -> the surface, as a notebook +""" + +import sys +from pathlib import Path + +from ..ir import Graph, Meta + + +def _modelgen(): + """modelgen, from wherever this instance keeps station tools. + + Imported lazily and by path rather than as a hard dependency: docgen belongs + to atlas and may depend on a station tool, but it should not fail to import + because one is missing. + """ + here = Path(__file__).resolve() + for parent in here.parents: + tools = parent / "station" / "tools" + if (tools / "modelgen").is_dir(): + if str(tools) not in sys.path: + sys.path.insert(0, str(tools)) + from modelgen.loader.extract.openapi import OpenAPIExtractor + + return OpenAPIExtractor + raise ImportError( + "modelgen not found — docgen reads OpenAPI through " + "station/tools/modelgen/loader/extract/openapi.py, which parses the spec " + "and resolves $ref. It is not reimplemented here." + ) + + +def _type_name(hint) -> str: + if hint is None: + return "Any" + if isinstance(hint, str): + return hint + return getattr(hint, "__name__", str(hint)) + + +def extract(spec_path, source: str = "openapi") -> Graph: + """An OpenAPI file -> IR.""" + OpenAPIExtractor = _modelgen() + path = Path(spec_path) + extractor = OpenAPIExtractor(path) + models, enums = extractor.extract() + endpoints = extractor.endpoints() + + g = Graph(Meta(source=source, root=path.name)) + known = {m.name for m in models} + + for model in models: + attrs = {} + if getattr(model, "docstring", None): + attrs["doc"] = model.docstring.strip().split("\n")[0] + g.node(model.name, "table", model.name, attrs=attrs) + for field in model.fields: + a = {"type": _type_name(field.type_hint)} + if getattr(field, "optional", False): + a["nullable"] = True + if field.name in ("id", "uuid"): + a["pk"] = True + target = _type_name(field.type_hint) + if target in known and target != model.name: + a["references"] = target + g.edge(model.name, target, "foreign_key", attrs={"label": field.name}) + g.node( + f"{model.name}.{field.name}", "column", field.name, + parent=model.name, attrs=a, + ) + + for e in endpoints: + eid = f"{e.method} {e.path}" + attrs = { + "method": e.method, + "path": e.path, + "status": getattr(e, "status", None) or 200, + } + for key in ("summary", "operation_id", "envelope_key"): + value = getattr(e, key, None) + if value: + attrs[key] = value + if getattr(e, "response_is_list", False): + attrs["returns_list"] = True + if getattr(e, "path_params", None): + attrs["path_params"] = list(e.path_params) + if getattr(e, "request_model", None): + attrs["request_model"] = e.request_model + if getattr(e, "response_model", None): + attrs["response_model"] = e.response_model + + g.node(eid, "endpoint", eid, attrs=attrs) + # `accepts` and `returns` rather than one `uses`: which direction a + # shape travels is the thing a reader wants to know. + if getattr(e, "request_model", None) in known: + g.edge(eid, e.request_model, "accepts") + if getattr(e, "response_model", None) in known: + g.edge(eid, e.response_model, "returns") + + return g diff --git a/soleprint/atlas2/docgen/extractors/openapi_main.py b/soleprint/atlas2/docgen/extractors/openapi_main.py new file mode 100644 index 0000000..0ec4df1 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/openapi_main.py @@ -0,0 +1,29 @@ +""" python3 -m docgen.extractors.openapi --spec petstore.yaml [-o ir.json]""" + +import argparse +import json +import sys +from pathlib import Path + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.extractors.openapi") + p.add_argument("--spec", "-s", required=True, type=Path) + p.add_argument("--output", "-o", type=Path) + args = p.parse_args(argv) + from .openapi import extract + + try: + ir = extract(args.spec) + except (OSError, ImportError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + text = json.dumps(ir.to_dict(), indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + eps = sum(1 for n in ir.nodes if n.kind == "endpoint") + print(f"{len(ir.nodes)} nodes ({eps} endpoints), {len(ir.edges)} edges -> {args.output}") + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/extractors/python/__init__.py b/soleprint/atlas2/docgen/extractors/python/__init__.py new file mode 100644 index 0000000..ddabf3e --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/python/__init__.py @@ -0,0 +1,29 @@ +""" +Python source -> IR, by `ast`. + + from docgen.extractors.python import extract + ir = extract(Path("app/")) + +Deterministic. A diagram built from this cannot be out of date with the code, +which is the whole reason the structural path has no model in it. + +Two passes, because `ast` resolves nothing on its own — see `collect.py` and +`resolve.py`. +""" + +from pathlib import Path + +from .collect import collect +from .resolve import to_ir + + +def extract(root, exclude=(), source="python"): + """Walk `root`, return an IR Graph. Never raises on a bad file.""" + root = Path(root).resolve() + if not root.is_dir(): + raise NotADirectoryError(f"not a directory: {root}") + modules = collect(root, exclude=exclude) + return to_ir(modules, root=root.name, source=source) + + +__all__ = ["extract", "collect", "to_ir"] diff --git a/soleprint/atlas2/docgen/extractors/python/__main__.py b/soleprint/atlas2/docgen/extractors/python/__main__.py new file mode 100644 index 0000000..9e18544 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/python/__main__.py @@ -0,0 +1,37 @@ +""" python3 -m docgen.extractors.python --root PATH [--exclude NAME ...]""" + +import argparse +import json +import sys +from pathlib import Path + +from . import extract + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.extractors.python") + p.add_argument("--root", "-s", required=True, type=Path, help="Tree to read.") + p.add_argument("--output", "-o", type=Path, help="Where to write. Default stdout.") + p.add_argument("--exclude", action="append", default=[], help="Directory name to skip.") + args = p.parse_args(argv) + + try: + ir = extract(args.root, exclude=tuple(args.exclude)) + except (NotADirectoryError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + text = json.dumps(ir.to_dict(), indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + skipped = sum(1 for n in ir.nodes if n.attrs.get("error")) + print(f"{len(ir.nodes)} nodes, {len(ir.edges)} edges -> {args.output}" + + (f" ({skipped} file(s) unparsed)" if skipped else "")) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/extractors/python/collect.py b/soleprint/atlas2/docgen/extractors/python/collect.py new file mode 100644 index 0000000..0cf7380 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/python/collect.py @@ -0,0 +1,235 @@ +""" +Pass one: read each module, record what it defines and what it imports. + +Nothing is resolved here. `class User(Base)` is recorded as the literal string +`"Base"`, because that is genuinely all `ast` knows — it has no idea the name +came from `from .db import Base` three lines up. Resolving it needs every +module's import table, which is why there is a second pass. + +Keeping the two apart is what makes the analysis testable: pass one is a pure +function of one file, pass two is a pure function of the collected tables. + +## What `ast` gives, and what it costs + +`ast.NodeVisitor` dispatches on node type. Every `visit_*` must end in +`generic_visit(node)` or traversal stops there and nested definitions are lost — +a class inside a function, a method inside a class. That one missing call is the +classic silent hole in an AST walker, so it is at the end of every visitor here. + +Every node carries `lineno`, which lands in `attrs` and is what later lets a UI +link a box to a line. + +`ast.parse` uses the **running interpreter's grammar**. A file using syntax newer +than this Python raises `SyntaxError`; it is recorded as a skipped file rather +than crashing the run, because one unparseable file should not cost you the +other four hundred. +""" + +import ast +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class Definition: + """Something a module defines, before it has an id.""" + + name: str # local name, as written + qualname: str # dotted within the module: "User.save" + kind: str # class | function + lineno: int + end_lineno: int = 0 + doc: str | None = None + bases: list[str] = field(default_factory=list) # raw, unresolved + + +@dataclass +class Module: + """One file's worth of collected facts.""" + + name: str # dotted, root-relative + path: str # root-relative posix path + doc: str | None = None + lines: int = 0 + package: str = "" # the package it lives in + defines: list[Definition] = field(default_factory=list) + imports: dict[str, str] = field(default_factory=dict) # local -> dotted target + import_order: list[str] = field(default_factory=list) # modules imported, in order + error: str | None = None + + +class _Collector(ast.NodeVisitor): + """Walks one module. Instance state tracks the enclosing scope.""" + + def __init__(self, module: Module): + self.module = module + self._scope: list[str] = [] # enclosing class/function names + + # -- definitions ------------------------------------------------------ + + def _define(self, node, kind: str, bases=()): + qualname = ".".join([*self._scope, node.name]) + self.module.defines.append( + Definition( + name=node.name, + qualname=qualname, + kind=kind, + lineno=node.lineno, + # How many lines a construct occupies is structure, not styling: + # it is what a density map sizes a block by, and what "this class + # is 400 lines" means. `ast` carries it, so there is no reason to + # record only where something starts. + end_lineno=getattr(node, "end_lineno", node.lineno) or node.lineno, + doc=_first_line(ast.get_docstring(node)), + bases=list(bases), + ) + ) + + def visit_ClassDef(self, node: ast.ClassDef): + self._define(node, "class", bases=[_name_of(b) for b in node.bases]) + self._scope.append(node.name) + self.generic_visit(node) + self._scope.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef): + self._define(node, "function") + self._scope.append(node.name) + self.generic_visit(node) + self._scope.pop() + + # `async def` is a different AST node with the same meaning here. + visit_AsyncFunctionDef = visit_FunctionDef + + # -- imports ---------------------------------------------------------- + + def visit_Import(self, node: ast.Import): + for alias in node.names: + # `import a.b.c` binds `a`; `import a.b.c as x` binds `x` to a.b.c. + local = alias.asname or alias.name.split(".")[0] + target = alias.name + self.module.imports[local] = target + self.module.import_order.append(target) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom): + base = _resolve_relative(self.module, node.module, node.level) + for alias in node.names: + if alias.name == "*": + # A star import binds names this pass cannot know. Recorded as a + # module edge; the names it brought in stay unresolved, which is + # the honest outcome rather than a guess. + self.module.import_order.append(base) + continue + local = alias.asname or alias.name + self.module.imports[local] = f"{base}.{alias.name}" if base else alias.name + if base: + self.module.import_order.append(base) + self.generic_visit(node) + + +def _first_line(doc: str | None) -> str | None: + if not doc: + return None + line = doc.strip().split("\n", 1)[0].strip() + return line or None + + +def _name_of(node: ast.expr) -> str: + """The dotted source text of a name expression, or "" if it is not one. + + `Base` -> "Base"; `db.Base` -> "db.Base"; `Generic[T]` -> "Generic". + Anything genuinely computed returns "" and is dropped — a base class that is + a function call is not a name any resolver could follow. + """ + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + prefix = _name_of(node.value) + return f"{prefix}.{node.attr}" if prefix else "" + if isinstance(node, ast.Subscript): + return _name_of(node.value) + return "" + + +def _resolve_relative(module: Module, target: str | None, level: int) -> str: + """`from ..x import y` inside a.b.c -> "a.x". + + level 0 is absolute. level 1 is the current package, level 2 its parent, and + so on. Getting this wrong silently attaches edges to the wrong module, so it + is computed from the package rather than from the module name. + """ + if not level: + return target or "" + parts = module.package.split(".") if module.package else [] + if level > 1: + parts = parts[: -(level - 1)] if level - 1 <= len(parts) else [] + base = ".".join(parts) + if target: + return f"{base}.{target}" if base else target + return base + + +def module_name(path: Path, root: Path, prefix: str = "") -> tuple[str, str]: + """(dotted module name, package) for a file, relative to root. + + `prefix` is the root's own name, supplied when the root directory is itself + a package. Without it, pointing at `histgen/` makes `histgen/__init__.py` + resolve to the empty string — an unnamed root that every sibling then fails + to claim as its parent. Found by running this over the tools next door, + which is the case a fixture tree does not cover. + """ + rel = path.relative_to(root) + parts = list(rel.parts) + is_init = parts[-1] == "__init__.py" + if is_init: + parts = parts[:-1] # a package is named by its directory + else: + parts[-1] = parts[-1][: -len(".py")] + if prefix: + parts = [prefix, *parts] + name = ".".join(parts) + # An `__init__.py` *is* its package, so `from .x import y` inside it resolves + # against itself, not against its parent. Taking parts[:-1] here sent every + # relative import in a package root one level too high, where it resolved to + # nothing and became a bogus `external` node sitting next to the real module + # of the same name. + package = name if is_init else ".".join(parts[:-1]) + return name, package + + +def collect_file(path: Path, root: Path, prefix: str = "") -> Module: + """Everything pass two needs from one file. Never raises on bad input.""" + name, package = module_name(path, root, prefix) + module = Module(name=name, path=path.relative_to(root).as_posix(), package=package) + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as e: + module.error = f"unreadable: {e}" + return module + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as e: + # Newer syntax than this interpreter, or a genuinely broken file. One + # bad file must not cost the run. + module.error = f"syntax: line {e.lineno}: {e.msg}" + return module + + module.doc = _first_line(ast.get_docstring(tree)) + module.lines = source.count("\n") + 1 + _Collector(module).visit(tree) + return module + + +def collect(root: Path, exclude: tuple[str, ...] = ()) -> list[Module]: + """Every .py under root, in sorted order so the result is stable.""" + skip = {"__pycache__", ".git", ".venv", "venv", "node_modules", "site-packages"} + skip.update(exclude) + # A root that is itself a package is named by its own directory, the way it + # would be imported. A plain directory of packages is not. + prefix = root.name if (root / "__init__.py").exists() else "" + modules = [] + for path in sorted(root.rglob("*.py")): + if any(part in skip for part in path.relative_to(root).parts): + continue + modules.append(collect_file(path, root, prefix)) + return modules diff --git a/soleprint/atlas2/docgen/extractors/python/resolve.py b/soleprint/atlas2/docgen/extractors/python/resolve.py new file mode 100644 index 0000000..844466e --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/python/resolve.py @@ -0,0 +1,154 @@ +""" +Pass two: turn collected names into ids, and collected facts into an IR. + +This is the part that decides whether the output is analysis or decoration. +`ast` resolves nothing — pass one recorded the literal string `"Base"`. Here we +know every module's import table, so `"Base"` in `app.models` becomes +`app.db.Base` and the edge points at a real node. + +## Unresolved names become nodes, never nothing + +A name that cannot be resolved — a third-party import, a star-import binding, a +dynamically built base — becomes a node with `kind: "external"` and keeps its +edge. Dropping it would be the worse failure: the diagram would silently lose a +dependency and look complete, and nobody would know to go looking. An `external` +box is visible and can be styled as the boundary it is. + +## What is deliberately not attempted + +`calls` edges. Resolving `self.foo()` or `obj.method()` needs type inference, +and a call graph that is quietly 60% right is worse than none — it reads as +authoritative. `imports` and `inherits` are both syntactically decidable from +the import table, which is why they are here and `calls` is not. +""" + +from .collect import Module +from ...ir import Graph, Meta + + +def _id_for(module: Module, qualname: str) -> str: + return f"{module.name}.{qualname}" if module.name else qualname + + +def _resolve(name: str, module: Module, known: set[str]) -> str | None: + """A raw name, as an id — or None when it cannot be resolved. + + Order matters, and mirrors Python's own: an imported name shadows a + module-level definition of the same name, because the import ran later. + """ + if not name: + return None + + head, _, rest = name.partition(".") + + # 1. the whole name was imported: `from .db import Base` -> Base + if name in module.imports: + return module.imports[name] + + # 2. a dotted name whose head was imported: `import db` -> db.Base + if head in module.imports: + target = module.imports[head] + return f"{target}.{rest}" if rest else target + + # 3. defined in this module + local = _id_for(module, name) + if local in known: + return local + + # 4. a sibling in the same package: `models.User` inside `app.db` + if module.package: + sibling = f"{module.package}.{name}" + if sibling in known: + return sibling + + return None + + +def to_ir(modules: list[Module], root: str, source: str = "python") -> Graph: + """Collected modules -> a validated-shaped IR graph.""" + g = Graph(Meta(source=source, root=root)) + + # -- nodes we own ------------------------------------------------------ + known: set[str] = set() + for m in modules: + known.add(m.name) + for d in m.defines: + known.add(_id_for(m, d.qualname)) + + module_names = {m.name for m in modules} + + for m in modules: + attrs = {"file": m.path} + if m.lines: + attrs["lines"] = m.lines + if m.doc: + attrs["doc"] = m.doc + if m.error: + # Kept rather than dropped: a file that could not be parsed is a + # hole in the analysis, and the IR should say so out loud. + attrs["error"] = m.error + # Containment is the *enclosing* package, which is not the same as the + # package relative imports resolve against: inside `a/b/__init__.py`, + # `from .x` means `a.b.x` (so package is `a.b`) while the module is + # contained by `a`. Using one for both floats every package root out of + # its own parent. + enclosing = m.name.rsplit(".", 1)[0] if "." in m.name else None + g.node( + m.name, + "module", + m.name.rsplit(".", 1)[-1], + parent=enclosing if enclosing in module_names else None, + attrs=attrs, + ) + + for d in m.defines: + nid = _id_for(m, d.qualname) + parent_qual = d.qualname.rsplit(".", 1)[0] if "." in d.qualname else None + parent = _id_for(m, parent_qual) if parent_qual else m.name + a = {"file": m.path, "line": d.lineno, "lines": max(1, d.end_lineno - d.lineno + 1)} + if d.doc: + a["doc"] = d.doc + g.node(nid, d.kind, d.name, parent=parent, attrs=a) + + # -- edges, and the external nodes they force into existence ----------- + external: dict[str, str] = {} + + def _point_at(raw: str, module: Module) -> str | None: + """The id an edge should target, creating an `external` node if needed.""" + resolved = _resolve(raw, module, known) + if resolved and resolved in known: + return resolved + target = resolved or raw + if target not in known: + external.setdefault(target, target) + return target + + for m in modules: + for target in dict.fromkeys(m.import_order): # dedupe, keep order + if not target: + continue + dest = target if target in known else None + if dest is None: + external.setdefault(target, target) + dest = target + if dest != m.name: + g.edge(m.name, dest, "imports") + + for d in m.defines: + if d.kind != "class": + continue + src = _id_for(m, d.qualname) + for base in d.bases: + dest = _point_at(base, m) + if dest and dest != src: + g.edge(src, dest, "inherits") + + for eid in sorted(external): + g.node( + eid, + "external", + eid.rsplit(".", 1)[-1], + attrs={"unresolved": True}, + ) + + return g diff --git a/soleprint/atlas2/docgen/ir/__init__.py b/soleprint/atlas2/docgen/ir/__init__.py new file mode 100644 index 0000000..526538f --- /dev/null +++ b/soleprint/atlas2/docgen/ir/__init__.py @@ -0,0 +1,6 @@ +"""The IR: what a graph is. Structure only, no visual information.""" + +from .model import Edge, Graph, Meta, Node, SCHEMA_VERSION +from .validate import IRError, check, validate + +__all__ = ["Graph", "Node", "Edge", "Meta", "SCHEMA_VERSION", "check", "validate", "IRError"] diff --git a/soleprint/atlas2/docgen/ir/__main__.py b/soleprint/atlas2/docgen/ir/__main__.py new file mode 100644 index 0000000..c052842 --- /dev/null +++ b/soleprint/atlas2/docgen/ir/__main__.py @@ -0,0 +1,8 @@ +""" python3 -m docgen.ir — validate a document at the boundary.""" + +import sys + +from .validate import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/ir/model.py b/soleprint/atlas2/docgen/ir/model.py new file mode 100644 index 0000000..74a1bbd --- /dev/null +++ b/soleprint/atlas2/docgen/ir/model.py @@ -0,0 +1,146 @@ +""" +The IR, as Python. Mirrors `schema.json`, which is the contract. + +Stdlib dataclasses, not Pydantic. The IR's whole value is being a plain JSON +document that anything can read — a format that needs a library installed to be +opened is not a format, it is an API. Validation is a function at the boundary +(`validate.py`), not a property of the type. + +The duplication between this file and `schema.json` is deliberate and is the one +place it is allowed. `validate.py::check_model_matches_schema` asserts the two +agree, so they cannot drift silently. + + ir = Graph(meta=Meta(source="python", root="app/")) + ir.node("app.models.User", "class", "User", parent="app.models") + ir.edge("app.models.User", "app.db.Base", "inherits") + json.dumps(ir.to_dict()) + +## What must never appear here + +No colour, no shape, no size, no position. `shape="cylinder"` is not a field — +it is `kind="datastore"` plus a style rule, which is what lets the same IR render +in a theme that has no cylinders. The previous iteration of this model carried +`cls`, `shape` and `style`, and that is exactly the mistake this replaces. +""" + +from dataclasses import dataclass, field +from typing import Any + +SCHEMA_VERSION = "1" + + +@dataclass +class Meta: + """Where this graph came from. + + `generated_at` defaults to None on purpose. A timestamp makes two + extractions of an unchanged tree differ, which destroys the diff emitter's + only useful property. Set it when the run time is the fact being recorded, + not by habit. + """ + + source: str + root: str + schema_version: str = SCHEMA_VERSION + generated_at: str | None = None + + def to_dict(self) -> dict: + return { + "source": self.source, + "root": self.root, + "schema_version": self.schema_version, + "generated_at": self.generated_at, + } + + +@dataclass +class Node: + """One thing. `kind` is the only field style and layout may key on.""" + + id: str + kind: str + label: str = "" + parent: str | None = None + attrs: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + if not self.label: + self.label = self.id.rsplit(".", 1)[-1] + + def to_dict(self) -> dict: + return { + "id": self.id, + "kind": self.kind, + "label": self.label, + "parent": self.parent, + "attrs": dict(self.attrs), + } + + +@dataclass +class Edge: + """One relationship. Containment is not one — that is `Node.parent`.""" + + source: str + target: str + kind: str + attrs: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "source": self.source, + "target": self.target, + "kind": self.kind, + "attrs": dict(self.attrs), + } + + +@dataclass +class Graph: + """The whole document.""" + + meta: Meta + nodes: list[Node] = field(default_factory=list) + edges: list[Edge] = field(default_factory=list) + + # -- building --------------------------------------------------------- + + def node(self, id: str, kind: str, label: str = "", **kw) -> Node: + n = Node(id, kind, label, **kw) + self.nodes.append(n) + return n + + def edge(self, source: str, target: str, kind: str, **kw) -> Edge: + e = Edge(source, target, kind, **kw) + self.edges.append(e) + return e + + def has(self, node_id: str) -> bool: + return any(n.id == node_id for n in self.nodes) + + # -- serialising ------------------------------------------------------ + + def to_dict(self) -> dict: + """Sorted, so the same tree twice produces the same bytes. + + Sorting here rather than asking every extractor to emit in order: the + stability guarantee belongs to the format, not to each producer's + traversal order, and an extractor that visits files in directory order + is otherwise at the mercy of the filesystem. + """ + return { + "meta": self.meta.to_dict(), + "nodes": [n.to_dict() for n in sorted(self.nodes, key=lambda n: n.id)], + "edges": [ + e.to_dict() + for e in sorted(self.edges, key=lambda e: (e.source, e.target, e.kind)) + ], + } + + @classmethod + def from_dict(cls, data: dict) -> "Graph": + meta = Meta(**data["meta"]) + g = cls(meta=meta) + g.nodes = [Node(**n) for n in data["nodes"]] + g.edges = [Edge(**e) for e in data["edges"]] + return g diff --git a/soleprint/atlas2/docgen/ir/schema.json b/soleprint/atlas2/docgen/ir/schema.json new file mode 100644 index 0000000..f1fbe9e --- /dev/null +++ b/soleprint/atlas2/docgen/ir/schema.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://soleprint.local/atlas2/docgen/ir/schema.json", + "title": "docgen graph IR", + "description": "What a graph is. Structure only: no colour, no shape, no position. If a field would change between light and dark theme, it does not belong here. Extractors write this; emitters read it; neither knows the other exists.", + "type": "object", + "required": ["meta", "nodes", "edges"], + "additionalProperties": false, + + "properties": { + "meta": { + "type": "object", + "required": ["source", "root", "schema_version"], + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "description": "Which extractor produced this: python, db, ..." + }, + "root": { + "type": "string", + "description": "What it was pointed at. Relative where possible, so the IR is not machine-specific." + }, + "schema_version": { + "type": "string", + "description": "This document's version. Bumped when the shape changes, so a consumer can refuse what it cannot read." + }, + "generated_at": { + "type": ["string", "null"], + "description": "ISO-8601, or null. Null is the default and the honest one: a timestamp makes two extractions of the same tree differ, which breaks diffing. Set it only when the run time is itself the fact being recorded." + } + } + }, + + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "label"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Fully qualified and stable across runs. Stability is what makes two graphs from two commits diffable." + }, + "kind": { + "type": "string", + "description": "The hinge of the whole system, and the ONLY field style and layout may key on. Small closed vocabulary per domain: module/class/function, table/column/view. `external` means a name that could not be resolved." + }, + "label": { + "type": "string", + "description": "Human-facing short name. Distinct from id on purpose." + }, + "parent": { + "type": ["string", "null"], + "description": "Containment only — a module contains a class. Not a relationship; relationships are edges. Must name another node or be null." + }, + "attrs": { + "type": "object", + "description": "Open bag for domain data. file/line live here and are what let a UI link a box to a line. Anything not needed by every consumer goes here.", + "default": {} + } + } + } + }, + + "edges": { + "type": "array", + "items": { + "type": "object", + "required": ["source", "target", "kind"], + "additionalProperties": false, + "properties": { + "source": { "type": "string", "description": "A node id." }, + "target": { "type": "string", "description": "A node id." }, + "kind": { + "type": "string", + "description": "Small closed vocabulary per domain: imports, inherits, calls / foreign_key, references." + }, + "attrs": { "type": "object", "default": {} } + } + } + } + }, + + "$comment": "Field names are shaped toward Cytoscape's {data: {id, source, target}} so an interactive canvas is nearly free. The banned-field list lives in validate.py, not here, because JSON Schema can say what is allowed but not why." +} diff --git a/soleprint/atlas2/docgen/ir/validate.py b/soleprint/atlas2/docgen/ir/validate.py new file mode 100644 index 0000000..49d4832 --- /dev/null +++ b/soleprint/atlas2/docgen/ir/validate.py @@ -0,0 +1,236 @@ +""" +Check an IR document at the boundary. + +Called wherever the IR crosses between layers: after an extractor writes one, +before an emitter reads one. That is the whole point of having one format — the +check is in one place instead of every consumer defending itself. + + python3 -m docgen.ir path/to/ir.json + +Stdlib only. This is not a general JSON Schema engine and does not want to be; +it reads the field lists **out of `schema.json`** so the two cannot drift, then +checks the handful of things that actually go wrong. `jsonschema` would validate +the shape and still miss every item below the first section, which are the ones +that produce a broken diagram. + +## What it catches that a schema cannot + +- an edge naming a node that does not exist +- a `parent` naming a node that does not exist, or a containment cycle +- duplicate ids +- **a visual field smuggled into the IR** — the architectural rule, as a check +""" + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +SCHEMA_PATH = HERE / "schema.json" + +# Not "fields we forgot to allow" — fields that mean the layering has broken. +# A colour in the IR means an extractor decided what something looks like, and +# from then on the graph renders one way forever. Named here rather than in +# schema.json because a schema can say a key is disallowed but not why. +VISUAL_KEYS = { + "color", "colour", "fill", "fillcolor", "bgcolor", "background", + "shape", "style", "penwidth", "stroke", "width", "height", + "font", "fontname", "fontsize", "fontcolor", + "pos", "x", "y", "rank", "layout", "theme", "class", "cls", +} + + +class IRError(ValueError): + """An IR document that a consumer cannot safely read.""" + + +def _schema() -> dict: + return json.loads(SCHEMA_PATH.read_text()) + + +def _props(schema: dict, *path: str) -> dict: + node = schema + for step in path: + node = node["properties"][step] if "properties" in node else node[step] + return node + + +def _fields(schema: dict, section: str) -> tuple[set, set]: + """(required, allowed) for `nodes`/`edges` items, read from the schema.""" + item = schema["properties"][section]["items"] + return set(item.get("required", [])), set(item.get("properties", {})) + + +def check(data: dict, *, strict_visual: bool = True) -> list[str]: + """Every problem, as a list. Empty means the document is sound. + + A list rather than raising on the first: an extractor with four dangling + edges should report four, not make you run it four times. + """ + problems: list[str] = [] + schema = _schema() + + # -- top level -------------------------------------------------------- + for key in schema["required"]: + if key not in data: + problems.append(f"missing top-level {key!r}") + if problems: + return problems # nothing below is meaningful without these + + meta_schema = schema["properties"]["meta"] + for key in meta_schema["required"]: + if key not in data["meta"]: + problems.append(f"meta is missing {key!r}") + for key in data["meta"]: + if key not in meta_schema["properties"]: + problems.append(f"meta has unknown key {key!r}") + + version = data["meta"].get("schema_version") + expected = _props(schema, "meta")["properties"]["schema_version"] + if version is not None and not isinstance(version, str): + problems.append(f"meta.schema_version must be a string, got {type(version).__name__}") + + # -- nodes ------------------------------------------------------------ + node_required, node_allowed = _fields(schema, "nodes") + seen: set[str] = set() + for i, n in enumerate(data["nodes"]): + where = f"nodes[{i}]" + if not isinstance(n, dict): + problems.append(f"{where} is not an object") + continue + for key in node_required - set(n): + problems.append(f"{where} is missing {key!r}") + for key in set(n) - node_allowed: + problems.append(f"{where} has unknown key {key!r}") + nid = n.get("id") + if isinstance(nid, str): + if nid in seen: + problems.append(f"{where} id {nid!r} is declared more than once") + seen.add(nid) + + # -- edges ------------------------------------------------------------ + edge_required, edge_allowed = _fields(schema, "edges") + for i, e in enumerate(data["edges"]): + where = f"edges[{i}]" + if not isinstance(e, dict): + problems.append(f"{where} is not an object") + continue + for key in edge_required - set(e): + problems.append(f"{where} is missing {key!r}") + for key in set(e) - edge_allowed: + problems.append(f"{where} has unknown key {key!r}") + for end in ("source", "target"): + ref = e.get(end) + if isinstance(ref, str) and ref not in seen: + problems.append(f"{where} {end} names unknown node {ref!r}") + + # -- containment ------------------------------------------------------ + parents = { + n["id"]: n.get("parent") + for n in data["nodes"] + if isinstance(n, dict) and isinstance(n.get("id"), str) + } + for nid, parent in parents.items(): + if parent is None: + continue + if parent not in seen: + problems.append(f"node {nid!r} has parent {parent!r}, which is not a node") + continue + # A containment cycle makes any tree walk non-terminating, and the index + # emitter is a tree walk. + slow, fast = nid, parent + while fast is not None and fast in parents: + if slow == fast: + problems.append(f"node {nid!r} is in a containment cycle") + break + fast = parents[fast] + if fast is None or fast not in parents: + break + fast = parents[fast] + slow = parents[slow] + + # -- the architectural rule ------------------------------------------- + if strict_visual: + for i, n in enumerate(data["nodes"]): + if isinstance(n, dict): + for key in set(n.get("attrs") or {}) & VISUAL_KEYS: + problems.append( + f"nodes[{i}].attrs.{key!r} is a visual field — " + "style belongs in style/*.json keyed on `kind`, not in the IR" + ) + for i, e in enumerate(data["edges"]): + if isinstance(e, dict): + for key in set(e.get("attrs") or {}) & VISUAL_KEYS: + problems.append( + f"edges[{i}].attrs.{key!r} is a visual field — " + "style belongs in style/*.json keyed on `kind`, not in the IR" + ) + + return problems + + +def validate(data: dict, **kw) -> dict: + """check(), but raises. For use at a boundary where carrying on is wrong.""" + problems = check(data, **kw) + if problems: + raise IRError(f"{len(problems)} problem(s):\n " + "\n ".join(problems)) + return data + + +def check_model_matches_schema() -> list[str]: + """The sanctioned duplication, asserted. + + `model.py` mirrors `schema.json` by hand. This is what stops the two from + drifting: every field the schema declares must exist on the dataclass, and + every dataclass field must be declared in the schema. + """ + from .model import Edge, Meta, Node + + schema = _schema() + problems = [] + pairs = [ + ("meta", Meta, set(schema["properties"]["meta"]["properties"])), + ("nodes", Node, _fields(schema, "nodes")[1]), + ("edges", Edge, _fields(schema, "edges")[1]), + ] + for name, cls, declared in pairs: + actual = set(cls.__dataclass_fields__) + for missing in declared - actual: + problems.append(f"schema declares {name}.{missing!r}; {cls.__name__} has no such field") + for extra in actual - declared: + problems.append(f"{cls.__name__} has {extra!r}; schema does not declare it") + return problems + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if not argv: + print("usage: python3 -m docgen.ir ", file=sys.stderr) + return 2 + + drift = check_model_matches_schema() + if drift: + print("model.py and schema.json disagree:", file=sys.stderr) + for d in drift: + print(f" {d}", file=sys.stderr) + return 1 + + path = Path(argv[0]) + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {path}: {e}", file=sys.stderr) + return 1 + + problems = check(data) + if problems: + print(f"{path}: {len(problems)} problem(s)", file=sys.stderr) + for p in problems: + print(f" {p}", file=sys.stderr) + return 1 + + print( + f"{path}: ok — {len(data['nodes'])} nodes, {len(data['edges'])} edges, " + f"schema v{data['meta'].get('schema_version')}" + ) + return 0 diff --git a/soleprint/atlas2/docgen/lab/__init__.py b/soleprint/atlas2/docgen/lab/__init__.py new file mode 100644 index 0000000..f9a6580 --- /dev/null +++ b/soleprint/atlas2/docgen/lab/__init__.py @@ -0,0 +1,10 @@ +"""A sanctioned place to try a dependency against real IRs before adopting it. + +Nothing in `ir/`, `extractors/`, `ops/` or `emitters/` may import from here. When +an experiment earns its place it graduates into `ops/` behind an IR->IR +signature, and *then* the dependency is declared. + +First candidate: networkx — transitive reduction, cycle detection, dominators. +The question to answer is which part is actually attractive, not whether the +whole library should be adopted on faith. +""" diff --git a/soleprint/atlas2/docgen/lab/pg_probe.py b/soleprint/atlas2/docgen/lab/pg_probe.py new file mode 100644 index 0000000..f07ae61 --- /dev/null +++ b/soleprint/atlas2/docgen/lab/pg_probe.py @@ -0,0 +1,149 @@ +""" +EXPERIMENT — a live PostgreSQL schema, as a graphgen-compatible `schema.json`. + +**Not a supported path, and deliberately not an extractor.** Reflecting a live +database is `modelgen from-db --url ...`, which already does it across every +SQLAlchemy dialect and writes exactly the file this writes. This exists because +SQLAlchemy is not installed on this machine and the IR still needed testing +against a real schema rather than a fixture — which is what `lab/` is for. + +If a psql-based path ever turns out to be worth keeping, it belongs in modelgen +beside the other extractors, not here and not in `extractors/`. + + python3 -m docgen.lab.pg_probe --db money26 --port 5433 -o schema.json + +## No credentials on the command line + +There is no `--password` and no DSN argument, on purpose. `psql` is invoked with +host/port/dbname/user and left to find credentials the way it normally does — +`~/.pgpass`, peer auth, `PGPASSWORD` in the environment. A connection string +passed as an argument lands in shell history and in `ps`, visible to every other +user on the box, and `modelgen/__main__.py:202` additionally prints it to +stdout, so it lands in CI logs too. That is worth not reproducing. + +Nothing here reads a row. Only `information_schema` and `pg_catalog`, which is +structure — table names, column names, types, keys. No table data is selected, +so a schema can be drawn without the contents being touched. +""" + +import argparse +import json +import subprocess +import sys + +# One query, returning JSON, because assembling this from three result sets in +# Python is more code and more ways to get the joins wrong. +QUERY = r""" +SELECT json_build_object( + 'tables', ( + SELECT COALESCE(json_agg(t), '[]'::json) FROM ( + SELECT c.relname AS name, + obj_description(c.oid) AS doc, + (SELECT COALESCE(json_agg(f ORDER BY f->>'ord'), '[]'::json) FROM ( + SELECT json_build_object( + 'name', a.attname, + 'ord', a.attnum, + 'type', format_type(a.atttypid, a.atttypmod), + 'notnull', a.attnotnull, + 'pk', COALESCE(( + SELECT true FROM pg_constraint pk + WHERE pk.conrelid = c.oid AND pk.contype = 'p' + AND a.attnum = ANY (pk.conkey)), false), + 'fk', ( + SELECT ref.relname FROM pg_constraint fk + JOIN pg_class ref ON ref.oid = fk.confrelid + WHERE fk.conrelid = c.oid AND fk.contype = 'f' + AND a.attnum = ANY (fk.conkey) + LIMIT 1) + ) AS f + FROM pg_attribute a + WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + ) fields) AS fields + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'v', 'm') + ORDER BY c.relname + ) t + ) +) AS payload; +""" + + +def probe(db: str, host: str = "localhost", port: int = 5432, user: str | None = None) -> dict: + """Read structure via psql. Returns the graphgen-compatible schema dict.""" + cmd = ["psql", "-h", host, "-p", str(port), "-d", db, "-tAq", "-c", QUERY] + if user: + cmd[1:1] = ["-U", user] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + # psql puts the reason on stderr; pass it through rather than guessing. + raise RuntimeError(proc.stderr.strip() or f"psql exited {proc.returncode}") + + payload = json.loads(proc.stdout.strip()) + models = {} + for table in payload["tables"]: + fields = {} + for f in table["fields"]: + if f["fk"]: + type_str = f"FK:{f['fk']}" + else: + type_str = _simplify(f["type"]) + entry = {"type": type_str} + if f["pk"]: + entry["pk"] = True + if not f["notnull"]: + entry["nullable"] = True + fields[f["name"]] = entry + model = {"fields": fields} + if table.get("doc"): + model["doc"] = table["doc"] + models[table["name"]] = model + return {"models": models} + + +# Postgres type names are more precise than a diagram needs; the IR keeps what +# was read, and this only shortens the common ones so a box stays readable. +_SIMPLE = { + "integer": "int", "bigint": "int", "smallint": "int", + "character varying": "str", "text": "str", "character": "str", + "boolean": "bool", "double precision": "float", "real": "float", + "timestamp without time zone": "datetime", "timestamp with time zone": "datetime", + "date": "date", "time without time zone": "time", + "jsonb": "json", "uuid": "uuid", +} + + +def _simplify(pg_type: str) -> str: + base = pg_type.split("(")[0].strip() + return _SIMPLE.get(base, base) + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.lab.pg_probe") + p.add_argument("--db", required=True) + p.add_argument("--host", default="localhost") + p.add_argument("--port", type=int, default=5432) + p.add_argument("--user") + p.add_argument("--output", "-o") + args = p.parse_args(argv) + + try: + schema = probe(args.db, args.host, args.port, args.user) + except (RuntimeError, json.JSONDecodeError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + text = json.dumps(schema, indent=2) + "\n" + if args.output: + with open(args.output, "w") as fh: + fh.write(text) + tables = len(schema["models"]) + cols = sum(len(m["fields"]) for m in schema["models"].values()) + print(f"{tables} tables, {cols} columns -> {args.output}") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/notebook/__init__.py b/soleprint/atlas2/docgen/notebook/__init__.py new file mode 100644 index 0000000..5ffbbc3 --- /dev/null +++ b/soleprint/atlas2/docgen/notebook/__init__.py @@ -0,0 +1,14 @@ +""" +The notebook layer: a sequence, a hand-written overlay, and the merge. + + from docgen.notebook import from_ir, merge, scaffold + + base, _ = merge(from_ir(ir), load("overlay.json")) + +`spec.py` holds the format. The `.ipynb` writer is `emitters/notebook.py`, which +renders a merged spec and knows nothing about where it came from. +""" + +from .spec import SPEC_VERSION, dump, from_ir, load, merge, scaffold + +__all__ = ["from_ir", "merge", "scaffold", "load", "dump", "SPEC_VERSION"] diff --git a/soleprint/atlas2/docgen/notebook/spec.py b/soleprint/atlas2/docgen/notebook/spec.py new file mode 100644 index 0000000..04cf7e4 --- /dev/null +++ b/soleprint/atlas2/docgen/notebook/spec.py @@ -0,0 +1,226 @@ +""" +The notebook, as a sequence — before it is a `.ipynb`. + +A **notebook spec** is a declarative, ordered list of steps: what the document +does, in order, with no Jupyter in it. It is to a notebook what an OpenAPI +document is to an API — the thing you can read, diff, version and generate +*from*, rather than the artifact you end up opening. + + IR ──► spec ──(+ overlay)──► spec ──► .ipynb + generated hand-written merged emitted + +## Why there are two files and not one + +Generation alone gives you a document that is never stale and never says +anything a parser could not work out. Hand-authoring gives you insight and a +document that rots. The split is the only arrangement that gets both: + + base regenerated every time, never edited, holds what was extracted + overlay small, hand-written, the only file anyone edits, re-applied on + every build + +So a regeneration cannot lose someone's work, and someone's work cannot go +stale silently — if the base moves under an overlay, the overlay's step ids stop +matching and that is reported rather than quietly dropped. + +**Extraction must work with the overlay absent**, which is the same rule the +brief sets for LLM annotation. The overlay is an addition, never a dependency. + +## What an overlay is for + +The honest reason it exists: **a spec says what endpoints are, not how to use +them.** Order of calls, which fields actually matter, a payload that is real +rather than shaped, the GraphQL endpoint sitting beside the REST ones, the +non-RESTful verb that takes a body nobody would guess. None of that is derivable, +and all of it is what makes a walkthrough worth reading. + + annotate add prose to a generated step — the common case + replace override the generated code where the generated guess is wrong + insert a step generation could not know about + drop hide a step that is noise here + order pin the sequence, when the order is the point + +`replace` is the one that matters most: it is how real usage gets into the +document today, and it keeps working unchanged when a usage extractor exists. +""" + +import json +from pathlib import Path + +SPEC_VERSION = "1" + +# Step kinds. Small and closed, the same discipline the IR's `kind` follows. +KINDS = ("md", "code", "params", "call") + + +def _step(id: str, kind: str, **fields) -> dict: + step = {"id": id, "kind": kind} + step.update({k: v for k, v in fields.items() if v is not None}) + return step + + +def from_ir(ir: dict, base_url: str = "https://api.example.invalid", + env_var: str = "API_TOKEN") -> dict: + """The generated half: everything extraction can know, in order.""" + meta = ir.get("meta", {}) + root = meta.get("root", "api") + + fields_of: dict[str, list] = {} + for n in ir["nodes"]: + if n["kind"] == "column" and n.get("parent"): + fields_of.setdefault(n["parent"], []).append(n) + + endpoints = sorted( + (n for n in ir["nodes"] if n["kind"] == "endpoint"), + key=lambda n: ((n.get("attrs") or {}).get("path", ""), + (n.get("attrs") or {}).get("method", "")), + ) + + steps = [ + _step("intro", "md", title=root, + text=( + f"Generated from `{root}`. The generated steps below are " + "rebuilt on every run and must not be edited here — put " + "changes in the overlay, which is re-applied each time." + )), + _step("setup", "params", title="Parameters", + base_url=base_url, env_var=env_var), + _step("client", "code", title="The client", builtin="client"), + ] + + for n in endpoints: + a = n.get("attrs") or {} + steps.append( + _step( + n["id"], "call", + title=f'{a.get("method", "GET")} {a.get("path", "/")}', + summary=a.get("summary"), + method=a.get("method", "GET"), + path=a.get("path", "/"), + status=a.get("status"), + request_model=a.get("request_model"), + response_model=a.get("response_model"), + returns_list=a.get("returns_list"), + path_params=a.get("path_params"), + body_fields=[ + {"name": f.get("label") or f["id"].rsplit(".", 1)[-1], + **(f.get("attrs") or {})} + for f in fields_of.get(a.get("request_model") or "", []) + ] or None, + ) + ) + + shapes = sorted(n["id"] for n in ir["nodes"] if n["kind"] == "table") + if shapes: + steps.append( + _step("shapes", "md", title="Shapes", + text="The models these endpoints carry.", + table=[ + {"name": s, + "fields": [f.get("label") or f["id"].rsplit(".", 1)[-1] + for f in fields_of.get(s, [])]} + for s in shapes + ]) + ) + + return { + "meta": { + "source": meta.get("source", ""), + "root": root, + "spec_version": SPEC_VERSION, + }, + "steps": steps, + } + + +def scaffold(spec: dict) -> dict: + """A blank overlay for this spec — every step id, nothing filled in. + + Handing someone the list of ids is the difference between an overlay being + written and an overlay being meant to be written. + """ + return { + "note": ( + "Hand-written. Re-applied on every generation, so edits here survive " + "and edits to the notebook do not. Keys are step ids from the spec; " + "an id that no longer exists is reported, never silently ignored." + ), + "steps": { + s["id"]: {"_kind": s["kind"], "_title": s.get("title", "")} + for s in spec["steps"] + }, + "insert": [], + "drop": [], + "order": [], + } + + +def merge(spec: dict, overlay: dict | None) -> tuple[dict, list[str]]: + """Apply an overlay. Returns (spec, problems). + + Problems are returned rather than raised: an overlay pointing at a step that + no longer exists means the base moved underneath it, which is worth saying + out loud and is not a reason to refuse to build the document. + """ + if not overlay: + return spec, [] + + problems: list[str] = [] + steps = [dict(s) for s in spec["steps"]] + by_id = {s["id"]: s for s in steps} + + for sid, patch in (overlay.get("steps") or {}).items(): + if sid not in by_id: + problems.append(f"overlay targets step {sid!r}, which the spec no longer has") + continue + step = by_id[sid] + for key, value in patch.items(): + if key.startswith("_") or value in (None, "", [], {}): + continue # scaffold hints and unfilled slots are not edits + step[key] = value + + for sid in overlay.get("drop") or []: + if sid not in by_id: + problems.append(f"overlay drops step {sid!r}, which the spec no longer has") + continue + steps = [s for s in steps if s["id"] != sid] + by_id.pop(sid, None) + + for extra in overlay.get("insert") or []: + step = {k: v for k, v in extra.items() if k != "after"} + step.setdefault("kind", "md") + if "id" not in step: + problems.append("an inserted step has no id; it cannot be re-applied reliably") + continue + after = extra.get("after") + if after is None: + steps.append(step) + elif after in {s["id"] for s in steps}: + at = next(i for i, s in enumerate(steps) if s["id"] == after) + 1 + steps.insert(at, step) + else: + problems.append(f"inserted step {step['id']!r} goes after {after!r}, which is gone") + steps.append(step) + by_id[step["id"]] = step + + order = overlay.get("order") or [] + if order: + missing = [sid for sid in order if sid not in by_id] + problems += [f"overlay orders step {sid!r}, which the spec no longer has" + for sid in missing] + pinned = [by_id[sid] for sid in order if sid in by_id] + rest = [s for s in steps if s["id"] not in set(order)] + steps = pinned + rest + + return {"meta": dict(spec["meta"]), "steps": steps}, problems + + +def load(path) -> dict: + return json.loads(Path(path).read_text()) + + +def dump(spec: dict, path) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(spec, indent=2, sort_keys=True) + "\n") + return path diff --git a/soleprint/atlas2/docgen/ops/__init__.py b/soleprint/atlas2/docgen/ops/__init__.py new file mode 100644 index 0000000..9a690ec --- /dev/null +++ b/soleprint/atlas2/docgen/ops/__init__.py @@ -0,0 +1,27 @@ +""" +Views over the IR. Filtering is not an emitter concern — it belongs here, once, +so the index, the diagram and the diff all narrow the same way. + + python3 -m docgen.ops ir.json --drop-stdlib --only class -o smaller.json +""" + +from .filter import ( + classify, + collapse_to_depth, + drop_builtins, + drop_external, + drop_kinds, + drop_stdlib, + neighbourhood, + only_kinds, + overview, + shape, + split, + subtree, +) + +__all__ = [ + "overview", + "drop_stdlib", "drop_external", "drop_builtins", "drop_kinds", "only_kinds", + "subtree", "neighbourhood", "collapse_to_depth", "shape", "split", "classify", +] diff --git a/soleprint/atlas2/docgen/ops/__main__.py b/soleprint/atlas2/docgen/ops/__main__.py new file mode 100644 index 0000000..470f02a --- /dev/null +++ b/soleprint/atlas2/docgen/ops/__main__.py @@ -0,0 +1,101 @@ +""" python3 -m docgen.ops [views...] [-o out.json] + +Views compose, left to right, in the order given on the command line.""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from . import filter as F + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.ops") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path) + p.add_argument("--overview", action="store_true", + help="The default view for this source type. Usually what you want.") + p.add_argument("--drop-stdlib", action="store_true", help="Remove stdlib externals.") + p.add_argument("--drop-builtins", action="store_true", help="Remove builtin externals.") + p.add_argument("--drop-external", action="store_true", help="Remove every unresolved name.") + p.add_argument("--only", action="append", default=[], help="Keep only this kind. Repeatable.") + p.add_argument("--drop", action="append", default=[], help="Remove this kind. Repeatable.") + p.add_argument("--subtree", help="Just this node id and its contents.") + p.add_argument("--around", help="This node id and its neighbours.") + p.add_argument("--hops", type=int, default=1) + p.add_argument("--depth", type=int, help="Collapse to this containment depth.") + p.add_argument("--split", action="store_true", + help="Write one document per subsystem into OUT/ (a directory).") + p.add_argument("--shape", action="store_true", + help="Report what this will look like, and write nothing.") + args = p.parse_args(argv) + + try: + ir = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + + before = (len(ir["nodes"]), len(ir["edges"])) + + if args.overview: + ir = F.overview(ir) + if args.drop_stdlib: + ir = F.drop_stdlib(ir) + if args.drop_builtins: + ir = F.drop_builtins(ir) + if args.drop_external: + ir = F.drop_external(ir) + if args.drop: + ir = F.drop_kinds(ir, args.drop) + if args.only: + ir = F.only_kinds(ir, args.only) + if args.subtree: + ir = F.subtree(ir, args.subtree) + if args.around: + ir = F.neighbourhood(ir, args.around, hops=args.hops) + if args.depth is not None: + ir = F.collapse_to_depth(ir, args.depth) + + if args.shape: + sh = F.shape(ir) + for k, v in sh.items(): + print(f" {k:<14} {v}") + return 0 + + if args.split: + if not args.output: + print("Error: --split needs -o DIRECTORY", file=sys.stderr) + return 1 + args.output.mkdir(parents=True, exist_ok=True) + for name, part in F.split(ir).items(): + (args.output / f"{name}.json").write_text(json.dumps(part, indent=2) + "\n") + sh = F.shape(part) + print(f" {name:<16} {sh['nodes']:>4} nodes {sh['edges']:>4} edges " + f"-> {args.output / (name + '.json')}") + return 0 + + problems = check(ir) + if problems: + # A view that produces an invalid document is a bug in the view, and it + # must not be written out for an emitter to trip over later. + print(f"Error: the view produced an invalid IR ({len(problems)}):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + text = json.dumps(ir, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + print(f" {before[0]} nodes, {before[1]} edges -> " + f"{len(ir['nodes'])} nodes, {len(ir['edges'])} edges -> {args.output}") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/soleprint/atlas2/docgen/ops/filter.py b/soleprint/atlas2/docgen/ops/filter.py new file mode 100644 index 0000000..1354018 --- /dev/null +++ b/soleprint/atlas2/docgen/ops/filter.py @@ -0,0 +1,468 @@ +""" +Views: IR in, smaller IR out. + +A view is a *filter over the graph*, not a rendering option. Putting it here +rather than behind an emitter flag is the difference between one implementation +and one per output — the index, the DOT and the diff all want "just this +subsystem, two hops out, without the stdlib", and none of them should own it. + +Everything here is a pure function returning a new IR document. They compose: + + ir = drop_stdlib(ir) + ir = neighbourhood(ir, "docgen.ir", hops=2) + +## Why this exists at all + +The first real diagram produced from this pipeline was a 3000px-wide strip: four +modules of actual content and sixty `sys`/`json`/`typing` boxes, all peers. The +emitter was correct and the picture was useless. That is a missing view, not a +broken renderer, and it is the clearest argument for keeping this layer separate +— the fix belongs to every consumer at once. + +## What is not here + +Graph algorithms. Transitive reduction, cycle detection, dominators and +shortest paths are `networkx`'s, and reimplementing them is the classic way to +acquire a quiet bug. `lab/` is where that dependency gets tried against real +IRs before anything here depends on it. +""" + +import builtins +import sys + +# Exact, and needs no hand-maintained list that goes stale one release later. +STDLIB = set(getattr(sys, "stdlib_module_names", ())) + + +def _rebuild(ir: dict, keep: set[str]) -> dict: + """A new document holding only `keep`, with dangling references repaired. + + A node whose parent was filtered out is re-parented to its nearest surviving + ancestor rather than dropped — losing a class because its module was + filtered would be a surprise, and an orphan fails validation. + """ + by_id = {n["id"]: n for n in ir["nodes"]} + + def surviving_parent(nid: str): + parent = by_id.get(nid, {}).get("parent") + while parent is not None and parent not in keep: + parent = by_id.get(parent, {}).get("parent") + return parent + + nodes = [] + for n in ir["nodes"]: + if n["id"] not in keep: + continue + node = dict(n) + node["parent"] = surviving_parent(n["id"]) + nodes.append(node) + + def lift(nid: str): + """The surviving node that should carry an edge touching `nid`.""" + cur = nid + while cur is not None and cur not in keep: + cur = by_id.get(cur, {}).get("parent") + return cur + + # Edges are **lifted, not dropped**. A class in module A inheriting from a + # class in module B is a dependency of A on B; if you then look at the + # module level, dropping the edge says the two modules are unrelated, which + # is false. Collapsing docgen to its packages lost 69 of 77 edges this way + # and the picture was worse for it — not simpler, wrong. + merged: dict[tuple, dict] = {} + for e in ir["edges"]: + src, dst = lift(e["source"]), lift(e["target"]) + if src is None or dst is None or src == dst: + # Both ends inside one surviving node: an internal relationship, + # which is a fact about that node rather than a line between two. + continue + key = (src, dst, e["kind"]) + if key in merged: + merged[key]["attrs"]["weight"] = merged[key]["attrs"].get("weight", 1) + 1 + continue + attrs = dict(e.get("attrs") or {}) + lifted = src != e["source"] or dst != e["target"] + if lifted: + # `weight` is how many underlying relationships this line stands + # for. Semantic, not visual — whether it becomes a thicker stroke is + # the style layer's call, not this one's. + attrs["weight"] = 1 + attrs.pop("label", None) # a single field name no longer applies + merged[key] = {"source": src, "target": dst, "kind": e["kind"], "attrs": attrs} + + return {"meta": dict(ir["meta"]), "nodes": nodes, "edges": list(merged.values())} + + +def drop_kinds(ir: dict, kinds) -> dict: + """Remove every node of these kinds, and their descendants.""" + kinds = set(kinds) + doomed = {n["id"] for n in ir["nodes"] if n["kind"] in kinds} + # A surviving child of a removed node would be an orphan with a real parent + # id pointing at nothing, so descendants go too. + changed = True + while changed: + changed = False + for n in ir["nodes"]: + if n["id"] not in doomed and n.get("parent") in doomed: + doomed.add(n["id"]) + changed = True + return _rebuild(ir, {n["id"] for n in ir["nodes"]} - doomed) + + +def only_kinds(ir: dict, kinds) -> dict: + """Keep only these kinds. Ancestors are kept so containment survives.""" + kinds = set(kinds) + by_id = {n["id"]: n for n in ir["nodes"]} + keep = set() + for n in ir["nodes"]: + if n["kind"] not in kinds: + continue + keep.add(n["id"]) + parent = n.get("parent") + while parent and parent not in keep: + keep.add(parent) + parent = by_id.get(parent, {}).get("parent") + return _rebuild(ir, keep) + + +def drop_stdlib(ir: dict) -> dict: + """Remove external nodes that are the standard library. + + The usual first view: `pathlib` and `typing` are true dependencies and + almost never the thing being looked at. Third-party externals stay, because + *those* are the dependency surface worth seeing. + """ + keep = set() + for n in ir["nodes"]: + if n["kind"] == "external" and n["id"].split(".")[0] in STDLIB: + continue + keep.add(n["id"]) + return _rebuild(ir, keep) + + +def drop_external(ir: dict) -> dict: + """Remove every unresolved name. What is left is this tree talking to itself.""" + return drop_kinds(ir, {"external"}) + + +def subtree(ir: dict, root_id: str) -> dict: + """Just this node and everything inside it.""" + children = {} + for n in ir["nodes"]: + children.setdefault(n.get("parent"), []).append(n["id"]) + keep, stack = set(), [root_id] + while stack: + nid = stack.pop() + if nid in keep: + continue + keep.add(nid) + stack.extend(children.get(nid, [])) + return _rebuild(ir, keep) + + +def neighbourhood(ir: dict, node_id: str, hops: int = 1, *, undirected: bool = True) -> dict: + """This node and everything within `hops` edges of it. + + The view behind "what does this touch, and what touches it". Containment + ancestors are kept so the result still nests. + """ + by_id = {n["id"]: n for n in ir["nodes"]} + out, inc = {}, {} + for e in ir["edges"]: + out.setdefault(e["source"], set()).add(e["target"]) + inc.setdefault(e["target"], set()).add(e["source"]) + + frontier, keep = {node_id}, {node_id} + for _ in range(hops): + nxt = set() + for nid in frontier: + nxt |= out.get(nid, set()) + if undirected: + nxt |= inc.get(nid, set()) + nxt -= keep + keep |= nxt + frontier = nxt + + for nid in list(keep): + parent = by_id.get(nid, {}).get("parent") + while parent and parent not in keep: + keep.add(parent) + parent = by_id.get(parent, {}).get("parent") + return _rebuild(ir, keep) + + +def collapse_to_depth(ir: dict, depth: int) -> dict: + """Keep nodes at or above `depth`, dropping what is inside them. + + Different from the emitter's `--max-depth`, and the difference is the point: + that one still draws the deep nodes and merely stops nesting them, while + this removes them from the graph, so an index built afterwards agrees with + the diagram. + """ + by_id = {n["id"]: n for n in ir["nodes"]} + + def level(nid: str) -> int: + n, d = by_id.get(nid), 0 + while n and n.get("parent"): + d += 1 + n = by_id.get(n["parent"]) + return d + + return _rebuild(ir, {n["id"] for n in ir["nodes"] if level(n["id"]) <= depth}) + + +def drop_builtins(ir: dict) -> dict: + """Remove externals that are builtins — `ValueError`, `RuntimeError`. + + `sys.stdlib_module_names` does not cover these: they are names in + `builtins`, not modules, so `drop_stdlib` leaves them. At overview level + "this exception subclasses ValueError" is true and not what anyone came to + find out. + """ + keep = { + n["id"] for n in ir["nodes"] + if not (n["kind"] == "external" and hasattr(builtins, n["id"].split(".")[-1]) + and "." not in n["id"]) + } + return _rebuild(ir, keep) + + +def overview(ir: dict) -> dict: + """The default view — what you want without having to ask for it. + + Dispatches on `meta.source`, because a good default for a codebase and a + good default for a schema are not the same picture: + + python modules, no stdlib, no builtin exceptions. Third-party + externals stay: those are the dependency surface. + db tables and their foreign keys. A schema's 254 columns are a + reference, not a diagram. + + **Modules, not a depth cut.** Depth looks like the obvious knob and is the + wrong one on a real tree: a directory without `__init__.py` is not a package, + so its modules have no parent and sit at depth 0. soleprint has **173 such + roots**, which left `collapse_to_depth(2)` holding 566 functions and 142 + classes — a cut that reduced the count and not the noise. Selecting by + `kind` does not care how the directories happen to be arranged. + + Edges are lifted rather than dropped on the way (see `_rebuild`), so + collapsing detail turns relationships into higher-level ones instead of + losing them. That is the whole difference between a simpler picture and a + wrong one. + + An unknown source gets `drop_stdlib` and nothing else — a conservative + default beats a confident wrong one. + """ + source = (ir.get("meta") or {}).get("source", "") + if source == "db": + return only_kinds(ir, {"table"}) + if source == "python": + # `external` is kept alongside `module`: those nodes are the dependency + # surface, and dropping them is how a diagram comes to look complete + # while saying nothing about what the code reaches outside itself. + return only_kinds(drop_builtins(drop_stdlib(ir)), {"module", "external"}) + return drop_stdlib(ir) + + +def shape(ir: dict) -> dict: + """What this graph will look like before anything renders it. + + Aspect ratio is not a rendering accident, it is a property of the graph: + a layered engine puts everything at the same dependency level side by side, + so the widest level *is* the width. soleprint's overview is 7 levels by 109 + nodes, and no engine draws that well — dot, ELK and dagre all layer, and + the sheet is the sheet. + + So it is worth knowing *before* writing a 375 KB image, which is why this + computes it from the IR rather than measuring the output. + + Measured, one diagram per subsystem: + + <= 20 nodes ~1.6:1 readable + 70-106 nodes ~7:1 a strip + 261 nodes ~14:1 unusable + + Roughly twenty nodes is where it stops being a diagram. + """ + ids = {n["id"] for n in ir["nodes"]} + out, inc, adj = {}, {}, {i: set() for i in ids} + for e in ir["edges"]: + out.setdefault(e["source"], set()).add(e["target"]) + inc.setdefault(e["target"], set()).add(e["source"]) + adj[e["source"]].add(e["target"]) + adj[e["target"]].add(e["source"]) + + seen, components = set(), 0 + isolated = 0 + for i in ids: + if i in seen: + continue + components += 1 + stack, size = [i], 0 + while stack: + x = stack.pop() + if x in seen: + continue + seen.add(x) + size += 1 + stack.extend(adj[x] - seen) + if size == 1: + isolated += 1 + + # Longest-path ranking, which is what a layered engine assigns. + rank: dict[str, int] = {} + + def rank_of(n, guard=frozenset()): + if n in rank: + return rank[n] + if n in guard: + return 0 + parents = inc.get(n, ()) + value = 1 + max([rank_of(p, guard | {n}) for p in parents], default=-1) + rank[n] = value + return value + + for i in ids: + rank_of(i) + widths: dict[int, int] = {} + for level in rank.values(): + widths[level] = widths.get(level, 0) + 1 + + return { + "nodes": len(ids), + "edges": len(ir["edges"]), + "components": components, + "isolated": isolated, + "levels": len(widths), + "widest_level": max(widths.values(), default=0), + } + + +def split(ir: dict, depth: int = 0) -> dict[str, dict]: + """One IR per subsystem — the answer to a graph too big for one picture. + + Splits on the `depth`-th segment of the id, so `depth=0` gives one document + per top-level package. Edges that cross between subsystems are kept on both + sides, because "this is what my subsystem reaches" is the useful half of a + crossing edge and dropping it would understate the coupling. + """ + groups: dict[str, set] = {} + for n in ir["nodes"]: + parts = n["id"].split(".") + key = ".".join(parts[: depth + 1]) + groups.setdefault(key, set()).add(n["id"]) + + out: dict[str, dict] = {} + for key, keep in sorted(groups.items()): + if len(keep) < 2: + continue + touching = { + e["source"] for e in ir["edges"] if e["target"] in keep + } | {e["target"] for e in ir["edges"] if e["source"] in keep} + visible = keep | (touching & {n["id"] for n in ir["nodes"]}) + out[key] = _rebuild(ir, visible) + return out + + +def classify(ir: dict) -> dict: + """What kind of graph is this, and what should draw it. + + The lesson that produced this: **a diagram that fights its layout engine is + usually the wrong kind of diagram.** soleprint's module graph rendered 14:1 + through `dot`, and no Graphviz flag fixed it because a 7-level, 109-wide + sheet has no good rectangular form. The same database rendered 235:1 through + `dot` and 0.9:1 through `erd` — not because one engine is better, but + because a schema is a set of peer entities with references, and drawing it in + dependency ranks was never the right shape. + + So: read the structure, then pick the form. Returns the kind, the emitter + that suits it, and why — the reason matters, because "use the index" is + advice somebody will override unless they know what it is based on. + + erd entities with references -> emitters/erd + layered a DAG that is taller than + it is wide -> emitters/dot + tree containment, few cross edges -> emitters/dot + flat many peers, no structure -> emitters/index (a list) + sheet too wide at every level -> emitters/index, or split + """ + sh = shape(ir) + source = (ir.get("meta") or {}).get("source", "") + kinds = {n["kind"] for n in ir["nodes"]} + + if source == "db" or "table" in kinds: + return { + **sh, + "kind": "erd", + "emitter": "erd", + "options": {}, + "why": "entities with references — cards in columns, not ranks", + } + + # Wider than about twenty at any level and it stops being readable, whatever + # draws it. Measured: <=20 nodes renders ~1.6:1, 70-106 about 7:1, 261 14:1. + if sh["widest_level"] > 20: + return { + **sh, + "kind": "sheet", + "emitter": "index", + "options": {}, + "why": ( + f"{sh['widest_level']} nodes sit at one level; any layered engine " + "draws that as a strip. Split it, scope it, or read it as an index" + ), + } + + linked = sh["nodes"] - sh["isolated"] + if sh["nodes"] and sh["isolated"] > sh["nodes"] * 0.6: + return { + **sh, + "kind": "flat", + "emitter": "index", + "options": {}, + "why": ( + f"{sh['isolated']} of {sh['nodes']} have no relationships — " + "that is a list, and a diagram of it says less than the list does" + ), + } + + # A pipeline is levels-dominant and narrow: a chain with some fan-out, which + # is the shape of an Airflow DAG, a build, an ETL run. Ranks genuinely suit + # it — but read left to right, the way every scheduler's own UI draws it, + # because a pipeline is a sequence and sequences read across. + if source in ("airflow", "dag", "pipeline") or "task" in kinds: + return { + **sh, + "kind": "pipeline", + "emitter": "dot", + "options": {"rankdir": "LR"}, + "why": "a sequence with fan-out — ranks suit it, read left to right", + } + if sh["levels"] >= 4 and sh["widest_level"] <= 8 and linked > 3: + return { + **sh, + "kind": "pipeline", + "emitter": "dot", + "options": {"rankdir": "LR"}, + "why": ( + f"{sh['levels']} levels only {sh['widest_level']} wide — a chain, " + "and a chain reads across rather than down" + ), + } + + if sh["levels"] <= 2 and linked > 8: + return { + **sh, + "kind": "hub", + "emitter": "dot", + "options": {}, + "why": "shallow and wide — one or two things everything points at", + } + + return { + **sh, + "kind": "layered" if sh["levels"] > 2 else "tree", + "emitter": "dot", + "options": {}, + "why": f"{sh['levels']} levels, widest {sh['widest_level']} — ranks suit this", + } diff --git a/soleprint/atlas2/docgen/selftest.py b/soleprint/atlas2/docgen/selftest.py new file mode 100644 index 0000000..0764898 --- /dev/null +++ b/soleprint/atlas2/docgen/selftest.py @@ -0,0 +1,942 @@ +""" +Prove the pipeline, offline, on a tree it builds itself. + + python3 selftest.py # or: make check + +No network, nothing installed. The render steps skip with a message when +graphviz is absent rather than failing — a machine without `dot` can still check +extraction, the IR and the style layer, and saying so is more useful than a red +mark about the host. + +**A check that only ever proves things work is not worth running.** So the +negative cases carry equal weight, and the four checks that are the *design* +rather than a regression are marked below. If anything ever gets cut, those are +the ones to keep: + + the IR carries no visual field extractors cannot decide appearance + an emitter never reads a source file the layering, from the other side + style names slots, not colours one colour language, not three + ids are stable across runs without it, diffing is noise + +## Golden tests go on the IR, never on the SVG + +Graphviz measures label text with the host's fonts to size nodes, so identical +input produces different geometry on a machine with different fontconfig. The IR +is deterministic; the SVG is not. Pinning the wrong one gives a suite that fails +on someone else's laptop for no reason anyone can act on. +""" + +import ast +import json +import re +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) + +PKG = HERE.name +ir_mod = __import__(f"{PKG}.ir", fromlist=["*"]) +ir_validate = __import__(f"{PKG}.ir.validate", fromlist=["*"]) +py_ex = __import__(f"{PKG}.extractors.python", fromlist=["*"]) +style_mod = __import__(f"{PKG}.style", fromlist=["*"]) +dot_mod = __import__(f"{PKG}.emitters.dot", fromlist=["*"]) +index_mod = __import__(f"{PKG}.emitters.index", fromlist=["*"]) +ops_mod = __import__(f"{PKG}.ops", fromlist=["*"]) +erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"]) +nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"]) +spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"]) +db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"]) + +check_ir = ir_mod.check +Style, StyleError = style_mod.Style, style_mod.StyleError + +PASS, FAIL, SKIP = [], [], [] + +# A docstring that could not plausibly be a style value or a module name, so +# finding it somewhere it should not be means something really read it. +MARKER = "Quarterly Revenue Ledger" + +FIXTURE = { + "app/__init__.py": '"""The app package."""\nfrom .db import Base\n', + "app/db.py": '"""Database plumbing."""\n\n\nclass Base:\n """Declarative base."""\n', + "app/models.py": ( + '"""Domain models."""\n' + "from .db import Base\n" + "import json\n" + "from third_party.orm import Mixin\n" + "\n\n" + "class User(Base):\n" + f' """{MARKER}."""\n' + "\n" + " def save(self):\n" + " pass\n" + "\n\n" + "class Admin(User, Mixin):\n" + " pass\n" + ), + "app/sub/deep.py": "from ..db import Base\n\n\nclass Deep(Base):\n class Inner:\n pass\n", + "app/broken.py": "def oops(:\n", # must not cost the run +} + + +def check(name, condition, detail=""): + (PASS if condition else FAIL).append(name) + print(f" {'ok ' if condition else 'FAIL'} {name}" + + (f"\n {detail}" if detail and not condition else "")) + return condition + + +def _err(fn): + """The message from a call that is expected to fail.""" + try: + fn() + except Exception as e: # noqa: BLE001 - the message is the assertion + return str(e) + return "" + + +def skip(name, why): + SKIP.append(name) + print(f" -- {name} ({why})") + + +def build_tree(root: Path): + for rel, text in FIXTURE.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + + +tmp = tempfile.TemporaryDirectory(prefix="docgen-selftest-") +ROOT = Path(tmp.name) / "fx" +build_tree(ROOT) + +ir = py_ex.extract(ROOT).to_dict() +ids = {n["id"] for n in ir["nodes"]} +edges = {(e["source"], e["target"], e["kind"]) for e in ir["edges"]} + + +# -------------------------------------------------------------------------- +print("\n1. extraction — is it analysis, or is it guessing") + +check("the IR validates", check_ir(ir) == [], str(check_ir(ir)[:3])) + +# The claim the whole thing rests on: `ast` hands over the string "Base", and +# two-pass resolution turns it into the id of the thing it actually names. +check( + "a relative import resolves to the real id", + ("app.models.User", "app.db.Base", "inherits") in edges, + f"got {sorted(e for e in edges if e[2] == 'inherits')}", +) +check( + "a relative import two levels up resolves", + ("app.sub.deep.Deep", "app.db.Base", "inherits") in edges, +) +check( + "a base defined in the same module resolves", + ("app.models.Admin", "app.models.User", "inherits") in edges, +) +check( + "a package's own __init__ resolves against itself", + ("app", "app.db", "imports") in edges, + "from .db inside app/__init__.py must mean app.db, not db", +) +check( + "no raw source name leaks in as an id", + "Base" not in ids and "User" not in ids, + f"unresolved names present as bare ids: {sorted(ids & {'Base', 'User'})}", +) + +# Dropping an edge you cannot resolve is the worse failure: the diagram looks +# complete and is quietly missing a dependency. +externals = {n["id"] for n in ir["nodes"] if n["kind"] == "external"} +check( + "an unresolvable name survives as `external`", + "third_party.orm.Mixin" in externals, + f"got {sorted(externals)}", +) +check( + "and it keeps its edge", + ("app.models.Admin", "third_party.orm.Mixin", "inherits") in edges, +) + +check( + "nested definitions are reached", + "app.sub.deep.Deep.Inner" in ids and "app.models.User.save" in ids, + "a missing generic_visit() silently loses everything below a definition", +) +check( + "containment is recorded", + next(n for n in ir["nodes"] if n["id"] == "app.models.User.save")["parent"] + == "app.models.User", +) +# Spans, not just a start line. "This class is 400 lines" is structure, and a +# density map sizes a block by it — `ast` carries end_lineno, so recording only +# where something begins throws away half the fact for nothing. +spans = [n for n in ir["nodes"] if n["kind"] in ("class", "function")] +check( + "a construct records how many lines it spans", + all((n.get("attrs") or {}).get("lines", 0) >= 1 for n in spans), + f"missing on {[n['id'] for n in spans if not (n.get('attrs') or {}).get('lines')][:3]}", +) +check( + "a multi-line class spans more than one", + any((n.get("attrs") or {}).get("lines", 0) > 1 for n in spans), +) +check( + "a module records its length", + all((n.get("attrs") or {}).get("lines", 0) >= 1 + for n in ir["nodes"] if n["kind"] == "module" + and not (n.get("attrs") or {}).get("error")), +) +check( + "a span is not a visual field", + check_ir(ir) == [], + "`lines` is a fact about the code, not about how it is drawn", +) + +check( + "source anchors are carried", + all( + (n.get("attrs") or {}).get("file") + for n in ir["nodes"] + if n["kind"] in ("class", "function") + ), +) + +broken = [n for n in ir["nodes"] if (n.get("attrs") or {}).get("error")] +check( + "an unparseable file is recorded, not fatal", + len(broken) == 1 and "broken.py" in broken[0]["attrs"]["file"], + f"got {[(n['id'], n['attrs'].get('error')) for n in broken]}", +) +check("...and the rest of the tree still extracted", len(ids) > 10) + + +# -------------------------------------------------------------------------- +print("\n2. the design rules") + +# (design) Without this the diff emitter reports noise and nobody trusts it. +again = py_ex.extract(ROOT).to_dict() +check( + "ids are stable — the same tree twice is the same bytes", + json.dumps(ir, sort_keys=True) == json.dumps(again, sort_keys=True), +) + +# (design) An extractor that sets a colour decides how the graph looks forever. +visual = set() +for n in ir["nodes"]: + visual |= set(n.get("attrs") or {}) & ir_validate.VISUAL_KEYS +for e in ir["edges"]: + visual |= set(e.get("attrs") or {}) & ir_validate.VISUAL_KEYS +check("no visual field reaches the IR", not visual, f"found {sorted(visual)}") + +check( + "the IR is readable without this package", + isinstance(json.loads(json.dumps(ir)), dict), + "it must be plain JSON, not something that needs a library to open", +) + +# (design) The layering, checked from the emitter side. Parsed, not grepped: +# a docstring mentioning `ast` is prose, an import is a dependency. +offenders = [] +for src in sorted((HERE / "emitters").glob("*.py")): + tree = ast.parse(src.read_text(), str(src)) + for node in ast.walk(tree): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom): + names = [node.module or ""] + for name in names: + if name.split(".")[0] in {"ast", "sqlalchemy", "griffe"}: + offenders.append(f"{src.name}:{node.lineno} imports {name}") +check( + "no emitter reads source — it has never heard of Python or SQL", + not offenders, + "; ".join(offenders), +) + +extractor_offenders = [] +for src in sorted((HERE / "extractors").rglob("*.py")): + tree = ast.parse(src.read_text(), str(src)) + for node in ast.walk(tree): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom): + names = [node.module or ""] + if any(n and ("emitters" in n or "style" in n) for n in names): + extractor_offenders.append(f"{src.name}:{node.lineno}") +check( + "no extractor knows about style or emitters", + not extractor_offenders, + "; ".join(extractor_offenders), +) + + +# -------------------------------------------------------------------------- +print("\n3. style — one colour language") + +styles = Style.available() +check("a style ships", "lucid" in styles, f"found {styles}") + +lucid = Style.load("lucid") # the default: dark +print_theme = Style.load("lucid", theme="lucid") # the print palette + +check("it carries more than one theme", set(lucid.themes()) >= {"lucid", "dark"}) +check( + "an unknown kind falls back rather than crashing", + lucid.node("no-such-kind-at-all") == lucid.node("default") != {}, +) + +# (design) A rule naming a hex is a rule outside the shared vocabulary. +raw = json.loads((HERE / "style" / "lucid.json").read_text()) +leaked = [] +for section in ("nodes", "groups", "edges", "graph"): + for kind, rule in (raw.get(section) or {}).items(): + if not isinstance(rule, dict): + continue + for key, value in rule.items(): + if isinstance(value, str) and re.fullmatch(r"#[0-9A-Fa-f]{3,8}", value): + leaked.append(f"{section}.{kind}.{key}={value}") +check( + "style rules name slots, never colours", + not leaked, + "; ".join(leaked) + " <- the colour language has stopped being one language", +) + +# A theme that defines only some of the slots renders half a diagram in the +# right colours and the rest in whatever DOT does with an empty string. +half_bound = [] +for theme in lucid.themes(): + try: + Style.load("lucid", theme=theme) + except StyleError as e: + half_bound.append(f"{theme}: {e}") +check("every slot resolves in every theme", not half_bound, "; ".join(half_bound)) + +# The point of binding to spr's models: a diagram and the page around it must +# not drift apart. docs/graphs/README.md states the rule. +SYSTEM_ACCENTS = { + "artery": ("artery/index.html", "#b91c1c"), + "atlas": ("atlas/index.html", "#15803d"), + "station": ("station/index.html", "#1d4ed8"), +} +spr_root = HERE.parent.parent +mismatched = [] +for slot, (page, expected) in SYSTEM_ACCENTS.items(): + got = lucid.slot(slot).lower() + if got != expected: + mismatched.append(f"{slot}: style has {got}, {page} sets {expected}") + src = spr_root / page + if src.exists() and expected not in src.read_text(): + mismatched.append(f"{slot}: {page} no longer sets {expected}") +check( + "the dark theme's slots are spr's own --system-accent values", + not mismatched, + "; ".join(mismatched), +) + +check( + "domain -> slot is a mapping, not a colour choice", + lucid.domain_slot("atlas") == "atlas" and lucid.domain_slot("station") == "station", +) +check( + "an undomained group still gets a deterministic slot", + [lucid.domain_slot(None, i) for i in range(4)] + == [lucid.domain_slot(None, i) for i in range(4)], +) + +check( + "the style records where DOT runs out", + {"header-bar", "dasharray", "sequence-badge"} <= set(lucid.limits()), + f"got {sorted(lucid.limits())}", +) + + +# -------------------------------------------------------------------------- +print("\n4. emitters") + +dot_text = dot_mod.emit(ir, lucid) +check("DOT is produced", dot_text.startswith("digraph ir {")) +check( + "every IR edge reaches the DOT", + dot_text.count("->") == len(ir["edges"]), + f"{dot_text.count('->')} of {len(ir['edges'])} — cluster endpoints need lhead/ltail", +) +check("the profile's colours are applied", lucid.node("class")["border"] in dot_text) +check( + "ids and kinds reach the output for a front end to bind to", + 'id="app.models.User"' in dot_text and 'class="class"' in dot_text, +) +check("source anchors become links", "href=" in dot_text) + +print_text = dot_mod.emit(ir, print_theme) +check( + "one IR, two looks, no re-extraction", + print_text != dot_text and print_theme.slot("surface-0") in print_text, +) + +md = index_mod.to_markdown(ir) +check("the index names what exists", "app.models" in md or "models" in md) +check("the index carries the docstrings", MARKER in md) +check( + "the index reports the dependency surface", + "third_party.orm.Mixin" in md, + "externals are the thing a reader most often wants and a diagram scatters", +) +check("the index reports what could not be parsed", "Not parsed" in md) + +sidebar = index_mod.to_sidebar(ir) +check("the sidebar is nested, not flat", any(i.get("children") for i in sidebar["items"])) + + +# -------------------------------------------------------------------------- +print("\n5. views") + +no_std = ops_mod.drop_stdlib(ir) +check( + "dropping the stdlib removes json and keeps third-party", + "json" not in {n["id"] for n in no_std["nodes"]} + and "third_party.orm.Mixin" in {n["id"] for n in no_std["nodes"]}, +) +check("a view still validates", check_ir(no_std) == [], str(check_ir(no_std)[:3])) + +classes = ops_mod.only_kinds(ir, {"class"}) +check( + "keeping one kind keeps its ancestors, so containment survives", + check_ir(classes) == [] and "app.models" in {n["id"] for n in classes["nodes"]}, +) + +around = ops_mod.neighbourhood(ir, "app.db.Base", hops=1) +check( + "a neighbourhood keeps what points at it", + "app.models.User" in {n["id"] for n in around["nodes"]} and check_ir(around) == [], +) + +shallow = ops_mod.collapse_to_depth(ir, 1) +check( + "collapsing to a depth drops what is inside", + "app.models.User.save" not in {n["id"] for n in shallow["nodes"]} + and check_ir(shallow) == [], +) + +# The difference between a simpler picture and a wrong one. `app.models.User` +# inherits from `app.db.Base`; look at the module level and that is a dependency +# of app.models on app.db. Dropping the edge says they are unrelated. +mods = ops_mod.only_kinds(ir, {"module"}) +mod_edges = {(e["source"], e["target"], e["kind"]) for e in mods["edges"]} +check( + "an edge is lifted to the surviving node, not dropped", + ("app.models", "app.db", "inherits") in mod_edges, + f"got {sorted(mod_edges)}", +) +check( + "a lifted edge says how many it stands for", + any((e.get("attrs") or {}).get("weight") for e in mods["edges"]), +) +check( + "lifting never leaves a self-loop", + not [e for e in mods["edges"] if e["source"] == e["target"]], + "both ends inside one node is a fact about that node, not a line", +) +check( + "a label that no longer applies is dropped with the lift", + not any( + (e.get("attrs") or {}).get("label") + for e in mods["edges"] + if (e.get("attrs") or {}).get("weight") + ), +) + +# The default view: what you get without knowing which filters to ask for. +ov = ops_mod.overview(ir) +ov_kinds = {n["kind"] for n in ov["nodes"]} +check( + "the python overview is modules and what is outside", + ov_kinds <= {"module", "external"} and check_ir(ov) == [], + f"got {sorted(ov_kinds)}", +) +check( + "it drops the stdlib but keeps third-party", + "json" not in {n["id"] for n in ov["nodes"]} + and "third_party.orm.Mixin" in {n["id"] for n in ov["nodes"]}, +) +# Depth is the tempting knob and the wrong one: a directory without +# `__init__.py` is not a package, so its modules have no parent and sit at +# depth 0. soleprint has 173 such roots. +fragmented = ops_mod.overview( + {"meta": {"source": "python", "root": "x", "schema_version": "1"}, + "nodes": [{"id": "loose", "kind": "module", "label": "loose", "parent": None, "attrs": {}}, + {"id": "loose.C", "kind": "class", "label": "C", "parent": "loose", "attrs": {}}], + "edges": []} +) +check( + "a parentless module still reduces to its module", + [n["id"] for n in fragmented["nodes"]] == ["loose"], + "depth-based collapsing would have kept the class; kind-based does not care", +) + + +# -------------------------------------------------------------------------- +print("\n5b. shape — knowing the picture before rendering it") + +# Aspect ratio is a property of the graph, not of the renderer. A layered +# engine puts one dependency level in one row, so the widest level is the +# width. Measured on soleprint: 7 levels x 109 nodes rendered 14:1, and no +# layout flag helped — LR merely rotated it (1818x11183), ratio=compress +# squashed it to an unreadable 1008x75. +wide = { + "meta": {"source": "python", "root": "x", "schema_version": "1"}, + "nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}} + for i in range(40)] + + [{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}], + "edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}} + for i in range(40)], +} +sh = ops_mod.shape(wide) +check("shape counts the widest level", sh["widest_level"] == 40, str(sh)) +check("shape counts levels", sh["levels"] == 2, str(sh)) +check("shape finds isolated nodes", ops_mod.shape(ir)["isolated"] >= 0) +check( + "shape needs no renderer", + isinstance(sh["nodes"], int) and "width" not in sh, + "it is computed from the IR, so it can warn before writing a 375 KB image", +) + +parts = ops_mod.split(ir) +check("split produces one document per subsystem", len(parts) >= 1, str(list(parts))) +check("each part still validates", all(check_ir(p) == [] for p in parts.values())) +check( + "a crossing edge is kept on the side that reaches out", + any(p["edges"] for p in parts.values()), + "dropping it would understate the coupling", +) + + +# -------------------------------------------------------------------------- +print("\n6. render") + +if not dot_mod.have_graphviz(): + skip("render", "graphviz not installed — sudo apt install graphviz") +else: + svg = dot_mod.render(dot_text) + check("SVG comes out", svg.startswith(b"" in svg) + check("the SVG is addressable", b'id="app.models.User"' in svg) + check( + "kinds survive as classes for CSS to reach", + b'class="node class"' in svg and b'class="edge inherits"' in svg, + ) + print_svg = dot_mod.render(print_text) + check("the two themes really differ", print_svg != svg) + + # `fontname="Arial Bold"` passes through as a family no browser has, so it + # renders neither Arial nor bold. Only the PostScript spelling produces a + # real weight — and a correctly measured box to hold it. + check( + "labels are actually bold, not nominally bold", + b'font-weight="bold"' in svg, + "no font-weight in the SVG — the face name did not map to a weight", + ) + check( + "the font resolves to a real stack", + b'font-family="Helvetica,sans-Serif"' in svg, + ) + + +# -------------------------------------------------------------------------- +print("\n7. a second domain — the schema checkpoint") + +# The brief makes this a checkpoint: if tables and foreign keys need the schema +# contorted, the schema is wrong and must be fixed before more extractors. +SCHEMA = { + "models": { + "Customer": {"doc": "A buyer.", "fields": {"id": {"type": "int", "pk": True}, + "email": {"type": "str"}}}, + "Invoice": {"fields": {"id": {"type": "int", "pk": True}, + "customer_id": {"type": "FK:Customer"}, + "due_at": {"type": "datetime", "nullable": True}}}, + "Tag": {"fields": {"id": {"type": "int", "pk": True}, + "invoices": {"type": "M2M:Invoice"}}}, + "Orphan": {"fields": {"ref": {"type": "FK:NotDefinedHere"}}}, + } +} +db_ir = db_ex.from_schema_dict(SCHEMA, root="fixture").to_dict() +db_ids = {n["id"] for n in db_ir["nodes"]} +db_edges = {(e["source"], e["target"], e["kind"]) for e in db_ir["edges"]} +db_kinds = {n["kind"] for n in db_ir["nodes"]} + +check("the db IR validates", check_ir(db_ir) == [], str(check_ir(db_ir)[:3])) +check( + "it needed no new top-level field", + set(db_ir) == set(ir), + f"{sorted(set(db_ir) ^ set(ir))} — the schema did not survive the second domain", +) +check("tables and columns are kinds, not fields", db_kinds >= {"table", "column"}) +check( + "a column is contained by its table", + next(n for n in db_ir["nodes"] if n["id"] == "Invoice.customer_id")["parent"] == "Invoice", +) +check("a foreign key is an edge", ("Invoice", "Customer", "foreign_key") in db_edges) +check("a many-to-many is a different edge kind", ("Tag", "Invoice", "references") in db_edges) +check( + "domain detail lives in attrs, not in new columns", + next(n for n in db_ir["nodes"] if n["id"] == "Customer.id")["attrs"].get("pk") is True, +) +check( + "a key naming a table the schema does not define survives as external", + "NotDefinedHere" in db_ids + and next(n for n in db_ir["nodes"] if n["id"] == "NotDefinedHere")["kind"] == "external", +) + +# "Done means: adding a source type costs one adapter and every existing emitter +# works on it unchanged." This is that claim, executed. +db_md = index_mod.to_markdown(db_ir) +check("the index emitter needed no change for a new domain", "Customer" in db_md) +db_dot = dot_mod.emit(db_ir, lucid) +check("the dot emitter needed no change either", db_dot.startswith("digraph ir {")) +check( + "an unstyled kind still renders, via `default`", + lucid.node("table") == lucid.node("default"), + "no `table` rule exists yet, and the diagram is still legible", +) +check( + "the same view filters both domains", + check_ir(ops_mod.only_kinds(db_ir, {"table"})) == [], +) + + +# -------------------------------------------------------------------------- +print("\n8. the structure picks the drawing") + +# The lesson this encodes: a diagram that fights its layout engine is usually +# the wrong kind of diagram. The same 24-table schema rendered 32034x136 through +# `dot` (235:1) and 1740x1860 through `erd` (0.9:1) — not because one engine is +# better, but because a schema is peer entities with references, and drawing it +# in dependency ranks was never its shape. +verdict_db = ops_mod.classify(db_ir) +check("a schema is classified as entities", verdict_db["kind"] == "erd", str(verdict_db)) +check("...and routed to the card emitter", verdict_db["emitter"] == "erd") + +verdict_code = ops_mod.classify(ops_mod.overview(ir)) +check( + "a small module graph still goes to ranks", + verdict_code["emitter"] == "dot", + str(verdict_code), +) + +sheet = { + "meta": {"source": "python", "root": "x", "schema_version": "1"}, + "nodes": [{"id": f"n{i}", "kind": "module", "label": f"n{i}", "parent": None, "attrs": {}} + for i in range(40)] + + [{"id": "hub", "kind": "module", "label": "hub", "parent": None, "attrs": {}}], + "edges": [{"source": f"n{i}", "target": "hub", "kind": "imports", "attrs": {}} + for i in range(40)], +} +verdict_sheet = ops_mod.classify(sheet) +check( + "a 40-wide level is called a sheet, not a diagram", + verdict_sheet["kind"] == "sheet" and verdict_sheet["emitter"] == "index", + str(verdict_sheet), +) +check("every verdict explains itself", all( + ops_mod.classify(g)["why"] for g in (db_ir, sheet, ir) +), "advice without a reason gets overridden") + +# The property that makes the card layout work, and the one dot cannot offer. +svg = erd_mod.emit(db_ir, lucid) +import re as _re +m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg) +w, h = int(m.group(1)), int(m.group(2)) +check( + f"the ER layout stays near-square ({w}x{h})", + 0.25 < w / h < 4, + "sqrt(n) columns should bound the aspect ratio whatever the table count", +) + +wide_schema = { + "meta": {"source": "db", "root": "x", "schema_version": "1"}, + "nodes": [{"id": f"t{i}", "kind": "table", "label": f"t{i}", "parent": None, "attrs": {}} + for i in range(60)] + + [{"id": f"t{i}.c", "kind": "column", "label": "c", "parent": f"t{i}", + "attrs": {"type": "int"}} for i in range(60)], + "edges": [], +} +svg60 = erd_mod.emit(wide_schema, lucid) +m = _re.search(r'width="(\d+)pt" height="(\d+)pt"', svg60) +w60, h60 = int(m.group(1)), int(m.group(2)) +check( + f"...and still does at 60 tables ({w60}x{h60})", + 0.25 < w60 / h60 < 4, + "this is the whole reason it is not a rank-based layout", +) +check( + "the ER geometry is deterministic", + erd_mod.emit(db_ir, lucid) == svg, + "nothing here measures a font, so it renders identically on any machine", +) +check( + "it draws no tables only when there are none", + "no tables" in ( + _err(lambda: erd_mod.emit(ops_mod.overview(ir), lucid)) + ), + "pointing it at a code graph should say so, not emit an empty canvas", +) + + +# -------------------------------------------------------------------------- +print("\n9. the notebook is a build artifact, not a source file") + +# The whole argument: a hand-confected notebook drifts from whatever it +# documents and there is no way to tell by looking. Generated from the spec the +# server is built from, "is this current" becomes a question about the build. +SPEC_IR = { + "meta": {"source": "openapi", "root": "petstore.yaml", "schema_version": "1"}, + "nodes": [ + {"id": "Pet", "kind": "table", "label": "Pet", "parent": None, "attrs": {}}, + {"id": "Pet.id", "kind": "column", "label": "id", "parent": "Pet", + "attrs": {"type": "int", "pk": True}}, + {"id": "Pet.name", "kind": "column", "label": "name", "parent": "Pet", + "attrs": {"type": "str"}}, + {"id": "Pet.neutered", "kind": "column", "label": "neutered", "parent": "Pet", + "attrs": {"type": "bool"}}, + {"id": "GET /pets", "kind": "endpoint", "label": "GET /pets", "parent": None, + "attrs": {"method": "GET", "path": "/pets", "summary": "List pets", + "status": 200, "response_model": "Pet", "returns_list": True}}, + {"id": "POST /pets", "kind": "endpoint", "label": "POST /pets", "parent": None, + "attrs": {"method": "POST", "path": "/pets", "summary": "Create a pet", + "status": 201, "request_model": "Pet", "response_model": "Pet"}}, + ], + "edges": [{"source": "POST /pets", "target": "Pet", "kind": "accepts", "attrs": {}}], +} +check("the spec IR validates", check_ir(SPEC_IR) == [], str(check_ir(SPEC_IR)[:2])) + +base = spec_mod.from_ir(SPEC_IR) +book_spec, _ = spec_mod.merge(base, None) +book = nb_mod.build(book_spec) +text = nb_mod.emit(book_spec) +check("it is nbformat 4", book["nbformat"] == 4 and book["nbformat_minor"] == 5) +check( + "regenerating gives byte-identical bytes", + nb_mod.emit(book_spec) == text, + "a notebook that changes every build cannot be reviewed", +) +check("nothing has run in it", all( + c["outputs"] == [] and c["execution_count"] is None + for c in book["cells"] if c["cell_type"] == "code" +)) +intro = "".join(book["cells"][0]["source"]).lower() +check( + "it says not to edit it, and where to put changes instead", + "not be edited" in intro and "overlay" in intro, + f"the frame only holds if the artifact states it — got: {intro[:120]!r}", +) +body = "".join("".join(c["source"]) for c in book["cells"]) +check("every endpoint in the spec has a section", "GET `/pets`" in body and "POST `/pets`" in body) +check("the facts come from the spec", "returns **Pet**" in body and "`201`" in body) +check( + "no credential is written into it", + "Bearer sk-" not in body and "password" not in body.lower(), +) +check( + "it needs nothing installed", + "pip install" not in body and "import requests" not in body, +) + +# Compiling is not enough. `json.dumps` writes `false`/`true`/`null`, which are +# valid *identifiers* in Python — a generated body full of them compiles and +# then raises NameError on the first run. This is the check that caught it. +import io as _io +import contextlib as _ctx + +ns, ran, failure = {}, 0, None +with _ctx.redirect_stdout(_io.StringIO()): + for c in book["cells"]: + if c["cell_type"] != "code": + continue + src = "".join(c["source"]) + if "urlopen" in src or "call(" in src and "def call" not in src: + # Anything that would reach the network is compiled, not run. + try: + compile(src, c["id"], "exec") + ran += 1 + except SyntaxError as e: + failure = f"{c['id']}: {e}" + break + continue + try: + exec(compile(src, c["id"], "exec"), ns) + ran += 1 + except Exception as e: # noqa: BLE001 - any failure is the finding + failure = f"{c['id']}: {type(e).__name__}: {e}" + break +check(f"its cells run, not merely compile ({ran})", failure is None, failure or "") + +literal = next( + ("".join(c["source"]) for c in book["cells"] + if c["cell_type"] == "code" and "BODY" in "".join(c["source"])), "" +) +check( + "a generated body is a Python literal, not JSON", + "False" in literal and "false" not in literal, + f"got: {literal[:120]}", +) +check( + "the primary key is left out of a create body", + "'id'" not in literal, + "the server assigns it", +) + +# The chain, if modelgen is next door: spec -> IR -> notebook, nothing by hand. +try: + oa = __import__(f"{PKG}.extractors.openapi", fromlist=["*"]) + spec = HERE.parent.parent / "station/tools/shuntgen/fixtures/petstore.yaml" + if not spec.exists(): + raise ImportError("no petstore fixture") + real = oa.extract(spec).to_dict() +except (ImportError, Exception) as e: # noqa: BLE001 + skip("openapi -> IR", str(e).splitlines()[0][:60]) +else: + check("a real spec extracts", check_ir(real) == [], str(check_ir(real)[:2])) + eps = [n for n in real["nodes"] if n["kind"] == "endpoint"] + check(f"its endpoints become nodes ({len(eps)})", len(eps) >= 3) + check( + "its schemas reuse the db vocabulary", + {"table", "column"} <= {n["kind"] for n in real["nodes"]}, + "so the ER emitter draws an API's data model without knowing it is one", + ) + check( + "and the ER emitter does draw it", + erd_mod.emit(ops_mod.only_kinds(real, {"table"}), lucid).startswith(" 1) + +bare, drift3 = spec_mod.merge(spec_mod.from_ir(SPEC_IR), None) +check( + "extraction works with the overlay absent", + drift3 == [] and len(bare["steps"]) > 1, + "the overlay is an addition, never a dependency", +) + +blank = spec_mod.scaffold(spec_mod.from_ir(SPEC_IR)) +check( + "a scaffold hands over every step id", + set(blank["steps"]) == {st["id"] for st in spec_mod.from_ir(SPEC_IR)["steps"]}, + "listing the ids is the difference between an overlay being written and meant to be", +) +check( + "a blank scaffold changes nothing", + nb_mod.emit(spec_mod.merge(spec_mod.from_ir(SPEC_IR), blank)[0]) == nb_mod.emit(bare), +) + + +# -------------------------------------------------------------------------- +print("\n10. defaults") + +check( + "dark is the default theme", + Style.load("lucid").theme == "dark", + "a diagram lands in a dark docs page far more often than in print", +) +check("print is one flag away", Style.load("lucid", theme="lucid").theme == "lucid") + +pipeline = { + "meta": {"source": "airflow", "root": "etl", "schema_version": "1"}, + "nodes": [{"id": x, "kind": "task", "label": x, "parent": None, "attrs": {}} + for x in ("extract", "transform", "load", "notify")], + "edges": [{"source": a, "target": b, "kind": "depends", "attrs": {}} + for a, b in (("extract", "transform"), ("transform", "load"), + ("load", "notify"))], +} +verdict = ops_mod.classify(pipeline) +check( + "a DAG is recognised as a pipeline", + verdict["kind"] == "pipeline", + str(verdict), +) +check( + "...and is drawn left to right", + verdict.get("options", {}).get("rankdir") == "LR", + "a sequence reads across, which is how every scheduler's own UI draws it", +) +check( + "a task has its own style, not a module's", + lucid.node("task") != lucid.node("module"), + "a schedule should not render as if it were code structure", +) +check( + "the emitter honours the rankdir it is given", + "rankdir=LR" in dot_mod.emit(pipeline, lucid, rankdir="LR"), +) +check("every verdict carries options", "options" in ops_mod.classify(ir)) + + +# -------------------------------------------------------------------------- +tmp.cleanup() +print() +print(f"{len(PASS)} passed, {len(FAIL)} failed, {len(SKIP)} skipped") +if FAIL: + print("\nfailed:") + for name in FAIL: + print(f" {name}") +sys.exit(1 if FAIL else 0) diff --git a/soleprint/atlas2/docgen/style/__init__.py b/soleprint/atlas2/docgen/style/__init__.py new file mode 100644 index 0000000..3a7f29a --- /dev/null +++ b/soleprint/atlas2/docgen/style/__init__.py @@ -0,0 +1,179 @@ +""" +Style: what a `kind` looks like. Loaded by emitters only. + +A style file names **slots**, never colours. `"border": "atlas"` is a rule; the +theme binds `atlas` to `#43A047` in print and `#15803d` on the docs site. That +indirection is the whole reason this is one colour language rather than three: +`tokens.css`, `docs/graphs/themes/*.gvpr` and this file all use the same slot +names, so a diagram and the page around it match by construction. + +`docs/graphs/README.md` states the rule these files obey: + + Match the palette to soleprint/common/theme/themes/.css so a diagram + and the page around it are the same visual language. + + style = Style.load("lucid") + style.node("class")["border"] -> "#1E88E5" (resolved for the theme) + style.node("nonesuch") -> the `default` entry, never a crash + +An unknown `kind` falls back to `default`. That matters more than it looks: it +means a new extractor with a new vocabulary renders plainly and legibly on day +one instead of failing, and nobody is forced to write a style file before they +can see anything. +""" + +import json +from pathlib import Path + +HERE = Path(__file__).resolve().parent + +# Keys whose value is a slot name to be resolved against the theme. Anything +# not listed here is passed through as-is, which is how numbers, shapes and +# booleans survive. +COLOUR_KEYS = ("fill", "border", "text", "color", "header-fill", "bgcolor", "fontcolor") + +# Keys whose value names a geometry entry rather than a colour. +GEOMETRY_KEYS = ("font-size", "font") + +# Prose, not rules. Stripped before a rule reaches an emitter so it cannot be +# mistaken for an attribute. +NOTE_KEYS = ("note",) + + +class StyleError(ValueError): + """A style file that an emitter cannot apply.""" + + +class Style: + """One style file, resolved against one theme.""" + + def __init__(self, data: dict, theme: str | None = None, name: str = ""): + self.name = name + self.data = data + self.theme = theme or data.get("default_theme") + themes = data.get("themes", {}) + if self.theme not in themes: + raise StyleError( + f"{name}: no theme {self.theme!r} — have {', '.join(sorted(themes)) or 'none'}" + ) + self.slots: dict[str, str] = dict(themes[self.theme].get("slots", {})) + self.geometry: dict = { + k: v for k, v in data.get("geometry", {}).items() if k not in NOTE_KEYS + and not k.endswith("-note") + } + problems = self.validate() + if problems: + raise StyleError(f"{name} [{self.theme}]:\n " + "\n ".join(problems)) + + # -- loading ---------------------------------------------------------- + + @classmethod + def load(cls, ref: str | Path, theme: str | None = None) -> "Style": + """A shipped style by name, or any JSON file by path.""" + path = Path(ref) + if not path.suffix and not path.exists(): + path = HERE / f"{ref}.json" + if not path.exists(): + available = sorted(p.stem for p in HERE.glob("*.json")) + raise StyleError( + f"no style {str(ref)!r} — shipped: {', '.join(available) or 'none'}" + "\n(a path to any JSON file works too)" + ) + return cls(json.loads(path.read_text()), theme=theme, name=path.stem) + + @classmethod + def available(cls) -> list[str]: + return sorted(p.stem for p in HERE.glob("*.json")) + + def themes(self) -> list[str]: + return sorted(self.data.get("themes", {})) + + # -- checking --------------------------------------------------------- + + def validate(self) -> list[str]: + """Every slot a rule mentions must exist in this theme. + + Run at load, so a half-bound theme fails at the boundary rather than + rendering most of a diagram in the right colours and the rest in + whatever DOT does with an empty string. + """ + problems = [] + for section in ("nodes", "groups", "edges"): + for kind, rule in self.data.get(section, {}).items(): + if not isinstance(rule, dict): + problems.append(f"{section}.{kind} is not an object") + continue + for key in COLOUR_KEYS: + slot = rule.get(key) + if slot is not None and slot not in self.slots: + problems.append( + f"{section}.{kind}.{key} names slot {slot!r}, " + f"which theme {self.theme!r} does not define" + ) + for key in GEOMETRY_KEYS: + ref = rule.get(key) + if ref is not None and ref not in self.geometry: + problems.append( + f"{section}.{kind}.{key} names geometry {ref!r}, which is not defined" + ) + graph = self.data.get("graph", {}) + for key in COLOUR_KEYS: + slot = graph.get(key) + if slot is not None and slot not in self.slots: + problems.append(f"graph.{key} names slot {slot!r}, undefined in {self.theme!r}") + return problems + + # -- reading ---------------------------------------------------------- + + def _resolve(self, rule: dict) -> dict: + out = {} + for key, value in rule.items(): + if key in NOTE_KEYS: + continue + if key in COLOUR_KEYS and isinstance(value, str): + out[key] = self.slots.get(value, value) + elif key in GEOMETRY_KEYS and isinstance(value, str): + out[key] = self.geometry.get(value, value) + else: + out[key] = value + return out + + def _lookup(self, section: str, kind: str) -> dict: + rules = self.data.get(section, {}) + return self._resolve(rules.get(kind) or rules.get("default") or {}) + + def node(self, kind: str) -> dict: + return self._lookup("nodes", kind) + + def group(self, kind: str) -> dict: + return self._lookup("groups", kind) + + def edge(self, kind: str) -> dict: + return self._lookup("edges", kind) + + def graph(self) -> dict: + return self._resolve(self.data.get("graph", {})) + + def geom(self, key, default=None): + return self.geometry.get(key, default) + + def slot(self, name: str, default: str = "") -> str: + return self.slots.get(name, default) + + def domain_slot(self, domain: str | None, index: int = 0) -> str: + """Which slot a group with this domain uses. + + The IR says *which spr model* a group is; this says which slot that maps + to. Where there is no domain, the rotation is indexed by the caller's + sorted position, so the assignment is deterministic — two runs of the + same graph colour the same group the same way. + """ + table = self.data.get("domain_slots", {}) + if domain and domain in table: + return table[domain] + rotation = table.get("rotation") or ["accent"] + return rotation[index % len(rotation)] + + def limits(self) -> dict: + """Where this style asks for more than the target can express.""" + return {k: v for k, v in self.data.get("limits", {}).items() if k not in NOTE_KEYS} diff --git a/soleprint/atlas2/docgen/style/lucid.json b/soleprint/atlas2/docgen/style/lucid.json new file mode 100644 index 0000000..e40bc11 --- /dev/null +++ b/soleprint/atlas2/docgen/style/lucid.json @@ -0,0 +1,336 @@ +{ + "name": "lucid", + "note": "Visual language for generated diagrams. Keyed on `kind`, which is the only IR field style may read. Rules name SLOTS, never hexes; a theme binds slot -> colour. That indirection is the point: it is the same vocabulary `soleprint/common/theme/tokens.css` and `docs/graphs/themes/*.gvpr` already use, so a diagram and the page around it are one visual language. docs/graphs/README.md states the rule.", + "default_theme": "dark", + "themes": { + "lucid": { + "note": "Print and light, for documents and anything that gets exported. Extracted from real architecture diagrams, then re-bound onto spr's slots. Ask for it with `--theme lucid`.", + "slots": { + "surface-0": "#FFFFFF", + "surface-1": "#FAFAFA", + "surface-2": "#F5F7FA", + "border": "#BDBDBD", + "border-strong": "#9E9E9E", + "text": "#333333", + "text-muted": "#616E7C", + "text-dim": "#9AA5B1", + "text-on-fill": "#000000", + "accent": "#FFB74D", + "accent-soft": "#FFF3E0", + "artery": "#E53935", + "artery-soft": "#FFEBEE", + "atlas": "#43A047", + "atlas-soft": "#E8F5E9", + "station": "#1E88E5", + "station-soft": "#E3F2FD", + "ok": "#43A047", + "error": "#E53935", + "muted": "#E0E0E0", + "muted-soft": "#FAFAFA", + "alt-1": "#B39DDB", + "alt-1-soft": "#EDE7F6", + "alt-2": "#FFF176", + "alt-2-soft": "#FFFDE7" + } + }, + "dark": { + "note": "The default. Every value is a tokens.css variable or a --system-accent, so a diagram embedded in a page matches it without anyone choosing. artery/atlas/station are exactly what artery/index.html:30, atlas/index.html:25 and station/index.html:29 set. `--theme lucid` switches to the print palette.", + "slots": { + "surface-0": "#0a0a0a", + "surface-1": "#141414", + "surface-2": "#1a1a1a", + "border": "#333333", + "border-strong": "#4a4a4a", + "text": "#e5e5e5", + "text-muted": "#a3a3a3", + "text-dim": "#666666", + "text-on-fill": "#0a0a0a", + "accent": "#d4a574", + "accent-soft": "#242424", + "artery": "#b91c1c", + "artery-soft": "#fca5a5", + "atlas": "#15803d", + "atlas-soft": "#86efac", + "station": "#1d4ed8", + "station-soft": "#93c5fd", + "ok": "#3ecf8e", + "error": "#f06565", + "muted": "#555568", + "muted-soft": "#141414", + "alt-1": "#4f9cf9", + "alt-1-soft": "#141414", + "alt-2": "#f5a623", + "alt-2-soft": "#141414" + } + } + }, + "geometry": { + "note": "Pinned to tokens.css rather than retyped. The supplied spec and spr's tokens agreed on six of these independently: radius 4-6px, label 10-11px, header 12-14px, padding 8px, hairline 1px, and a Segoe-UI-through-Arial sans stack.", + "radius-sm": 4, + "radius": 6, + "hairline": 1, + "font": "Helvetica", + "font-note": "PostScript-style names, because they are the only spelling Graphviz maps to a real SVG font-weight. `fontname=\"Arial Bold\"` passes through verbatim as font-family=\"Arial Bold\" \u2014 a family no browser has, so it renders neither Arial nor bold. `Helvetica-Bold` emits font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\", which is a real stack with a real weight. It also measures wider (128pt vs 116pt for the same label), so boxes are sized for the text that actually renders instead of overflowing it.", + "font-size-sm": 10, + "font-size-base": 11, + "font-size-header": 13, + "padding": "0.20,0.10", + "font-bold": "Helvetica-Bold" + }, + "graph": { + "bgcolor": "surface-0", + "fontcolor": "text", + "rankdir": "TB", + "nodesep": 0.5, + "ranksep": 0.6, + "pad": 0.3 + }, + "nodes": { + "default": { + "shape": "box", + "rounded": true, + "fill": "surface-0", + "border": "border", + "text": "text", + "note": "Every unknown kind lands here rather than crashing. A new extractor renders plainly and legibly on day one.", + "bold": true + }, + "component": { + "shape": "box", + "rounded": true, + "fill": "surface-0", + "border": "border", + "text": "text", + "bold": true + }, + "datastore": { + "shape": "cylinder", + "rounded": false, + "fill": "surface-0", + "border": "border", + "text": "text", + "bold": true + }, + "bucket": { + "shape": "trapezium", + "rounded": false, + "fill": "surface-0", + "border": "border", + "text": "text", + "note": "An approximation. DOT has no S3-bucket silhouette; trapezium is the nearest native shape.", + "bold": true + }, + "module": { + "shape": "box", + "rounded": true, + "fill": "surface-2", + "border": "border-strong", + "text": "text", + "bold": true + }, + "class": { + "shape": "box", + "rounded": true, + "fill": "surface-0", + "border": "station", + "text": "text", + "bold": true + }, + "function": { + "shape": "box", + "rounded": true, + "fill": "surface-0", + "border": "border", + "text": "text-muted", + "font-size": "font-size-sm", + "bold": true + }, + "external": { + "shape": "box", + "rounded": true, + "fill": "muted-soft", + "border": "muted", + "text": "text-muted", + "dashed": true, + "note": "A name that could not be resolved. Drawn as the boundary it is, and never dropped \u2014 a missing dependency that looks like a complete diagram is the worse failure.", + "bold": true + }, + "task": { + "shape": "box", + "rounded": true, + "fill": "surface-2", + "border": "station", + "text": "text", + "bold": true, + "note": "A unit of work in a pipeline \u2014 an Airflow task, a build step. Distinct from `module` so a schedule does not render as if it were code structure." + }, + "sensor": { + "shape": "hexagon", + "rounded": false, + "fill": "surface-2", + "border": "accent", + "text": "text", + "bold": true, + "note": "A task that waits on something outside the pipeline." + } + }, + "groups": { + "default": { + "rounded": false, + "dashed": true, + "fill": "surface-2", + "border": "border", + "text": "text-muted", + "bold": true + }, + "boundary": { + "rounded": false, + "dashed": false, + "fill": "surface-0", + "border": "border-strong", + "header-fill": "accent", + "text": "text-on-fill", + "bold": true, + "note": "Type A. The header is a filled bar across the top of the container in the source spec; DOT gives a label plus a cluster bgcolor and no separate bar. See limits.header-bar." + }, + "zone": { + "rounded": false, + "dashed": true, + "fill": "surface-2", + "border": "border", + "text": "text-muted", + "note": "Type B. Dashed area boundary.", + "bold": true + }, + "artery": { + "rounded": false, + "dashed": true, + "fill": "artery-soft", + "border": "artery", + "text": "text-muted", + "bold": true + }, + "atlas": { + "rounded": false, + "dashed": true, + "fill": "atlas-soft", + "border": "atlas", + "text": "text-muted", + "bold": true + }, + "station": { + "rounded": false, + "dashed": true, + "fill": "station-soft", + "border": "station", + "text": "text-muted", + "bold": true + }, + "pipeline": { + "rounded": false, + "dashed": true, + "fill": "surface-2", + "border": "border", + "text": "text-muted", + "bold": true, + "note": "One DAG, when several are drawn together." + } + }, + "edges": { + "default": { + "color": "text", + "arrowhead": "normal", + "arrowsize": 0.7, + "text": "text-muted" + }, + "imports": { + "color": "border-strong", + "arrowhead": "normal", + "arrowsize": 0.6, + "text": "text-dim" + }, + "inherits": { + "color": "station", + "arrowhead": "empty", + "arrowsize": 0.8, + "text": "text-muted" + }, + "calls": { + "color": "text", + "arrowhead": "normal", + "arrowsize": 0.7, + "text": "text-muted" + }, + "async": { + "color": "text", + "arrowhead": "normal", + "arrowsize": 0.7, + "dashed": true, + "text": "text-muted" + }, + "flow": { + "color": "artery", + "arrowhead": "normal", + "arrowsize": 0.7, + "dashed": true, + "text": "artery" + }, + "ok": { + "color": "ok", + "arrowhead": "normal", + "arrowsize": 0.7, + "dashed": true, + "text": "ok" + }, + "foreign_key": { + "color": "station", + "arrowhead": "normal", + "arrowsize": 0.7, + "text": "text-muted" + }, + "references": { + "color": "border-strong", + "arrowhead": "normal", + "arrowsize": 0.6, + "dashed": true, + "text": "text-dim" + }, + "depends": { + "color": "border-strong", + "arrowhead": "normal", + "arrowsize": 0.7, + "text": "text-muted" + }, + "triggers": { + "color": "accent", + "arrowhead": "normal", + "arrowsize": 0.8, + "text": "accent" + } + }, + "domain_slots": { + "note": "How a group picks its colour without the IR ever naming one. The IR says which spr model a group belongs to (attrs.domain \u2014 semantic); this maps that to a slot. Same mechanism as --system-accent. Where a group has no domain, the emitter assigns the rotation below by sorted id, so the choice is deterministic and two runs give the same bytes.", + "artery": "artery", + "atlas": "atlas", + "station": "station", + "soleprint": "accent", + "rotation": [ + "accent", + "alt-1", + "alt-2", + "station", + "atlas", + "artery" + ] + }, + "limits": { + "note": "Where DOT stops. Recorded rather than worked around; the full spec is kept here so a richer emitter needs no re-authoring. See the plan's 'Use DOT until it hits its limits'.", + "header-bar": "Type A's filled 100%-width header rectangle. DOT clusters have a label and a bgcolor, not a header bar. Needs an HTML-like label table.", + "dasharray": "stroke-dasharray is not parameterised \u2014 style=dashed is one pattern, so 4,4 and 5,5 collapse. Async and thematic edges must differ by colour, not by dash.", + "corner-radius": "style=rounded is binary, so radius-sm (4) and radius (6) render identically.", + "icon-above-label": "Icon centered above a centered label needs an HTML-like label or image+labelloc; text only here.", + "sequence-badge": "A numbered circle on an edge. xlabel carries the number; the circle, its fill and its border are not expressible.", + "bucket-shape": "trapezium approximates the S3 bucket silhouette.", + "ranged-widths": "The spec gives 1-2px and 1-1.5px; penwidth is scalar, so these pin to geometry.hairline." + } +} diff --git a/soleprint/station/tools/docgen/.gitignore b/soleprint/station/tools/docgen/.gitignore new file mode 100644 index 0000000..8b93c3e --- /dev/null +++ b/soleprint/station/tools/docgen/.gitignore @@ -0,0 +1,4 @@ +# Everything this makes. Regenerate with `make notebook`, `make graph`. +out/ +__pycache__/ +*.pyc diff --git a/soleprint/station/tools/docgen/Makefile b/soleprint/station/tools/docgen/Makefile new file mode 100644 index 0000000..3828819 --- /dev/null +++ b/soleprint/station/tools/docgen/Makefile @@ -0,0 +1,88 @@ +# docgen — one target per thing this makes. +# +# The folder is meant to be copied out of soleprint and used on its own, so +# everything here is derived from where this file sits rather than written down: +# copy the directory anywhere, `cd` into it, and `make` works. Renaming it works +# too, since the package name comes from the directory. +# +# This tool is a library, not a CLI. There is no `python -m docgen` and no +# argparse — the prompt is explicit that it is consumed from a notebook. So each +# target below is a one-line `python3 -c` against the public surface, which +# doubles as the shortest possible worked example of calling it. +# +# make check prove it works, on fixtures it builds +# make notebook the three .ipynb variants -> out/ +# make graph the example graph, every profile -> out/ +# make extract DIR=~/diagrams a folder of exports -> tokens + a profile +# make profiles what profiles are available +# make doctor what this machine has +# +# The logic lives in the Python, never here. + +HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +PKG := $(notdir $(HERE)) +PARENT := $(patsubst %/,%,$(dir $(HERE))) +PY ?= python3 + +# Run the package from its parent, which is what an import needs and what lets +# this work without installing anything. +RUN := PYTHONPATH=$(PARENT) $(PY) -c + +OUT ?= $(HERE)/out +DIR ?= +NAME ?= extracted + +.PHONY: help check notebook graph extract profiles doctor clean + +help: ## List every target + @echo "docgen — graph generation and document output, with the style as data" + @echo + @grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-12s\033[0m %s\n", $$1, $$2}' + @echo + @echo " OUT=/path where output goes (default: $(PKG)/out)" + @echo " DIR=/path the folder of exported diagrams, for extract" + @echo " NAME=lucid-real what to call the extracted profile" + @echo + @echo " It is a library. From a notebook:" + @echo " from $(PKG) import Profile, emit, render # graphgen owns Graph" + +check: ## Prove the whole thing works, needing nothing installed and no network + @$(PY) $(HERE)/selftest.py + +notebook: ## Emit vanilla, executable and live .ipynb into OUT + @$(RUN) "from $(PKG).export import write_all; \ +from $(PKG).export.specs import vanilla; \ +[print(' ', p) for p in write_all(vanilla.build(), '$(OUT)')]" + +graph: ## Render graphgen's example graph through every shipped profile into OUT + @$(RUN) "from $(PKG).demo import render_all; render_all('$(OUT)')" + +extract: ## A folder of exported SVG/PDF diagrams -> tokens.json + a profile + @test -n "$(DIR)" || { echo "Error: set DIR=/path/to/diagrams" >&2; exit 1; } + @test -d "$(DIR)" || { echo "Error: no such folder: $(DIR)" >&2; exit 1; } + @$(RUN) "from $(PKG).style import from_folder, summarise; \ +r = from_folder('$(DIR)', '$(OUT)', '$(NAME)'); \ +print(summarise(r['data'])); \ +print(); print(' tokens ', r['tokens']); print(' profile ', r['profile']); \ +print(); print(' Drop the profile into $(PKG)/profiles/ to use it by name,'); \ +print(' or pass its path: Profile.load(\"%s\")' % r['profile'])" + +profiles: ## What style profiles are available + @$(RUN) "from $(PKG) import Profile; \ +[print(' %-12s %s' % (n, Profile.load(n).data.get('note','')[:88])) for n in Profile.available()]" + +doctor: ## Report whether this machine can run it + @printf 'python : '; $(PY) --version 2>&1 || echo MISSING + @printf 'dot : '; (dot -V 2>&1) || echo 'MISSING — sudo apt install graphviz (needed to render)' + @printf 'pdftocairo : '; (pdftocairo -v 2>&1 | head -1) || echo 'MISSING — sudo apt install poppler-utils (only for PDF sources)' + @printf 'lxml : '; $(PY) -c 'import lxml.etree; print(lxml.etree.__version__)' 2>/dev/null || echo 'MISSING — pip install lxml (only for extraction)' + @printf 'package : %s (from %s)\n' '$(PKG)' '$(PARENT)' + @printf 'graphgen : '; test -d '$(PARENT)/graphgen' \ + && echo 'beside this folder — `make graph` will work' \ + || echo 'absent — everything works except `make graph` (it owns the model)' + @$(RUN) "import $(PKG), $(PKG).style, $(PKG).export" >/dev/null 2>&1 \ + && echo 'import : ok' || echo 'import : FAILED — is the folder intact?' + +clean: ## Delete OUT. Nothing else is ever written to + @rm -rf "$(OUT)" && echo "Removed $(OUT)" diff --git a/soleprint/station/tools/docgen/README.md b/soleprint/station/tools/docgen/README.md new file mode 100644 index 0000000..857cbf3 --- /dev/null +++ b/soleprint/station/tools/docgen/README.md @@ -0,0 +1,256 @@ +# docgen + +The interface to Graphviz and DOT: styling and export. + +Eight repos under `semester/` draw their architecture with Graphviz, and every +one of them hand-writes `.dot` with the palette inlined and commits the `.svg` +beside it — `sms`, `unt`, `cht`, `mpr`, `mly`, `nvi`, `eth`, and `spr`. Three +different dark palettes between them, eight answers to the same question. Only +`spr/docs/graphs/` separates what a diagram *means* from what it *looks like*, +and it does that in `gvpr` rather than in anything reusable. + +This is that separation, as a library. + +```python +from docgen import Profile, emit, render +from graphgen import Graph # what a graph *is* lives next door + +g = Graph("overview", title="System Overview", rankdir="LR") +g.node("api", "API", cls="station") +g.node("db", "Database", shape="cylinder") +g.edge("api", "db", "reads") + +svg = render(emit(g, Profile.load("lucid"))) +``` + +The same `g` through `Profile.load("default")` is the same diagram in the +docs-site palette. Nothing about `g` changes, because nothing about `g` was ever +about colour. + +## The split + + docgen how a graph is drawn. DOT, Graphviz, style profiles, and the + documents the result goes into. + graphgen what a graph is. Nodes, edges, groups, and the sources + graphs come from. + +**docgen imports nothing from graphgen.** A graph is read structurally — see +[`shape.py`](shape.py), which names every attribute `emit()` touches — so either +folder can be copied out and used with the other absent. Prove it: + +```bash +cp -r docgen /tmp/alone && cd /tmp/alone/docgen && make check +# 49 passed, 0 failed, 1 skipped <- the skip is the graphgen seam +``` + +docgen deliberately ships **no graph class**. If it did, people would use it, and +there would be two graph models — which is exactly what this separation exists to +prevent. The one place that rule is relaxed is scaffolding: `demo.py` and +`selftest.py` may import graphgen, because demonstrating and testing the drawing +needs something to draw. + +It is a **library**, not a CLI — the first thing that uses it is a notebook. The +Makefile is the front door for the things that are worth running from a shell. + +```bash +make check # prove it, on fixtures it builds — nothing installed, no network +make doctor # what this machine has +make graph # the example graph, every profile -> out/ +make notebook # the three .ipynb variants -> out/ +make extract DIR=~/diagrams +make help # every target +``` + +## The three concerns + +``` +model ──► emit ──► render + ▲ + profile ← every visual value, as data +``` + +| | | +|---|---| +| `shape.py` | the contract: what docgen needs a graph to look like. Protocols only — nothing instantiable. | +| `profile.py` | every visual value, from JSON. Fill, stroke, penwidth, fonts, arrowsize, splines, separations. | +| `dot.py` | a graph and a profile, into DOT. The dullest module here on purpose. | +| `render.py` | DOT into SVG, via the `dot` binary. | + +The model those three read — `graphgen/graph.py` — carries **meaning only**: a node +is `cls="artery"`, never `fillcolor="#c0ffee"`. + +The class vocabulary is the one `spr/docs/graphs/README.md` already documents — +`accent`, `accent-text`, `ok`, `artery`, `atlas`, `station`, `muted` — so a graph +built in graphgen and a `.dot` written by hand mean the same thing by the same +word. Anything untagged renders neutral, which is where most nodes should stay. +The selftest asserts the two vocabularies still agree, because a class graphgen +can set that no profile styles renders neutral and tells nobody why. + +### Three things a profile cannot touch + +Structure wearing a visual field's clothing. A profile that overrode these would +be destroying meaning rather than restyling it. + +| | | +|---|---| +| `shape` | a cylinder is a datastore, not a decoration | +| `style="invis"` | layout scaffolding — filling it in would draw it | +| `style="dashed"` | a weaker relationship. The profile *composes* with it (`filled,rounded,dashed`); it never replaces it | + +## Profiles + +`lucid` and `default` ship. `make profiles` lists them, `Profile.load(name)` +loads one, and `Profile.load("/path/to/anything.json")` loads a file — so code +that got a profile name out of a config does not have to know which it got. + +`lucid` is first because the output has to import cleanly into Lucid and Google +Drawings. It is **not** the built-in one, and it is not in the code: the shipped +`lucid.json` is a plausible default derived from values already visible in +`spr/docs/graphs/themes/lucid.gvpr`, and the real one — extracted from company +diagrams — drops in as a file with no code change. + +`selftest.py` asserts that. It takes every colour in every profile and greps the +package's own source for it, because the failure this design exists to prevent is +somebody reaching for a literal `"#5A6C86"` in `dot.py` on a Tuesday and the whole +arrangement quietly becoming decorative. + +An unknown key in a profile is **refused, not ignored** — a setting plainly +written in a file and silently not applied is a genuinely bad thing to debug, +because the file says it is on. + +## Extraction + +The real style values come from real diagrams. Point `style/` at a folder of +exports: + +```bash +make extract DIR=~/exported-diagrams NAME=house +``` + +``` + fill + 87 #1f2933 + 43 #ffffff + 28 #616e7c + + stroke + 85 #9aa5b1 + 32 #3a7dff +``` + +Frequency-sorted, because that is what turns a heap of values into a palette: the +top two or three fills *are* the palette, and the modal stroke width *is* the +house line weight. SVG directly, PDF via `pdftocairo -svg`. Out comes a +`tokens.json` and a profile. + +Two rules, and they are not stylistic preferences: + +- **Text content is never read.** `` elements are visited for their style + attributes and nothing else. Style values are visual metadata; the semantics of + the diagram are not needed to derive a palette, so they are not looked at. The + selftest renders a graph with a distinctive label and asserts it appears + nowhere in the token output. +- **Fully offline.** Nothing here opens a socket. Confidential source diagrams + stay off any network path, and off the Lucid API path. + +Parsed with `lxml`, not grepped — `fill` appears as an attribute, inside an +inline `style`, and inside a `