From 160ee31b8c632d943c233c0505f500fd6b503f17 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Sun, 13 Sep 2026 21:41:29 -0300 Subject: [PATCH] docgen: drop the superseded first pass, port the style harvester MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The IR supersedes both intermediate designs (requirements D6, D7): station/tools/docgen and the graph model that briefly lived in graphgen. Shipping them beside atlas2/docgen would mean two graph models, which is the thing the architecture argues against. Carried across rather than lost: - style/extract.py and tokens.py, the offline theme harvester (R31). Rewritten to emit a *theme* — a slot-to-hex binding — rather than a whole style file, since what harvesting recovers is which colour a slot should be, not what a kind should look like. - graphgen/README.md, rewritten for what graphgen actually is now: the schema explorer. It fixes the blank station-index entry at run.py:304. Two bugs found while doing it: - Canvas and ink are the two lightness extremes, not the two most common values. In a Graphviz SVG every label carries a fill, so the ink outnumbers the canvas 87 to 43 and the old rule produced a theme whose text was invisible against its own background. - A name defined in both branches of an if/else produced a duplicate id, which failed validation on docgen's own source. Disambiguated by line. --- soleprint/atlas2/docgen/Makefile | 36 +- soleprint/atlas2/docgen/docs/docs.css | 135 +++ .../atlas2/docgen/docs/img/architecture.svg | 800 ++++++++++++++++++ soleprint/atlas2/docgen/docs/img/erd.svg | 103 +++ soleprint/atlas2/docgen/docs/img/minimap.svg | 298 +++++++ soleprint/atlas2/docgen/docs/index.html | 793 +++++++++++++++++ soleprint/atlas2/docgen/docs/viewer.html | 119 +++ soleprint/atlas2/docgen/emitters/__main__.py | 8 +- .../atlas2/docgen/emitters/cli_explore.py | 47 + .../atlas2/docgen/emitters/cli_minimap.py | 50 ++ soleprint/atlas2/docgen/emitters/erd.py | 48 +- soleprint/atlas2/docgen/emitters/explore.py | 315 +++++++ soleprint/atlas2/docgen/emitters/minimap.py | 277 ++++++ soleprint/atlas2/docgen/explore_test.js | 67 ++ .../atlas2/docgen/extractors/__main__.py | 5 +- soleprint/atlas2/docgen/extractors/code.py | 260 ++++++ .../atlas2/docgen/extractors/code_main.py | 39 + .../docgen/extractors/python/resolve.py | 8 + soleprint/atlas2/docgen/ops/filter.py | 18 +- soleprint/atlas2/docgen/selftest.py | 213 +++++ soleprint/atlas2/docgen/style/__init__.py | 14 + .../tools => atlas2}/docgen/style/extract.py | 0 soleprint/atlas2/docgen/style/lucid.json | 10 + soleprint/atlas2/docgen/style/tokens.py | 210 +++++ soleprint/station/tools/docgen/.gitignore | 4 - soleprint/station/tools/docgen/Makefile | 88 -- soleprint/station/tools/docgen/README.md | 256 ------ soleprint/station/tools/docgen/__init__.py | 60 -- soleprint/station/tools/docgen/demo.py | 40 - soleprint/station/tools/docgen/dot.py | 232 ----- .../station/tools/docgen/export/__init__.py | 6 - soleprint/station/tools/docgen/export/doc.py | 71 -- .../station/tools/docgen/export/notebook.py | 102 --- .../tools/docgen/export/specs/__init__.py | 1 - .../tools/docgen/export/specs/vanilla.py | 292 ------- .../docgen/out/system_overview.default.dot | 107 +++ .../docgen/out/system_overview.default.svg | 209 +++++ .../docgen/out/system_overview.lucid.dot | 107 +++ .../docgen/out/system_overview.lucid.svg | 209 +++++ soleprint/station/tools/docgen/profile.py | 176 ---- .../tools/docgen/profiles/default.json | 56 -- .../station/tools/docgen/profiles/lucid.json | 56 -- soleprint/station/tools/docgen/render.py | 107 --- soleprint/station/tools/docgen/selftest.py | 585 ------------- soleprint/station/tools/docgen/shape.py | 131 --- .../station/tools/docgen/style/__init__.py | 7 - .../station/tools/docgen/style/tokens.py | 161 ---- soleprint/station/tools/graphgen/README.md | 87 +- soleprint/station/tools/graphgen/examples.py | 71 -- soleprint/station/tools/graphgen/graph.py | 185 ---- soleprint/station/tools/graphgen/selftest.py | 131 --- 51 files changed, 4504 insertions(+), 2906 deletions(-) create mode 100644 soleprint/atlas2/docgen/docs/docs.css create mode 100644 soleprint/atlas2/docgen/docs/img/architecture.svg create mode 100644 soleprint/atlas2/docgen/docs/img/erd.svg create mode 100644 soleprint/atlas2/docgen/docs/img/minimap.svg create mode 100644 soleprint/atlas2/docgen/docs/index.html create mode 100644 soleprint/atlas2/docgen/docs/viewer.html create mode 100644 soleprint/atlas2/docgen/emitters/cli_explore.py create mode 100644 soleprint/atlas2/docgen/emitters/cli_minimap.py create mode 100644 soleprint/atlas2/docgen/emitters/explore.py create mode 100644 soleprint/atlas2/docgen/emitters/minimap.py create mode 100644 soleprint/atlas2/docgen/explore_test.js create mode 100644 soleprint/atlas2/docgen/extractors/code.py create mode 100644 soleprint/atlas2/docgen/extractors/code_main.py rename soleprint/{station/tools => atlas2}/docgen/style/extract.py (100%) create mode 100644 soleprint/atlas2/docgen/style/tokens.py delete mode 100644 soleprint/station/tools/docgen/.gitignore delete mode 100644 soleprint/station/tools/docgen/Makefile delete mode 100644 soleprint/station/tools/docgen/README.md delete mode 100644 soleprint/station/tools/docgen/__init__.py delete mode 100644 soleprint/station/tools/docgen/demo.py delete mode 100644 soleprint/station/tools/docgen/dot.py delete mode 100644 soleprint/station/tools/docgen/export/__init__.py delete mode 100644 soleprint/station/tools/docgen/export/doc.py delete mode 100644 soleprint/station/tools/docgen/export/notebook.py delete mode 100644 soleprint/station/tools/docgen/export/specs/__init__.py delete mode 100644 soleprint/station/tools/docgen/export/specs/vanilla.py create mode 100644 soleprint/station/tools/docgen/out/system_overview.default.dot create mode 100644 soleprint/station/tools/docgen/out/system_overview.default.svg create mode 100644 soleprint/station/tools/docgen/out/system_overview.lucid.dot create mode 100644 soleprint/station/tools/docgen/out/system_overview.lucid.svg delete mode 100644 soleprint/station/tools/docgen/profile.py delete mode 100644 soleprint/station/tools/docgen/profiles/default.json delete mode 100644 soleprint/station/tools/docgen/profiles/lucid.json delete mode 100644 soleprint/station/tools/docgen/render.py delete mode 100644 soleprint/station/tools/docgen/selftest.py delete mode 100644 soleprint/station/tools/docgen/shape.py delete mode 100644 soleprint/station/tools/docgen/style/__init__.py delete mode 100644 soleprint/station/tools/docgen/style/tokens.py delete mode 100644 soleprint/station/tools/graphgen/examples.py delete mode 100644 soleprint/station/tools/graphgen/graph.py delete mode 100644 soleprint/station/tools/graphgen/selftest.py diff --git a/soleprint/atlas2/docgen/Makefile b/soleprint/atlas2/docgen/Makefile index c1d76ed..acb3c21 100644 --- a/soleprint/atlas2/docgen/Makefile +++ b/soleprint/atlas2/docgen/Makefile @@ -28,10 +28,11 @@ SCHEMA ?= STYLE ?= lucid THEME ?= DEPTH ?= 2 +SCALE ?= 0.55 THEME_ARG := $(if $(THEME),--theme $(THEME)) -.PHONY: help check ir db graph index site view self doctor clean +.PHONY: help check ir db code graph index site minimap explore docs view self doctor clean help: ## List every target @echo "docgen — static analysis of a tree, and the artifacts that fall out of it" @@ -70,6 +71,35 @@ site: view ## OUT/view.json -> a self-contained docs site in OUT/site --style $(STYLE) $(THEME_ARG) @echo " open $(OUT)/site/index.html" +docs: ## Regenerate the figures in docs/ — docgen documented by docgen + @mkdir -p docs/img + @$(RUN) $(PKG).extractors.python --root $(HERE) -o /tmp/$(PKG)-docs.json >/dev/null + @$(RUN) $(PKG).ops /tmp/$(PKG)-docs.json --overview -o /tmp/$(PKG)-docs-view.json >/dev/null + @$(RUN) $(PKG).emitters dot /tmp/$(PKG)-docs-view.json -o docs/img/architecture.svg -q + @$(RUN) $(PKG).emitters minimap /tmp/$(PKG)-docs.json -o docs/img/minimap.svg --scale 0.5 --width 860 + @$(RUN) $(PKG).emitters erd $(OUT)/ir.json -o docs/img/erd.svg 2>/dev/null \ + || echo " (erd figure kept — needs a schema IR at $(OUT)/ir.json to refresh)" + @PYTHONPATH=$(PARENT) $(PY) -c "from $(PKG).emitters.site import VIEWER, _slots, _fill; \ +from $(PKG).style import Style; import pathlib; \ +pathlib.Path('$(HERE)/docs/viewer.html').write_text( \ + _fill(VIEWER.replace('__TITLE__', 'docgen docs'), _slots(Style.load('lucid'))))" + @echo " open $(HERE)/docs/index.html" + +explore: ## OUT/ir.json -> OUT/explore/ — navigate on one side, explore on the other + @$(RUN) $(PKG).emitters explore $(OUT)/ir.json -o $(OUT)/explore \ + --style $(STYLE) $(THEME_ARG) --scale $(SCALE) + @echo " open $(OUT)/explore/explore.html" + +minimap: ## OUT/ir.json -> OUT/minimap.svg — what is where, read from the colours + @$(RUN) $(PKG).emitters minimap $(OUT)/ir.json -o $(OUT)/minimap.svg \ + --style $(STYLE) $(THEME_ARG) --scale $(SCALE) + +code: ## Extract C#/TypeScript from SRC (needs tree-sitter) + @test -n "$(SRC)" || { echo "Error: set SRC=/path/to/tree" >&2; exit 1; } + @mkdir -p $(OUT) + @$(RUN) $(PKG).extractors code --root "$(SRC)" -o $(OUT)/ir.json + @$(RUN) $(PKG).ir $(OUT)/ir.json + index: ## OUT/ir.json -> OUT/index.md and OUT/sidebar.json @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/index.md @$(RUN) $(PKG).emitters index $(OUT)/ir.json -o $(OUT)/sidebar.json @@ -79,12 +109,16 @@ self: ## Run the whole pipeline over soleprint itself — the honest end-to-end @$(MAKE) --no-print-directory index OUT=$(OUT) @$(MAKE) --no-print-directory graph OUT=$(OUT) @$(MAKE) --no-print-directory site OUT=$(OUT) + @$(MAKE) --no-print-directory minimap OUT=$(OUT) + @$(MAKE) --no-print-directory explore OUT=$(OUT) @echo @echo " Read $(OUT)/index.md, or open $(OUT)/site/index.html" doctor: ## Report whether this machine can run it @printf 'python : '; $(PY) --version 2>&1 || echo MISSING @printf 'dot : '; (dot -V 2>&1) || echo 'MISSING — sudo apt install graphviz (only to render)' + @printf 'tree-sit : '; $(PY) -c 'import tree_sitter, tree_sitter_c_sharp, tree_sitter_typescript; print("ok — C# and TypeScript available")' 2>/dev/null || echo 'absent — Python only. pip install tree_sitter tree_sitter_c_sharp tree_sitter_typescript' + @printf 'networkx : '; $(PY) -c 'import networkx; print(networkx.__version__ + " — for lab/ experiments")' 2>/dev/null || echo 'absent — only used in lab/' @printf 'package : %s (from %s)\n' '$(PKG)' '$(PARENT)' @printf 'styles : '; $(RUN) $(PKG).style 2>/dev/null \ || $(RUN) $(PKG) 2>/dev/null \ diff --git a/soleprint/atlas2/docgen/docs/docs.css b/soleprint/atlas2/docgen/docs/docs.css new file mode 100644 index 0000000..a86d7cd --- /dev/null +++ b/soleprint/atlas2/docgen/docs/docs.css @@ -0,0 +1,135 @@ +/* docgen docs. The layout five demos under semester/ converged on: a sticky + sidebar beside a bounded content column. Colours are tokens.css values, the + same ones the diagrams are drawn in. */ +:root { + --bg: #0a0a0a; + --surface: #141414; + --surface-2: #1a1a1a; + --border: #333333; + --border-strong: #4a4a4a; + --text: #e5e5e5; + --muted: #a3a3a3; + --dim: #666666; + --accent: #d4a574; + --station: #1d4ed8; + --atlas: #15803d; + --artery: #b91c1c; +} + +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + background: var(--bg); + color: var(--text); + font-family: "Segoe UI", Inter, system-ui, -apple-system, Arial, sans-serif; + font-size: 14px; + line-height: 1.72; +} + +.layout { display: flex; min-height: 100vh; } + +/* ── sidebar ─────────────────────────────────────────────────────────── */ +.sidebar { + width: 232px; flex-shrink: 0; + background: var(--surface); + border-right: 1px solid var(--border); + position: sticky; top: 0; height: 100vh; overflow-y: auto; + padding: 1.4rem 0 3rem; + scrollbar-width: none; +} +.sidebar::-webkit-scrollbar { display: none; } + +.sidebar-header { padding: 0 1.15rem 1rem; border-bottom: 1px solid var(--border); } +.sidebar-header b { display: block; font-size: 14px; color: var(--text); } +.sidebar-header small { display: block; color: var(--dim); font-size: 11px; margin-top: 3px; } + +.sidebar nav { padding-top: .75rem; } +.sidebar .group { + color: var(--dim); font-size: 9.5px; text-transform: uppercase; + letter-spacing: .07em; padding: 1rem 1.15rem .3rem; +} +.sidebar a { + display: block; padding: 3px 1.15rem; + color: var(--muted); text-decoration: none; font-size: 12.5px; + border-left: 2px solid transparent; +} +.sidebar a:hover { color: var(--text); background: var(--surface-2); } +.sidebar a.active { color: var(--accent); border-left-color: var(--accent); } + +/* ── content ─────────────────────────────────────────────────────────── */ +.content { flex: 1; min-width: 0; max-width: 820px; padding: 2.5rem 3.25rem 6rem; } + +h1 { font-size: 26px; letter-spacing: -.01em; margin-bottom: .35rem; } +.lede { color: var(--muted); font-size: 15px; margin-bottom: 2.5rem; } + +h2 { + font-size: 18px; margin: 3rem 0 .9rem; padding-top: 1.6rem; + border-top: 1px solid var(--border); +} +h2:first-of-type { border-top: none; padding-top: 0; } +h3 { font-size: 14px; margin: 1.9rem 0 .5rem; color: var(--text); } +h4 { font-size: 12.5px; margin: 1.3rem 0 .35rem; color: var(--muted); font-weight: 600; } + +p { margin-bottom: .95rem; color: var(--muted); } +p strong, li strong { color: var(--text); font-weight: 600; } +em { color: var(--text); font-style: italic; } + +ul, ol { margin: 0 0 1rem 1.15rem; color: var(--muted); } +li { margin-bottom: .3rem; } + +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } + +code { + font-family: ui-monospace, "Cascadia Mono", Consolas, "SF Mono", monospace; + font-size: 12px; background: var(--surface); color: var(--text); + padding: 1px 5px; border-radius: 3px; +} +pre { + background: var(--surface); border: 1px solid var(--border); + border-radius: 7px; padding: 13px 16px; overflow-x: auto; margin: .9rem 0 1.2rem; +} +pre code { background: none; padding: 0; font-size: 12px; line-height: 1.62; color: var(--muted); } +pre .c { color: var(--dim); } +pre .k { color: var(--accent); } + +table { border-collapse: collapse; margin: 1rem 0 1.4rem; width: 100%; font-size: 13px; } +th, td { text-align: left; padding: 7px 16px 7px 0; border-bottom: 1px solid var(--border); + color: var(--muted); vertical-align: top; } +th { color: var(--dim); font-size: 10px; text-transform: uppercase; letter-spacing: .06em; + font-weight: 600; } +td code { white-space: nowrap; } + +blockquote { + border-left: 2px solid var(--border-strong); padding: .15rem 0 .15rem 1.1rem; + margin: 1.1rem 0; color: var(--dim); font-style: italic; +} + +/* a claim worth not losing in the prose */ +.note { + border: 1px solid var(--border); border-left: 3px solid var(--accent); + background: var(--surface); border-radius: 6px; + padding: .85rem 1.1rem; margin: 1.2rem 0; font-size: 13px; +} +.note b { color: var(--accent); } +.note.warn { border-left-color: var(--artery); } +.note.warn b { color: var(--artery); } + +/* figures — inline and scaled, click for the viewer */ +figure { margin: 1.3rem 0 1.7rem; } +figure a { display: block; border: 1px solid var(--border); border-radius: 8px; + overflow: hidden; background: var(--surface-2); } +figure a:hover { border-color: var(--accent); } +figure img { display: block; width: 100%; height: auto; } +figcaption { color: var(--dim); font-size: 11px; margin-top: .45rem; } + +.pill { + display: inline-block; font-size: 10px; letter-spacing: .04em; + border: 1px solid var(--border-strong); border-radius: 20px; + padding: 1px 9px; color: var(--dim); margin-left: .5em; vertical-align: middle; +} +.pill.on { color: var(--atlas); border-color: var(--atlas); } +.pill.opt { color: var(--accent); border-color: var(--accent); } + +.cols { display: flex; gap: 2rem; flex-wrap: wrap; } +.cols > div { flex: 1 1 250px; } diff --git a/soleprint/atlas2/docgen/docs/img/architecture.svg b/soleprint/atlas2/docgen/docs/img/architecture.svg new file mode 100644 index 0000000..e4139be --- /dev/null +++ b/soleprint/atlas2/docgen/docs/img/architecture.svg @@ -0,0 +1,800 @@ + + + + + + +ir + + +cluster_docgen + +docgen + + +cluster_docgen_emitters + +emitters + + +cluster_docgen_extractors + +extractors + + +cluster_docgen_extractors_python + +python + + +cluster_docgen_ir + +ir + + +cluster_docgen_lab + +lab + + +cluster_docgen_notebook + +notebook + + +cluster_docgen_ops + +ops + + + +docgen.emitters.__main__ + + +__main__ + + + + + +docgen.emitters.auto + + +auto + + + + + +docgen.emitters.__main__->docgen.emitters.auto + + + + + +docgen.emitters.cli_dot + + +cli_dot + + + + + +docgen.emitters.__main__->docgen.emitters.cli_dot + + + + + +docgen.emitters.cli_erd + + +cli_erd + + + + + +docgen.emitters.__main__->docgen.emitters.cli_erd + + + + + +docgen.emitters.cli_explore + + +cli_explore + + + + + +docgen.emitters.__main__->docgen.emitters.cli_explore + + + + + +docgen.emitters.cli_index + + +cli_index + + + + + +docgen.emitters.__main__->docgen.emitters.cli_index + + + + + +docgen.emitters.cli_minimap + + +cli_minimap + + + + + +docgen.emitters.__main__->docgen.emitters.cli_minimap + + + + + +docgen.emitters.cli_notebook + + +cli_notebook + + + + + +docgen.emitters.__main__->docgen.emitters.cli_notebook + + + + + +docgen.emitters.cli_site + + +cli_site + + + + + +docgen.emitters.__main__->docgen.emitters.cli_site + + + + + +docgen.emitters.dot + + +dot + + + + + +docgen.emitters.auto->docgen.emitters.dot + + + + + +docgen.emitters.erd + + +erd + + + + + +docgen.emitters.auto->docgen.emitters.erd + + + + + +docgen.emitters.index + + +index + + + + + +docgen.emitters.auto->docgen.emitters.index + + + + + +docgen.ir.__main__ + + +__main__ + + + + + +docgen.emitters.auto->docgen.ir.__main__ + + + + + +docgen.ops.__main__ + + +__main__ + + + + + +docgen.emitters.auto->docgen.ops.__main__ + + + + + +docgen.style + + +style + + + + + +docgen.emitters.auto->docgen.style + + + + + +docgen.emitters.cli_dot->docgen.emitters.dot + + + + + +docgen.emitters.cli_dot->docgen.ir.__main__ + + + + + +docgen.emitters.cli_dot->docgen.ops.__main__ + + + + + +docgen.emitters.cli_dot->docgen.style + + + + + +docgen.emitters.cli_erd->docgen.emitters.erd + + + + + +docgen.emitters.cli_erd->docgen.ir.__main__ + + + + + +docgen.emitters.cli_erd->docgen.style + + + + + +docgen.emitters.explore + + +explore + + + + + +docgen.emitters.cli_explore->docgen.emitters.explore + + + + + +docgen.emitters.cli_explore->docgen.ir.__main__ + + + + + +docgen.emitters.cli_explore->docgen.style + + + + + +docgen.emitters.cli_index->docgen.emitters.index + + + + + +docgen.emitters.cli_index->docgen.ir.__main__ + + + + + +docgen.emitters.minimap + + +minimap + + + + + +docgen.emitters.cli_minimap->docgen.emitters.minimap + + + + + +docgen.emitters.cli_minimap->docgen.ir.__main__ + + + + + +docgen.emitters.cli_minimap->docgen.style + + + + + +docgen.emitters.notebook + + +notebook + + + + + +docgen.emitters.cli_notebook->docgen.emitters.notebook + + + + + +docgen.emitters.cli_notebook->docgen.ir.__main__ + + + + + +docgen.notebook.spec + + +spec + + + + + +docgen.emitters.cli_notebook->docgen.notebook.spec + + + + + +docgen.emitters.cli_site->docgen.emitters.dot + + + + + +docgen.emitters.cli_site->docgen.emitters.erd + + + + + +docgen.emitters.site + + +site + + + + + +docgen.emitters.cli_site->docgen.emitters.site + + + + + +docgen.emitters.cli_site->docgen.ir.__main__ + + + + + +docgen.emitters.cli_site->docgen.ops.__main__ + + + + + +docgen.emitters.cli_site->docgen.style + + + + + +docgen.emitters.explore->docgen.emitters.dot + + + + + +docgen.emitters.explore->docgen.emitters.erd + + + + + +docgen.emitters.explore->docgen.emitters.minimap + + + + + +docgen.emitters.explore->docgen.ops.__main__ + + + + + +docgen.emitters.site->docgen.emitters.index + + + + + +docgen.extractors.__main__ + + +__main__ + + + + + +docgen.extractors.code_main + + +code_main + + + + + +docgen.extractors.__main__->docgen.extractors.code_main + + + + + +docgen.extractors.db_main + + +db_main + + + + + +docgen.extractors.__main__->docgen.extractors.db_main + + + + + +docgen.extractors.openapi_main + + +openapi_main + + + + + +docgen.extractors.__main__->docgen.extractors.openapi_main + + + + + +docgen.extractors.usage_main + + +usage_main + + + + + +docgen.extractors.__main__->docgen.extractors.usage_main + + + + + +docgen.extractors.code + + +code + + + + + +docgen.extractors.code->docgen.ir.__main__ + + + + + +tree_sitter + +tree_sitter + + + +docgen.extractors.code->tree_sitter + + + + + +docgen.extractors.code_main->docgen.extractors.code + + + + + +docgen.extractors.db + + +db + + + + + +docgen.extractors.db->docgen.ir.__main__ + + + + + +docgen.extractors.db_main->docgen.extractors.db + + + + + +docgen.extractors.openapi + + +openapi + + + + + +docgen.extractors.openapi->docgen.ir.__main__ + + + + + +modelgen.loader.extract.openapi + +openapi + + + +docgen.extractors.openapi->modelgen.loader.extract.openapi + + + + + +docgen.extractors.openapi_main->docgen.extractors.openapi + + + + + +docgen.extractors.python.__main__ + + +__main__ + + + + + +docgen.extractors.python.collect + + +collect + + + + + +docgen.extractors.python.__main__->docgen.extractors.python.collect + + + + + +docgen.extractors.python.resolve + + +resolve + + + + + +docgen.extractors.python.__main__->docgen.extractors.python.resolve + + + + + +docgen.extractors.python.resolve->docgen.extractors.python.collect + + + + + +docgen.extractors.python.resolve->docgen.ir.__main__ + + + + + +docgen.extractors.usage + + +usage + + + + + +docgen.extractors.usage->docgen.ir.__main__ + + + + + +docgen.extractors.usage_main->docgen.extractors.usage + + + + + +docgen.ir.model + + +model + + + + + +docgen.ir.__main__->docgen.ir.model + + + + + +docgen.ir.validate + + +validate + + + + + +docgen.ir.__main__->docgen.ir.validate + + + + + +docgen.ir.__main__->docgen.ir.validate + + + + + +docgen.ir.validate->docgen.ir.model + + + + + +docgen.lab.pg_probe + + +pg_probe + + + + + +docgen.ops.__main__->docgen.ir.__main__ + + + + + +docgen.ops.filter + + +filter + + + + + +docgen.ops.__main__->docgen.ops.filter + + + + + +docgen.selftest + + +selftest + + + + + diff --git a/soleprint/atlas2/docgen/docs/img/erd.svg b/soleprint/atlas2/docgen/docs/img/erd.svg new file mode 100644 index 0000000..b4d2ff7 --- /dev/null +++ b/soleprint/atlas2/docgen/docs/img/erd.svg @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + +Customer +A fixture customer (obviously… +created_at +datetime + +email +str + +PK +id +int + +name +str + + + + + +Invoice +An invoice issued to a custom… +FK +customer_id +Customer + +due_at +datetime + +PK +id +int + +issued_at +datetime + +number +str + +status +str + + + + + +LineItem +A single billable line on an … +description +str + +PK +id +int + +FK +invoice_id +Invoice + +quantity +int + +unit_price +Decimal + + + + + +Payment +A payment recorded against an… +amount +Decimal + +PK +id +int + +FK +invoice_id +Invoice + +method +str + +paid_at +datetime + + diff --git a/soleprint/atlas2/docgen/docs/img/minimap.svg b/soleprint/atlas2/docgen/docs/img/minimap.svg new file mode 100644 index 0000000..78880f0 --- /dev/null +++ b/soleprint/atlas2/docgen/docs/img/minimap.svg @@ -0,0 +1,298 @@ + + + +docgen.emitters.site — 408 lines +_slots — function, 13 lines +_fill — function, 4 lines +_sidebar — function, 17 lines +_sections — function, 15 lines +emit — function, 86 lines +write — function, 9 lines +docgen.emitters.explore — 316 lines +_is_schema — function, 4 lines +_neighbourhood_svgs — function, 48 lines +render_one — function, 2 lines +render_one — function, 2 lines +_facts — function, 27 lines +emit — function, 172 lines +write — function, 17 lines +docgen.emitters.dot — 292 lines +RenderError — class, 2 lines +_esc — function, 2 lines +_attrs — function, 3 lines +_style_words — function, 9 lines +_node_attrs — function, 24 lines +emit — function, 105 lines +write — function, 29 lines +_within — function, 8 lines +_endpoints — function, 41 lines +walk — function, 4 lines +collapsed — function, 2 lines +first_leaf — function, 4 lines +_safe — function, 2 lines +_q — function, 2 lines +have_graphviz — function, 2 lines +render — function, 29 lines +docgen.emitters.notebook — 279 lines +_cell — function, 13 lines +_example — function, 26 lines +_params_cell — function, 18 lines +_call_cell — function, 40 lines +_call_md — function, 26 lines +build — function, 52 lines +emit — function, 3 lines +write — function, 5 lines +docgen.emitters.minimap — 278 lines +_files — function, 57 lines +declared — function, 18 lines +build — function, 11 lines +_bands — function, 7 lines +_blocks — function, 19 lines +emit — function, 120 lines +marks_to_labels — function, 3 lines +docgen.emitters.erd — 260 lines +_truncate — function, 3 lines +_tables — function, 20 lines +_card_height — function, 3 lines +layout — function, 20 lines +_field_y — function, 3 lines +emit — function, 146 lines +docgen.emitters.index — 164 lines +_tree — function, 8 lines +_anchor — function, 5 lines +to_markdown — function, 84 lines +walk — function, 27 lines +to_sidebar — function, 26 lines +build — function, 13 lines +docgen.emitters.cli_dot — 87 lines +main — function, 73 lines +docgen.emitters.auto — 80 lines +main — function, 56 lines +docgen.emitters.cli_notebook — 74 lines +main — function, 57 lines +docgen.emitters.cli_site — 74 lines +main — function, 57 lines +docgen.emitters.cli_erd — 51 lines +main — function, 38 lines +docgen.emitters.cli_minimap — 51 lines +main — function, 37 lines +docgen.emitters.cli_explore — 48 lines +main — function, 35 lines +docgen.emitters.cli_index — 44 lines +main — function, 32 lines +docgen.emitters.__main__ — 37 lines +main — function, 27 lines +docgen.extractors.code — 261 lines +MissingParser — class, 2 lines +_parser — function, 21 lines +_name — function, 10 lines +_walk — function, 34 lines +extract_file — function, 27 lines +_count_errors — function, 5 lines +extract — function, 54 lines +docgen.extractors.usage — 248 lines +_template — function, 27 lines +_body — function, 10 lines +_shape — function, 15 lines +_graphql — function, 11 lines +extract — function, 112 lines +docgen.extractors.db — 145 lines +from_schema_dict — function, 67 lines +_relation — function, 10 lines +_plain_type — function, 6 lines +_dedupe — function, 9 lines +extract — function, 5 lines +docgen.extractors.openapi — 124 lines +_modelgen — function, 21 lines +_type_name — function, 6 lines +extract — function, 60 lines +docgen.extractors.code_main — 40 lines +main — function, 31 lines +docgen.extractors.usage_main — 33 lines +main — function, 24 lines +docgen.extractors.db_main — 32 lines +main — function, 21 lines +docgen.extractors.openapi_main — 30 lines +main — function, 21 lines +docgen.extractors.python — 30 lines +extract — function, 7 lines +docgen.extractors.__main__ — 26 lines +main — function, 16 lines +docgen.selftest — 1333 lines +check — function, 5 lines +_err — function, 7 lines +skip — function, 3 lines +build_tree — function, 5 lines +_entry — function, 7 lines +docgen.style — 180 lines +StyleError — class, 2 lines +Style — class, 133 lines +__init__ — function, 17 lines +load — function, 12 lines +available — function, 2 lines +themes — function, 2 lines +validate — function, 32 lines +_resolve — function, 12 lines +_lookup — function, 3 lines +node — function, 2 lines +group — function, 2 lines +edge — function, 2 lines +graph — function, 2 lines +geom — function, 2 lines +slot — function, 2 lines +domain_slot — function, 13 lines +limits — function, 3 lines +docgen.ops — 28 lines +docgen.notebook — 15 lines +docgen.emitters — 12 lines +docgen.lab — 11 lines +docgen.ir — 7 lines +docgen.extractors — 2 lines +docgen.extractors.python.collect — 236 lines +Definition — class, 10 lines +Module — class, 12 lines +_Collector — class, 67 lines +__init__ — function, 3 lines +_define — function, 17 lines +visit_ClassDef — function, 5 lines +visit_FunctionDef — function, 5 lines +visit_Import — function, 8 lines +visit_ImportFrom — function, 14 lines +_first_line — function, 5 lines +_name_of — function, 15 lines +_resolve_relative — function, 16 lines +module_name — function, 26 lines +collect_file — function, 21 lines +collect — function, 13 lines +docgen.extractors.python.resolve — 163 lines +_id_for — function, 2 lines +_resolve — function, 32 lines +to_ir — function, 96 lines +_point_at — function, 9 lines +docgen.extractors.python.__main__ — 38 lines +main — function, 23 lines +docgen.ir.validate — 237 lines +IRError — class, 2 lines +_schema — function, 2 lines +_props — function, 5 lines +_fields — function, 4 lines +check — function, 106 lines +validate — function, 6 lines +check_model_matches_schema — function, 23 lines +main — function, 32 lines +docgen.ir.model — 147 lines +Meta — class, 21 lines +to_dict — function, 7 lines +Node — class, 21 lines +__post_init__ — function, 3 lines +to_dict — function, 8 lines +Edge — class, 15 lines +to_dict — function, 7 lines +Graph — class, 48 lines +node — function, 4 lines +edge — function, 4 lines +has — function, 2 lines +to_dict — function, 16 lines +from_dict — function, 6 lines +docgen.ir.__main__ — 9 lines +docgen.ops.filter — 485 lines +_rebuild — function, 57 lines +surviving_parent — function, 5 lines +lift — function, 6 lines +drop_kinds — function, 14 lines +only_kinds — function, 14 lines +drop_stdlib — function, 13 lines +drop_external — function, 3 lines +subtree — function, 13 lines +neighbourhood — function, 45 lines +collapse_to_depth — function, 18 lines +level — function, 6 lines +drop_builtins — function, 14 lines +overview — function, 35 lines +shape — function, 72 lines +rank_of — function, 9 lines +split — function, 24 lines +classify — function, 102 lines +docgen.ops.__main__ — 102 lines +main — function, 84 lines +docgen — 2 lines +docgen.lab.pg_probe — 150 lines +probe — function, 30 lines +_simplify — function, 3 lines +main — function, 25 lines +docgen.notebook.spec — 264 lines +_step — function, 4 lines +from_ir — function, 108 lines +_order — function, 5 lines +scaffold — function, 20 lines +merge — function, 58 lines +load — function, 2 lines +dump — function, 5 lines +docgen.emitters +docgen.emitters +docgen.extractors +docgen.extractors +docgen +docgen +docgen.extractors.python +docgen.ir +docgen.ir +docgen.ops +(root) +docgen.lab +docgen.notebook +site +explore +dot +notebook +minimap +erd +index +cli_dot +auto +cli_noteboo +cli_site +cli_erd +cli_minimap +cli_explore +cli_index +__main__ +code +usage +db +openapi +code_main +usage_main +db_main +openapi_mai +python +__main__ +selftest +style +ops +notebook +emitters +lab +ir +extractors +collect +resolve +__main__ +validate +model +__main__ +filter +__main__ +docgen +pg_probe +spec + +module + +class + +interface + +function +45 files · 6,933 lines · 1px ≈ 2.0 lines + diff --git a/soleprint/atlas2/docgen/docs/index.html b/soleprint/atlas2/docgen/docs/index.html new file mode 100644 index 0000000..40b01c2 --- /dev/null +++ b/soleprint/atlas2/docgen/docs/index.html @@ -0,0 +1,793 @@ + + + + + +docgen + + + +
+ + + +
+ +

docgen

+

+ Turn source artifacts — a codebase, a database, an API spec, a recording of + real traffic — into one canonical graph format, then render that format to + whatever the audience needs. The point is not the diagram. The point is the + format in the middle. +

+ + +

What it is

+ +

+ Eight demos under semester/ draw their architecture with + Graphviz. Every one of them hand-writes a .dot file with the + palette inlined and commits the .svg beside it. Three different + dark palettes between them, eight answers to the same question, and every one + of those diagrams is out of date the moment somebody moves a file. +

+

+ docgen is the one answer. It reads the source rather than being told about it, + so a diagram cannot drift from the thing it describes, and it separates + what a graph is from how it looks so one extraction feeds + a diagram, an index, a notebook and a browsable site without being redone. +

+ +
+ Nothing here writes a parser or a graph algorithm. Parsers are adopted + (ast, tree-sitter, SQLAlchemy reflection via modelgen), algorithms + are networkx's. What docgen owns is the adapters, the schema, the style tables + and the emitters — all small, and all the places where the value is that + we made the call. +
+ + +

The idea

+ +

+ N sources and M outputs need N×M converters if you join them directly, + or N+M if you put a hub in the middle. The hub is an + intermediate representation — the compiler term, and the same bargain: + both sides depend on the IR and neither on the other. +

+

+ It is lossy on purpose. It throws away every token of syntax and keeps + "a class named User inherits from Base". That is the part that is + worth versioning, worth diffing, and worth drawing. +

+

+ The practical consequence is the thing to judge it on: adding a source + costs one extractor and every emitter works on it unchanged; adding an output + costs one emitter and every extractor feeds it unchanged. When the + OpenAPI reader was written it emitted schemas using the same vocabulary the + database reader uses — and the ER diagram drew an API's data model without + anyone teaching it what an API was. +

+ + +

Five minutes

+ +

Three commands, and they compose. That is the whole interface.

+ +
# 1. read something
+python3 -m docgen.extractors.python --root ../station/tools/histgen -o ir.json
+
+# 2. narrow it to a useful view
+python3 -m docgen.ops ir.json --overview -o view.json
+
+# 3. draw whatever its structure asks for
+python3 -m docgen.emitters auto view.json -o out/
+ +

Or through the Makefile, which is a thin wrapper over exactly those:

+ +
make ir SRC=/path/to/repo OUT=out   # extract
+make explore OUT=out                # the two-pane navigator
+make site OUT=out                   # a docs site with a sidebar
+make self                           # run the whole thing over soleprint
+ +

+ Everything is offline and self-contained. No server, no CDN, no build step — + the outputs open over file://. +

+ + +

The three concerns

+ +

+ DOT collapses three separate questions into one file format, which is why a + hand-written .dot is never reusable: you cannot change the palette + without editing the structure, and you cannot change the structure without + re-deciding the layout. docgen keeps them apart. +

+ + + + + + +
concernquestionowner
structurewhat the graph isir/schema.json
meaningwhat things mean visuallystyle/*.json, keyed on kind
placementwhere things gothe emitter, and only there
+ +

+ An extractor has never heard of SVG, colours or layout. An emitter has never + heard of Python, ast or SQL. Both halves of that are checked by + parsing the source and looking at what it imports, because a rule nobody + enforces is a rule that lasts about a month. +

+ +
+ + docgen's own module structure + +
+ docgen read by docgen. extractors/ reaches only ir; + emitters/ reaches ir and style; + ir/ reaches nothing outside itself; lab/ has no + edges at all. Click to open the viewer — then click again for actual size. +
+
+ + +

The IR

+ +

+ Plain JSON. Three keys, and it has survived four domains without gaining a + fourth. +

+ +
{
+  "meta":  { "source": "python", "root": "app/", "schema_version": "1" },
+  "nodes": [ { "id": "app.models.User", "kind": "class", "label": "User",
+               "parent": "app.models",
+               "attrs": { "file": "app/models.py", "line": 12, "lines": 40 } } ],
+  "edges": [ { "source": "app.models.User", "target": "app.db.Base",
+               "kind": "inherits", "attrs": {} } ]
+}
+ + + + + + + + + + + +
fieldmeaning
idFully qualified and stable across runs. Stability is what makes + two extractions from two commits diffable; without it a diff reports + noise and nobody trusts it.
kindThe hinge of the whole system, and the only field style and + layout may read. A small closed vocabulary per domain — + module/class/function, + table/column, endpoint, + task.
parentContainment, and nothing else. A module contains a class. Relationships + are edges.
attrsAn open bag for whatever one domain cares about. + file/line/lines are what let a + box link to the line it came from, and what the minimap sizes by.
+ +
+ No visual information, ever. If a field would change between a light and + a dark theme, it does not belong in the IR. shape: "cylinder" is + not a field — it is kind: "datastore" plus a style rule, and that + is exactly what lets the same IR render in a theme that has no cylinders. The + test suite sweeps every emitted document for colour-like keys. +
+ +

Stdlib dataclasses, not Pydantic

+

+ The IR's whole value is being a plain document anything can open. A format that + needs a library installed to be read is an API, not a format. Validation is + therefore a function called at the boundary rather than a property of the type, + and it reads its field lists out of schema.json so the schema and + the dataclasses cannot drift apart. +

+
python3 -m docgen.ir ir.json
+

+ It catches what a schema cannot: an edge naming a node that does not exist, a + containment cycle, a duplicate id, and a visual field smuggled into + attrs. +

+ + +

Shape decides the drawing

+ +

+ A diagram that fights its layout engine is usually the wrong kind of + diagram. The clearest evidence: the same 24-table database rendered + 32034×136 through Graphviz — a 235:1 strip — and + 1740×1860 through the ER emitter. Not because one engine is + better, but because a schema is a set of peer entities with references, and + laying it out in dependency ranks was never its shape. +

+

+ So ops.classify() reads the structure and names the emitter, + with the reason attached — advice without a reason gets overridden the first + time it is inconvenient. +

+ + + + + + + + +
kinddrawn bywhen
erdcards in columnsentities with references
pipelineranks, left to righta chain with fan-out — an Airflow DAG, a build
layered / treeranks, top downranks genuinely suit it
sheetthe indexone level is wider than ~20 — a strip in any engine
flatthe indexmost nodes have no relationships: that is a list
+ +
$ python3 -m docgen.emitters auto view.json -o out/
+  sheet    -> index
+           109 nodes sit at one level; any layered engine draws that as a
+           strip. Split it, scope it, or read it as an index
+ +

Why twenty

+

+ Measured, one diagram per subsystem: at or under 20 nodes the output lands + around 1.6:1; at 70–106 nodes about 7:1; at 261 nodes 14:1. Aspect ratio is a + property of the graph, not of the renderer — a layered engine puts one + dependency level in one row, so the widest level is the width. +

+

+ Every Graphviz lever was tried before concluding this. ratio=compress + squashed a graph to an unreadable 1008×75; rankdir=LR merely + rotated a 14:1 into a 1:6; packing disconnected components gained nothing. + The fix was never a flag. It was to stop asking for one picture of everything — + which is what explore does. +

+ + +

Extractors

+ +

+ Deterministic parsing only. No model in the structural path. + A diagram built from an AST cannot be out of date with the code. A diagram + built from a model's reading of the code is wrong the moment the model has a + bad day — which is the problem this exists to fix. +

+ + + + + + + + + + + + + +
readerreadsgives
pythona tree of .pymodules, classes, functions; imports and inherits edges
code tree-sitterC#, TypeScript, TSXnamespaces, classes, interfaces, methods — structure only
dba graphgen-compatible schema.jsontables, columns, foreign keys
openapian OpenAPI / Swagger documentendpoints and the shapes they carry
usagea HAR recordingwhat was actually called, in what order
+ +

Two passes, because ast resolves nothing

+

+ Given class User(Base), Python's ast hands over the + literal string "Base". It has no idea that came from + from .db import Base three lines up. So pass one collects, per + module, what it defines and what it imports; pass two resolves local names to + fully qualified ids. The edge then points at app.db.Base — a real + node — rather than at a box called Base that means nothing. +

+ +
+ Unresolved names become nodes, never nothing. A third-party import or a + dynamically-built base becomes a node of kind: "external" and + keeps its edge. Dropping it would be the worse failure: the diagram + would look complete and have quietly lost a dependency. Gathered up, those + nodes are the project's real dependency surface. +
+ +

C# and TypeScript

+

+ Handled by tree-sitter, which is why generics, nested types and a brace inside + a string are non-events rather than special cases. The test suite asserts that + last one specifically, because it is exactly where a hand-rolled scanner breaks. +

+

+ This reader produces no edges. Resolving a C# + using to the thing it names is a different and much larger job, + and the consumer that needs this — the minimap — needs none of it. An + extractor that quietly produced half a dependency graph would be worse than one + producing none, because the half would look whole. +

+ + +

Usage, not just the spec

+ +

+ An OpenAPI document says what endpoints are. It does not say how to + use them — least of all when they are not RESTful, or when a GraphQL endpoint + sits alongside. So docgen also reads a HAR: the recording format that + browser devtools, mitmproxy, Charles and Insomnia all export. +

+ + + + + + + + +
what traffic knowswhat a spec cannot
the order of callsa spec is a set; usage is a sequence
which parameters are always senta spec lists twenty optional ones
which statuses really happenthe 422 everybody hits is in no document
endpoints not in the documentGraphQL operations, found by body shape and named
which id formats a route takesnumeric and uuid on one route
+ +
+ No credential and no payload value reaches the IR — only field names and + types. A HAR is full of live bearer tokens and cookies, and a generated + document gets committed. The test suite plants a token in its fixture and fails + if it appears anywhere in the output. +
+ +

+ Two limits, stated rather than glossed: path templating is a guess + (attrs.observed_paths keeps what was actually seen beside it), and + consecutive is not caused-by — the edge weight is what separates a habit from + an accident, and one recording will not tell you which. +

+ + +

Views

+ +

+ The first real diagram out of this pipeline was a 3000px-wide strip: four + modules of actual content and sixty sys/json/typing + boxes as their peers. The emitter was correct and the picture was useless. + That is a missing view, not a broken renderer — and the fix belongs to + every consumer at once, because the index, the diagram and the diff all want + the same narrowing. +

+ +
python3 -m docgen.ops ir.json --overview -o view.json
+python3 -m docgen.ops ir.json --around docgen.ir --hops 2 -o view.json
+python3 -m docgen.ops ir.json --split -o parts/
+python3 -m docgen.ops ir.json --shape          # what will this look like?
+ +

+ All of them are IR→IR, all composable, and each produces a document that + still validates. --overview is the default and dispatches on the + source: a codebase reduces to its modules and its outside dependencies, a + schema to its tables and their keys. +

+ +
+ Edges are lifted when a view collapses detail, never dropped. A class in + module A inheriting from a class in module B is a dependency of A on + B. Collapsing docgen to its packages once kept 8 of 77 edges — those pictures + were not simpler, they were wrong. Lifted edges carry a + weight saying how many they stand for. +
+ +

Depth is the tempting knob and the wrong one

+

+ A directory without an __init__.py is not a package, so its + modules have no parent and sit at depth 0. soleprint has 173 such roots, + and a depth-2 cut still held 566 functions and 142 classes. Selecting by + kind does not care how the directories happen to be arranged. +

+ + +

Emitters

+ + + + + + + + + + + + +
emitteroutputaudience
indexmarkdown, sidebar JSONanyone — no graph literacy required
dotDOT → Graphviz → SVGdependency structure
erdSVG, written directlya schema, as cards
minimapSVG, written directlywhat is where, at a glance
notebook.ipynba runnable walkthrough
sitea static docs sitereading
explorea two-pane navigatorfinding your way
autowhichever of the above fitsnot having to choose
+ +

+ The non-visual ones matter most for reach. A sorted, described + list of what exists is readable by someone who will never open a diagram, and + it also reports the dependency surface and any file that failed to parse. + It is built second, not last — it is what proves the IR is not secretly + diagram-shaped. +

+ +

ERD — and where the layout came from

+

+ Not invented here. station/tools/graphgen/templates/index.html, + the Supabase-style schema explorer already in this repo, had solved it: +

+
const cols = Math.max(2, Math.ceil(Math.sqrt(sorted.length * 1.2)));
+

+ Columns from the square root of the table count. The aspect + ratio is chosen rather than emergent, so the result stays near-square + at 4 tables or 400. That is the one thing a rank-based engine cannot offer. + Three more things it gets right: a table is a card with its columns; + an edge leaves the column holding the key and lands on the target's primary + key; and the geometry is computed rather than measured, so it renders + identically on any machine. +

+ +
+ + an entity-relationship diagram + +
A schema from the sample room. Same emitter, same style file as + every other diagram here.
+
+ +

Minimap

+

+ Sublime's minimap shrinks the characters. This draws the + structure at full scale: one file is a column, one line is a fixed + number of pixels, every construct a block sized by its span and coloured by + what it is. No text inside a block — the shape is the message. +

+

+ The claim is that the pattern comes from the colours alone, so nesting is drawn + by inset rather than by hue. On soleprint you can see that modelgen is + class-based, histgen is function-based and tester is mixed, without reading a + line. +

+ +
+ + a structural minimap of docgen + +
docgen's own files. Blue class, amber interface, green function, + dark for everything that is not a declaration — imports, constants, prose.
+
+ + +

Explore

+ +

+ The minimap on its own shows shape and no meaning: a block says "a 30-line + class", not which class or what it touches. So it is not the artifact. + It is the selector. +

+ +
make explore OUT=out     # then open out/explore/explore.html
+ +
+
+

Left — navigate

+

The whole thing at once. Scan by colour, click a block.

+
+
+

Right — explore

+

What that is, what it reaches, what reaches it, and the neighbourhood + drawn small enough to read. Every neighbour is a link, so you walk + outward from wherever you started.

+
+
+ +
+ This is what retires the 14:1 sheet. The whole graph is never drawn. The + overview pane carries the overview, and only the neighbourhood of a selection + is rendered — a handful of nodes, which lays out fine every time. Overview and + detail stop competing for one picture. +
+ +

+ The same split applies to a database: every table at once with no column + detail, then click one to get its columns plus the tables its keys reach. + The two differ exactly where they should — a module's neighbourhood + deliberately leaves its contents out, because those are the hundred functions + that made the sheet unreadable, while a table's brings them in, because a table + without its columns is not a table. +

+ +

The selection basket

+

+ Shift-click accumulates blocks. The basket is a copyable list of paths with a + line count — enough to hand to distill, and enough to see that the + selection got too big before spending the context on it. Navigating a + tree quickly in order to decide what to feed a model is a real use, and this is + the part that serves it. +

+ + +

Notebooks

+ +

+ A notebook is normally a source file somebody confects by hand: prose, code and + stored output braided together, diffing badly, drifting from whatever it + documents the moment either moves, with no way to tell by looking. +

+

+ Here a notebook is a build artifact. The source is the OpenAPI + document — the same file the server is built from — and the notebook is + regenerated from it. Nobody edits the .ipynb, the same way nobody + edits a .o. "Is this document current" stops being a question + about somebody's diligence and becomes a question about whether the build ran. +

+ +
+ This is the disagreement with jupytext. Jupytext fixes the diffing — + it makes a notebook editable as text — and leaves the actual problem: you still + hand-author it, so it still rots. +
+ +

Generated base, hand-written overlay

+

+ Generation alone gives a document that is never stale and never says anything a + parser could not work out. Hand-authoring alone gives insight and a document + that rots. Two files is the only arrangement that gets both: +

+ +
IR ──► spec ──(+ overlay)──► merged spec ──► .ipynb
+   generated   hand-written       merged       emitted
+ +

+ The spec is an ordered list of steps with no Jupyter in it — a Swagger + for notebooks, readable and diffable. The overlay is the only file + anyone edits, and it is re-applied on every build. It can + annotate, replace, insert, + drop and order. +

+

+ replace is the one that matters. It is how real usage gets into a + document that a spec could not describe — the call that is always made with + status=available, the GraphQL endpoint that is not in the OpenAPI + file at all — and it keeps working unchanged once a usage recording supplies + the same facts automatically. +

+ +
+ Three properties hold it together: regenerating re-applies the overlay + byte-for-byte; when the base moves underneath it the mismatch is + reported, never silently dropped; and extraction works with the overlay + absent — it is an addition, never a dependency. +
+ +
python3 -m docgen.emitters notebook ir.json --scaffold overlay.json
+python3 -m docgen.emitters notebook ir.json --overlay overlay.json -o walkthrough.ipynb
+ + +

Style & colour

+ +

+ A style rule names a slot, never a colour. + "border": "atlas" is the rule; a theme binds atlas to + #43A047 in print and #15803d on the docs site. +

+

+ That indirection is the whole point. common/theme/tokens.css, + docs/graphs/themes/*.gvpr and style/lucid.json use + the same slot names, so a diagram and the page around it match by construction + — which is the rule docs/graphs/README.md already states. The dark + theme's artery, atlas and station slots + are exactly the --system-accent values the three system pages set, + and the test suite fails if they drift apart. +

+

+ Dark is the default, because a generated diagram lands in a dark docs page far + more often than in a document. --theme lucid gives the print + palette — and gives it to the page as well as the diagram, since both + are baked from the same slots. +

+ +
+ An unknown kind falls back to default rather than + crashing. That matters more than it sounds: a new extractor with a new + vocabulary renders plainly and legibly on day one, instead of requiring + somebody to write a style file before they can see anything. +
+ +

Where DOT stops

+

+ The emitter writes what DOT expresses natively and stops at the boundary rather + than growing machinery. The limits are recorded in the style file itself: a + cluster has a label and a fill but not a header bar; + stroke-dasharray is not parameterised, so 4,4 and + 5,5 collapse; rounded is binary, so 4px and 6px are + identical. Those mark where a richer emitter would begin — and the style file + carries the full specification regardless, so that emitter needs no + re-authoring. +

+

+ One limit was worth solving: DOT cannot use a cluster as an edge + endpoint, so every module-to-module import silently vanished. The native answer + is compound=true with lhead/ltail — draw + between a representative leaf and clip the line at the cluster border. +

+ + +

Commands

+ +

Make

+ + + + + + + + + + + + + + +
targetdoes
make checkthe whole test suite, offline, nothing installed
make doctorwhat this machine has and what it is missing
make ir SRC=…extract Python into OUT/ir.json
make code SRC=…extract C#/TypeScript tree-sitter
make db SCHEMA=…extract a database schema
make viewthe default view for that source type
make graphdraw whatever the structure asks for
make indexmarkdown index and sidebar JSON
make minimapwhat is where, read from the colours
make explorethe two-pane navigator
make sitea self-contained docs site
make selfthe whole pipeline over soleprint itself
+ +

+ Variables: SRC, OUT, SCHEMA, + STYLE, THEME, SCALE, DEPTH, + PY. The Makefile derives its own package name from where it sits, + so the folder can be copied anywhere and renamed and still work. +

+ +

Modules

+
python3 -m docgen.extractors.python --root SRC -o ir.json
+python3 -m docgen.extractors code    --root SRC -o ir.json
+python3 -m docgen.extractors db      --schema schema.json -o ir.json
+python3 -m docgen.extractors openapi --spec spec.yaml -o ir.json
+python3 -m docgen.extractors usage   --har session.har -o ir.json
+
+python3 -m docgen.ir ir.json                      # validate
+python3 -m docgen.ops ir.json --overview -o view.json
+
+python3 -m docgen.emitters auto     view.json -o out/
+python3 -m docgen.emitters index    ir.json -o index.md
+python3 -m docgen.emitters dot      view.json -o graph.svg --theme lucid
+python3 -m docgen.emitters erd      ir.json -o schema.svg
+python3 -m docgen.emitters minimap  ir.json -o map.svg --scale 0.5
+python3 -m docgen.emitters notebook ir.json -o book.ipynb --overlay overlay.json
+python3 -m docgen.emitters site     view.json -o site/
+python3 -m docgen.emitters explore  ir.json -o explore/
+ + +

Dependencies

+ +

+ The core is standard library only. Everything else is optional + and reported by make doctor; when something is missing you lose + exactly one capability and get told what to install. +

+ + + + + + + + + + + + + +
needsforwithout it
graphviz (binary)rendering DOT to SVGERD, minimap, index and notebooks still work
tree_sitter + grammarsC#, TypeScript, TSXPython only
networkxlab/ experimentsnothing — nothing depends on it yet
nodetesting the browser pagesthose checks skip
psqlthe lab/ schema probeuse modelgen's from-db instead
+ +
+ lab/ is where a dependency gets tried before anything depends + on it. Nothing in ir/, extractors/, + ops/ or emitters/ may import from it. When an + experiment earns its place it graduates into ops/ behind an + IR→IR signature, and then the dependency is declared. +
+ + +

Testing

+ +
make check      # 191 checks, offline, no network
+ +

+ Four of those are the design rather than regressions, and they are the + ones to keep if anything is ever cut: +

+
    +
  • No visual field reaches the IR — extractors cannot decide appearance.
    • +
  • No emitter reads a source file — the layering, checked from the other side by parsing imports.
    • +
  • Style names slots, not colours — one colour language rather than three.
    • +
  • Ids are stable across runs — without it, diffing is noise.
    • +
+ +
+ Golden tests go on the IR, never on the SVG. Graphviz measures label + text with the host's fonts to size nodes, so identical input produces different + geometry on a machine with different fontconfig. The IR is deterministic; the + SVG is not. Pinning the wrong one gives a suite that fails on somebody else's + laptop for no reason anyone can act on. +
+ +

+ The browser pages are JavaScript, so they are tested as JavaScript: a stub DOM + under node drives the viewer's zoom and 1:1 toggle, and the + explorer's select-and-walk. Both skip cleanly where node is absent. +

+

+ Self-hosting is the honest end-to-end check, and it is where the real bugs came + from — two name-resolution faults and a duplicate-id crash that no fixture had + reached. make self runs the whole pipeline over soleprint; if the + index does not read like the system, something is wrong. +

+ + +

Limits & non-goals

+ +

Things deliberately not done, with the reason, so they are not re-litigated:

+ +
    +
  • No layout engine. No positioning, no neato -n2, no ELK. + Aspect ratio was solved by choosing the right emitter and by not drawing + everything at once.
    • +
  • No calls edges. Resolving self.foo() needs + type inference, and a call graph that is quietly 60% right is worse than + none because it reads as authoritative.
    • +
  • No edges from the C#/TypeScript reader. Structure only — half a + dependency graph would look whole.
    • +
  • No model in the structural path. Annotation — summarising a module, + naming a cluster — is a later layer, cached to its own file keyed by node + id, merged at emit time, and extraction must work with it absent.
    • +
  • No configuration knobs until two real consumers disagree.
    • +
+ +

Known gaps, stated plainly:

+
    +
  • The C# reader is verified against a written fixture, not a real + repository. That is the next check that matters.
    • +
  • The diff emitter is not built. Stable ids exist to make it possible, and + two IRs from two commits is the cheapest useful thing left.
    • +
  • Minimap blocks carry no names. Rendering them legible only at 1:1, through + the viewer, is the other half of reading a file without reading every line.
    • +
+ +
+
+ + + + diff --git a/soleprint/atlas2/docgen/docs/viewer.html b/soleprint/atlas2/docgen/docs/viewer.html new file mode 100644 index 0000000..01a1e0f --- /dev/null +++ b/soleprint/atlas2/docgen/docs/viewer.html @@ -0,0 +1,119 @@ + + + + + +docgen docs + + + +
+← docs +
100%fit· click 1:1 · drag · wheel
+ + + diff --git a/soleprint/atlas2/docgen/emitters/__main__.py b/soleprint/atlas2/docgen/emitters/__main__.py index 35fd2dc..efc9246 100644 --- a/soleprint/atlas2/docgen/emitters/__main__.py +++ b/soleprint/atlas2/docgen/emitters/__main__.py @@ -6,7 +6,7 @@ import sys def main(argv=None): argv = sys.argv[1:] if argv is None else argv if not argv: - print("usage: python3 -m docgen.emitters [-o OUT] [--style NAME] [--theme NAME]", + print("usage: python3 -m docgen.emitters [-o OUT] [--style NAME] [--theme NAME]", file=sys.stderr) return 2 name, rest = argv[0], argv[1:] @@ -22,8 +22,12 @@ def main(argv=None): from .cli_notebook import main as run elif name == "site": from .cli_site import main as run + elif name == "minimap": + from .cli_minimap import main as run + elif name == "explore": + from .cli_explore import main as run else: - print(f"Error: no emitter {name!r} — have: auto, dot, index, erd, notebook, site", file=sys.stderr) + print(f"Error: no emitter {name!r} — have: auto, dot, index, erd, notebook, site, minimap, explore", file=sys.stderr) return 1 return run(rest) diff --git a/soleprint/atlas2/docgen/emitters/cli_explore.py b/soleprint/atlas2/docgen/emitters/cli_explore.py new file mode 100644 index 0000000..0b2d6f4 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_explore.py @@ -0,0 +1,47 @@ +""" python3 -m docgen.emitters explore -o DIR [--scale 0.55] [--hops 1]""" + +import argparse +import json +import sys +from pathlib import Path + +from ..ir import check +from ..style import Style, StyleError +from .explore import write + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters explore") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path, required=True, help="Directory.") + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None) + p.add_argument("--scale", type=float, default=0.55) + p.add_argument("--width", type=int, default=1100) + p.add_argument("--hops", type=int, default=1) + p.add_argument("--title", default="") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)}):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + try: + style = Style.load(args.style, theme=args.theme) + path = write(data, style, args.output, scale=args.scale, width=args.width, + hops=args.hops, title=args.title) + except (StyleError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + graphs = len(list((args.output / "graphs").glob("*.svg"))) if (args.output / "graphs").exists() else 0 + print(f" explore {path} {graphs} neighbourhood diagram(s)") + return 0 diff --git a/soleprint/atlas2/docgen/emitters/cli_minimap.py b/soleprint/atlas2/docgen/emitters/cli_minimap.py new file mode 100644 index 0000000..52cbfd5 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/cli_minimap.py @@ -0,0 +1,50 @@ +""" python3 -m docgen.emitters minimap [-o out.svg] [--scale 0.55]""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from ..ir import check +from ..style import Style, StyleError +from .minimap import emit + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.emitters minimap") + p.add_argument("ir", type=Path) + p.add_argument("--output", "-o", type=Path) + p.add_argument("--style", default="lucid") + p.add_argument("--theme", default=None) + p.add_argument("--scale", type=float, default=0.55, help="Pixels per source line.") + p.add_argument("--width", type=int, default=1180, help="Wrap a shelf past this.") + args = p.parse_args(argv) + + try: + data = json.loads(args.ir.read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: could not read {args.ir}: {e}", file=sys.stderr) + return 1 + problems = check(data) + if problems: + print(f"Error: {args.ir} is not a valid IR ({len(problems)}):", file=sys.stderr) + for pr in problems[:5]: + print(f" {pr}", file=sys.stderr) + return 1 + + try: + style = Style.load(args.style, theme=args.theme) + svg = emit(data, style, scale=args.scale, target_width=args.width) + except (StyleError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(svg) + m = re.search(r'width="(\d+)pt" height="(\d+)pt"', svg) + print(f" minimap {args.output} {m.group(1)}x{m.group(2)}" if m else "") + else: + sys.stdout.write(svg) + return 0 diff --git a/soleprint/atlas2/docgen/emitters/erd.py b/soleprint/atlas2/docgen/emitters/erd.py index 0b96d48..d736c1f 100644 --- a/soleprint/atlas2/docgen/emitters/erd.py +++ b/soleprint/atlas2/docgen/emitters/erd.py @@ -79,12 +79,12 @@ def _tables(ir: dict) -> list[dict]: return sorted(out, key=lambda t: t["id"]) -def _card_height(table: dict) -> int: +def _card_height(table: dict, columns: bool = True) -> int: header = HDR_H_DOC if table["doc"] else HDR_H - return header + len(table["fields"]) * FIELD_H + return header + (len(table["fields"]) * FIELD_H if columns else 0) -def layout(tables: list[dict], edges: list[dict]) -> dict[str, tuple[int, int]]: +def layout(tables: list[dict], edges: list[dict], columns: bool = True) -> dict[str, tuple[int, int]]: """Place cards in √n columns, referenced tables first. Sorting by "is the target of a foreign key" puts the tables everything @@ -102,7 +102,7 @@ def layout(tables: list[dict], edges: list[dict]) -> dict[str, tuple[int, int]]: for i, table in enumerate(ordered): col = i % cols pos[table["id"]] = (col * col_w + PAD, cursor[col]) - cursor[col] += _card_height(table) + ROW_GAP + cursor[col] += _card_height(table, columns) + ROW_GAP return pos @@ -111,8 +111,14 @@ def _field_y(table: dict, index: int, top: int) -> float: return top + header + (max(index, 0) + 0.5) * FIELD_H -def emit(ir: dict, style) -> str: - """IR (a db document) + Style -> SVG text.""" +def emit(ir: dict, style, *, columns: bool = True) -> str: + """IR (a db document) + Style -> SVG text. + + `columns=False` draws the header of every card and none of its contents — + the whole schema at a glance, which is what you want before you know which + table you care about. Two hundred tables with their columns is a reference; + two hundred names is a map. + """ tables = _tables(ir) if not tables: raise ValueError( @@ -121,11 +127,11 @@ def emit(ir: dict, style) -> str: ) edges = [e for e in ir["edges"] if e["kind"] in ("foreign_key", "references")] by_id = {t["id"]: t for t in tables} - pos = layout(tables, edges) + pos = layout(tables, edges, columns) s = style.slot width = max(x for x, _ in pos.values()) + CARD_W + PAD - height = max(y + _card_height(by_id[t]) for t, (_, y) in pos.items()) + PAD + height = max(y + _card_height(by_id[t], columns) for t, (_, y) in pos.items()) + PAD out = [ '', @@ -161,8 +167,12 @@ def emit(ir: dict, style) -> str: leaving_right = dx_ >= sx x1 = sx + CARD_W if leaving_right else sx x2 = dx_ if leaving_right else dx_ + CARD_W - y1 = _field_y(src, from_idx, sy_top) - y2 = _field_y(dst, to_idx, dy_top) + if columns: + y1 = _field_y(src, from_idx, sy_top) + y2 = _field_y(dst, to_idx, dy_top) + else: + y1 = sy_top + _card_height(src, False) / 2 + y2 = dy_top + _card_height(dst, False) / 2 ctrl = min(max(abs(x2 - x1) * 0.5, 50), 180) c1 = x1 + ctrl if leaving_right else x1 - ctrl @@ -178,11 +188,12 @@ def emit(ir: dict, style) -> str: for table in tables: x, y = pos[table["id"]] header = HDR_H_DOC if table["doc"] else HDR_H - h = _card_height(table) - out.append(f'') + h = _card_height(table, columns) + out.append(f'') out.append( f'' + f'fill="{s("surface-0")}" stroke="{s("border")}" stroke-width="1" ' + f'data-id="{escape(table["id"])}" data-kind="table" class="blk"/>' ) # Header band, clipped to the card's rounded top by drawing a rounded # rect and squaring its bottom with a second one. @@ -191,10 +202,11 @@ def emit(ir: dict, style) -> str: f'h {CARD_W - 2 * RADIUS} a {RADIUS},{RADIUS} 0 0 1 {RADIUS},{RADIUS} ' f'v {header - RADIUS} h {-CARD_W} z" fill="{s("surface-2")}"/>' ) - out.append( - f'' - ) + if columns: + out.append( + f'' + ) out.append( f'' @@ -207,7 +219,7 @@ def emit(ir: dict, style) -> str: f'{escape(_truncate(table["doc"], CARD_W - 24))}' ) - for i, field in enumerate(table["fields"]): + for i, field in enumerate(table["fields"] if columns else []): fy = y + header + i * FIELD_H attrs = field.get("attrs") or {} name = field.get("label") or field["id"].rsplit(".", 1)[-1] diff --git a/soleprint/atlas2/docgen/emitters/explore.py b/soleprint/atlas2/docgen/emitters/explore.py new file mode 100644 index 0000000..63d09e6 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/explore.py @@ -0,0 +1,315 @@ +""" +Two panes: a minimap to navigate by, and a detail pane to explore with. + +The minimap on its own showed shape and no meaning — a block said "a 30-line +class" and not *which* class, or what it touched. It is not the artifact. It is +the **selector**. + + left the whole tree as coloured blocks. Scan, then click. + right what that block is, what it reaches, what reaches it — and the + neighbourhood drawn, small enough to read. + +**This is also what solves the 14:1 problem.** The whole-graph diagram was +unusable because 109 nodes sat at one dependency level, and no engine draws that +well. Here the whole graph is never drawn: the minimap carries the overview, and +only the neighbourhood of a selection is rendered — a handful of nodes, which +lays out fine every time. Overview and detail stop competing for one picture. + +## Selecting for an LLM + +The other reason to navigate a tree quickly is to decide what to feed a model. +Blocks can be added to a basket, and the basket is a copyable list of file paths +plus a line count — enough to hand to `distill` or paste, and enough to see that +the selection got too big before spending the context on it. + +## Offline and static + +The neighbourhood diagrams are rendered at build time, one small SVG per module, +so the page needs no layout engine, no server and no network. Everything is +computed here and read there. +""" + +import json +import re +from html import escape +from pathlib import Path + +MAX_NEIGHBOURS = 24 # past this, a list reads better than a picture + + +def _is_schema(ir: dict) -> bool: + return (ir.get("meta") or {}).get("source") == "db" or any( + n["kind"] == "table" for n in ir["nodes"] + ) + + +def _neighbourhood_svgs(ir: dict, style, out_dir: Path, hops: int) -> dict[str, str]: + """One small diagram per navigable thing, pre-rendered. id -> filename. + + A schema walks table by table: the selected table **with its columns**, plus + the tables its keys reach, each clickable to step further. A codebase walks + module by module through its imports. Same operation, different drawing — + which is the `classify` rule applied one level down. + """ + from ..ops import neighbourhood + + schema = _is_schema(ir) + if schema: + from .erd import emit as draw + def render_one(view): + return draw(view, style).encode() + wanted = "table" + else: + from .dot import emit as dot_emit, have_graphviz, render + if not have_graphviz(): + return {} + def render_one(view): + return render(dot_emit(view, style)) + wanted = "module" + + graphs_dir = out_dir / "graphs" + graphs_dir.mkdir(parents=True, exist_ok=True) + made: dict[str, str] = {} + + for node in ir["nodes"]: + if node["kind"] != wanted: + continue + # A table without its columns is not a table, so a schema's + # neighbourhood carries contents; a module's does not, because its + # contents are the hundred functions that made the sheet unreadable. + view = neighbourhood(ir, node["id"], hops=hops, with_contents=schema) + if len(view["nodes"]) < 2: + continue + if not schema and len(view["nodes"]) > MAX_NEIGHBOURS: + continue + if schema and sum(1 for n in view["nodes"] if n["kind"] == "table") > 12: + continue + name = re.sub(r"[^A-Za-z0-9_.-]", "_", node["id"]) + ".svg" + try: + (graphs_dir / name).write_bytes(render_one(view)) + except Exception: # noqa: BLE001 - one bad graph must not cost the page + continue + made[node["id"]] = f"graphs/{name}" + return made + + +def _facts(ir: dict) -> dict: + """Everything the detail pane needs, keyed by id.""" + out: dict[str, dict] = {} + for n in ir["nodes"]: + a = n.get("attrs") or {} + out[n["id"]] = { + "label": n.get("label") or n["id"], + "kind": n["kind"], + "parent": n.get("parent"), + "file": a.get("file"), + "line": a.get("line"), + "lines": a.get("lines"), + "doc": a.get("doc"), + "error": a.get("error"), + "out": [], + "in": [], + "members": [], + } + for n in ir["nodes"]: + if n.get("parent") in out: + out[n["parent"]]["members"].append(n["id"]) + for e in ir["edges"]: + if e["source"] in out: + out[e["source"]]["out"].append([e["target"], e["kind"]]) + if e["target"] in out: + out[e["target"]]["in"].append([e["source"], e["kind"]]) + return out + + +def emit(ir: dict, style, minimap_svg: str, graphs: dict[str, str], + title: str = "") -> str: + s = style.slot + meta = ir.get("meta", {}) + name = title or meta.get("root", "explore") + facts = _facts(ir) + + # The minimap goes inline rather than in an : a block has to be + # clickable, and an image is one opaque rectangle. + inner = minimap_svg[minimap_svg.index(" + + + + +{escape(name)} — explore + + + +
+ + +
+ + + +""" + + +def write(ir: dict, style, out_dir, *, scale: float = 0.55, width: int = 1100, + hops: int = 1, title: str = "") -> Path: + from .minimap import emit as minimap_emit + + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + if _is_schema(ir): + # The whole schema, no columns: a map rather than a reference. The + # minimap's line-span geometry means nothing for a table. + from .erd import emit as erd_emit + svg = erd_emit(ir, style, columns=False) + else: + svg = minimap_emit(ir, style, scale=scale, target_width=width) + graphs = _neighbourhood_svgs(ir, style, out_dir, hops) + path = out_dir / "explore.html" + path.write_text(emit(ir, style, svg, graphs, title=title)) + return path diff --git a/soleprint/atlas2/docgen/emitters/minimap.py b/soleprint/atlas2/docgen/emitters/minimap.py new file mode 100644 index 0000000..cb31af3 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/minimap.py @@ -0,0 +1,277 @@ +""" +IR -> a structural minimap. What is where, readable without reading. + +Sublime's minimap shrinks the *characters*. This draws the *structure* at full +scale: one file is one column, one line is a fixed number of pixels, and every +construct is a block sized by the lines it actually occupies and coloured by +what it is. Nothing is summarised and no text is rendered — the point is to see +the shape of a codebase without reading a line of it. + + a tall solid block one long class + a column of thin stripes many small functions + a wide pale gap module-level code, imports, comments + one block filling a file the 800-line thing everybody avoids + +**The pattern has to come from the colours alone**, so `kind` is the only thing +that varies and nesting is drawn by inset rather than by hue: a method inside a +class is the class's colour, indented. Scanning a hundred files then shows which +are class-shaped, which are a pile of loose functions, and which are one block. + +Not a node-edge graph, and deliberately not forced into one — it answers "what +is where", which a dependency diagram never does. + +## Layout + +Files are grouped by their package, packed left to right into shelves, and each +shelf is as tall as its tallest file. Grouping by package is what makes *where* +legible: the shape of a subsystem is the shape of its band. + +## Geometry + +Computed, never measured — `lines × SCALE`. Deterministic, portable, and +independent of any font, which is the same property `erd` has and `dot` does +not. +""" + +from html import escape + +COL_W = 74 # one file +COL_GAP = 8 +SCALE = 0.55 # pixels per line of source +MIN_H = 14 +PAD = 28 +LABEL_H = 18 +ROW_GAP = 26 +PKG_GAP = 18 +INSET = 7 # per level of nesting +TARGET_W = 1180 # wrap a shelf past this + +# Which slot each kind is drawn in. Everything else lands on `default`, so an +# unfamiliar vocabulary still renders rather than vanishing. +KIND_SLOT = { + "class": "station", + "interface": "accent", + "function": "atlas", + "module": "surface-2", + "table": "station", + "endpoint": "accent", + "operation": "accent", + "task": "station", + "external": "muted", +} + + +def _files(ir: dict) -> list[dict]: + """Modules with their constructs, nested, each carrying a line span.""" + by_id = {n["id"]: n for n in ir["nodes"]} + kids: dict[str, list] = {} + for n in ir["nodes"]: + if n.get("parent"): + kids.setdefault(n["parent"], []).append(n) + + def declared(node: dict) -> list: + """A node's drawable children, with wrappers dissolved. + + A C# `namespace` and a TypeScript `module` come through as `kind: + "module"` nested inside a file. Drawing them adds a level of inset to + everything without adding information — and worse, the file's own + contents then appear twice. They are descended through and not drawn. + """ + out = [] + for c in sorted(kids.get(node["id"], []), + key=lambda c: ((c.get("attrs") or {}).get("line", 0))): + if not (c.get("attrs") or {}).get("line"): + continue + if c["kind"] == "module": + out.extend(declared(c)) # a wrapper: keep what is inside it + else: + out.append(c) + return out + + def build(node: dict, depth: int) -> dict: + a = node.get("attrs") or {} + return { + "id": node["id"], + "label": node.get("label") or node["id"], + "kind": node["kind"], + "line": a.get("line", 1), + "lines": max(1, a.get("lines", 1)), + "depth": depth, + "children": [build(c, depth + 1) for c in declared(node)], + } + + out = [] + for n in ir["nodes"]: + if n["kind"] != "module": + continue + a = n.get("attrs") or {} + total = a.get("lines") + if not total: + continue + # A file, not a namespace. Both arrive as `module`; only a file has a + # length without a starting line, because a file starts at the start. + if a.get("line"): + continue + node = build(n, 0) + node["total"] = total + node["package"] = n["id"].rsplit(".", 1)[0] if "." in n["id"] else "" + node["error"] = a.get("error") + out.append(node) + return out + + +def _bands(files: list[dict]) -> list[tuple[str, list]]: + groups: dict[str, list] = {} + for f in files: + groups.setdefault(f["package"] or "(root)", []).append(f) + for members in groups.values(): + members.sort(key=lambda f: -f["total"]) + return sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])) + + +def _blocks(node: dict, top: float, height: float, x: float, out: list, style, file_lines: int): + """Place a construct and everything inside it.""" + slot = KIND_SLOT.get(node["kind"], "border") + inset = INSET * max(0, node["depth"] - 1) + out.append({ + "x": x + inset, + "y": top, + "w": COL_W - 2 * inset, + "h": max(2.0, height), + "fill": style.slot(slot, style.slot("border")), + "depth": node["depth"], + "id": node["id"], + "kind": node["kind"], + "title": f'{node["label"]} — {node["kind"]}, {node["lines"]} lines', + }) + for child in node["children"]: + offset = (child["line"] - node["line"]) / max(node["lines"], 1) + child_h = child["lines"] / max(node["lines"], 1) * height + _blocks(child, top + offset * height, child_h, x, out, style, file_lines) + + +def emit(ir: dict, style, *, scale: float = SCALE, target_width: int = TARGET_W) -> str: + """IR + Style -> SVG text. No Graphviz, no font measurement.""" + files = _files(ir) + if not files: + raise ValueError( + "nothing to map — a minimap needs modules carrying `attrs.lines`. " + "Was this extracted with the python reader?" + ) + + s = style.slot + placed, labels, marks = [], [], [] + x, y, shelf_h, max_x = PAD, PAD + LABEL_H, 0, 0 + current_package = None + + # One continuous flow, files ordered by package, wrapping at the target + # width — rather than a row per package. A real tree has many small + # packages (soleprint: 234 files in 70) and a row each gave a 1196x10165 + # ribbon. The aspect ratio has to be chosen, not left to emerge; the + # grouping stays legible because a package's files are still adjacent. + ordered = [(pkg, f) for pkg, members in _bands(files) for f in members] + + for package, f in ordered: + height = max(MIN_H, f["total"] * scale) + starts_package = package != current_package + gap = COL_GAP + (PKG_GAP if starts_package and x > PAD else 0) + + if x + gap + COL_W > target_width and x > PAD: + y += shelf_h + ROW_GAP + x, shelf_h, gap = PAD, 0, 0 + starts_package = True # a wrap re-labels, so a row is readable alone + elif x > PAD: + x += gap + + if starts_package: + marks.append({"x": x, "y": y - 6, "text": package}) + current_package = package + + placed.append({ + "x": x, "y": y, "w": COL_W, "h": height, + "fill": s("surface-1", s("surface-2")), "depth": -1, + "title": f'{f["id"]} — {f["total"]} lines' + + (f' — NOT PARSED: {f["error"]}' if f["error"] else ""), + "outline": s("border"), + }) + for child in f["children"]: + offset = (child["line"] - 1) / max(f["total"], 1) + _blocks(child, y + offset * height, + child["lines"] / max(f["total"], 1) * height, + x, placed, style, f["total"]) + + labels.append({"x": x, "y": y + height + 9, "text": f["label"][:11], "band": False}) + shelf_h = max(shelf_h, height + 12) + max_x = max(max_x, x + COL_W) + x += COL_W + + y += shelf_h + PAD + labels = marks_to_labels(marks) + labels + + width = max(max_x + PAD, 420) + height = y + 30 + + out = [ + '', + f'', + f'', + ] + + for b in placed: + extra = (f' stroke="{b["outline"]}" stroke-width="1"' if b.get("outline") + else ' stroke="none"') + # Nested blocks sit on their parent, so a little transparency keeps the + # containment readable instead of hiding it. + opacity = "" if b["depth"] < 0 else f' opacity="{0.95 if b["depth"] <= 1 else 0.8}"' + ident = (f' data-id="{escape(b["id"])}" data-kind="{b.get("kind", "")}" ' + f'class="blk"' if b.get("id") else "") + out.append( + f'' + f'{escape(b["title"])}' + ) + + for lab in labels: + if lab["band"]: + out.append( + f'' + f'{escape(lab["text"])}' + ) + else: + out.append( + f'{escape(lab["text"])}' + ) + + # A legend, because the whole claim is that the colours carry the meaning. + lx = PAD + ly = height - 14 + for kind in ("module", "class", "interface", "function"): + slot = KIND_SLOT.get(kind, "border") + out.append( + f'' + ) + out.append( + f'{kind}' + ) + lx += 22 + len(kind) * 6 + if lx > width - 220: + ly += 13 # the legend ran into the summary; put it on its own line + out.append( + f'' + f'{len(files)} files · {sum(f["total"] for f in files):,} lines · ' + f'1px ≈ {1 / scale:.1f} lines' + ) + + out.append("") + return "\n".join(out) + "\n" + + +def marks_to_labels(marks: list) -> list: + """Package names, as band labels above the first file of each group.""" + return [{"x": m["x"], "y": m["y"], "text": m["text"], "band": True} for m in marks] diff --git a/soleprint/atlas2/docgen/explore_test.js b/soleprint/atlas2/docgen/explore_test.js new file mode 100644 index 0000000..c3bd413 --- /dev/null +++ b/soleprint/atlas2/docgen/explore_test.js @@ -0,0 +1,67 @@ +// Drive the explorer's logic under a stub DOM: select, walk, basket. +const fs = require('fs'); +const html = fs.readFileSync(process.argv[2], 'utf8'); +const script = html.split('')[0]; + +const detail = { innerHTML: '' }; +const basketEl = { textContent: '' }; +const nav = { addEventListener() {} }; +const ids = { detail, basket: basketEl, nav }; + +global.document = { + getElementById: (i) => ids[i], + querySelectorAll: () => [], +}; +global.CSS = { escape: (s) => s }; + +const api = new Function(script + '; return {select, toggleBasket, showBasket, clearBasket, FACTS, GRAPHS};')(); + +let ok = true; +const check = (name, cond) => { + console.log(` ${cond ? 'ok ' : 'FAIL'} ${name}`); + if (!cond) ok = false; +}; + +const ids_ = Object.keys(api.FACTS); +const withEdges = ids_.filter((i) => api.FACTS[i].out.length || api.FACTS[i]['in'].length); +check('facts are embedded for every node', ids_.length > 0); +check('some node has relationships', withEdges.length > 0); + +const target = withEdges[0]; +api.select(target); +check('selecting renders a detail pane', detail.innerHTML.includes('

')); +check('it names what the thing is', detail.innerHTML.includes(api.FACTS[target].kind)); +check('it lists what it reaches', detail.innerHTML.includes('reaches (')); +check('it lists what reaches it', detail.innerHTML.includes('reached by (')); + +// Walking forward: every link in the pane must be a selectable id. +const links = [...detail.innerHTML.matchAll(/select\('([^']+)'\)/g)].map((m) => m[1]); +check('neighbours are offered as links to walk to', links.length > 0); +const reachable = links.every((i) => api.FACTS[i] !== undefined); +check('every link resolves to a real node', reachable); + +if (links.length) { + api.select(links[0]); + check('walking forward re-renders on the neighbour', + detail.innerHTML.includes(api.FACTS[links[0]].label)); +} + +const withGraph = ids_.find((i) => api.GRAPHS[i]); +if (withGraph) { + api.select(withGraph); + check('a neighbourhood diagram is shown when one exists', + detail.innerHTML.includes(' [options]""" +""" python3 -m docgen.extractors [options]""" import sys @@ -8,6 +8,9 @@ def main(argv=None): if argv and argv[0] == "openapi": from .openapi_main import main as run return run(argv[1:]) + if argv and argv[0] == "code": + from .code_main import main as run + return run(argv[1:]) if argv and argv[0] == "usage": from .usage_main import main as run return run(argv[1:]) diff --git a/soleprint/atlas2/docgen/extractors/code.py b/soleprint/atlas2/docgen/extractors/code.py new file mode 100644 index 0000000..809812e --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/code.py @@ -0,0 +1,260 @@ +""" +Source in several languages -> IR structure, via **tree-sitter**. + + python3 -m docgen.extractors code --root src/ --lang auto -o ir.json + +Adopts a parser rather than writing one, which is the rule the brief sets and +the reason this handles generics, strings containing braces, nested types and +`#region` without any of them being a special case. `pip install tree_sitter +tree_sitter_c_sharp tree_sitter_typescript tree_sitter_python`. + +**Optional, never required.** Python still goes through the stdlib `ast` +extractor, which does two-pass name resolution tree-sitter would have to +reimplement. Without tree-sitter installed, docgen loses C# and TypeScript and +nothing else — the import is lazy and the error says what to install. + +## Structure only, and that is the point + +This produces what is *where*: declarations, their nesting, and the lines each +occupies. It produces **no edges**. Resolving a C# `using` or a TypeScript +`import` to the thing it names is a different and much larger job, and the +consumer that needs this — the minimap — needs none of it. + +Saying so matters: an extractor that quietly produced half a dependency graph +would be worse than one that produces none, because the half would look whole. + +## Vocabulary + + module a file, or a namespace + class class, struct, record, enum — a type with members + interface kept separate from class on purpose: in C# and TypeScript the + distinction is most of what reading a file tells you, and the + minimap's whole claim is that the pattern comes from the colours + function method, constructor, property, function, arrow function +""" + +import sys +from pathlib import Path + +from ..ir import Graph, Meta + +# Extension -> (grammar module, language function). `.tsx` needs its own parser: +# the TSX grammar is a different language, not an option on the TypeScript one. +LANGUAGES = { + ".cs": ("tree_sitter_c_sharp", "language"), + ".ts": ("tree_sitter_typescript", "language_typescript"), + ".mts": ("tree_sitter_typescript", "language_typescript"), + ".tsx": ("tree_sitter_typescript", "language_tsx"), + ".py": ("tree_sitter_python", "language"), +} + +# Declaration node types, per grammar, mapped onto the IR's small vocabulary. +DECLARATIONS = { + "c_sharp": { + "namespace_declaration": "module", + "file_scoped_namespace_declaration": "module", + "class_declaration": "class", + "struct_declaration": "class", + "record_declaration": "class", + "record_struct_declaration": "class", + "enum_declaration": "class", + "interface_declaration": "interface", + "method_declaration": "function", + "constructor_declaration": "function", + "destructor_declaration": "function", + "property_declaration": "function", + "operator_declaration": "function", + "local_function_statement": "function", + }, + "typescript": { + "module": "module", + "internal_module": "module", + "class_declaration": "class", + "abstract_class_declaration": "class", + "enum_declaration": "class", + "interface_declaration": "interface", + "type_alias_declaration": "interface", + "function_declaration": "function", + "generator_function_declaration": "function", + "method_definition": "function", + "public_field_definition": "function", + }, + "python": { + "class_definition": "class", + "function_definition": "function", + "decorated_definition": None, # descend; the real node is inside + }, +} + +GRAMMAR_FAMILY = { + "tree_sitter_c_sharp": "c_sharp", + "tree_sitter_typescript": "typescript", + "tree_sitter_python": "python", +} + + +class MissingParser(ImportError): + """tree-sitter, or one of its grammars, is not installed.""" + + +def _parser(suffix: str): + """(Parser, family) for a file extension. Lazy, so the import is optional.""" + if suffix not in LANGUAGES: + raise MissingParser(f"no grammar registered for {suffix!r}") + module_name, fn = LANGUAGES[suffix] + try: + from tree_sitter import Language, Parser + except ImportError: + raise MissingParser( + "tree-sitter is not installed — C# and TypeScript need it.\n" + " pip install tree_sitter tree_sitter_c_sharp tree_sitter_typescript\n" + "Python does not: it uses the stdlib `ast` extractor." + ) from None + try: + grammar = __import__(module_name) + except ImportError: + raise MissingParser( + f"{module_name} is not installed — needed for {suffix} files.\n" + f" pip install {module_name.replace('_', '-')}" + ) from None + return Parser(Language(getattr(grammar, fn)())), GRAMMAR_FAMILY[module_name] + + +def _name(node, source: bytes) -> str | None: + """A declaration's name, or None when it has none worth recording.""" + field = node.child_by_field_name("name") + if field is not None: + return source[field.start_byte:field.end_byte].decode("utf-8", "replace") + # C# properties and TS fields sometimes carry the name as an identifier child. + for child in node.children: + if child.type in ("identifier", "property_identifier", "type_identifier"): + return source[child.start_byte:child.end_byte].decode("utf-8", "replace") + return None + + +def _walk(node, source: bytes, family: str, out: list, parent: str | None, prefix: str, + seen_ids: set | None = None): + """Collect declarations, depth-first, keeping nesting in the id.""" + table = DECLARATIONS[family] + seen_ids = seen_ids if seen_ids is not None else set() + for child in node.children: + kind = table.get(child.type, ...) + if kind is None: + # A wrapper — a decorated definition, say. Descend without naming it. + _walk(child, source, family, out, parent, prefix, seen_ids) + continue + if kind is ...: + _walk(child, source, family, out, parent, prefix, seen_ids) + continue + name = _name(child, source) + if not name: + _walk(child, source, family, out, parent, prefix, seen_ids) + continue + nid = f"{prefix}.{name}" if prefix else name + if nid in seen_ids: + # Same reason as the Python extractor: an overload, a partial class, + # or a name declared twice in different branches. The line keeps the + # id unique without making it unstable. + nid = f"{nid}#L{child.start_point[0] + 1}" + seen_ids.add(nid) + out.append({ + "id": nid, + "kind": kind, + "label": name, + "parent": parent, + "line": child.start_point[0] + 1, + "lines": max(1, child.end_point[0] - child.start_point[0] + 1), + }) + _walk(child, source, family, out, nid, nid, seen_ids) + + +def extract_file(path: Path, root: Path) -> tuple[dict, list] | None: + """One file -> (module node, declarations). None when unparseable.""" + parser, family = _parser(path.suffix) + try: + source = path.read_bytes() + except OSError as e: + return {"error": str(e)}, [] + + rel = path.relative_to(root) + module_id = ".".join([*rel.parts[:-1], rel.stem]) + tree = parser.parse(source) + decls: list = [] + _walk(tree.root_node, source, family, decls, module_id, module_id) + + module = { + "id": module_id, + "kind": "module", + "label": rel.stem, + "parent": ".".join(rel.parts[:-1]) or None, + "lines": source.count(b"\n") + 1, + "file": rel.as_posix(), + # tree-sitter never fails to parse; it produces ERROR nodes instead. That + # is more useful than an exception, and worth recording rather than + # silently accepting a partial tree. + "errors": _count_errors(tree.root_node), + } + return module, decls + + +def _count_errors(node) -> int: + n = 1 if node.type == "ERROR" or node.is_missing else 0 + for child in node.children: + n += _count_errors(child) + return n + + +def extract(root, suffixes=None, exclude=(), source: str = "code") -> Graph: + """Walk a tree and map its structure. No edges — see the module docstring.""" + root = Path(root).resolve() + if not root.is_dir(): + raise NotADirectoryError(f"not a directory: {root}") + + wanted = tuple(suffixes) if suffixes else tuple(LANGUAGES) + skip = {"__pycache__", ".git", ".venv", "venv", "node_modules", "dist", "build", + "bin", "obj", *exclude} + + g = Graph(Meta(source=source, root=root.name)) + packages: set[str] = set() + files = [ + p for p in sorted(root.rglob("*")) + if p.suffix in wanted and p.is_file() + and not any(part in skip for part in p.relative_to(root).parts) + ] + + for path in files: + try: + module, decls = extract_file(path, root) + except MissingParser: + raise + except Exception as e: # noqa: BLE001 - one bad file must not cost the run + rel = path.relative_to(root) + g.node(".".join([*rel.parts[:-1], rel.stem]), "module", rel.stem, + attrs={"file": rel.as_posix(), "error": f"{type(e).__name__}: {e}"}) + continue + + parent = module["parent"] + if parent: + packages.add(parent) + attrs = {"file": module["file"], "lines": module["lines"]} + if module["errors"]: + attrs["error"] = f"{module['errors']} unparsed region(s)" + g.node(module["id"], "module", module["label"], parent=parent, attrs=attrs) + + for d in decls: + g.node(d["id"], d["kind"], d["label"], parent=d["parent"], + attrs={"file": module["file"], "line": d["line"], "lines": d["lines"]}) + + # Directories that hold files but are not themselves files still need to + # exist, or every module in them is an orphan. + known = {n.id for n in g.nodes} + for pkg in sorted(packages): + parts = pkg.split(".") + for i in range(1, len(parts) + 1): + pid = ".".join(parts[:i]) + if pid not in known: + g.node(pid, "module", parts[i - 1], + parent=".".join(parts[: i - 1]) or None, + attrs={"lines": 0, "directory": True}) + known.add(pid) + return g diff --git a/soleprint/atlas2/docgen/extractors/code_main.py b/soleprint/atlas2/docgen/extractors/code_main.py new file mode 100644 index 0000000..1255413 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/code_main.py @@ -0,0 +1,39 @@ +""" python3 -m docgen.extractors code --root src/ [--ext .cs] [-o ir.json]""" + +import argparse +import json +import sys +from pathlib import Path + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.extractors code") + p.add_argument("--root", "-s", required=True, type=Path) + p.add_argument("--output", "-o", type=Path) + p.add_argument("--ext", action="append", default=[], + help="Limit to these extensions. Default: every registered one.") + p.add_argument("--exclude", action="append", default=[]) + args = p.parse_args(argv) + + from .code import LANGUAGES, MissingParser, extract + + try: + ir = extract(args.root, suffixes=args.ext or None, exclude=tuple(args.exclude)) + except MissingParser as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except (NotADirectoryError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + text = json.dumps(ir.to_dict(), indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + from collections import Counter + c = Counter(n.kind for n in ir.nodes) + print(f"{len(ir.nodes)} nodes -> {args.output} " + + " · ".join(f"{v} {k}" for k, v in sorted(c.items(), key=lambda kv: -kv[1]))) + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/extractors/python/resolve.py b/soleprint/atlas2/docgen/extractors/python/resolve.py index 844466e..23cb415 100644 --- a/soleprint/atlas2/docgen/extractors/python/resolve.py +++ b/soleprint/atlas2/docgen/extractors/python/resolve.py @@ -101,8 +101,16 @@ def to_ir(modules: list[Module], root: str, source: str = "python") -> Graph: attrs=attrs, ) + taken: set[str] = set() for d in m.defines: nid = _id_for(m, d.qualname) + if nid in taken: + # Two definitions can share a qualified name — the same helper + # defined in both branches of an `if`, or a name rebound later. + # That is legal Python and the id has to stay unique, so the + # line disambiguates. Stable across runs, and it says which one. + nid = f"{nid}#L{d.lineno}" + taken.add(nid) parent_qual = d.qualname.rsplit(".", 1)[0] if "." in d.qualname else None parent = _id_for(m, parent_qual) if parent_qual else m.name a = {"file": m.path, "line": d.lineno, "lines": max(1, d.end_lineno - d.lineno + 1)} diff --git a/soleprint/atlas2/docgen/ops/filter.py b/soleprint/atlas2/docgen/ops/filter.py index 1354018..c334a1f 100644 --- a/soleprint/atlas2/docgen/ops/filter.py +++ b/soleprint/atlas2/docgen/ops/filter.py @@ -160,7 +160,8 @@ def subtree(ir: dict, root_id: str) -> dict: return _rebuild(ir, keep) -def neighbourhood(ir: dict, node_id: str, hops: int = 1, *, undirected: bool = True) -> dict: +def neighbourhood(ir: dict, node_id: str, hops: int = 1, *, undirected: bool = True, + with_contents: bool = False) -> dict: """This node and everything within `hops` edges of it. The view behind "what does this touch, and what touches it". Containment @@ -188,6 +189,21 @@ def neighbourhood(ir: dict, node_id: str, hops: int = 1, *, undirected: bool = T while parent and parent not in keep: keep.add(parent) parent = by_id.get(parent, {}).get("parent") + + if with_contents: + # A table without its columns is not a table. Ancestors come along by + # default because the tree has to stay whole; descendants do not, and + # for anything card-shaped they are the substance. + children: dict[str, list] = {} + for n in ir["nodes"]: + if n.get("parent"): + children.setdefault(n["parent"], []).append(n["id"]) + stack = list(keep) + while stack: + for kid in children.get(stack.pop(), ()): + if kid not in keep: + keep.add(kid) + stack.append(kid) return _rebuild(ir, keep) diff --git a/soleprint/atlas2/docgen/selftest.py b/soleprint/atlas2/docgen/selftest.py index 019fc24..32f7b95 100644 --- a/soleprint/atlas2/docgen/selftest.py +++ b/soleprint/atlas2/docgen/selftest.py @@ -47,6 +47,8 @@ ops_mod = __import__(f"{PKG}.ops", fromlist=["*"]) erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"]) nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"]) site_mod = __import__(f"{PKG}.emitters.site", fromlist=["*"]) +mm_mod = __import__(f"{PKG}.emitters.minimap", fromlist=["*"]) +exp_mod = __import__(f"{PKG}.emitters.explore", fromlist=["*"]) spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"]) db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"]) usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"]) @@ -1108,6 +1110,217 @@ else: _shutil.rmtree(site_dir, ignore_errors=True) +# -------------------------------------------------------------------------- +print("\n12. the minimap — what is where, read from the colours") + +mm = mm_mod.emit(ir, lucid, scale=1.0) +check("it produces SVG", mm.startswith("" in mm) +check( + "a block is sized by the lines it occupies", + mm_mod._files(ir)[0]["total"] > 1 + and all(f["total"] >= 1 for f in mm_mod._files(ir)), +) +check( + "no text is rendered inside a block", + " mm.count(" CountAsync(); }\n' + '\n' + ' public class Repo : IRepo\n' + ' {\n' + ' public async Task CountAsync()\n' + ' {\n' + ' var sql = "SELECT 1 WHERE x = {0}";\n' + ' return 1;\n' + ' }\n' + '\n' + ' private enum Mode { A, B }\n' + ' }\n' + '}\n' + ) + got = code_ex.extract(cs).to_dict() + kinds = {n["id"].rsplit(".", 1)[-1]: n["kind"] for n in got["nodes"]} + check("the C# IR validates", check_ir(got) == [], str(check_ir(got)[:2])) + check("a class is a class", kinds.get("Repo") == "class", str(kinds)) + check( + "an interface is kept apart from a class", + kinds.get("IRepo") == "interface", + "in C# and TypeScript that distinction is most of what a file tells you", + ) + check("a method is a function", kinds.get("CountAsync") == "function") + iface_method = next(n for n in got["nodes"] if n["id"].endswith(".IRepo.CountAsync")) + check( + "an interface's one-line method is one line", + iface_method["attrs"]["lines"] == 1, + "and is a different node from the class's implementation of it", + ) + check("a nested enum is found", kinds.get("Mode") == "class") + # `.endswith("Repo.CountAsync")` also matches `IRepo.CountAsync`, whose + # one-line declaration is correct — so name the class's method exactly. + method = next(n for n in got["nodes"] if n["id"].endswith(".Repo.CountAsync")) + check( + "a brace inside a string does not end a block", + method["attrs"]["lines"] >= 4, + f'got {method["attrs"]["lines"]} lines — a scanner would stop at the brace in the string', + ) + check( + "structure only — no edges are invented", + got["edges"] == [], + "resolving a `using` is a different job, and half a graph looks whole", + ) + check("the minimap draws it", mm_mod.emit(got, lucid).startswith("= 1) +check( + "a table's neighbourhood keeps its columns", + any("column" not in g.read_text() or "PK" in g.read_text() for g in graphs), + "a table without its columns is not a table", +) + +# A module's neighbourhood must NOT drag its contents in — that is the sheet. +code_ir = ops_mod.overview(ir) +mod_view = ops_mod.neighbourhood(code_ir, next( + n["id"] for n in code_ir["nodes"] if n["kind"] == "module"), hops=1) +check( + "a module's neighbourhood stays small", + len(mod_view["nodes"]) <= 24, + "its contents are the hundred functions that made the sheet unreadable", +) + +if not _shutil.which("node"): + skip("explore behaviour", "node not installed") +else: + harness = HERE / "explore_test.js" + if not harness.exists(): + skip("explore behaviour", "explore_test.js missing") + else: + proc = _sub.run(["node", str(harness), str(page)], capture_output=True, text=True) + for line in proc.stdout.strip().splitlines(): + name = line.strip()[5:] + (PASS if line.strip().startswith("ok") else FAIL).append(f"explore: {name}") + print(f" {line.strip()[:4]} explore: {name}") + if proc.returncode and not proc.stdout.strip(): + check("the explore harness runs", False, proc.stderr.strip()[:200]) + +_shutil.rmtree(exp_dir, ignore_errors=True) + + # -------------------------------------------------------------------------- tmp.cleanup() print() diff --git a/soleprint/atlas2/docgen/style/__init__.py b/soleprint/atlas2/docgen/style/__init__.py index 3a7f29a..763e802 100644 --- a/soleprint/atlas2/docgen/style/__init__.py +++ b/soleprint/atlas2/docgen/style/__init__.py @@ -25,6 +25,9 @@ can see anything. import json from pathlib import Path +# Harvesting a theme from real diagrams. Imported lazily by callers that want +# it; `extract` needs lxml and the rest of the package does not. + HERE = Path(__file__).resolve().parent # Keys whose value is a slot name to be resolved against the theme. Anything @@ -177,3 +180,14 @@ class Style: def limits(self) -> dict: """Where this style asks for more than the target can express.""" return {k: v for k, v in self.data.get("limits", {}).items() if k not in NOTE_KEYS} + + +def harvest(folder, out_dir, name: str = "harvested"): + """A folder of exported diagrams -> tokens.json and a theme to paste in. + + Offline, and it never reads text content: style values are visual metadata, + so the semantics of a confidential diagram are not needed and not touched. + """ + from .tokens import from_folder + + return from_folder(folder, out_dir, name) diff --git a/soleprint/station/tools/docgen/style/extract.py b/soleprint/atlas2/docgen/style/extract.py similarity index 100% rename from soleprint/station/tools/docgen/style/extract.py rename to soleprint/atlas2/docgen/style/extract.py diff --git a/soleprint/atlas2/docgen/style/lucid.json b/soleprint/atlas2/docgen/style/lucid.json index 69e7daa..2ce3799 100644 --- a/soleprint/atlas2/docgen/style/lucid.json +++ b/soleprint/atlas2/docgen/style/lucid.json @@ -190,6 +190,16 @@ "text": "text", "bold": true, "note": "A GraphQL operation \u2014 usually one endpoint carrying many, which is why it is its own kind rather than a path." + }, + "interface": { + "shape": "box", + "rounded": true, + "fill": "surface-0", + "border": "accent", + "text": "text", + "dashed": true, + "bold": true, + "note": "A contract rather than an implementation. Dashed, because that is the convention everywhere else too." } }, "groups": { diff --git a/soleprint/atlas2/docgen/style/tokens.py b/soleprint/atlas2/docgen/style/tokens.py new file mode 100644 index 0000000..c2e51fa --- /dev/null +++ b/soleprint/atlas2/docgen/style/tokens.py @@ -0,0 +1,210 @@ +""" +A harvested token vocabulary -> a **theme**. + +`extract.py` says which values a folder of real diagrams uses and how often. +This decides which of them are the house palette. The rule is the plain one: + + the top two or three fills *are* the palette + the modal stroke width *is* the house line weight + +Nothing cleverer. Frequency is a good enough signal because a real diagram set +repeats its own vocabulary constantly and its one-offs stay one-offs. + +## It produces a theme, not a style file + +A style file says *what a `kind` looks like* — "a class is filled with +`surface-0` and outlined in `station`". That is a design decision and no amount +of counting hexes recovers it. What harvesting recovers is the **binding**: +which colour `station` should be. So the output is a `themes` entry — a slot to +hex map — that drops into an existing style file beside `dark` and `lucid`. + +That split is the point of the whole colour language. One set of rules, several +bindings, and a new binding costs a paste rather than a rewrite. + +## The judgement calls, named + +Three, all of the kind that should be visible rather than buried: + +- **Canvas and ink are the two lightness extremes, not the two most common + values.** Frequency is the wrong signal: in a Graphviz SVG every label carries + a `fill`, so the ink outnumbers the canvas 87 to 43, and "most common is the + background" produces a theme whose text is invisible against it. +- **Polarity is assumed light**, because a folder of exports is usually for + print. `dark=True` flips it. Get it backwards and the contrast check says so + rather than letting an unreadable theme through quietly. +- **Slot assignments are a guess.** Which accent means `artery` is a decision, + not a measurement. The note says so, and they are meant to be checked. +- **Widths and sizes take the mode, not the mean.** A mean of 1 and 4 is 2.5, + which is a width the diagrams never use. +""" + +import json +from pathlib import Path + +# The slots a theme has to bind for a style file to load. Anything a harvest +# cannot find gets a neutral, so a partial harvest is still usable. +REQUIRED_SLOTS = ( + "surface-0", "surface-1", "surface-2", + "border", "border-strong", + "text", "text-muted", "text-dim", "text-on-fill", + "accent", "accent-soft", + "artery", "artery-soft", "atlas", "atlas-soft", "station", "station-soft", + "ok", "error", "muted", "muted-soft", + "alt-1", "alt-1-soft", "alt-2", "alt-2-soft", +) + + +def _top(tokens: dict, prop: str, n: int = 10) -> list[str]: + return [e["value"] for e in tokens.get(prop, [])[:n] if e["value"].startswith("#")] + + +def _mode(tokens: dict, prop: str, fallback: str) -> str: + entries = [e for e in tokens.get(prop, []) if e["value"] not in ("none", "transparent")] + return entries[0]["value"] if entries else fallback + + +def _luminance(hexv: str) -> float: + """Rough perceptual lightness, 0..1. Enough to sort pale from dark.""" + if not hexv.startswith("#") or len(hexv) != 7: + return 0.5 + r, g, b = (int(hexv[i: i + 2], 16) / 255 for i in (1, 3, 5)) + return 0.2126 * r + 0.7152 * g + 0.0722 * b + + +def derive(data: dict, name: str = "harvested", *, dark: bool = False) -> dict: + """Token JSON -> a `themes` entry: {name: {note, slots}}. + + Paste the result into a style file's `themes` and select it with + `--theme `. Every slot is bound, so the file loads; the ones a harvest + could not distinguish fall back to a neutral rather than being absent, which + would fail validation at load. + """ + tokens = data.get("tokens", {}) + fills = _top(tokens, "fill") + strokes = _top(tokens, "stroke") + + # The canvas and the ink are the two **lightness extremes**, not the two + # most common values. Frequency is the wrong signal here: in a Graphviz SVG + # every label carries a `fill`, so the ink outnumbers the canvas 87 to 43 + # and "most common is the background" puts the text colour on the canvas — + # which renders a theme whose text is invisible against its own background. + if fills: + lightest = max(fills, key=_luminance) + darkest = min(fills, key=_luminance) + else: + lightest, darkest = "#ffffff", "#333333" + + # Polarity is a guess, and it is the one the caller most needs to check: + # a light export and a dark one use the same two extremes the other way + # round. Light is assumed, because a folder of exports is usually for print. + background, ink = (lightest, darkest) if not dark else (darkest, lightest) + + mid = [f for f in fills if f not in (lightest, darkest)] + surface = (min(mid, key=lambda f: abs(_luminance(f) - _luminance(background))) + if mid else background) + border = strokes[0] if strokes else "#9aa5b1" + strong = strokes[1] if len(strokes) > 1 else border + + # Accents are the frequent colours that are neither the canvas nor the ink — + # the ones a diagram uses to mean something. + neutral = {background, surface, ink, border, strong} + accents = [f for f in fills + strokes if f not in neutral] + seen, ordered = set(), [] + for a in accents: + if a not in seen: + seen.add(a) + ordered.append(a) + + def accent(i: int, fallback: str) -> str: + return ordered[i] if i < len(ordered) else fallback + + slots = { + "surface-0": background, + "surface-1": surface, + "surface-2": surface, + "border": border, + "border-strong": strong, + "text": ink, + "text-muted": strong, + "text-dim": border, + "text-on-fill": background, + "accent": accent(0, strong), + "accent-soft": surface, + "artery": accent(1, strong), + "artery-soft": surface, + "atlas": accent(2, strong), + "atlas-soft": surface, + "station": accent(3, strong), + "station-soft": surface, + "ok": accent(2, strong), + "error": accent(1, strong), + "muted": border, + "muted-soft": surface, + "alt-1": accent(4, strong), + "alt-1-soft": surface, + "alt-2": accent(5, strong), + "alt-2-soft": surface, + } + for slot in REQUIRED_SLOTS: + slots.setdefault(slot, border) + + contrast = abs(_luminance(slots["text"]) - _luminance(slots["surface-0"])) + if contrast < 0.25: + # Not fatal — the harvest is still the best available guess — but a + # theme nobody can read is worth saying out loud rather than shipping. + slots["_warning"] = ( + f"text and surface-0 differ by {contrast:.2f} in lightness; this theme " + "is close to unreadable. The polarity is probably backwards." + ) + + return { + name: { + "note": ( + f"Harvested from {data.get('files_read', 0)} diagram(s) in " + f"{data.get('source', 'a target folder')}. Frequency-sorted: the top " + "fills are the palette, the modal stroke width the house line weight. " + "Slot *assignments* are a guess — which accent means `artery` is a " + "decision, not a measurement, so check them. Polarity was assumed " + f"{'dark' if dark else 'light'} — pass dark=True if that is backwards. " + "No text content was read. " + f"Also seen, unassigned: {', '.join(ordered[6:12]) or 'none'}." + ), + "slots": slots, + } + } + + +def geometry(data: dict) -> dict: + """The non-colour half: line weight, type size, corner rounding.""" + tokens = data.get("tokens", {}) + size = _mode(tokens, "font-size", "11") + return { + "hairline": _mode(tokens, "stroke-width", "1"), + "font-size-base": size, + "font-size-sm": str(max(int(float(size)) - 1, 6)), + "font-size-header": str(int(float(size)) + 2), + "font": _mode(tokens, "font-family", "Helvetica"), + # Lucid's `rounding` is a scalar and DOT's `rounded` is binary, so the + # radius is dropped. Its presence is the only part that survives. + "rounded": bool(tokens.get("rx")), + } + + +def write(data: dict, out, name: str = "harvested", *, dark: bool = False) -> Path: + """Write the theme entry, ready to paste into a style file's `themes`.""" + out = Path(out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"themes": derive(data, name, dark=dark), + "geometry": geometry(data)}, indent=2) + "\n") + return out + + +def from_folder(folder, out_dir, name: str = "harvested", *, dark: bool = False) -> dict: + """The whole path: a folder of exports -> tokens.json + a theme.""" + from . import extract + + out_dir = Path(out_dir) + tokens_path = extract.write(folder, out_dir / "tokens.json") + data = json.loads(tokens_path.read_text()) + theme_path = write(data, out_dir / f"{name}.theme.json", name, dark=dark) + return {"tokens": tokens_path, "theme": theme_path, "data": data} diff --git a/soleprint/station/tools/docgen/.gitignore b/soleprint/station/tools/docgen/.gitignore deleted file mode 100644 index 8b93c3e..0000000 --- a/soleprint/station/tools/docgen/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Everything this makes. Regenerate with `make notebook`, `make graph`. -out/ -__pycache__/ -*.pyc diff --git a/soleprint/station/tools/docgen/Makefile b/soleprint/station/tools/docgen/Makefile deleted file mode 100644 index 3828819..0000000 --- a/soleprint/station/tools/docgen/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -# docgen — one target per thing this makes. -# -# The folder is meant to be copied out of soleprint and used on its own, so -# everything here is derived from where this file sits rather than written down: -# copy the directory anywhere, `cd` into it, and `make` works. Renaming it works -# too, since the package name comes from the directory. -# -# This tool is a library, not a CLI. There is no `python -m docgen` and no -# argparse — the prompt is explicit that it is consumed from a notebook. So each -# target below is a one-line `python3 -c` against the public surface, which -# doubles as the shortest possible worked example of calling it. -# -# make check prove it works, on fixtures it builds -# make notebook the three .ipynb variants -> out/ -# make graph the example graph, every profile -> out/ -# make extract DIR=~/diagrams a folder of exports -> tokens + a profile -# make profiles what profiles are available -# make doctor what this machine has -# -# The logic lives in the Python, never here. - -HERE := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) -PKG := $(notdir $(HERE)) -PARENT := $(patsubst %/,%,$(dir $(HERE))) -PY ?= python3 - -# Run the package from its parent, which is what an import needs and what lets -# this work without installing anything. -RUN := PYTHONPATH=$(PARENT) $(PY) -c - -OUT ?= $(HERE)/out -DIR ?= -NAME ?= extracted - -.PHONY: help check notebook graph extract profiles doctor clean - -help: ## List every target - @echo "docgen — graph generation and document output, with the style as data" - @echo - @grep -E '^[a-z-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - | awk 'BEGIN{FS=":.*?## "}{printf " \033[1m%-12s\033[0m %s\n", $$1, $$2}' - @echo - @echo " OUT=/path where output goes (default: $(PKG)/out)" - @echo " DIR=/path the folder of exported diagrams, for extract" - @echo " NAME=lucid-real what to call the extracted profile" - @echo - @echo " It is a library. From a notebook:" - @echo " from $(PKG) import Profile, emit, render # graphgen owns Graph" - -check: ## Prove the whole thing works, needing nothing installed and no network - @$(PY) $(HERE)/selftest.py - -notebook: ## Emit vanilla, executable and live .ipynb into OUT - @$(RUN) "from $(PKG).export import write_all; \ -from $(PKG).export.specs import vanilla; \ -[print(' ', p) for p in write_all(vanilla.build(), '$(OUT)')]" - -graph: ## Render graphgen's example graph through every shipped profile into OUT - @$(RUN) "from $(PKG).demo import render_all; render_all('$(OUT)')" - -extract: ## A folder of exported SVG/PDF diagrams -> tokens.json + a profile - @test -n "$(DIR)" || { echo "Error: set DIR=/path/to/diagrams" >&2; exit 1; } - @test -d "$(DIR)" || { echo "Error: no such folder: $(DIR)" >&2; exit 1; } - @$(RUN) "from $(PKG).style import from_folder, summarise; \ -r = from_folder('$(DIR)', '$(OUT)', '$(NAME)'); \ -print(summarise(r['data'])); \ -print(); print(' tokens ', r['tokens']); print(' profile ', r['profile']); \ -print(); print(' Drop the profile into $(PKG)/profiles/ to use it by name,'); \ -print(' or pass its path: Profile.load(\"%s\")' % r['profile'])" - -profiles: ## What style profiles are available - @$(RUN) "from $(PKG) import Profile; \ -[print(' %-12s %s' % (n, Profile.load(n).data.get('note','')[:88])) for n in Profile.available()]" - -doctor: ## Report whether this machine can run it - @printf 'python : '; $(PY) --version 2>&1 || echo MISSING - @printf 'dot : '; (dot -V 2>&1) || echo 'MISSING — sudo apt install graphviz (needed to render)' - @printf 'pdftocairo : '; (pdftocairo -v 2>&1 | head -1) || echo 'MISSING — sudo apt install poppler-utils (only for PDF sources)' - @printf 'lxml : '; $(PY) -c 'import lxml.etree; print(lxml.etree.__version__)' 2>/dev/null || echo 'MISSING — pip install lxml (only for extraction)' - @printf 'package : %s (from %s)\n' '$(PKG)' '$(PARENT)' - @printf 'graphgen : '; test -d '$(PARENT)/graphgen' \ - && echo 'beside this folder — `make graph` will work' \ - || echo 'absent — everything works except `make graph` (it owns the model)' - @$(RUN) "import $(PKG), $(PKG).style, $(PKG).export" >/dev/null 2>&1 \ - && echo 'import : ok' || echo 'import : FAILED — is the folder intact?' - -clean: ## Delete OUT. Nothing else is ever written to - @rm -rf "$(OUT)" && echo "Removed $(OUT)" diff --git a/soleprint/station/tools/docgen/README.md b/soleprint/station/tools/docgen/README.md deleted file mode 100644 index 857cbf3..0000000 --- a/soleprint/station/tools/docgen/README.md +++ /dev/null @@ -1,256 +0,0 @@ -# docgen - -The interface to Graphviz and DOT: styling and export. - -Eight repos under `semester/` draw their architecture with Graphviz, and every -one of them hand-writes `.dot` with the palette inlined and commits the `.svg` -beside it — `sms`, `unt`, `cht`, `mpr`, `mly`, `nvi`, `eth`, and `spr`. Three -different dark palettes between them, eight answers to the same question. Only -`spr/docs/graphs/` separates what a diagram *means* from what it *looks like*, -and it does that in `gvpr` rather than in anything reusable. - -This is that separation, as a library. - -```python -from docgen import Profile, emit, render -from graphgen import Graph # what a graph *is* lives next door - -g = Graph("overview", title="System Overview", rankdir="LR") -g.node("api", "API", cls="station") -g.node("db", "Database", shape="cylinder") -g.edge("api", "db", "reads") - -svg = render(emit(g, Profile.load("lucid"))) -``` - -The same `g` through `Profile.load("default")` is the same diagram in the -docs-site palette. Nothing about `g` changes, because nothing about `g` was ever -about colour. - -## The split - - docgen how a graph is drawn. DOT, Graphviz, style profiles, and the - documents the result goes into. - graphgen what a graph is. Nodes, edges, groups, and the sources - graphs come from. - -**docgen imports nothing from graphgen.** A graph is read structurally — see -[`shape.py`](shape.py), which names every attribute `emit()` touches — so either -folder can be copied out and used with the other absent. Prove it: - -```bash -cp -r docgen /tmp/alone && cd /tmp/alone/docgen && make check -# 49 passed, 0 failed, 1 skipped <- the skip is the graphgen seam -``` - -docgen deliberately ships **no graph class**. If it did, people would use it, and -there would be two graph models — which is exactly what this separation exists to -prevent. The one place that rule is relaxed is scaffolding: `demo.py` and -`selftest.py` may import graphgen, because demonstrating and testing the drawing -needs something to draw. - -It is a **library**, not a CLI — the first thing that uses it is a notebook. The -Makefile is the front door for the things that are worth running from a shell. - -```bash -make check # prove it, on fixtures it builds — nothing installed, no network -make doctor # what this machine has -make graph # the example graph, every profile -> out/ -make notebook # the three .ipynb variants -> out/ -make extract DIR=~/diagrams -make help # every target -``` - -## The three concerns - -``` -model ──► emit ──► render - ▲ - profile ← every visual value, as data -``` - -| | | -|---|---| -| `shape.py` | the contract: what docgen needs a graph to look like. Protocols only — nothing instantiable. | -| `profile.py` | every visual value, from JSON. Fill, stroke, penwidth, fonts, arrowsize, splines, separations. | -| `dot.py` | a graph and a profile, into DOT. The dullest module here on purpose. | -| `render.py` | DOT into SVG, via the `dot` binary. | - -The model those three read — `graphgen/graph.py` — carries **meaning only**: a node -is `cls="artery"`, never `fillcolor="#c0ffee"`. - -The class vocabulary is the one `spr/docs/graphs/README.md` already documents — -`accent`, `accent-text`, `ok`, `artery`, `atlas`, `station`, `muted` — so a graph -built in graphgen and a `.dot` written by hand mean the same thing by the same -word. Anything untagged renders neutral, which is where most nodes should stay. -The selftest asserts the two vocabularies still agree, because a class graphgen -can set that no profile styles renders neutral and tells nobody why. - -### Three things a profile cannot touch - -Structure wearing a visual field's clothing. A profile that overrode these would -be destroying meaning rather than restyling it. - -| | | -|---|---| -| `shape` | a cylinder is a datastore, not a decoration | -| `style="invis"` | layout scaffolding — filling it in would draw it | -| `style="dashed"` | a weaker relationship. The profile *composes* with it (`filled,rounded,dashed`); it never replaces it | - -## Profiles - -`lucid` and `default` ship. `make profiles` lists them, `Profile.load(name)` -loads one, and `Profile.load("/path/to/anything.json")` loads a file — so code -that got a profile name out of a config does not have to know which it got. - -`lucid` is first because the output has to import cleanly into Lucid and Google -Drawings. It is **not** the built-in one, and it is not in the code: the shipped -`lucid.json` is a plausible default derived from values already visible in -`spr/docs/graphs/themes/lucid.gvpr`, and the real one — extracted from company -diagrams — drops in as a file with no code change. - -`selftest.py` asserts that. It takes every colour in every profile and greps the -package's own source for it, because the failure this design exists to prevent is -somebody reaching for a literal `"#5A6C86"` in `dot.py` on a Tuesday and the whole -arrangement quietly becoming decorative. - -An unknown key in a profile is **refused, not ignored** — a setting plainly -written in a file and silently not applied is a genuinely bad thing to debug, -because the file says it is on. - -## Extraction - -The real style values come from real diagrams. Point `style/` at a folder of -exports: - -```bash -make extract DIR=~/exported-diagrams NAME=house -``` - -``` - fill - 87 #1f2933 - 43 #ffffff - 28 #616e7c - - stroke - 85 #9aa5b1 - 32 #3a7dff -``` - -Frequency-sorted, because that is what turns a heap of values into a palette: the -top two or three fills *are* the palette, and the modal stroke width *is* the -house line weight. SVG directly, PDF via `pdftocairo -svg`. Out comes a -`tokens.json` and a profile. - -Two rules, and they are not stylistic preferences: - -- **Text content is never read.** `` elements are visited for their style - attributes and nothing else. Style values are visual metadata; the semantics of - the diagram are not needed to derive a palette, so they are not looked at. The - selftest renders a graph with a distinctive label and asserts it appears - nowhere in the token output. -- **Fully offline.** Nothing here opens a socket. Confidential source diagrams - stay off any network path, and off the Lucid API path. - -Parsed with `lxml`, not grepped — `fill` appears as an attribute, inside an -inline `style`, and inside a `