From b693e324ef7f9afc6e36218471d7cd542f148da0 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Tue, 22 Sep 2026 04:54:49 -0300 Subject: [PATCH] init docgen --- soleprint/atlas2/docgen/.gitignore | 1 + soleprint/atlas2/docgen/Makefile | 11 +- soleprint/atlas2/docgen/README.md | 52 +- soleprint/atlas2/docgen/book/build.py | 133 ++- soleprint/atlas2/docgen/book/checks.py | 107 +- soleprint/atlas2/docgen/book/config.py | 330 +++++- soleprint/atlas2/docgen/book/document.py | 448 ++++++++ soleprint/atlas2/docgen/book/notes.py | 161 +++ soleprint/atlas2/docgen/cli/__init__.py | 4 + soleprint/atlas2/docgen/cli/book.py | 5 +- soleprint/atlas2/docgen/cli/check.py | 9 +- soleprint/atlas2/docgen/cli/contract.py | 33 + soleprint/atlas2/docgen/cli/run.py | 33 +- soleprint/atlas2/docgen/cli/scaffold.py | 87 ++ .../atlas2/docgen/docgen.project.example.toml | 53 + .../atlas2/docgen/docs/img/architecture.svg | 966 +++++++++++------- soleprint/atlas2/docgen/docs/img/minimap.svg | 778 ++++++++------ soleprint/atlas2/docgen/docs/index.html | 353 ++++++- soleprint/atlas2/docgen/docs/models.md | 259 +++++ soleprint/atlas2/docgen/emitters/docsite.py | 269 +++++ soleprint/atlas2/docgen/emitters/dot.py | 27 +- soleprint/atlas2/docgen/emitters/erd.py | 22 +- soleprint/atlas2/docgen/emitters/markdown.py | 145 +++ soleprint/atlas2/docgen/emitters/notebook.py | 49 +- soleprint/atlas2/docgen/emitters/paged.py | 118 +++ soleprint/atlas2/docgen/emitters/site.py | 45 + .../atlas2/docgen/fixtures/notes/_intro.md | 3 + .../docgen/fixtures/notes/schema/overview.md | 5 + soleprint/atlas2/docgen/notebook/spec.py | 2 +- soleprint/atlas2/docgen/ops/pages.py | 382 +++++++ soleprint/atlas2/docgen/selftest.py | 447 ++++++++ 31 files changed, 4521 insertions(+), 816 deletions(-) create mode 100644 soleprint/atlas2/docgen/book/document.py create mode 100644 soleprint/atlas2/docgen/book/notes.py create mode 100644 soleprint/atlas2/docgen/cli/contract.py create mode 100644 soleprint/atlas2/docgen/cli/scaffold.py create mode 100644 soleprint/atlas2/docgen/docgen.project.example.toml create mode 100644 soleprint/atlas2/docgen/docs/models.md create mode 100644 soleprint/atlas2/docgen/emitters/docsite.py create mode 100644 soleprint/atlas2/docgen/emitters/markdown.py create mode 100644 soleprint/atlas2/docgen/emitters/paged.py create mode 100644 soleprint/atlas2/docgen/fixtures/notes/_intro.md create mode 100644 soleprint/atlas2/docgen/fixtures/notes/schema/overview.md create mode 100644 soleprint/atlas2/docgen/ops/pages.py diff --git a/soleprint/atlas2/docgen/.gitignore b/soleprint/atlas2/docgen/.gitignore index f58baf9..b055b03 100644 --- a/soleprint/atlas2/docgen/.gitignore +++ b/soleprint/atlas2/docgen/.gitignore @@ -3,3 +3,4 @@ out/ __pycache__/ *.pyc .venv/ +.docgen/ diff --git a/soleprint/atlas2/docgen/Makefile b/soleprint/atlas2/docgen/Makefile index ac3585d..d963bac 100644 --- a/soleprint/atlas2/docgen/Makefile +++ b/soleprint/atlas2/docgen/Makefile @@ -43,6 +43,7 @@ SLUG ?= # en_US.UTF-8 from the environment and --reader rejects it. READER ?= python OVERLAY ?= +NOTES ?= CONFIG ?= docgen.toml ONLY ?= CHECK ?= @@ -54,7 +55,7 @@ SLUG_ARG := $(if $(SLUG),--slug $(SLUG)) OVER_ARG := $(if $(OVERLAY),--overlay $(OVERLAY)) ONLY_ARGS := $(foreach n,$(subst $(comma), ,$(ONLY)),--only $(n)) -.PHONY: help sync lock book run check ir db code view graph index site minimap explore docs self doctor clean +.PHONY: help sync lock book run scaffold check ir db code view graph index site minimap explore docs self doctor clean help: ## List every target @echo "docgen — static analysis of a tree, and the artifacts that fall out of it" @@ -68,6 +69,7 @@ help: ## List every target @echo " BOOK=/path where a book goes, and which book to check" @echo " READER=python|code ast, or tree-sitter SLUG=name what to call the book" @echo " OVERLAY=overlay.json hand-written notebook additions, re-applied every build" + @echo " NOTES=notes/ markdown filed by page: notes//.md" @echo " CONFIG=docgen.toml a run file ONLY=a,b just these books" @echo " CHECK=1 with run: each book's own level after building it" @echo @@ -92,11 +94,14 @@ book: ## SRC (or SCHEMA/OPENAPI/HAR) -> one operation, measured at both ends $(if $(OPENAPI),--openapi "$(OPENAPI)") \ $(if $(HAR),--har "$(HAR)") \ -o "$(if $(BOOK),$(BOOK),$(OUT)/book)" \ - $(STYLE_ARGS) $(SLUG_ARG) $(OVER_ARG) + $(STYLE_ARGS) $(SLUG_ARG) $(OVER_ARG) $(if $(NOTES),--notes "$(NOTES)") -run: ## CONFIG (a run file) -> every book it lists; ONLY=a,b for some +run: ## CONFIG (a run file or a project file) -> every book it lists; ONLY=a,b for some @$(CLI) run "$(CONFIG)" $(ONLY_ARGS) $(if $(CHECK),--check) +scaffold: ## CONFIG -> an empty note for every page and step, never overwriting + @$(CLI) scaffold "$(CONFIG)" $(ONLY_ARGS) + check: ## Prove docgen (or one book, with BOOK=) @$(CLI) check $(if $(BOOK),"$(BOOK)") diff --git a/soleprint/atlas2/docgen/README.md b/soleprint/atlas2/docgen/README.md index c07f87d..b02ddf8 100644 --- a/soleprint/atlas2/docgen/README.md +++ b/soleprint/atlas2/docgen/README.md @@ -17,17 +17,25 @@ extractors/ → graph IR (JSON) → emitters/ make sync # optional: .venv with every group, from uv.lock make check # prove it, on a tree it builds itself make book SRC=/path/to/repo # one whole operation, measured at both ends -make run CONFIG=docgen.toml # every book a run file lists +make run CONFIG=docgen.toml # every book a run file lists, or a project's document +make scaffold CONFIG=docgen.toml # an empty note for every page and step +python3 -m docgen contract # the instructions to hand a model make help ``` -The full documentation is `docs/index.html` — open it in a browser. +The full documentation is `docs/index.html` — open it in a browser. Start with +**Your first project** for a step-by-step walkthrough. + +**For a model** writing `docgen.toml` and notes: `python3 -m docgen contract` prints +`docs/models.md` — every key, every rule, every error with its fix. Attach it (with +explode's contract, if replies go through explode). The suite checks it against the +loaders, so it cannot say something docgen does not accept. ## Layout ``` cli/ the command line — every command, and nothing else -book/ an operation: larder measure, steps, web output, run files +book/ an operation: larder measure, steps, pages, notes, documents, run files extractors/ source -> IR, one per source type ir/ the contract: schema.json, the dataclasses, the validator ops/ IR -> a smaller IR @@ -69,6 +77,40 @@ stops the others; unknown keys are refused; a rebuild replaces the previous build's outputs and never touches a hand-written `checks.py`. `docgen.example.toml` runs against the shipped fixtures. +## Project files, pages and notes + +A **project file** is one document built from several sources — the shape a model +writes unprompted: + +```toml +[project] +name = "orders-api" + +[extract.openapi] +spec = "doc/contracts/openapi.yaml" + +[extract.code] +root = "src" +ext = [".cs"] + +[emit.notebook] +output = "../docs_out/orders_api_notebook" +``` + +Each `[extract.*]` is a book — a *part* of every output — and each `[emit.*]` an +output made of all of them. Keys docgen deliberately doesn't do (`[extract.db] url`, +shell/YAML/SQL `ext`, unknown readers) are refused with what to do instead. + +A book's diagram is split into **pages** by structure — overview first, one page per +subsystem — measured until each fits its column. On the site the diagrams are inline, +so a box opens its page; in a notebook, a page index links every figure, and each +figure is an SVG cell attachment that renders without running anything. + +Hand-written text goes in **notes**: `notes//.md` lands above that page. +`make scaffold` writes an empty stub for every spot; a note whose page disappeared is +reported and fails the check level. `docgen.project.example.toml` builds the fixtures +into one document with a sample note. + Or as three composable commands, which is what the Makefile is wrapping — run from the directory above this one: @@ -240,8 +282,8 @@ about either. ## Testing ```bash -make check # docgen's own suite: 287 after make sync, 272 with nothing -make check BOOK=out/book/x # one book's own level — generated and hand-written checks +make check # docgen's own suite: 346 after make sync, 331 with nothing +make check BOOK=out/book/x # one book's own level — or a project's .docgen/ make doctor # the machine; never fails ``` diff --git a/soleprint/atlas2/docgen/book/build.py b/soleprint/atlas2/docgen/book/build.py index 6df0abd..0122b96 100644 --- a/soleprint/atlas2/docgen/book/build.py +++ b/soleprint/atlas2/docgen/book/build.py @@ -73,7 +73,7 @@ EXTRACTORS = { TAKES_EXCLUDE = {"python", "code"} -def extract(kind: str, source, *, exclude=(), identity=None): +def extract(kind: str, source, *, exclude=(), identity=None, suffixes=None): """(IR dict, Larder) for one source type, dispatched through EXTRACTORS.""" from importlib import import_module @@ -91,6 +91,8 @@ def extract(kind: str, source, *, exclude=(), identity=None): kwargs = {"identity": identity or str(source)} if kind in TAKES_EXCLUDE: kwargs["exclude"] = exclude + if kind == "code" and suffixes: + kwargs["suffixes"] = suffixes g = fn(source, **kwargs) ir = g.to_dict() @@ -102,7 +104,7 @@ def extract(kind: str, source, *, exclude=(), identity=None): return ir, Larder.from_dict(measured) -def spec_from(book: Book, ir: dict) -> dict: +def spec_from(book: Book, ir: dict, *, prelude_dir: str | None = None) -> dict: """The book's ledger -> a notebook spec, with the two measures at the ends. Reuses `notebook.spec`'s step vocabulary rather than inventing one, so @@ -128,7 +130,10 @@ def spec_from(book: Book, ir: dict) -> dict: ), ), spec_mod._step("prelude", "code", title="Reading a step on its own", - code=PRELUDE), + # In its own book directory the notebook reads steps from + # where it is opened; elsewhere it points back at the book. + code=PRELUDE if prelude_dir is None else PRELUDE.replace( + "BOOK = Path.cwd()", f"BOOK = Path.cwd() / {prelude_dir!r}", 1)), ] for s in book.steps: @@ -191,7 +196,9 @@ def _load_cell(step_id: str, artifact: str) -> str: if artifact.endswith(".json"): return ( f'data = load("{artifact}")\n' - 'if data:\n' + 'if isinstance(data, list):\n' + ' print(f"{len(data)} item(s)")\n' + 'elif data:\n' ' if "nodes" in data:\n' ' print(f\'{len(data["nodes"])} nodes, {len(data.get("edges", []))} edges\')\n' ' else:\n' @@ -222,7 +229,20 @@ def _load_cell(step_id: str, artifact: str) -> str: # remove first. Named, never globbed: a book directory also holds the hand-written # `checks.py` and `overlay.json`, and a misconfigured output path could point at # somebody's source tree — so nothing outside this list is ever deleted. -GENERATED = ("book.json", "notebook.ipynb", "steps", "site", "explore") +GENERATED = ("book.json", "notebook.ipynb", "steps", "site", "explore", "document.json") + +# The steps a book runs, by id — fixed, so a note can be filed under one before +# the book has ever been built, and a skipped step does not make its note drift. +STEP_LABELS = ( + ("ir", "the graph, extracted"), + ("view", "the default view for this source"), + ("pages", "the diagram, split into pages that fit"), + ("graph", "the whole view in one drawing"), + ("index", "readable without a diagram"), + ("sidebar", "navigation, for whatever renders it"), + ("minimap", "what is where, read from the colours"), + ("explore", "navigate on one side, explore on the other"), +) def clear(out) -> list[str]: @@ -253,12 +273,19 @@ def clear(out) -> list[str]: return removed -def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid", - theme: str | None = None, exclude=(), overlay=None, quiet: bool = False) -> Book: - """The whole book, in one process. Returns it; `book.json` is written.""" +def build_book(kind: str, source, out, *, slug: str | None = None, exclude=(), + ops: dict | None = None, suffixes=None, quiet: bool = False) -> Book: + """One book's steps: larder, IR, view, pages, drawings. No site, no notebook. + + Those are a document's outputs and are assembled by `book/document.py`, which + may put several books into one of each. The book is returned open — its last + step is added once the outputs it belongs to exist. + """ from ..ir import check from ..ops import classify, overview + from ..ops.pages import for_view from ..style import Style + from .document import apply_ops out = Path(out) slug = slug or Path(str(source)).name or "book" @@ -270,7 +297,7 @@ def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid" if not quiet: print(text) - ir, larder = extract(kind, source, exclude=exclude) + ir, larder = extract(kind, source, exclude=exclude, suffixes=suffixes) if larder is None: larder = Larder(kind=kind, identity=str(source), unit="document", seen=0) book = Book(slug=slug, larder=larder, out=out) @@ -289,22 +316,31 @@ def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid" p.write_text(json.dumps(ir, indent=2) + "\n") book.step("ir", "the graph, extracted", p, note=f"{len(ir['nodes'])} nodes") - view = overview(ir) + view = apply_ops(overview(ir), ops or {}) + book.view = view p = steps_dir / "view.json" p.write_text(json.dumps(view, indent=2) + "\n") verdict = classify(view) book.step("view", "the default view for this source", p, note=f"{verdict['kind']} — {verdict['why']}") - style_obj = Style.load(style, theme=theme) + # Which pages exist — width-free, so it is decided once. Each output lays + # them out for its own column when it is assembled. + book.pages = for_view(ir, view) + p = steps_dir / "pages.json" + p.write_text(json.dumps( + [{k: v for k, v in page.items() if k != "ir"} for page in book.pages], + indent=2) + "\n") + book.step("pages", "the diagram, split into pages that fit", p, + note=f"{len(book.pages)} page(s)") + + style_obj = Style.load("lucid") - # The drawing, chosen by structure rather than by the caller — the same - # `draw()` the `emit auto` and `emit site` commands use, so the three can - # never disagree about what a graph wants. + # The whole view in one drawing — still worth having as a step on its own, + # and the same `draw()` the `emit auto` command uses. from ..emitters.auto import draw drawn = draw(view, style_obj) - graph_rel = None labels = {"erd": "drawn as an ERD", "dot": f"drawn as a {verdict['kind']}", "index": "not a diagram — written as a list"} if drawn["content"] is None: @@ -315,8 +351,6 @@ def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid" p.write_bytes(drawn["content"]) else: p.write_text(drawn["content"]) - if drawn["suffix"] == ".svg": - graph_rel = f"steps/{p.name}" book.step("graph", labels[drawn["emitter"]], p, note=drawn["why"]) from ..emitters.index import to_markdown, to_sidebar @@ -343,38 +377,41 @@ def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid" except Exception as e: # noqa: BLE001 book.step("explore", "explorer not built", skipped=f"{type(e).__name__}: {e}") - # -- the last step ---------------------------------------------------- - from ..emitters.site import write as site_write - site_dir = out / "site" - site_write(view, style_obj, site_dir, - graph=Path(graph_rel).name if graph_rel else None, - title=slug, book=book.to_dict()) - if graph_rel and (out / graph_rel).exists(): - (site_dir / Path(graph_rel).name).write_bytes((out / graph_rel).read_bytes()) - - # close() BEFORE the notebook spec is built, not after. The spec quotes the - # book measure, and the book measure only includes the site once the last - # step exists — build it the other way round and the notebook says 7 - # artifacts while book.json says 8, which is exactly what it did. - book.close(site=site_dir) - - from ..emitters.notebook import write as nb_write - from ..notebook import spec as spec_mod - spec, spec_problems = spec_mod.merge(spec_from(book, ir), overlay) - nb_write(spec, out / "notebook.ipynb") - # Recorded by name and not by size: a notebook cannot report its own byte - # count without changing it. - book.notebook = "notebook.ipynb" + return book - for pr in spec_problems: - say(f" overlay {pr}") - path = book.write() +def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid", + theme: str | None = None, exclude=(), overlay=None, notes=None, + quiet: bool = False) -> Book: + """One book, as a one-book document whose outputs live in its own directory. + + `book.json`, `steps/`, `site/` and `notebook.ipynb` land where they always did; + the site and notebook are now assembled the same way a many-book document's are. + """ + from .document import Document, Output, Part, build + + out = Path(out) + slug = slug or Path(str(source)).name or "book" + doc = Document( + name=slug, title=slug, build=out, inline=True, + notes=Path(notes) if notes else None, + parts=[Part(name=slug, kind=kind, source=Path(source), out=out, + exclude=tuple(exclude), overlay=overlay)], + outputs=[Output("site", out / "site", style=style, theme=theme), + Output("notebook", out / "notebook.ipynb", style=style, theme=theme)], + ) + build(doc, quiet=quiet) + book = doc.books[0] + + for pr in doc.problems: + if not quiet: + print(f" note {pr}") m = book.measure() - say(f" book {m['nodes']} nodes · {m['edges']} edges · " - f"{m['external']} external · {len(m['artifacts'])} artifacts") - for r in book.compare(): - say(f" {'ok ' if r['ok'] else 'LOST'} {r['claim']}") - say(f" open {site_dir / 'index.html'}") - say(f" {path}") + if not quiet: + print(f" book {m['nodes']} nodes · {m['edges']} edges · " + f"{m['external']} external · {len(m['artifacts'])} artifacts") + for r in book.compare(): + print(f" {'ok ' if r['ok'] else 'LOST'} {r['claim']}") + print(f" open {out / 'site' / 'index.html'}") + print(f" {out / 'book.json'}") return book diff --git a/soleprint/atlas2/docgen/book/checks.py b/soleprint/atlas2/docgen/book/checks.py index 8c69b78..3c56c5c 100644 --- a/soleprint/atlas2/docgen/book/checks.py +++ b/soleprint/atlas2/docgen/book/checks.py @@ -90,8 +90,9 @@ class Loaded: class Report: """rig's reporting shape: sections, one line per decision, nothing aborts.""" - def __init__(self): + def __init__(self, subject: str = "book"): self.passed, self.failed, self.skipped = 0, [], 0 + self.subject = subject def note(self, text: str) -> None: print(f"\n{text}") @@ -116,7 +117,7 @@ class Report: # it summarises — which is how this was found, piping to `tail`. sys.stdout.flush() if not self.failed: - print(f"{self.passed} checks passed — this book still holds" + print(f"{self.passed} checks passed — this {self.subject} still holds" + (f", {self.skipped} skipped" if self.skipped else "")) return 0 print(f"FAILED — {len(self.failed)} of {self.passed + len(self.failed)}: " @@ -204,6 +205,12 @@ def generated(book: Loaded, r: Report) -> None: r.check("the page admits the book is incomplete", True, "This book is incomplete" in html) + # A book built inline — its own site and notebook in its own directory — + # also carries the document ledger, and with it the notes it used. + doc_ledger = book.dir / "document.json" + if doc_ledger.is_file(): + _notes_drift(json.loads(doc_ledger.read_text()), r) + r.note("the book is reproducible") # A timestamp would make two builds of an unchanged larder differ, which # destroys the only useful property a ledger has: that a diff means a real @@ -272,3 +279,99 @@ def _run_cells(cells: list, cwd: Path) -> tuple[int, str | None]: finally: os.chdir(previous) return ran, failure + + +# -- documents -------------------------------------------------------------------- + +def _notes_drift(ledger: dict, r: Report) -> None: + r.note("notes land where they are filed") + # A note filed under a page that no longer exists has nowhere to go. It is + # reported rather than dropped, because dropping it is how hand-written text + # goes stale without anyone noticing — and this is the check that fails on it. + drift = (ledger.get("notes") or {}).get("drift") or [] + r.check("no note is filed under a page or step that is gone", [], drift) + + +def document(build_dir, r: Report) -> None: + """A document's own assertions: its books, its notes, every output it wrote.""" + import re + + build_dir = Path(build_dir) + ledger = json.loads((build_dir / "document.json").read_text()) + books = ledger.get("books") or [] + + r.note("the document is made of its books") + r.check("it has at least one book", True, len(books) > 0) + for b in books: + r.check(f"book {b['name']} has its ledger", True, (Path(b["dir"]) / "book.json").is_file()) + + _notes_drift(ledger, r) + + total_pages = sum(len(b.get("pages") or []) for b in books) + for out in ledger.get("outputs") or []: + path = Path(out["path"]) + if out["emitter"] == "notebook": + r.note(f"the notebook — {path.name}") + if not path.is_file(): + r.check(f"{path} exists", True, False) + continue + nb = json.loads(path.read_text()) + text = ["".join(c.get("source") or []) for c in nb.get("cells") or []] + # Both ends of every part: the rule holds per book, not only overall. + # Matched by book name, because the document's own closing cell is + # also headed "— what came out" and a bare count mistook it for a part. + def heading(name, end): + return sum(1 for t in text if f"## {name} — what came {end}" in t) + + r.check("every book part opens with what came in", [], + [b["name"] for b in books if heading(b["name"], "in") != 1]) + r.check("every book part closes with what came out", [], + [b["name"] for b in books if heading(b["name"], "out") != 1]) + figures = [c for c in nb.get("cells") or [] if 'what came in")) + r.check("the index shows what came out, for every book", len(books), + html.count("
what came out
")) + manifest = path / ".docgen-output.json" + written = json.loads(manifest.read_text()).get("written", []) if manifest.is_file() else [] + r.check("the output lists what it wrote", True, bool(written)) + r.check("every file it lists is there", [], [w for w in written if not (path / w).is_file()]) + # Every link on every page resolves — the drill-down is only worth + # having if following it never lands on nothing. + broken = [] + for page in sorted(path.glob("*/*.html")): + for href in re.findall(r'href="([^"#?]+)', page.read_text()): + if href.startswith(("http:", "https:", "mailto:")): + continue + if not (page.parent / href).exists(): + broken.append(f"{page.relative_to(path)} -> {href}") + r.check("every link on every page opens something", [], broken[:10]) + pages = sum(1 for b in books for p in (b.get("pages") or []) + if (path / b["name"] / f"{p['slug']}.html").is_file()) + r.check("every page has its own HTML", total_pages, pages) + + +def document_failures(build_dir, *, quiet: bool = True) -> list[str]: + import contextlib + import io + + report = Report("document") + sink = io.StringIO() if quiet else None + with contextlib.redirect_stdout(sink) if sink else contextlib.nullcontext(): + document(build_dir, report) + return list(report.failed) diff --git a/soleprint/atlas2/docgen/book/config.py b/soleprint/atlas2/docgen/book/config.py index e6391a6..bff8674 100644 --- a/soleprint/atlas2/docgen/book/config.py +++ b/soleprint/atlas2/docgen/book/config.py @@ -71,8 +71,8 @@ SOURCES = {"root": "python", "schema": "db", "openapi": "openapi", "har": "usage READERS = ("python", "code") TOP_KEYS = {"reference", "defaults", "book"} -DEFAULT_KEYS = {"out", "style", "theme", "exclude", "reader"} -BOOK_KEYS = {"name", "out", "style", "theme", "exclude", "reader", "overlay", *SOURCES} +DEFAULT_KEYS = {"out", "style", "theme", "exclude", "reader", "notes"} +BOOK_KEYS = {"name", "out", "style", "theme", "exclude", "reader", "overlay", "notes", *SOURCES} # A book name becomes a directory name and a slug, so it is held to what is safe # as both — no separators, nothing that means something to a shell. @@ -99,6 +99,7 @@ class Entry: theme: str | None = None overlay: Path | None = None exclude: tuple = () + notes: Path | None = None def line(self) -> str: return f"{self.name:<18} {self.kind:<8} {self.source} -> {self.out}" @@ -137,18 +138,38 @@ def _within(inner: Path, outer: Path) -> bool: return inner == outer or outer in inner.parents -def load(path) -> RunFile: - """Read and resolve a run file. Raises ConfigError listing every problem.""" - path = Path(path) +def _read(path: Path) -> dict: if tomllib is None: raise ConfigError(path, ["run files need Python 3.11+ (tomllib)"]) try: - data = tomllib.loads(path.read_text()) + return tomllib.loads(path.read_text()) except OSError as e: raise ConfigError(path, [f"cannot read: {e}"]) from None except tomllib.TOMLDecodeError as e: raise ConfigError(path, [f"not valid TOML: {e}"]) from None + +def load(path): + """Read and resolve a run file of either shape. Raises ConfigError listing + every problem. + + Returns a `RunFile` for a `[[book]]` list, or a `ProjectFile` for a project. + A file with both is refused: which one it meant is exactly what nobody should + have to guess. + """ + path = Path(path) + data = _read(path) + if "project" in data and "book" in data: + raise ConfigError(path, [ + "this file has both [project] and [[book]] — use one shape per file: " + "[project] for one document built from several sources, [[book]] for a " + "list of independent books"]) + if "project" in data: + return load_project(path, data) + return _load_books(path, data) + + +def _load_books(path: Path, data: dict) -> "RunFile": base = path.resolve().parent problems: list[str] = [] @@ -246,10 +267,12 @@ def load(path) -> RunFile: if source_key == "root" and source.is_dir() and _within(out, source): problems.append(f"{where}: out {out} is inside the tree it reads ({source})") + notes = _path(pick("notes", "notes"), base) + books.append(Entry( name=name, kind=kind, source=source, out=out, style=pick("style", "lucid"), theme=pick("theme"), - overlay=overlay, exclude=tuple(exclude), + overlay=overlay, exclude=tuple(exclude), notes=notes, )) seen_names, seen_outs = {}, {} @@ -282,13 +305,30 @@ def apply_reference(runfile: RunFile) -> str | None: return str(runfile.reference) -def run(runfile: RunFile, names=None, *, check: bool = False, quiet: bool = True): +def _checks(directory, quiet: bool) -> list[str]: + import contextlib + import io + + from . import checks as checks_mod + + report = checks_mod.Report() + loaded = checks_mod.Loaded(directory) + sink = io.StringIO() if quiet else None + with contextlib.redirect_stdout(sink) if sink else contextlib.nullcontext(): + checks_mod.generated(loaded, report) + checks_mod.custom(loaded, report) + return list(report.failed) + + +def run(runfile, names=None, *, check: bool = False, quiet: bool = True): """Build every selected book, in order. One failure never stops the rest. - Returns one result per book: + Returns one result per book, plus one per output for a project: {"name", "out", "ok", "error", "lost": [claims], "checks_failed": [names]} """ - from . import checks as checks_mod + if isinstance(runfile, ProjectFile): + return run_project(runfile, names, check=check, quiet=quiet) + from .build import run as build apply_reference(runfile) @@ -303,7 +343,7 @@ def run(runfile: RunFile, names=None, *, check: bool = False, quiet: bool = True overlay = spec_mod.load(entry.overlay) book = build(entry.kind, entry.source, entry.out, slug=entry.name, style=entry.style, theme=entry.theme, exclude=entry.exclude, - overlay=overlay, quiet=quiet) + overlay=overlay, notes=entry.notes, quiet=quiet) result["lost"] = [r["claim"] for r in book.compare() if not r["ok"]] except Exception as e: # noqa: BLE001 - one bad book must not cost the run result["error"] = f"{type(e).__name__}: {e}" @@ -311,17 +351,265 @@ def run(runfile: RunFile, names=None, *, check: bool = False, quiet: bool = True continue if check: - import contextlib - import io - - report = checks_mod.Report() - loaded = checks_mod.Loaded(entry.out) - sink = io.StringIO() if quiet else None - with contextlib.redirect_stdout(sink) if sink else contextlib.nullcontext(): - checks_mod.generated(loaded, report) - checks_mod.custom(loaded, report) - result["checks_failed"] = list(report.failed) + result["checks_failed"] = _checks(entry.out, quiet) result["ok"] = not result["lost"] and not result["checks_failed"] results.append(result) return results + + +# -- the project shape ---------------------------------------------------------- +# +# One file per project, the shape a model writes unprompted: +# +# [project] name, title, notes, build, reference +# [extract.] one book per table; [[extract.]] for several +# [ops] applied to every book's view, before paging +# [emit.] notebook, site — each assembles every book +# +# It compiles to the same Document a [[book]] entry does, so everything after +# loading is one code path. + +PROJECT_KEYS = {"name", "title", "notes", "build", "reference"} +PROJECT_TOP = {"project", "extract", "ops", "emit"} +EXTRACT_KEYS = { + "python": {"name", "root", "exclude", "overlay"}, + "code": {"name", "root", "ext", "exclude", "overlay"}, + "db": {"name", "schema", "overlay"}, + "openapi": {"name", "spec", "overlay"}, + "usage": {"name", "har", "overlay"}, +} +EXTRACT_SOURCE = {"python": "root", "code": "root", "db": "schema", + "openapi": "spec", "usage": "har"} +OPS_KEYS = {"drop_stdlib", "drop_builtins", "drop_external", "only", "drop"} +EMITTERS = {"notebook", "site"} +EMIT_KEYS = {"output", "style", "theme", "width"} + +# Keys a model reaches for that docgen deliberately does not do, with what to do +# instead. Refused with the reason rather than with a bare "unknown key", because +# the reason is the useful half of the refusal. +NOT_SUPPORTED = { + ("db", "url"): ( + "docgen reads a schema dump and never connects to a database — and a URL in " + "a committed file is a credential in a committed file. Write the schema with " + "`modelgen from-db`, then set `schema = \"path/to/schema.json\"`" + ), +} + + +@dataclass +class ProjectFile: + path: Path + document: object # book.document.Document + reference: Path | None = None + + @property + def books(self): + return self.document.parts + + +def load_project(path: Path, data: dict) -> ProjectFile: + from ..extractors.code import LANGUAGES + from .document import Document, Output, Part + + base = path.resolve().parent + problems: list[str] = [] + + for key in sorted(set(data) - PROJECT_TOP): + problems.append(f"unknown top-level key {key!r} — a project file has: " + f"{', '.join(sorted(PROJECT_TOP))}") + + project = data.get("project") or {} + for key in sorted(set(project) - PROJECT_KEYS): + problems.append(f"[project] has unknown key {key!r} — have: {', '.join(sorted(PROJECT_KEYS))}") + name = project.get("name") + if not isinstance(name, str) or not NAME.match(name): + problems.append("[project] needs a name of letters, digits, '.', '_' or '-'") + name = "project" + title = project.get("title") or name + notes = _path(project.get("notes", "notes"), base) + build_root = _path(project.get("build", ".docgen"), base) + reference = _path(project["reference"], base) if "reference" in project else None + if reference is not None and not reference.is_dir(): + problems.append(f"[project] reference {reference} is not a directory") + + # -- extract ---------------------------------------------------------------- + extract = data.get("extract") or {} + if not isinstance(extract, dict) or not extract: + problems.append("no sources — add at least one [extract.] table " + f"({', '.join(sorted(EXTRACT_KEYS))})") + extract = {} + parts, names = [], set() + for kind, tables in extract.items(): + if kind not in EXTRACT_KEYS: + problems.append(f"[extract.{kind}]: no reader named {kind!r} — have: " + f"{', '.join(sorted(EXTRACT_KEYS))}") + continue + tables = tables if isinstance(tables, list) else [tables] + for i, raw in enumerate(tables): + where = f"[extract.{kind}]" + (f"[{i}]" if len(tables) > 1 else "") + if not isinstance(raw, dict): + problems.append(f"{where} must be a table") + continue + for key in sorted(set(raw) - EXTRACT_KEYS[kind]): + reason = NOT_SUPPORTED.get((kind, key)) + problems.append(f"{where} {key}: {reason}" if reason else + f"{where} has unknown key {key!r} — have: " + f"{', '.join(sorted(EXTRACT_KEYS[kind]))}") + book = raw.get("name") or (kind if len(tables) == 1 else None) + if not isinstance(book, str) or not NAME.match(book): + problems.append(f"{where} needs a name when there are several " + f"[[extract.{kind}]] tables") + continue + if book in names: + problems.append(f"{where}: a book named {book!r} already exists — set `name`") + continue + names.add(book) + + src_key = EXTRACT_SOURCE[kind] + if src_key not in raw: + problems.append(f"{where} needs `{src_key}`") + continue + source = _path(raw[src_key], base) + if src_key == "root": + if not source.is_dir(): + problems.append(f"{where}: root {source} is not a directory") + elif not source.is_file(): + problems.append(f"{where}: {src_key} {source} is not a file") + + exclude = raw.get("exclude", []) + if not isinstance(exclude, list) or not all(isinstance(x, str) for x in exclude): + problems.append(f"{where}: exclude must be a list of directory names") + exclude = [] + + ext = raw.get("ext") + if ext is not None: + if not isinstance(ext, list) or not all(isinstance(x, str) for x in ext): + problems.append(f"{where}: ext must be a list like [\".cs\", \".ts\"]") + else: + missing = [x for x in ext if x not in LANGUAGES] + if missing: + problems.append( + f"{where}: no grammar for {', '.join(missing)} — code reads " + f"{', '.join(sorted(LANGUAGES))}. Shell, YAML and SQL have no " + "reader yet; leave them out rather than expect them in the book") + + overlay = None + if "overlay" in raw: + overlay_path = _path(raw["overlay"], base) + if not overlay_path.is_file(): + problems.append(f"{where}: overlay {overlay_path} is not a file") + else: + import json + overlay = json.loads(overlay_path.read_text()) + + out = build_root / name / "books" / book + if src_key == "root" and source.is_dir() and _within(out, source): + problems.append(f"{where}: build directory {out} is inside the tree it reads " + f"({source}) — set [project] build outside it") + parts.append(Part(name=book, kind=kind, source=source, out=out, + exclude=tuple(exclude), overlay=overlay, + suffixes=tuple(ext) if ext else None)) + + # -- ops -------------------------------------------------------------------- + ops = data.get("ops") or {} + for key in sorted(set(ops) - OPS_KEYS): + problems.append(f"[ops] has unknown key {key!r} — have: {', '.join(sorted(OPS_KEYS))}") + for key in ("only", "drop"): + if key in ops and not (isinstance(ops[key], list) + and all(isinstance(x, str) for x in ops[key])): + problems.append(f"[ops] {key} must be a list of kinds") + for part in parts: + part.ops = {k: v for k, v in ops.items() if k in OPS_KEYS} + + # -- emit ------------------------------------------------------------------- + emit = data.get("emit") or {} + outputs = [] + if not emit: + problems.append("no outputs — add [emit.notebook] or [emit.site] with an `output`") + for emitter, raw in emit.items(): + where = f"[emit.{emitter}]" + if emitter not in EMITTERS: + problems.append(f"{where}: no output named {emitter!r} — have: " + f"{', '.join(sorted(EMITTERS))}") + continue + if not isinstance(raw, dict): + problems.append(f"{where} must be a table") + continue + for key in sorted(set(raw) - EMIT_KEYS): + problems.append(f"{where} has unknown key {key!r} — have: {', '.join(sorted(EMIT_KEYS))}") + if "output" not in raw: + problems.append(f"{where} needs `output`") + continue + width = raw.get("width") + if width is not None and (not isinstance(width, int) or width < 200): + problems.append(f"{where}: width must be a number of pixels, 200 or more") + width = None + target = _path(raw["output"], base) + for part in parts: + if part.kind in ("python", "code") and part.source.is_dir() \ + and _within(target, part.source): + problems.append(f"{where}: output {target} is inside the tree book " + f"{part.name!r} reads — the next run would read it") + outputs.append(Output(emitter=emitter, path=target, style=raw.get("style", "lucid"), + theme=raw.get("theme"), width=width)) + + if problems: + raise ConfigError(path, problems) + doc = Document(name=name, title=title, build=build_root / name, parts=parts, + outputs=outputs, notes=notes) + return ProjectFile(path=path, document=doc, reference=reference) + + +def select_parts(project: ProjectFile, names) -> list: + if not names: + return list(project.document.parts) + known = {p.name for p in project.document.parts} + unknown = [n for n in names if n not in known] + if unknown: + raise ConfigError(project.path, [ + f"no book named {n!r} — have: {', '.join(sorted(known))}" for n in unknown]) + return [p for p in project.document.parts if p.name in set(names)] + + +def run_project(project: ProjectFile, names=None, *, check: bool = False, quiet: bool = True): + """Build a project's document: every book, then every output. + + `--only` narrows the books that go into this build's outputs, which is useful + while writing notes for one of them — the outputs are then that subset, and + say so in their ledger. + """ + from dataclasses import replace + + from .document import build + + apply_reference(project) + doc = replace(project.document, parts=select_parts(project, names), books=[], problems=[]) + results = [] + try: + build(doc, quiet=quiet) + except Exception as e: # noqa: BLE001 - reported, not raised + return [{"name": doc.name, "out": str(doc.build), "ok": False, + "error": f"{type(e).__name__}: {e}", "lost": [], "checks_failed": []}] + + for part in doc.parts: + if part.name in doc.failed: + results.append({"name": part.name, "out": str(part.out), "ok": False, + "error": doc.failed[part.name], "lost": [], "checks_failed": []}) + for book in doc.books: + lost = [r["claim"] for r in book.compare() if not r["ok"]] + failed = _checks(book.out, quiet) if check else [] + results.append({"name": book.slug, "out": str(book.out), "ok": not lost and not failed, + "error": None, "lost": lost, "checks_failed": failed}) + if check and doc.books: + from . import checks as checks_mod + failed = checks_mod.document_failures(doc.build, quiet=quiet) + else: + failed = [] + # A failed book is already its own row; the document row carries what else + # went wrong — notes that drifted, an overlay that no longer matches. + rest = [p for p in doc.problems if not p.startswith("book ") or "was not built" not in p] + results.append({"name": f"{doc.name} (document)", "out": str(doc.build), + "ok": not rest and not failed and bool(doc.books), "error": None, + "lost": rest, "checks_failed": failed}) + return results diff --git a/soleprint/atlas2/docgen/book/document.py b/soleprint/atlas2/docgen/book/document.py new file mode 100644 index 0000000..4de04b5 --- /dev/null +++ b/soleprint/atlas2/docgen/book/document.py @@ -0,0 +1,448 @@ +""" +A document: several books, assembled into each output. + + Document + ├── books one per source — each keeps its own larder → steps → book + ├── notes hand-written markdown, filed by page (book/notes.py) + └── outputs a notebook, a site, … — each one made of every book, as parts + +A book is a templated part of the final output, not the output itself. A project +with an OpenAPI contract and a C# tree is one document of two books: one notebook +and one site, each with a part per book. A `[[book]]` run file is the same thing +with one book per document, which is what it always was. + +## Built once, rendered per output + +Extraction, the view and the page structure happen once per book. Rendering +does not, because what a page looks like depends on where it is going: a site +column is narrower than a notebook cell, a site's boxes are links and a +notebook's cannot be, and each output can have its own theme. So each output +lays the pages out for its own width and draws them itself. + +## Both ends, at both levels + +Every book part still opens with what came in and closes with what came out. A +document of several books also opens with every larder and closes with every +reconciliation, so the rule — any operation is measured at both ends — holds for +the whole as well as for each part. +""" + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +from . import Book +from .notes import Notes, data_uri, images +from .larder import Larder + +MANIFEST = ".docgen-output.json" + + +@dataclass +class Output: + emitter: str # "notebook" | "site" + path: Path + style: str = "lucid" + theme: str | None = None + width: int | None = None # px; each emitter has its own default + + +@dataclass +class Part: + """One book to build: a source, and where its artifacts go.""" + + name: str + kind: str + source: Path + out: Path + exclude: tuple = () + overlay: dict | None = None + ops: dict = field(default_factory=dict) + suffixes: tuple | None = None # code only: which extensions to read + + +@dataclass +class Document: + name: str + title: str + build: Path # where document.json goes + parts: list[Part] + outputs: list[Output] + notes: Path | None = None + inline: bool = False # a single [[book]]: outputs live inside its directory + books: list[Book] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + failed: dict = field(default_factory=dict) # book name -> why it was not built + + +# -- views -------------------------------------------------------------------- + +OPS = ("drop_stdlib", "drop_builtins", "drop_external", "only", "drop") + + +def apply_ops(view: dict, ops: dict) -> dict: + """A project's `[ops]`, on top of the default view. Unknown keys never get here.""" + from ..ops import filter as F + + if ops.get("drop_stdlib"): + view = F.drop_stdlib(view) + if ops.get("drop_builtins"): + view = F.drop_builtins(view) + if ops.get("drop_external"): + view = F.drop_external(view) + if ops.get("drop"): + view = F.drop_kinds(view, ops["drop"]) + if ops.get("only"): + view = F.only_kinds(view, ops["only"]) + return view + + +# -- output bookkeeping ------------------------------------------------------- + +def clear_output(directory: Path) -> list[str]: + """Remove exactly what the last build wrote into an output directory. + + A page that disappeared between builds would otherwise linger — still on disk, + still linked from nowhere, still found by search. The manifest lists every file + written; anything not in it, including a hand-written file someone dropped in, + is left alone. + """ + manifest = directory / MANIFEST + if not manifest.is_file(): + return [] + try: + written = json.loads(manifest.read_text()).get("written", []) + except (OSError, json.JSONDecodeError): + return [] + removed = [] + for rel in written: + target = (directory / rel) + if target.is_file() and directory in target.resolve().parents: + target.unlink() + removed.append(rel) + # Directories docgen made and has now emptied — never one with anything left. + for d in sorted({(directory / r).parent for r in written}, key=lambda p: -len(p.parts)): + if d != directory and d.is_dir() and not any(d.iterdir()): + d.rmdir() + manifest.unlink() + return removed + + +def write_manifest(directory: Path, paths: list[Path]) -> None: + rel = sorted({p.relative_to(directory).as_posix() for p in paths if p.is_file()}) + (directory / MANIFEST).write_text(json.dumps({"written": rel}, indent=2) + "\n") + + +# -- pages, rendered ---------------------------------------------------------- + +def rendered_pages(book: Book, style, width: int, *, url=None): + """[(page, svg or None)] laid out for `width`. `url(page)` makes boxes links.""" + from ..emitters.paged import fits_for, links_for, render + from ..ops.pages import layout + + fits, measured = fits_for(style, width) + laid = layout(book.pages, fits) + known = {n["id"] for n in book.ir["nodes"]} + out = [] + for page in laid: + links = None + if url is not None: + links = links_for(laid, page, url) + # A box with no page of its own opens its section of the reference + # page, where its docstring and file are — when it is a real node. + for n in page["ir"]["nodes"]: + if n["id"] not in links and n["id"] in known \ + and not (n.get("attrs") or {}).get("context"): + links[n["id"]] = f"reference.html#{n['id']}" + page["measured"] = measured + out.append((page, render(page, style, width, links=links))) + return out + + +def caption(page: dict) -> str: + nodes = page["ir"]["nodes"] + parts = [f"{len(nodes)} box{'es' if len(nodes) != 1 else ''}", + f"{len(page['ir']['edges'])} relationship{'s' if len(page['ir']['edges']) != 1 else ''}"] + context = sum(1 for n in nodes if (n.get("attrs") or {}).get("context")) + if context: + parts.append(f"{context} outside this subsystem") + if page.get("children"): + parts.append(f"opens into {len(page['children'])} page{'s' if len(page['children']) != 1 else ''}") + if (page.get("options") or {}).get("rankdir") == "LR": + parts.append("drawn left to right to fit the width") + if page.get("too_wide"): + parts.append("still wider than the page — this level has no smaller structure to split into") + return " · ".join(parts) + + +# -- the notebook --------------------------------------------------------------- + +def _note_step(text: str, base: Path) -> tuple[str, dict]: + """A note for a notebook cell: local images become cell attachments.""" + attachments = {} + for ref, name, mime, data in images(text, base): + import base64 + text = text.replace(f"]({ref})", f"](attachment:{name})") + attachments[name] = {mime: base64.b64encode(data).decode("ascii")} + return text, attachments + + +def _with_note(step: dict, notes: Notes, book: str | None, spot: str) -> dict: + text = notes.book(book, spot) if book else notes.document(spot) + if text is None: + return step + path = notes.path(book, spot) + text, attachments = _note_step(text, path.parent) + step = dict(step) + step["before"] = text + ("\n\n" + step["before"] if step.get("before") else "") + if attachments: + step["before_attachments"] = {**(step.get("before_attachments") or {}), **attachments} + return step + + +def notebook_spec(doc: Document, notes: Notes, output: Output, notebook_dir: Path) -> dict: + """Every book's part, in order, with the document's own ends when there are several.""" + from ..emitters.paged import NOTEBOOK_WIDTH + from ..notebook import spec as spec_mod + from ..ops.pages import slugify + from ..style import Style + from .build import spec_from + + style = Style.load(output.style, theme=output.theme) + width = output.width or NOTEBOOK_WIDTH + several = len(doc.books) > 1 + steps: list[dict] = [] + + missing = "" + if doc.failed: + missing = ("\n\n**This document is incomplete** — these books could not be built " + "and are not below:\n\n" + + "\n".join(f"- `{n}` — {why}" for n, why in sorted(doc.failed.items()))) + if several or doc.failed: + lines = "\n".join(f"- **{b.slug}** — {b.larder.line()}" for b in doc.books) + steps.append(spec_mod._step( + "document", "md", title=doc.title, + text=(f"Built from {len(doc.books)} book(s). Each part below opens with what " + f"came in and closes with what came out.\n\n{lines}{missing}"))) + + for i, book in enumerate(doc.books): + rel = os.path.relpath(book.out, notebook_dir) + base = spec_from(book, book.ir, prelude_dir=None if doc.inline else rel) + base_steps = base["steps"] + + figures = [] + index_lines = [] + for page, svg in rendered_pages(book, style, width): + anchor = f"page-{slugify(book.slug)}-{page['slug']}" + index_lines.append(f"{' ' * page['depth']}- [{page['title']}](#{anchor})" + + (" — *too wide*" if page.get("too_wide") else "")) + figures.append(spec_mod._step( + f"page-{page['slug']}", "figure", title=page["title"], anchor=anchor, + depth=page["depth"], svg=svg, name=f"{page['slug']}.svg", + caption=caption(page), + why="Graphviz is not installed" if svg is None else None)) + index = spec_mod._step("pages", "md", title=f"{book.slug} — pages", + text="Each page is one subsystem, drawn small enough to " + "read. Follow a link to jump to it.\n\n" + + "\n".join(index_lines)) + + ordered = [] + for step in base_steps: + if step["id"] == "book": + ordered.extend(figures) + ordered.append(step) + if step["id"] == "larder": + ordered.append(index) + + spec, problems = spec_mod.merge({"meta": {}, "steps": ordered}, + doc.parts_built[i].overlay) + doc.problems += [f"{book.slug}: overlay: {p}" for p in problems] + + # Notes are applied after the overlay, so an overlay that moves a step moves + # its note with it. A page's note is filed under the page's slug; every + # other step under its own id. `_with_note` prepends, so applying the book's + # `_intro` last puts it above the larder's own note. + part = [] + for n, step in enumerate(spec["steps"]): + sid = step["id"] + spot = sid[len("page-"):] if step.get("kind") == "figure" else slugify(sid) + step = _with_note(step, notes, book.slug, spot) + if n == 0: + step = _with_note(step, notes, book.slug, "_intro") + if several: + step = dict(step, id=f"{book.slug}/{sid}") + part.append(step) + steps.extend(part) + + if several: + lines = [] + for b in doc.books: + m = b.measure() + lines.append(f"- **{b.slug}** — {m['nodes']} nodes · {m['edges']} edges · " + f"{m['external']} external") + lines += [f" - {'✓' if r['ok'] else '✗'} {r['claim']}" for r in b.compare()] + steps.append(spec_mod._step("document-end", "md", title=f"{doc.title} — what came out", + text="\n".join(lines))) + if steps: + steps[0] = _with_note(steps[0], notes, None, "_intro") + end = notes.document("_end") + if end is not None: + text, att = _note_step(end, notes.path(None, "_end").parent) + steps.append(spec_mod._step("_end", "md", text=text, + **({"before_attachments": att} if att else {}))) + return {"version": spec_mod.SPEC_VERSION, "steps": steps} + + +# -- building ----------------------------------------------------------------- + +def build(doc: Document, *, quiet: bool = True) -> Document: + """Build every book, then every output. Returns the document, ledgers written.""" + from ..emitters.notebook import write as nb_write + from ..ops.pages import slugify + from ..style import Style + from .build import build_book + + def say(text): + if not quiet: + print(text) + + notes = Notes(doc.notes) + doc.books, doc.failed = [], {} + built_parts = [] + for part in doc.parts: + # One book that cannot be built must not cost the others — the same rule a + # run of independent books follows. The outputs are assembled from what + # was built, and the ledger names what was not and why, so a partial + # document is visibly partial rather than quietly short. + try: + book = build_book(part.kind, part.source, part.out, slug=part.name, + exclude=part.exclude, ops=part.ops, suffixes=part.suffixes, + quiet=quiet) + except Exception as e: # noqa: BLE001 - recorded, not raised + reason = f"{type(e).__name__}: {str(e).splitlines()[0] if str(e) else ''}" + doc.failed[part.name] = reason + doc.problems.append(f"book {part.name!r} was not built — {reason}") + say(f" FAIL {part.name}: {reason}") + continue + doc.books.append(book) + built_parts.append(part) + doc.parts_built = built_parts + + # Sites first: in a single inline book the site is its last step's artifact, + # and the notebook quotes the book measure — which only counts the site once + # the last step exists. + sites = [o for o in doc.outputs if o.emitter == "site"] + notebooks = [o for o in doc.outputs if o.emitter == "notebook"] + written_outputs = [] + + for output in sites: + from ..emitters import docsite + + site_dir = Path(output.path) + clear_output(site_dir) + style = Style.load(output.style, theme=output.theme) + width = output.width or docsite.CONTENT_WIDTH + books = [] + for book in doc.books: + pages = rendered_pages(book, style, width, url=lambda p: f"{p['slug']}.html") + page_notes = {} + for page, _ in pages: + text = notes.book(book.slug, page["slug"]) + if text is not None: + base = notes.path(book.slug, page["slug"]).parent + page_notes[page["slug"]] = (text, {ref: data_uri(mime, data) + for ref, _, mime, data in images(text, base)}) + intro = notes.book(book.slug, "_intro") + books.append({"book": book, "pages": pages, "notes": page_notes, + "intro": (intro, {}) if intro else None}) + doc_intro = notes.document("_intro") + doc_end = notes.document("_end") + files = docsite.write(site_dir, style, title=doc.title, books=books, + intro=doc_intro, end=doc_end, failed=doc.failed) + write_manifest(site_dir, files) + written_outputs.append({"emitter": "site", "path": str(site_dir), "width": width, + "files": len(files)}) + say(f" site {site_dir / 'index.html'}") + + for book in doc.books: + inside = [o for o in sites if doc.inline and Path(o.path).parent == book.out] + book.close(site=Path(inside[0].path) if inside else None) + + for output in notebooks: + path = Path(output.path) + if path.suffix != ".ipynb": + clear_output(path) + path.mkdir(parents=True, exist_ok=True) + target = path / f"{slugify(doc.name)}.ipynb" + else: + path.parent.mkdir(parents=True, exist_ok=True) + target = path + spec = notebook_spec(doc, notes, output, target.parent) + nb_write(spec, target) + if path.suffix != ".ipynb": + write_manifest(path, [target]) + if doc.inline and len(doc.books) == 1: + doc.books[0].notebook = os.path.relpath(target, doc.books[0].out) + written_outputs.append({"emitter": "notebook", "path": str(target), + "width": output.width or 960}) + say(f" notebook {target}") + + known = {b.slug: known_spots(b) for b in doc.books} + drift = notes.drift(known) + doc.problems += drift + + for book in doc.books: + book.write() + + ledger = { + "name": doc.name, + "title": doc.title, + "failed": doc.failed, + "books": [{"name": b.slug, "dir": str(b.out), + "pages": [{"id": p["id"], "slug": p["slug"], "parent": p["parent"]} + for p in b.pages]} + for b in doc.books], + "outputs": written_outputs, + "notes": {"root": str(doc.notes) if doc.notes else None, + "used": sorted(str(p.relative_to(doc.notes)) for p in notes.used) if doc.notes else [], + "drift": drift}, + "problems": doc.problems, + } + doc.build.mkdir(parents=True, exist_ok=True) + (doc.build / "document.json").write_text(json.dumps(ledger, indent=2) + "\n") + return doc + + +def _spots(ir: dict, pages: list[dict]) -> list[tuple[str, str]]: + """[(spot id, what it is)] for one book, in reading order.""" + from ..notebook import spec as spec_mod + from ..ops.pages import slugify + from .build import STEP_LABELS + + out = [("larder", "what came in"), ("pages", "the page index")] + out += [(p["slug"], f"page {p['title']}") for p in pages] + out += [(f"step-{sid}", f"step: {label}") for sid, label in STEP_LABELS] + # An API-shaped book also carries the endpoint walkthrough; each call is a + # spot of its own, under the slug of its id. + if any(n["kind"] in ("endpoint", "operation") for n in ir.get("nodes") or []): + out += [(slugify(s["id"]), f"{s['kind']}: {s.get('title', s['id'])}") + for s in spec_mod.from_ir(ir)["steps"] if s["id"] != "intro"] + out.append(("book", "what came out")) + return out + + +def known_spots(book: Book) -> set[str]: + """Every id a note for this book may be filed under.""" + return {spot for spot, _ in _spots(book.ir, book.pages)} + + +def plan_spots(part: Part) -> list[tuple[str, str]]: + """The spots a book will have, without building it — extraction and pages only.""" + from ..ops import overview + from ..ops.pages import for_view + from .build import extract + + ir, _ = extract(part.kind, part.source, exclude=part.exclude, suffixes=part.suffixes) + view = apply_ops(overview(ir), part.ops) + return _spots(ir, for_view(ir, view)) diff --git a/soleprint/atlas2/docgen/book/notes.py b/soleprint/atlas2/docgen/book/notes.py new file mode 100644 index 0000000..b509d3c --- /dev/null +++ b/soleprint/atlas2/docgen/book/notes.py @@ -0,0 +1,161 @@ +""" +Notes: hand-written markdown, filed by the page or step it belongs above. + + notes/ + ├── _intro.md top of the document + ├── _end.md bottom of the document + └── / + ├── _intro.md top of that book's part + ├── larder.md above the "what came in" step + ├── overview.md above the overview page + └── docgen.book.md above the page for `docgen.book` + +## Why files named by id + +Generation alone gives a document that is never stale and never says anything a +parser could not work out. Prose is the part only a person can add, and it has +to survive every rebuild — so it cannot live in the output. An overlay could +carry it, but as JSON strings with `\\n` escapes, addressed by ids you have to +guess. A folder of markdown files, one per spot, is something a person or a +model writes without reading a schema: to put text above a page, create the +file named after it. + +`python3 -m docgen scaffold` writes an empty stub for every spot that has none, +so the ids are listed rather than guessed. A stub is a single HTML comment, and a +note that is nothing but that comment renders nothing — so scaffolding a +hundred spots changes the output not at all until one is filled in. + +## Drift is reported, never dropped + +When a page is renamed or removed its note has nowhere to go. It is not quietly +discarded — that is how hand-written text goes stale without anyone noticing — +it is reported, and the book's own check level fails on it. The same rule the +notebook overlay follows. +""" + +import base64 +import mimetypes +import re +from pathlib import Path + +DOCUMENT_SPOTS = ("_intro", "_end") +BOOK_SPOTS = ("_intro",) +STUB = re.compile(r"\A\s*\s*", re.S) +IMAGE = re.compile(r"!\[([^\]]*)\]\(([^)\s]+)\)") + + +def body(text: str | None) -> str | None: + """A note's content, without its stub comment. None when nothing is left.""" + if text is None: + return None + text = STUB.sub("", text, count=1).strip() + return text or None + + +def stub(where: str) -> str: + return (f"\n") + + +class Notes: + """The notes folder of one document. A missing folder is simply no notes.""" + + def __init__(self, root): + self.root = Path(root) if root else None + self.used: set[Path] = set() + + def _read(self, path: Path) -> str | None: + if self.root is None or not path.is_file(): + return None + text = body(path.read_text()) + if text is not None: + self.used.add(path) + return text + + def document(self, spot: str) -> str | None: + return self._read(self.root / f"{spot}.md") if self.root else None + + def book(self, book: str, spot: str) -> str | None: + return self._read(self.root / book / f"{spot}.md") if self.root else None + + def path(self, book: str | None, spot: str) -> Path | None: + if self.root is None: + return None + return (self.root / book / f"{spot}.md") if book else (self.root / f"{spot}.md") + + def drift(self, known: dict[str, set[str]]) -> list[str]: + """Notes filed under a spot that no longer exists, as readable problems. + + `known` is {book name: every page slug and step id it has}. + """ + if self.root is None or not self.root.is_dir(): + return [] + problems = [] + for path in sorted(self.root.rglob("*.md")): + rel = path.relative_to(self.root) + if len(rel.parts) == 1: + if rel.stem not in DOCUMENT_SPOTS: + problems.append(f"note {rel} is not a document spot " + f"({', '.join(s + '.md' for s in DOCUMENT_SPOTS)})") + continue + book = rel.parts[0] + if book not in known: + problems.append(f"note {rel} is filed under book {book!r}, which this " + f"document does not have — have: {', '.join(sorted(known))}") + continue + spot = Path(*rel.parts[1:]).with_suffix("").as_posix() + if spot not in known[book] and spot not in BOOK_SPOTS: + if body(path.read_text()) is None: + continue # an empty stub for a vanished page says nothing + problems.append(f"note {rel} belongs above {spot!r}, which book " + f"{book!r} no longer has — rename or remove it") + return problems + + def scaffold(self, spots: dict[str, list[tuple[str, str]]]) -> list[Path]: + """Write a stub for every spot without a note. Never overwrites. + + `spots` is {book name: [(spot id, what it is)]}. Returns the files written. + """ + if self.root is None: + return [] + written = [] + for spot in DOCUMENT_SPOTS: + path = self.root / f"{spot}.md" + where = "top of the document" if spot == "_intro" else "end of the document" + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(stub(where)) + written.append(path) + for book, entries in spots.items(): + for spot, where in [("_intro", f"top of book {book}")] + list(entries): + path = self.root / book / f"{spot}.md" + if path.exists(): + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(stub(f"{book} › {where}")) + written.append(path) + return written + + +def images(text: str, base: Path) -> list[tuple[str, str, str, bytes]]: + """Local images a note links to: [(markdown ref, name, mime type, bytes)]. + + Resolved against the note's own folder. Remote and missing images are left as + written — a link that does not resolve here is the author's to see, not ours + to delete. + """ + out = [] + for m in IMAGE.finditer(text): + ref = m.group(2) + if re.match(r"^[a-z]+:", ref): + continue + path = (base / ref).resolve() + if not path.is_file(): + continue + mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + out.append((ref, path.name, mime, path.read_bytes())) + return out + + +def data_uri(mime: str, data: bytes) -> str: + return f"data:{mime};base64,{base64.b64encode(data).decode('ascii')}" diff --git a/soleprint/atlas2/docgen/cli/__init__.py b/soleprint/atlas2/docgen/cli/__init__.py index 44f1a18..e7188ec 100644 --- a/soleprint/atlas2/docgen/cli/__init__.py +++ b/soleprint/atlas2/docgen/cli/__init__.py @@ -6,6 +6,8 @@ The command line — every command docgen has, and nothing else. book one operation, measured at both ends run every book a run file lists check docgen's own suite, or one book's own level + scaffold empty notes for every page and step, never overwriting + contract the instructions to hand a model that writes docgen.toml extract source -> ir.json validate check an ir.json at the boundary view ir.json -> a narrower ir.json @@ -36,6 +38,8 @@ COMMANDS = { "book": ("book", "one operation, measured at both ends"), "run": ("run", "every book a run file lists"), "check": ("check", "docgen's own suite, or one book's own level"), + "scaffold": ("scaffold", "empty notes for every page and step, never overwriting"), + "contract": ("contract", "the instructions to hand a model that writes docgen.toml"), "extract": ("extract", "source -> ir.json"), "validate": ("validate", "check an ir.json at the boundary"), "view": ("view", "ir.json -> a narrower ir.json"), diff --git a/soleprint/atlas2/docgen/cli/book.py b/soleprint/atlas2/docgen/cli/book.py index d78ca21..f1db3af 100644 --- a/soleprint/atlas2/docgen/cli/book.py +++ b/soleprint/atlas2/docgen/cli/book.py @@ -37,6 +37,9 @@ def main(argv) -> int: help="A hand-written notebook overlay, re-applied on every build.") p.add_argument("--exclude", action="append", default=[], help="Directory name to skip. Repeatable.") + p.add_argument("--notes", type=Path, + help="A notes folder: /.md lands above that page. " + "See `scaffold`.") p.add_argument("--quiet", "-q", action="store_true") args = p.parse_args(argv) @@ -66,7 +69,7 @@ def main(argv) -> int: try: book = run(kind, source, args.output, slug=args.slug, style=args.style, theme=args.theme, exclude=tuple(args.exclude), overlay=overlay, - quiet=args.quiet) + notes=args.notes, quiet=args.quiet) except Exception as e: # noqa: BLE001 - the CLI reports, it does not traceback raise Abort(f"{type(e).__name__}: {e}") from None diff --git a/soleprint/atlas2/docgen/cli/check.py b/soleprint/atlas2/docgen/cli/check.py index cdd981b..606adda 100644 --- a/soleprint/atlas2/docgen/cli/check.py +++ b/soleprint/atlas2/docgen/cli/check.py @@ -34,7 +34,14 @@ def main(argv) -> int: # what a fresh one would do. return subprocess.call([sys.executable, str(HERE / "selftest.py")]) - from ..book.checks import Loaded, Report, custom, generated + from ..book.checks import Loaded, Report, custom, document, generated + + if not (args.book / "book.json").is_file() and (args.book / "document.json").is_file(): + # A project's build directory: the document's own level. + print(f"{args.book} — document") + r = Report("document") + document(args.book, r) + return r.total() try: book = Loaded(args.book) diff --git a/soleprint/atlas2/docgen/cli/contract.py b/soleprint/atlas2/docgen/cli/contract.py new file mode 100644 index 0000000..2205c4a --- /dev/null +++ b/soleprint/atlas2/docgen/cli/contract.py @@ -0,0 +1,33 @@ +""" + python3 -m docgen contract > docgen-contract.md + +Print the instructions to hand to a model that writes `docgen.toml` and notes — +`docs/models.md`, verbatim. + +One copy of the wording, printed rather than remembered, the same arrangement as +`explode.sh --contract`: what a model is asked to write and what docgen accepts +cannot drift apart, because `selftest.py` checks the file against the loaders — +every key named, every example loading. Attach it; do not paste it into a +message body, where markdown gets a say first. + +With `explode`, attach both: explode's contract says how to format the reply, +this one says what goes in the files. +""" + +import argparse +import sys +from pathlib import Path + +from .common import PROG, Abort + +CONTRACT = Path(__file__).resolve().parent.parent / "docs" / "models.md" + + +def main(argv) -> int: + p = argparse.ArgumentParser(prog=f"{PROG} contract", + description="Print the instructions for a model.") + p.parse_args(argv) + if not CONTRACT.is_file(): + raise Abort(f"{CONTRACT} is missing — is the docgen folder intact?") + sys.stdout.write(CONTRACT.read_text()) + return 0 diff --git a/soleprint/atlas2/docgen/cli/run.py b/soleprint/atlas2/docgen/cli/run.py index 5c9775b..e0321f5 100644 --- a/soleprint/atlas2/docgen/cli/run.py +++ b/soleprint/atlas2/docgen/cli/run.py @@ -30,13 +30,17 @@ def main(argv) -> int: help="Show each book's own output, not just the summary.") args = p.parse_args(argv) - from ..book.config import ConfigError, load, run + from ..book.config import ConfigError, ProjectFile, load, run, select_parts if not args.config.exists(): - raise Abort(f"no run file at {args.config} — see docgen.example.toml") + raise Abort(f"no run file at {args.config} — see docgen.example.toml " + "or docgen.project.example.toml") try: runfile = load(args.config) - selected = runfile.select(args.only) + if isinstance(runfile, ProjectFile): + selected = select_parts(runfile, args.only) + else: + selected = runfile.select(args.only) except ConfigError as e: raise Abort(str(e)) from None @@ -46,13 +50,26 @@ def main(argv) -> int: print(f"{runfile.path} — {len(selected)} of {len(runfile.books)} book(s)") if runfile.reference: print(f" reference {runfile.reference}") - for b in selected: - print(f" {b.line()}") + if isinstance(runfile, ProjectFile): + doc = runfile.document + print(f" project {doc.name} — {doc.title}") + print(f" build {doc.build}") + print(f" notes {doc.notes}") + for part in selected: + print(f" book {part.name:<16} {part.kind:<8} {part.source}") + for o in doc.outputs: + print(f" output {o.emitter:<16} {o.path}" + + (f" (width {o.width}px)" if o.width else "")) + else: + for b in selected: + print(f" {b.line()}") return 0 print(f"{runfile.path} — {len(selected)} book(s)") results = run(runfile, [b.name for b in selected], check=args.check, quiet=not args.verbose) + if not results: + raise Abort("nothing was built") width = max(len(r["name"]) for r in results) for r in results: @@ -70,10 +87,12 @@ def main(argv) -> int: print(f" {status} {r['name']:<{width}} {detail}") failed = [r for r in results if not r["ok"]] + documents = sum(1 for r in results if r["name"].endswith("(document)")) + what = f"{len(results) - documents} book(s)" + (f" in {documents} document" if documents else "") print() if failed: - print(f"{len(failed)} of {len(results)} book(s) did not hold" + print(f"{len(failed)} of {len(results)} did not hold ({what})" + ("" if args.verbose else " — --verbose for each book's own output")) return 1 - print(f"{len(results)} book(s) built" + (" and checked" if args.check else "")) + print(f"{what} built" + (" and checked" if args.check else "")) return 0 diff --git a/soleprint/atlas2/docgen/cli/scaffold.py b/soleprint/atlas2/docgen/cli/scaffold.py new file mode 100644 index 0000000..2dfee75 --- /dev/null +++ b/soleprint/atlas2/docgen/cli/scaffold.py @@ -0,0 +1,87 @@ +""" + python3 -m docgen scaffold [--only NAME ...] + python3 -m docgen scaffold --list # print the spots, write nothing + +Write an empty note for every spot a document has — the document's top and end, +each book's top, what came in, the page index, every page, every step, every +API call, what came out — so the ids are listed rather than guessed. + +Never overwrites: a spot that already has a note is left alone, so running it +again after the sources change only adds the spots that are new. A stub is a +single HTML comment and renders nothing until somebody writes below it. + +Extraction and page structure only — nothing is drawn and no output is written. +""" + +import argparse +from pathlib import Path + +from .common import PROG, Abort + + +def main(argv) -> int: + p = argparse.ArgumentParser(prog=f"{PROG} scaffold") + p.add_argument("config", type=Path, nargs="?", default=Path("docgen.toml")) + p.add_argument("--only", action="append", default=[], metavar="NAME", + help="Just these books. Repeatable.") + p.add_argument("--list", action="store_true", + help="Print every spot a note can be filed under, and write nothing — " + "the list to hand a model instead of letting it guess page ids.") + args = p.parse_args(argv) + + from ..book.config import ConfigError, ProjectFile, apply_reference, load, select_parts + from ..book.document import Part, plan_spots + from ..book.notes import Notes + + if not args.config.exists(): + raise Abort(f"no run file at {args.config}") + try: + loaded = load(args.config) + apply_reference(loaded) + if isinstance(loaded, ProjectFile): + groups = {loaded.document.notes: select_parts(loaded, args.only)} + else: + groups = {} + for e in loaded.select(args.only): + part = Part(name=e.name, kind=e.kind, source=e.source, out=e.out, + exclude=e.exclude) + groups.setdefault(e.notes, []).append(part) + except ConfigError as e: + raise Abort(str(e)) from None + + total = 0 + for root, parts in groups.items(): + spots = {} + for part in parts: + try: + spots[part.name] = plan_spots(part) + except Exception as e: # noqa: BLE001 - one bad source must not cost the rest + print(f" skip {part.name}: {type(e).__name__}: {str(e).splitlines()[0]}") + if args.list: + # Relative to the run file's folder, which is how a note's path is + # written in a reply — not as an absolute path on this machine. + base = args.config.resolve().parent + try: + shown = root.relative_to(base) + except ValueError: + shown = root + rows = [(f"{shown}/_intro.md", "top of the document"), + (f"{shown}/_end.md", "end of the document")] + for book, entries in spots.items(): + rows.append((f"{shown}/{book}/_intro.md", f"top of book {book}")) + for spot, where in entries: + has = " (has a note)" if Notes(root).book(book, spot) else "" + rows.append((f"{shown}/{book}/{spot}.md", where + has)) + width = max(len(r[0]) for r in rows) + for path, where in rows: + print(f"{path:<{width}} {where}") + continue + written = Notes(root).scaffold(spots) + total += len(written) + existing = sum(len(v) for v in spots.values()) + 2 + len(spots) - len(written) + print(f" notes {root} — {len(written)} new stub(s), {existing} already there") + for path in written[:12]: + print(f" {path.relative_to(root)}") + if len(written) > 12: + print(f" … and {len(written) - 12} more") + return 0 diff --git a/soleprint/atlas2/docgen/docgen.project.example.toml b/soleprint/atlas2/docgen/docgen.project.example.toml new file mode 100644 index 0000000..fa574f9 --- /dev/null +++ b/soleprint/atlas2/docgen/docgen.project.example.toml @@ -0,0 +1,53 @@ +# A project file: one document built from several sources. +# +# make run CONFIG=docgen.project.example.toml CHECK=1 +# make scaffold CONFIG=docgen.project.example.toml # a stub note for every spot +# python3 -m docgen run docgen.project.example.toml --list +# +# Every [extract.*] table is a book — a part of each output. Every [emit.*] table +# is an output made of all of them. Paths are relative to THIS FILE. Unknown keys +# are refused with a reason, never ignored. +# +# The other shape, a list of independent books, is docgen.example.toml. + +[project] +name = "docgen-fixtures" +title = "docgen's own fixtures" +notes = "fixtures/notes" # notes//.md lands above that page +build = ".docgen" # book artifacts; gitignore it +# reference = "../../" # the repo holding station/tools/modelgen, for OpenAPI + +# A tree of Python. Several trees: [[extract.python]] with a `name` each. +[extract.python] +name = "extractors" +root = "extractors" + +# A database schema, as `modelgen from-db` writes it. docgen reads the dump and +# never connects: `url = ...` is refused, since a URL in a committed file is a +# credential in a committed file. +[extract.db] +name = "schema" +schema = "fixtures/shop.json" + +# An OpenAPI document. Needs pyyaml and the reference repo's modelgen; without +# them this book fails with a one-line reason and the others still build. +[extract.openapi] +name = "api" +spec = "fixtures/orders.yaml" + +# Also available: +# [extract.code] root = "src", ext = [".cs", ".ts"] (tree-sitter) +# [extract.usage] har = "session.har" + +[ops] +drop_stdlib = true +drop_builtins = true + +[emit.notebook] +output = "out/project/notebook" # a directory: .ipynb goes inside +theme = "lucid" +width = 960 # px — pages are split until each fits + +[emit.site] +output = "out/project/site" +theme = "dark" diff --git a/soleprint/atlas2/docgen/docs/img/architecture.svg b/soleprint/atlas2/docgen/docs/img/architecture.svg index 9df61e1..001a373 100644 --- a/soleprint/atlas2/docgen/docs/img/architecture.svg +++ b/soleprint/atlas2/docgen/docs/img/architecture.svg @@ -4,72 +4,72 @@ - - + + ir - + cluster_docgen - -docgen + +docgen cluster_docgen_book - -book + +book cluster_docgen_cli - -cli + +cli cluster_docgen_emitters - -emitters + +emitters cluster_docgen_extractors - -extractors + +extractors cluster_docgen_extractors_python - -python + +python cluster_docgen_ir - -ir + +ir cluster_docgen_lab - -lab + +lab cluster_docgen_notebook - -notebook + +notebook cluster_docgen_ops - -ops + +ops cluster_docgen_style - -style + +style docgen.__main__ - -__main__ + +__main__ @@ -77,776 +77,1004 @@ docgen.cli.book - -book + +book docgen.__main__->docgen.cli.book - - + + docgen.book.build - -build + +build + + +docgen.book.document + + +document + + + + + +docgen.book.build->docgen.book.document + + + docgen.book.larder - -larder + +larder docgen.book.build->docgen.book.larder - - + + - + docgen.book.build->docgen.book.larder - - + + docgen.emitters.auto - -auto + +auto - + docgen.book.build->docgen.emitters.auto - - + + docgen.emitters.explore - -explore + +explore - + docgen.book.build->docgen.emitters.explore - - + + docgen.emitters.index - -index + +index - + docgen.book.build->docgen.emitters.index - - + + docgen.emitters.minimap - -minimap + +minimap - -docgen.book.build->docgen.emitters.minimap - - - - - -docgen.emitters.notebook - - -notebook - - - - -docgen.book.build->docgen.emitters.notebook - - - - - -docgen.emitters.site - - -site - - - - - -docgen.book.build->docgen.emitters.site - - +docgen.book.build->docgen.emitters.minimap + + docgen.ir.model - -model + +model - + docgen.book.build->docgen.ir.model - - + + docgen.notebook.spec - -spec + +spec - + docgen.book.build->docgen.notebook.spec - - + + docgen.ops.filter - -filter + +filter - + docgen.book.build->docgen.ops.filter - - + + + + + +docgen.ops.pages + + +pages + + + + + +docgen.book.build->docgen.ops.pages + + docgen.style.extract - -extract + +extract docgen.book.build->docgen.style.extract - - + + docgen.book.checks - -checks + +checks docgen.book.checks->docgen.book.build - - + + docgen.book.config - -config + +config docgen.book.config->docgen.__main__ - - + + docgen.book.config->docgen.book.build - - + + docgen.book.config->docgen.book.build - - + + - + +docgen.book.config->docgen.book.document + + + + + +docgen.extractors.code + + +code + + + + + +docgen.book.config->docgen.extractors.code + + + + + docgen.book.config->docgen.notebook.spec - - + + + + + +docgen.book.document->docgen.book.build + + + + + +docgen.book.document->docgen.book.build + + + + + +docgen.book.document->docgen.book.larder + + + + + +docgen.book.notes + + +notes + + + + + +docgen.book.document->docgen.book.notes + + + + + +docgen.book.document->docgen.emitters.auto + + + + + +docgen.emitters.notebook + + +notebook + + + + + +docgen.book.document->docgen.emitters.notebook + + + + + +docgen.emitters.paged + + +paged + + + + + +docgen.book.document->docgen.emitters.paged + + + + + +docgen.book.document->docgen.notebook.spec + + + + + +docgen.book.document->docgen.ops.filter + + + + + +docgen.book.document->docgen.ops.pages + + + + + +docgen.book.document->docgen.style.extract + + - + docgen.cli.book->docgen.book.build - - + + docgen.cli.common - -common + +common - + docgen.cli.book->docgen.cli.common - - + + - + docgen.cli.book->docgen.cli.common - - + + - + docgen.cli.book->docgen.notebook.spec - - + + docgen.cli.check - -check + +check - + docgen.cli.check->docgen.book.checks - - + + - + docgen.cli.check->docgen.cli.common - - + + - + docgen.cli.common->docgen.ir.model - - + + - + docgen.cli.common->docgen.style.extract - - + + + + + +docgen.cli.contract + + +contract + + + + + +docgen.cli.contract->docgen.cli.common + + docgen.cli.emit - -emit + +emit - + docgen.cli.emit->docgen.cli.common - - + + - + docgen.cli.emit->docgen.emitters.auto - - + + docgen.emitters.dot - -dot + +dot - + docgen.cli.emit->docgen.emitters.dot - - + + docgen.emitters.erd - -erd + +erd - + docgen.cli.emit->docgen.emitters.erd - - + + - + docgen.cli.emit->docgen.emitters.explore - - + + - + docgen.cli.emit->docgen.emitters.index - - + + - + docgen.cli.emit->docgen.emitters.minimap - - + + - + docgen.cli.emit->docgen.emitters.notebook - - + + + + + +docgen.emitters.site + + +site + + - + docgen.cli.emit->docgen.emitters.site - - + + - + docgen.cli.emit->docgen.notebook.spec - - + + - + docgen.cli.emit->docgen.ops.filter - - + + docgen.cli.extract - -extract + +extract - + docgen.cli.extract->docgen.cli.common - - - - - -docgen.extractors.code - - -code - - + + - + docgen.cli.extract->docgen.extractors.code - - + + docgen.extractors.db - -db + +db - + docgen.cli.extract->docgen.extractors.db - - + + docgen.extractors.openapi - -openapi + +openapi - + docgen.cli.extract->docgen.extractors.openapi - - + + docgen.extractors.python.collect - -collect + +collect - + docgen.cli.extract->docgen.extractors.python.collect - - + + docgen.extractors.usage - -usage + +usage - + docgen.cli.extract->docgen.extractors.usage - - + + docgen.cli.run - -run + +run - + docgen.cli.run->docgen.book.config - - + + - + docgen.cli.run->docgen.cli.common - - + + + + + +docgen.cli.scaffold + + +scaffold + + + + + +docgen.cli.scaffold->docgen.book.config + + + + + +docgen.cli.scaffold->docgen.book.document + + + + + +docgen.cli.scaffold->docgen.book.notes + + + + + +docgen.cli.scaffold->docgen.cli.common + + docgen.cli.validate - -validate + +validate - + docgen.cli.validate->docgen.cli.common - - + + docgen.ir.validate - -validate + +validate - + docgen.cli.validate->docgen.ir.validate - - + + docgen.cli.view - -view + +view - + docgen.cli.view->docgen.cli.common - - + + - + docgen.cli.view->docgen.ir.model - - + + - + docgen.cli.view->docgen.ops.filter - - + + - + docgen.emitters.auto->docgen.emitters.dot - - + + - + docgen.emitters.auto->docgen.emitters.erd - - + + - + docgen.emitters.auto->docgen.emitters.index - - + + - + docgen.emitters.auto->docgen.ops.filter - - + + + + + +docgen.emitters.docsite + + +docsite + + + + + +docgen.emitters.docsite->docgen.book.document + + + + + +docgen.emitters.markdown + + +markdown + + + + + +docgen.emitters.docsite->docgen.emitters.markdown + + + + + +docgen.emitters.docsite->docgen.emitters.paged + + + + + +docgen.emitters.docsite->docgen.emitters.site + + - + docgen.emitters.explore->docgen.emitters.dot - - + + - + docgen.emitters.explore->docgen.emitters.erd - - + + - + docgen.emitters.explore->docgen.emitters.minimap - - + + - + docgen.emitters.explore->docgen.ops.filter - - + + + + + +docgen.emitters.paged->docgen.emitters.dot + + + + + +docgen.emitters.paged->docgen.emitters.erd + + + + + +docgen.emitters.paged->docgen.ops.filter + + - + docgen.emitters.site->docgen.emitters.index - - + + - + docgen.extractors.code->docgen.book.larder - - + + - + docgen.extractors.code->docgen.ir.model - - + + tree_sitter - -tree_sitter + +tree_sitter - + docgen.extractors.code->tree_sitter - - + + - + docgen.extractors.db->docgen.book.larder - - + + - + docgen.extractors.db->docgen.ir.model - - + + - + docgen.extractors.openapi->docgen.__main__ - - + + - + docgen.extractors.openapi->docgen.book.larder - - + + - + docgen.extractors.openapi->docgen.ir.model - - + + modelgen.loader.extract.openapi - -openapi + +openapi - + docgen.extractors.openapi->modelgen.loader.extract.openapi - - + + yaml - -yaml + +yaml - + docgen.extractors.openapi->yaml - - + + docgen.extractors.python.resolve - -resolve + +resolve - + docgen.extractors.python.collect->docgen.extractors.python.resolve - - + + - + docgen.extractors.python.resolve->docgen.book.larder - - + + - + docgen.extractors.python.resolve->docgen.extractors.python.collect - - + + - + docgen.extractors.python.resolve->docgen.ir.model - - + + - + docgen.extractors.usage->docgen.book.larder - - + + - + docgen.extractors.usage->docgen.ir.model - - + + - + docgen.ir.model->docgen.ir.validate - - + + - + docgen.ir.validate->docgen.ir.model - - + + docgen.lab.pg_probe - -pg_probe + +pg_probe + + +docgen.ops.pages->docgen.ops.filter + + + docgen.reference - -reference + +reference @@ -854,49 +1082,49 @@ docgen.selftest - -selftest + +selftest lxml - -lxml + +lxml - + docgen.selftest->lxml - - + + docgen.style.tokens - -tokens + +tokens - + docgen.style.extract->docgen.style.tokens - - + + - + docgen.style.extract->lxml - - + + - + docgen.style.tokens->docgen.style.extract - - + + diff --git a/soleprint/atlas2/docgen/docs/img/minimap.svg b/soleprint/atlas2/docgen/docs/img/minimap.svg index dcdef6d..c132b2f 100644 --- a/soleprint/atlas2/docgen/docs/img/minimap.svg +++ b/soleprint/atlas2/docgen/docs/img/minimap.svg @@ -1,7 +1,7 @@ - - -docgen.selftest — 2328 lines + + +docgen.selftest — 2775 lines own_py_files — function, 3 lines check — function, 5 lines _err — function, 7 lines @@ -15,6 +15,12 @@ _runfiles — function, 195 lines write — function, 4 lines problems — function, 6 lines +_pages — function, 114 lines +_documents — function, 161 lines +_projects — function, 59 lines +load — function, 7 lines +_contract — function, 75 lines +named — function, 2 lines docgen.book — 316 lines Step — class, 20 lines to_dict — function, 10 lines @@ -54,353 +60,449 @@ describe — function, 10 lines on_path — function, 13 lines missing — function, 8 lines -docgen.cli — 76 lines -usage — function, 7 lines -main — function, 21 lines +docgen.cli — 80 lines +usage — function, 7 lines +main — function, 21 lines docgen.ops — 28 lines docgen.notebook — 15 lines docgen.emitters — 12 lines docgen.lab — 11 lines docgen.__main__ — 9 lines -docgen.ir — 7 lines -docgen.extractors — 2 lines -docgen.cli.emit — 253 lines -_parser — function, 7 lines -_size — function, 3 lines -auto — function, 16 lines -dot — function, 38 lines -erd — function, 11 lines -index — function, 14 lines -minimap — function, 14 lines -notebook — function, 41 lines -site — function, 31 lines -explore — function, 22 lines -main — function, 7 lines -docgen.cli.extract — 119 lines -_parser — function, 5 lines -_counts — function, 3 lines -python — function, 15 lines -code — function, 14 lines -db — function, 11 lines -openapi — function, 12 lines -usage — function, 15 lines -main — function, 7 lines -docgen.cli.view — 90 lines -main — function, 75 lines -docgen.cli.common — 80 lines -Abort — class, 2 lines -read_json — function, 5 lines -read_ir — function, 11 lines -load_style — function, 7 lines -style_args — function, 3 lines -emit — function, 16 lines -dump_ir — function, 3 lines -docgen.cli.run — 80 lines -main — function, 59 lines -docgen.cli.book — 74 lines -main — function, 52 lines -docgen.cli.check — 51 lines -main — function, 30 lines -docgen.cli.validate — 33 lines -main — function, 18 lines -docgen.emitters.site — 494 lines -_ledger — function, 54 lines -_slots — function, 13 lines -_fill — function, 4 lines -_sidebar — function, 17 lines -_sections — function, 15 lines -emit — function, 96 lines -write — function, 10 lines -docgen.emitters.explore — 316 lines -_is_schema — function, 4 lines -_neighbourhood_svgs — function, 48 lines -render_one — function, 2 lines -render_one — function, 2 lines -_facts — function, 27 lines -emit — function, 172 lines -write — function, 17 lines -docgen.emitters.dot — 292 lines -RenderError — class, 2 lines -_esc — function, 2 lines -_attrs — function, 3 lines -_style_words — function, 9 lines -_node_attrs — function, 24 lines -emit — function, 105 lines -write — function, 29 lines -_within — function, 8 lines -_endpoints — function, 41 lines -walk — function, 4 lines -collapsed — function, 2 lines -first_leaf — function, 4 lines -_safe — function, 2 lines -_q — function, 2 lines -have_graphviz — function, 2 lines -render — function, 29 lines -docgen.emitters.notebook — 279 lines -_cell — function, 13 lines -_example — function, 26 lines -_params_cell — function, 18 lines -_call_cell — function, 40 lines -_call_md — function, 26 lines -build — function, 52 lines -emit — function, 3 lines -write — function, 5 lines -docgen.emitters.minimap — 278 lines -_files — function, 57 lines -declared — function, 18 lines -build — function, 11 lines -_bands — function, 7 lines -_blocks — function, 19 lines -emit — function, 120 lines -marks_to_labels — function, 3 lines -docgen.emitters.erd — 260 lines -_truncate — function, 3 lines -_tables — function, 20 lines -_card_height — function, 3 lines -layout — function, 20 lines -_field_y — function, 3 lines -emit — function, 146 lines -docgen.emitters.index — 164 lines -_tree — function, 8 lines -_anchor — function, 5 lines -to_markdown — function, 84 lines -walk — function, 27 lines -to_sidebar — function, 26 lines -build — function, 13 lines -docgen.emitters.auto — 64 lines -draw — function, 36 lines -docgen.extractors.code — 282 lines -MissingParser — class, 2 lines -_parser — function, 21 lines -_name — function, 10 lines -_walk — function, 34 lines -extract_file — function, 27 lines -_count_errors — function, 5 lines -extract — function, 75 lines -docgen.extractors.usage — 269 lines -_template — function, 27 lines -_body — function, 10 lines -_shape — function, 15 lines -_graphql — function, 11 lines -extract — function, 133 lines -docgen.extractors.openapi — 191 lines -_modelgen — function, 28 lines -_type_name — function, 6 lines -_refs — function, 43 lines -extract — function, 76 lines -docgen.extractors.db — 161 lines -from_schema_dict — function, 82 lines -_relation — function, 10 lines -_plain_type — function, 6 lines -_dedupe — function, 9 lines -extract — function, 6 lines -docgen.extractors.python — 37 lines -extract — function, 14 lines -docgen.book.build — 381 lines -extract — function, 27 lines -spec_from — function, 68 lines -_load_cell — function, 44 lines -clear — function, 26 lines -run — function, 125 lines -say — function, 3 lines -docgen.book.config — 328 lines -ConfigError — class, 6 lines -__init__ — function, 3 lines -Entry — class, 14 lines -line — function, 2 lines -RunFile — class, 17 lines -select — function, 12 lines -_path — function, 6 lines -_within — function, 3 lines -load — function, 130 lines -pick — function, 3 lines -apply_reference — function, 11 lines -run — function, 43 lines -docgen.book.checks — 275 lines -Loaded — class, 26 lines -__init__ — function, 15 lines -larder — function, 2 lines -measure — function, 2 lines -Report — class, 37 lines -__init__ — function, 2 lines -note — function, 2 lines -check — function, 8 lines -skip — function, 3 lines -total — function, 15 lines -generated — function, 83 lines -custom — function, 26 lines -_run_cells — function, 33 lines -docgen.book.larder — 196 lines -_count — function, 13 lines -redact — function, 32 lines -Larder — class, 77 lines -__post_init__ — function, 7 lines -read — function, 3 lines -fail — function, 3 lines -to_dict — function, 13 lines -from_dict — function, 10 lines -line — function, 20 lines -of — function, 3 lines -docgen.extractors.python.collect — 236 lines -Definition — class, 10 lines -Module — class, 12 lines -_Collector — class, 67 lines -__init__ — function, 3 lines -_define — function, 17 lines -visit_ClassDef — function, 5 lines -visit_FunctionDef — function, 5 lines -visit_Import — function, 8 lines -visit_ImportFrom — function, 14 lines -_first_line — function, 5 lines -_name_of — function, 15 lines -_resolve_relative — function, 16 lines -module_name — function, 26 lines -collect_file — function, 21 lines -collect — function, 13 lines -docgen.extractors.python.resolve — 184 lines -_id_for — function, 2 lines -_resolve — function, 32 lines -larder_of — function, 17 lines -to_ir — function, 98 lines -_point_at — function, 9 lines -docgen.ir.validate — 278 lines -IRError — class, 2 lines -_schema — function, 2 lines -_props — function, 5 lines -_fields — function, 4 lines -check — function, 108 lines -_check_larder — function, 58 lines -validate — function, 6 lines -check_model_matches_schema — function, 23 lines -docgen.ir.model — 165 lines -Meta — class, 38 lines -to_dict — function, 18 lines -Node — class, 21 lines -__post_init__ — function, 3 lines -to_dict — function, 8 lines -Edge — class, 15 lines -to_dict — function, 7 lines -Graph — class, 48 lines -node — function, 4 lines -edge — function, 4 lines -has — function, 2 lines -to_dict — function, 16 lines -from_dict — function, 6 lines -docgen.style.extract — 222 lines -_values_from — function, 17 lines -_normalise — function, 45 lines -_svg_files — function, 25 lines -harvest — function, 43 lines -write — function, 7 lines -summarise — function, 13 lines -docgen.style.tokens — 211 lines -_top — function, 2 lines -_mode — function, 3 lines -_luminance — function, 6 lines -derive — function, 101 lines -accent — function, 2 lines -geometry — function, 14 lines -write — function, 7 lines -from_folder — function, 9 lines -docgen — 2 lines -docgen.lab.pg_probe — 150 lines -probe — function, 30 lines -_simplify — function, 3 lines -main — function, 25 lines -docgen.notebook.spec — 264 lines -_step — function, 4 lines -from_ir — function, 108 lines -_order — function, 5 lines -scaffold — function, 20 lines -merge — function, 58 lines -load — function, 2 lines -dump — function, 5 lines -docgen.ops.filter — 485 lines -_rebuild — function, 57 lines -surviving_parent — function, 5 lines -lift — function, 6 lines -drop_kinds — function, 14 lines -only_kinds — function, 14 lines -drop_stdlib — function, 13 lines -drop_external — function, 3 lines -subtree — function, 13 lines -neighbourhood — function, 45 lines -collapse_to_depth — function, 18 lines -level — function, 6 lines -drop_builtins — function, 14 lines -overview — function, 35 lines -shape — function, 72 lines -rank_of — function, 9 lines -split — function, 24 lines -classify — function, 102 lines +docgen.ir — 7 lines +docgen.extractors — 2 lines +docgen.emitters.site — 539 lines +_ledger — function, 54 lines +_slots — function, 15 lines +_fill — function, 4 lines +_sidebar — function, 17 lines +_sections — function, 15 lines +emit — function, 96 lines +write — function, 10 lines +docgen.emitters.notebook — 320 lines +_cell — function, 19 lines +_example — function, 26 lines +_params_cell — function, 18 lines +_call_cell — function, 40 lines +_call_md — function, 26 lines +_figure_md — function, 21 lines +_figure_attachments — function, 7 lines +build — function, 55 lines +emit — function, 3 lines +write — function, 5 lines +docgen.emitters.explore — 316 lines +_is_schema — function, 4 lines +_neighbourhood_svgs — function, 48 lines +render_one — function, 2 lines +render_one — function, 2 lines +_facts — function, 27 lines +emit — function, 172 lines +write — function, 17 lines +docgen.emitters.dot — 309 lines +RenderError — class, 2 lines +_esc — function, 2 lines +_attrs — function, 3 lines +_style_words — function, 9 lines +_node_attrs — function, 33 lines +emit — function, 113 lines +write — function, 31 lines +_within — function, 8 lines +_endpoints — function, 41 lines +walk — function, 4 lines +collapsed — function, 2 lines +first_leaf — function, 4 lines +_safe — function, 2 lines +_q — function, 2 lines +have_graphviz — function, 2 lines +render — function, 29 lines +docgen.emitters.minimap — 278 lines +_files — function, 57 lines +declared — function, 18 lines +build — function, 11 lines +_bands — function, 7 lines +_blocks — function, 19 lines +emit — function, 120 lines +marks_to_labels — function, 3 lines +docgen.emitters.erd — 276 lines +_truncate — function, 3 lines +_tables — function, 20 lines +_card_height — function, 3 lines +max_columns — function, 9 lines +layout — function, 25 lines +_field_y — function, 3 lines +emit — function, 146 lines +docgen.emitters.docsite — 270 lines +_note — function, 7 lines +_missing — function, 9 lines +_tree — function, 15 lines +walk — function, 7 lines +_page — function, 69 lines +write — function, 95 lines +put — function, 4 lines +docgen.emitters.index — 164 lines +_tree — function, 8 lines +_anchor — function, 5 lines +to_markdown — function, 84 lines +walk — function, 27 lines +to_sidebar — function, 26 lines +build — function, 13 lines +docgen.emitters.markdown — 146 lines +_url — function, 7 lines +_inline — function, 13 lines +keep — function, 3 lines +to_html — function, 91 lines +para — function, 3 lines +cells — function, 2 lines +docgen.emitters.paged — 119 lines +svg_width_px — function, 6 lines +_estimate_px — function, 10 lines +fits_for — function, 17 lines +fits — function, 6 lines +links_for — function, 15 lines +render — function, 19 lines +inline — function, 9 lines +docgen.emitters.auto — 64 lines +draw — function, 36 lines +docgen.cli.emit — 253 lines +_parser — function, 7 lines +_size — function, 3 lines +auto — function, 16 lines +dot — function, 38 lines +erd — function, 11 lines +index — function, 14 lines +minimap — function, 14 lines +notebook — function, 41 lines +site — function, 31 lines +explore — function, 22 lines +main — function, 7 lines +docgen.cli.extract — 119 lines +_parser — function, 5 lines +_counts — function, 3 lines +python — function, 15 lines +code — function, 14 lines +db — function, 11 lines +openapi — function, 12 lines +usage — function, 15 lines +main — function, 7 lines +docgen.cli.run — 99 lines +main — function, 78 lines +docgen.cli.view — 90 lines +main — function, 75 lines +docgen.cli.scaffold — 88 lines +main — function, 66 lines +docgen.cli.common — 80 lines +Abort — class, 2 lines +read_json — function, 5 lines +read_ir — function, 11 lines +load_style — function, 7 lines +style_args — function, 3 lines +emit — function, 16 lines +dump_ir — function, 3 lines +docgen.cli.book — 77 lines +main — function, 55 lines +docgen.cli.check — 58 lines +main — function, 37 lines +docgen.cli.contract — 34 lines +main — function, 8 lines +docgen.cli.validate — 33 lines +main — function, 18 lines +docgen.book.config — 616 lines +ConfigError — class, 6 lines +__init__ — function, 3 lines +Entry — class, 15 lines +line — function, 2 lines +RunFile — class, 17 lines +select — function, 12 lines +_path — function, 6 lines +_within — function, 3 lines +_read — function, 9 lines +load — function, 18 lines +_load_books — function, 121 lines +pick — function, 3 lines +apply_reference — function, 11 lines +_checks — function, 13 lines +run — function, 36 lines +ProjectFile — class, 8 lines +books — function, 2 lines +load_project — function, 151 lines +select_parts — function, 9 lines +run_project — function, 41 lines +docgen.book.document — 449 lines +Output — class, 6 lines +Part — class, 11 lines +Document — class, 11 lines +apply_ops — function, 15 lines +clear_output — function, 27 lines +write_manifest — function, 3 lines +rendered_pages — function, 22 lines +caption — function, 14 lines +_note_step — function, 8 lines +_with_note — function, 11 lines +notebook_spec — function, 91 lines +build — function, 116 lines +say — function, 3 lines +_spots — function, 16 lines +known_spots — function, 3 lines +plan_spots — function, 9 lines +docgen.book.build — 418 lines +extract — function, 29 lines +spec_from — function, 71 lines +_load_cell — function, 46 lines +clear — function, 26 lines +build_book — function, 105 lines +say — function, 3 lines +run — function, 35 lines +docgen.book.checks — 378 lines +Loaded — class, 26 lines +__init__ — function, 15 lines +larder — function, 2 lines +measure — function, 2 lines +Report — class, 38 lines +__init__ — function, 3 lines +note — function, 2 lines +check — function, 8 lines +skip — function, 3 lines +total — function, 15 lines +generated — function, 89 lines +custom — function, 26 lines +_run_cells — function, 33 lines +_notes_drift — function, 7 lines +document — function, 72 lines +heading — function, 2 lines +document_failures — function, 9 lines +docgen.book.larder — 196 lines +_count — function, 13 lines +redact — function, 32 lines +Larder — class, 77 lines +__post_init__ — function, 7 lines +read — function, 3 lines +fail — function, 3 lines +to_dict — function, 13 lines +from_dict — function, 10 lines +line — function, 20 lines +of — function, 3 lines +docgen.book.notes — 162 lines +body — function, 6 lines +stub — function, 3 lines +Notes — class, 78 lines +__init__ — function, 3 lines +_read — function, 7 lines +document — function, 2 lines +book — function, 2 lines +path — function, 4 lines +drift — function, 27 lines +scaffold — function, 24 lines +images — function, 18 lines +data_uri — function, 2 lines +docgen.extractors.code — 282 lines +MissingParser — class, 2 lines +_parser — function, 21 lines +_name — function, 10 lines +_walk — function, 34 lines +extract_file — function, 27 lines +_count_errors — function, 5 lines +extract — function, 75 lines +docgen.extractors.usage — 269 lines +_template — function, 27 lines +_body — function, 10 lines +_shape — function, 15 lines +_graphql — function, 11 lines +extract — function, 133 lines +docgen.extractors.openapi — 191 lines +_modelgen — function, 28 lines +_type_name — function, 6 lines +_refs — function, 43 lines +extract — function, 76 lines +docgen.extractors.db — 161 lines +from_schema_dict — function, 82 lines +_relation — function, 10 lines +_plain_type — function, 6 lines +_dedupe — function, 9 lines +extract — function, 6 lines +docgen.extractors.python — 37 lines +extract — function, 14 lines +docgen.extractors.python.collect — 236 lines +Definition — class, 10 lines +Module — class, 12 lines +_Collector — class, 67 lines +__init__ — function, 3 lines +_define — function, 17 lines +visit_ClassDef — function, 5 lines +visit_FunctionDef — function, 5 lines +visit_Import — function, 8 lines +visit_ImportFrom — function, 14 lines +_first_line — function, 5 lines +_name_of — function, 15 lines +_resolve_relative — function, 16 lines +module_name — function, 26 lines +collect_file — function, 21 lines +collect — function, 13 lines +docgen.extractors.python.resolve — 184 lines +_id_for — function, 2 lines +_resolve — function, 32 lines +larder_of — function, 17 lines +to_ir — function, 98 lines +_point_at — function, 9 lines +docgen.ir.validate — 278 lines +IRError — class, 2 lines +_schema — function, 2 lines +_props — function, 5 lines +_fields — function, 4 lines +check — function, 108 lines +_check_larder — function, 58 lines +validate — function, 6 lines +check_model_matches_schema — function, 23 lines +docgen.ir.model — 165 lines +Meta — class, 38 lines +to_dict — function, 18 lines +Node — class, 21 lines +__post_init__ — function, 3 lines +to_dict — function, 8 lines +Edge — class, 15 lines +to_dict — function, 7 lines +Graph — class, 48 lines +node — function, 4 lines +edge — function, 4 lines +has — function, 2 lines +to_dict — function, 16 lines +from_dict — function, 6 lines +docgen.ops.filter — 485 lines +_rebuild — function, 57 lines +surviving_parent — function, 5 lines +lift — function, 6 lines +drop_kinds — function, 14 lines +only_kinds — function, 14 lines +drop_stdlib — function, 13 lines +drop_external — function, 3 lines +subtree — function, 13 lines +neighbourhood — function, 45 lines +collapse_to_depth — function, 18 lines +level — function, 6 lines +drop_builtins — function, 14 lines +overview — function, 35 lines +shape — function, 72 lines +rank_of — function, 9 lines +split — function, 24 lines +classify — function, 102 lines +docgen.ops.pages — 383 lines +slugify — function, 8 lines +_key — function, 8 lines +_under — function, 5 lines +_representative — function, 13 lines +page_ir — function, 67 lines +layout — function, 20 lines +paginate — function, 61 lines +unique — function, 6 lines +opens — function, 3 lines +visit — function, 30 lines +paginate_tables — function, 76 lines +coverage — function, 28 lines +for_view — function, 17 lines +docgen.style.extract — 222 lines +_values_from — function, 17 lines +_normalise — function, 45 lines +_svg_files — function, 25 lines +harvest — function, 43 lines +write — function, 7 lines +summarise — function, 13 lines +docgen.style.tokens — 211 lines +_top — function, 2 lines +_mode — function, 3 lines +_luminance — function, 6 lines +derive — function, 101 lines +accent — function, 2 lines +geometry — function, 14 lines +write — function, 7 lines +from_folder — function, 9 lines +docgen — 2 lines +docgen.lab.pg_probe — 150 lines +probe — function, 30 lines +_simplify — function, 3 lines +main — function, 25 lines +docgen.notebook.spec — 264 lines +_step — function, 4 lines +from_ir — function, 108 lines +_order — function, 5 lines +scaffold — function, 20 lines +merge — function, 58 lines +load — function, 2 lines +dump — function, 5 lines docgen -docgen -docgen.cli -docgen.emitters -docgen.extractors -docgen.extractors -docgen.book -docgen.extractors.python -docgen.ir -docgen.style -(root) -docgen.lab -docgen.notebook -docgen.ops -selftest +docgen +docgen.emitters +docgen.emitters +docgen.cli +docgen.cli +docgen.book +docgen.extractors +docgen.extractors.python +docgen.ir +docgen.ops +docgen.style +(root) +docgen.lab +docgen.notebook +selftest book style reference -cli +cli ops notebook emitters lab __main__ -ir -extractors -emit -extract -view -common -run -book -check -validate -site -explore -dot -notebook -minimap -erd -index -auto -code -usage -openapi -db -python -build -config -checks -larder -collect -resolve -validate -model -extract -tokens -docgen -pg_probe -spec -filter - -module - -class - -interface - -function -47 files · 10,350 lines · 1px ≈ 2.0 lines +ir +extractors +site +notebook +explore +dot +minimap +erd +docsite +index +markdown +paged +auto +emit +extract +run +view +scaffold +common +book +check +contract +validate +config +document +build +checks +larder +notes +code +usage +openapi +db +python +collect +resolve +validate +model +filter +pages +extract +tokens +docgen +pg_probe +spec + +module + +class + +interface + +function +55 files · 13,028 lines · 1px ≈ 2.0 lines diff --git a/soleprint/atlas2/docgen/docs/index.html b/soleprint/atlas2/docgen/docs/index.html index 4df16ab..9931fb6 100644 --- a/soleprint/atlas2/docgen/docs/index.html +++ b/soleprint/atlas2/docgen/docs/index.html @@ -19,12 +19,15 @@ What it is The idea Five minutes + Your first project + Working with a model
Architecture
The three concerns The book The IR Shape decides the drawing + Pages
Reading
Extractors @@ -37,11 +40,13 @@ Emitters Explore Notebooks + Notes Style & colour
Reference
Standalone Run files + Project files Commands Dependencies Testing @@ -154,6 +159,145 @@ make self # docgen's book of soleprint the outputs open over file://.

+ +

Your first project

+ +

+ From nothing to a notebook and a site for a project of yours, with your own text + in the right places. docgen lives in its own folder, and the project only gets one + config file and, later, a folder of notes. +

+ +

1. Put docgen somewhere and check the machine

+ +
cp -r docgen ~/tools/docgen
+cd ~/tools/docgen
+make doctor          # what is installed; never fails
+make sync            # optional: C#/TypeScript, OpenAPI, all from uv.lock
+ +

+ From anywhere else, docgen runs as python3 -m docgen with its parent + folder on the path: +

+ +
export PYTHONPATH=~/tools             # the folder that holds docgen/
+python3 -m docgen --help
+ +

2. Write docgen.toml at the project's root

+ +
[project]
+name = "orders-service"
+
+[extract.code]
+root = "src"
+ext  = [".cs"]
+
+[emit.notebook]
+output = "../docs_out/orders_service_notebook"
+
+[emit.site]
+output = "../docs_out/orders_service_site"
+ +

+ Paths are relative to this file. Keep the outputs outside the source tree. See + project files for every key, and add + .docgen/ to the project's .gitignore. +

+ +

3. Check what it will do, then build

+ +
cd ~/work/orders-service
+python3 -m docgen run docgen.toml --list      # resolved paths; builds nothing
+python3 -m docgen run docgen.toml --check     # build, then check every book
+ +
docgen.toml  —  1 book(s)
+  ok    code                        …/.docgen/orders-service/books/code
+  ok    orders-service (document)   …/.docgen/orders-service
+
+1 book(s) in 1 document built and checked
+ +

+ A file docgen can't use is refused before anything is built, with every problem + listed at once and what to do about each. Open + ../docs_out/orders_service_site/index.html: each book shows what + came in and what came out, and its pages click through from the overview. +

+ +

4. Add your own text

+ +
python3 -m docgen scaffold docgen.toml --list   # every spot a note can go
+python3 -m docgen scaffold docgen.toml          # or: write an empty stub for each
+ +

+ Pick a spot and write markdown into notes/<book>/<spot>.md, + for example notes/code/overview.md above the overview page. Rebuild + and it's there, in the notebook and on the site. Stubs you leave empty add + nothing. +

+ +
+ If a note stops appearing, the page it was filed under was probably + renamed, for instance a package that moved. The rebuild says so and + --check fails with the note's name. Rename the file to the new spot + (scaffold --list shows them) or delete it. +
+ +

5. Rebuild whenever the code moves

+ +

+ python3 -m docgen run docgen.toml --check again. A rebuild replaces + exactly what the last one wrote and never touches your notes or anything you + added to the output folders. --only code rebuilds one book while + you're writing its notes. +

+ + +

Working with a model

+ +

+ A model can write the config and the notes. It shouldn't write anything else, + since every other file is generated. docgen ships the instructions to give it: +

+ +
python3 -m docgen contract > docgen-contract.md
+ +

+ That file is docs/models.md, printed verbatim. It lists every key + docgen accepts and the extensions it reads, has rules like "never a credential" + and "don't guess page ids", and ends with a table that maps each error docgen + can produce to the fix. docgen's own suite checks it against the code, so it + can't quietly go out of date: a key the loader accepts but the doc doesn't + name, or an example in it that doesn't load, fails make check. +

+ +

The loop

+ +
    +
  1. Give it the project and the contract. A digest of the project + (distill), plus docgen-contract.md as an + attachment rather than pasted. If replies go through explode, + attach its contract too: explode's says how to format the reply, docgen's + says what goes in the files.
    2. +
  2. Ask for docgen.toml first. Unpack it, then run + run docgen.toml --list. If it's refused, paste the errors back + verbatim. Each one is in the contract's repair table.
    3. +
  3. Build once, then list the spots. Run + scaffold docgen.toml --list and paste the output back. Page + names depend on the real directories, and a model that guesses them writes + notes that land nowhere.
    4. +
  4. Ask for notes for those spots. Unpack, rebuild with + --check, and paste back any note reported as filed under a page + that doesn't exist.
    5. +
+ +
+ What a model tends to write that docgen refuses. In a dry run, a model + asked for a live database URL (with its password), a seed reader, + and shell, YAML and SQL files as code. All three are refused with a reason, and + the URL is never echoed in the error. The contract tells the model up front, but + the refusal is what makes it safe. +
+

The three concerns

@@ -641,6 +785,69 @@ python3 -m docgen view ir.json --shape # what will this dark for everything that is not a declaration — imports, constants, prose. + +

Pages

+ +

+ A notebook doesn't scroll a wide image sideways: it scales it down to the cell + width until the labels can't be read. Zooming and panning is a workaround for a + picture that holds too much. So a book's diagram is split into pages, + each drawn small enough to fit the column it's going into. +

+ +
overview                 one box per top-level group, relationships lifted
+└── station              one box per child of station
+    └── station.tools    one box per child of station.tools, and so on
+ +

+ Pages follow structure, never scissors. Slicing one large image cuts boxes and + edges at the seams, and a slice means nothing on its own. A page is a subsystem: + its boxes are the scope's children, and anything outside the scope that they + touch appears as a context box, collapsed to the branch it sits on. A + page still says what its subsystem reaches without drawing the rest of the tree. +

+ +

Fitted by measuring, not estimating

+ +

+ Whether a page fits depends on its labels, fonts and spacing, so an estimate from + node counts is wrong the moment the names are long. Each page is rendered and its + width read back. If a page is too wide top-to-bottom, it's drawn left-to-right, + since height is free in a notebook and width isn't. If it still doesn't fit + (a level of many peers with no smaller structure to split into), the page is + kept and flagged too wide in its caption and in the index. Never cut, + never dropped. +

+ +

+ Measured on soleprint, whose module graph rendered at 14:1 as one picture: 69 + pages, and at the site's 800px column the widest flagged page is 988px. + Schemas page differently: cards in columns capped to the width, grouped by + tables joined by foreign keys, so a schema grows downward instead of sideways. +

+ +
+ Which pages exist doesn't depend on width, only how each is laid out + does. A notebook column and a site column get the same pages, which is what lets + a note filed under a page land in the same place in both. +
+ +

Clicking through, on the site

+ +

+ Every page is its own HTML file with the diagram inline, not in an + <img>, which swallows clicks. A box standing for a subsystem + opens that subsystem's page. A module box opens its section of the book's + reference page. A context box opens its own branch. Breadcrumbs lead back up. + The size is adjustable: fit scales to the column, 1:1 shows the + drawing at natural size with scrollbars, the choice is remembered, and + full screen opens the pan-and-zoom viewer. +

+

+ A notebook can't follow a link inside an image, so it gets a page index + instead: every page, indented by depth, each a link to its figure. +

+

Explore

@@ -747,6 +954,56 @@ python3 -m docgen view ir.json --shape # what will this 
python3 -m docgen emit notebook ir.json --scaffold overlay.json
 python3 -m docgen emit notebook ir.json --overlay overlay.json -o walkthrough.ipynb
+ +

Notes

+ +

+ Generation alone gives a document that's never stale and never says anything a + parser couldn't work out. Prose is the part only a person adds, and it has to + survive every rebuild, so it can't live in the output. It lives in a + notes folder, one markdown file per spot, named after the page or step it + belongs above: +

+ +
notes/
+├── _intro.md                   top of the document
+├── _end.md                     bottom of the document
+└── <book>/
+    ├── _intro.md               top of that book's part
+    ├── larder.md               above "what came in"
+    ├── overview.md             above the overview page
+    └── station.tools.md        above the page for station.tools
+ +

+ To put text above a page, create the file named after it. No schema, no JSON, + no ids to look up. That's something a person or a model can do without reading + anything first. Local images in a note are embedded: as a cell attachment in the + notebook, as a data URI on the site. +

+ +
make scaffold CONFIG=docgen.toml      # an empty stub for every spot, never overwriting
+ +

+ scaffold lists the spots rather than making you guess them. A stub + is a single HTML comment and renders nothing, so scaffolding a hundred spots + changes the output not at all until one is filled in. Run it again after the + sources change and it only adds the new spots. +

+ +
+ A note whose page is gone is reported, not dropped. Renaming a package + renames its page, and the note filed under the old name has nowhere to land. + Discarding it quietly is how hand-written text goes stale without anyone + noticing, so it appears in the document ledger and fails the check level. It's + the same rule the notebook overlay follows. +
+ +

+ The overlay stays for what notes can't do: drop a + step, reorder, replace generated code. Notes are applied after the overlay, so a + step that moves takes its note with it. +

+

Style & colour

@@ -872,10 +1129,10 @@ make doctor - - - - + + + +
contextchecksskipped
in the repo, synced287—
copied out, bare python3270tree-sitter, OpenAPI, the in-place case
copied out, synced279OpenAPI, the in-place case
copied out, synced, DOCGEN_REFERENCE set286
in the repo, synced346—
copied out, bare python3329tree-sitter, OpenAPI, the in-place case
copied out, synced338OpenAPI, the in-place case
copied out, synced, DOCGEN_REFERENCE set345 the in-place case — correctly, there is no repo above
@@ -973,6 +1230,80 @@ python3 -m docgen run docgen.toml --list # resolved paths because those are data that tools write.

+ +

Project files

+ +

+ A run file lists independent books. A project file describes one + document built from several sources: an API contract and the code + behind it, a schema and the service that uses it. It's the shape a model wrote + unprompted in a dry run, so docgen reads it as written: +

+ +
# <project>/docgen.toml
+[project]
+name  = "orders-api"
+title = "Orders API"
+notes = "notes"                     # default
+build = ".docgen"                   # book artifacts, default; gitignore it
+
+[extract.openapi]
+spec = "doc/contracts/openapi.yaml"
+
+[extract.code]
+root = "src"
+ext  = [".cs"]
+
+[ops]
+drop_stdlib   = true
+drop_builtins = true
+
+[emit.notebook]
+output = "../docs_out/orders_api_notebook"
+theme  = "lucid"
+
+[emit.site]
+output = "../docs_out/orders_api_site"
+width  = 800
+ +

+ Each [extract.*] table is a book, and a book is a part of + every output rather than an output of its own. Each [emit.*] table + is an output made of all of them: one notebook and one site, each with a part + per book. Several sources of one kind use [[extract.code]] with a + name each. [ops] applies to every book's view before + paging. +

+

+ Every part still opens with what came in and closes with what came out, and the + document opens with every larder and closes with every reconciliation. A book + that can't be built doesn't cost the others: the outputs are assembled from + what was built, and both say this document is incomplete and name what's + missing. +

+ +

Refused, with what to do instead

+ + + + + + + + + + + +
writtenwhy not
[extract.db] url = …docgen reads a schema dump and never connects, and a URL in a committed + file is a credential in a committed file. Write the schema with + modelgen from-db, then schema = …. The URL is never + repeated in the error.
ext = [".sh", ".yaml", ".sql"]No grammar for those yet. The error names the extensions that are read.
[extract.seed]No such reader. The error names the ones that exist.
[project] and [[book]] in one fileWhich one it meant is exactly what nobody should have to guess.
+ +

+ docgen.project.example.toml ships beside the run file example and + builds docgen's own fixtures into one document, with a sample note. +

+

Commands

@@ -981,7 +1312,9 @@ python3 -m docgen run docgen.toml --list # resolved paths targetdoes make sync.venv with every optional group, from uv.lock make book SRC=…one whole operation, measured at both ends - make run CONFIG=…every book a run file lists + make run CONFIG=…every book a run file lists, or a project's document + make scaffold CONFIG=…an empty note for every page and step + python3 -m docgen contractthe instructions for a model that writes the config and notes make checkdocgen's own suite, offline, nothing installed make check BOOK=…one book's own level — generated and custom make doctorwhat this machine has and what it is missing @@ -1028,6 +1361,10 @@ python3 -m docgen run docgen.toml --list # resolved paths python3 -m docgen run docgen.toml # every book a run file lists python3 -m docgen check # docgen's own suite python3 -m docgen check out/book/slug # that book's level +python3 -m docgen check .docgen/<project> # a project document's level +python3 -m docgen scaffold docgen.toml # stub notes for every spot +python3 -m docgen scaffold docgen.toml --list # the spots, written nowhere +python3 -m docgen contract # the instructions for a model python3 -m docgen extract python --root SRC -o ir.json python3 -m docgen extract code --root SRC -o ir.json @@ -1136,14 +1473,14 @@ make lock # after editing pyproject commandasks aboutfails? make doctorthe machine — what is installed never; it reports - make checkdocgen — 287 checks + make checkdocgen — 346 checks exit 1 - make check BOOK=<dir>that one book + make check BOOK=<dir>that one book, or a project's document exit 1

- All 287 after make sync, 272 with nothing optional installed — the + All 346 after make sync, 331 with nothing optional installed — the suite skips rather than fails when tree-sitter, lxml or the OpenAPI reader is absent. See standalone for the counts outside the repo. Among them, two that keep this page's structure true: nothing outside diff --git a/soleprint/atlas2/docgen/docs/models.md b/soleprint/atlas2/docgen/docs/models.md new file mode 100644 index 0000000..d92e257 --- /dev/null +++ b/soleprint/atlas2/docgen/docs/models.md @@ -0,0 +1,259 @@ +# docgen — instructions for a model + +You are helping document a software project with **docgen**. docgen reads source +artifacts (a code tree, a database schema dump, an OpenAPI document, a HAR recording) +and generates a notebook and a website from them. You do not generate those outputs. +You write the two kinds of file a person would otherwise write by hand: + +1. **`docgen.toml`**: which sources to read and which outputs to build. +2. **Notes**: short markdown files that docgen places above a specific page or step. + +Everything else is generated, and anything you write into a generated file is +overwritten on the next build. + +If your replies are unpacked by `explode`, its contract decides the reply format. This +file decides only what goes in the files. + +--- + +## Rules + +Follow all of these. A file that breaks one is refused when it is loaded, and the error +is pasted back to you. + +1. **Use only the keys listed below, spelled exactly as listed.** Unknown keys are + refused, never ignored. Do not invent keys, readers or outputs. +2. **Never put a database URL, password, token or any credential in any file.** docgen + does not connect to databases. It reads a schema file written by another tool. +3. **Every path is relative to the `docgen.toml` file itself.** +4. **Outputs must be outside every source tree** (`output = "../docs_out/..."` is the + usual choice). A build written inside a tree it reads is refused. +5. **Only list file extensions docgen can read** (see `ext` below). If the project's + interesting files are shell scripts, YAML or SQL, leave them out and say so in a + note. docgen does not read them yet. +6. **Do not guess page ids for notes.** Write a note only for a spot you were given, + either by the output of `docgen scaffold --list` or by the table in "Notes" below. A + note for a spot that does not exist fails the build's checks. +7. **Notes say what the code cannot.** Why something exists, what it replaced, how it is + meant to be used, what to watch out for. Do not restate what the diagram already shows + (names, counts, which module imports which). +8. If something you need to write a correct file is missing (the project's directory + layout, where the OpenAPI file is), ask for it instead of guessing. + +--- + +## `docgen.toml` for one project + +One file at the project root. It describes one **document** built from several sources. +Each `[extract.*]` table is a **book**, one part of the document. Each `[emit.*]` table +is an output that contains every book. + +```toml +[project] +name = "orders-service" +title = "Orders Service" + +[extract.openapi] +spec = "doc/contracts/openapi.yaml" + +[extract.code] +root = "src" +ext = [".cs"] + +[ops] +drop_stdlib = true +drop_builtins = true + +[emit.notebook] +output = "../docs_out/orders_service_notebook" +theme = "lucid" + +[emit.site] +output = "../docs_out/orders_service_site" +``` + +### `[project]` + +| key | required | value | +|---|---|---| +| `name` | yes | letters, digits, `.`, `_`, `-`; starts with a letter or digit | +| `title` | no | any text; defaults to `name` | +| `notes` | no | folder of notes; default `"notes"` | +| `build` | no | where intermediate files go; default `".docgen"` (should be gitignored) | +| `reference` | no | path to the spr repository; needed only to read OpenAPI | + +### `[extract.]`: the sources + +| reader | required key | optional keys | reads | +|---|---|---|---| +| `python` | `root` (a directory) | `name`, `exclude`, `overlay` | Python, with imports and inheritance | +| `code` | `root` (a directory) | `name`, `ext`, `exclude`, `overlay` | C#, TypeScript, Python structure (no edges) | +| `db` | `schema` (a file) | `name`, `overlay` | a schema JSON dump, as `modelgen from-db` writes | +| `openapi` | `spec` (a file) | `name`, `overlay` | an OpenAPI document | +| `usage` | `har` (a file) | `name`, `overlay` | a HAR recording of real API traffic | + +- `ext`: a list chosen from `".cs"`, `".ts"`, `".tsx"`, `".mts"`, `".py"`. Nothing else. +- `exclude`: a list of directory names to skip, e.g. `["tests", "migrations"]`. +- `name`: the book's name. It defaults to the reader's name. Use `[[extract.]]` + (double brackets) with a different `name` in each table to read two trees of the same + kind: + +```toml +[project] +name = "storefront" + +[[extract.code]] +name = "backend" +root = "src" +ext = [".cs"] + +[[extract.code]] +name = "frontend" +root = "web/src" +ext = [".ts", ".tsx"] + +[emit.site] +output = "../docs_out/storefront_site" +``` + +### `[ops]`: applied to every book before drawing + +| key | value | +|---|---| +| `drop_stdlib` | `true` removes standard-library imports | +| `drop_builtins` | `true` removes built-in names | +| `drop_external` | `true` removes every name the source does not define | +| `only` | a list of kinds to keep | +| `drop` | a list of kinds to remove | + +Kinds: `module`, `class`, `interface`, `function`, `external`, `table`, `column`, +`endpoint`, `operation`. + +### `[emit.]`: the outputs + +The outputs are `notebook` and `site`, nothing else. + +| key | required | value | +|---|---|---| +| `output` | yes | a directory. A notebook output may also be a path ending in `.ipynb` | +| `theme` | no | `"dark"` (default) or `"lucid"` (light, for print) | +| `style` | no | `"lucid"` (default) | +| `width` | no | pixels a diagram page must fit; default 960 for a notebook, 800 for a site | + +--- + +## `docgen.toml` for many independent books + +A different shape, for a list of unrelated books with one output folder each. Use it +only when asked. A file may not contain both `[project]` and `[[book]]`. + +```toml +[defaults] +out = "../docs_out" +exclude = ["tests"] + +[[book]] +name = "billing" +root = "src" + +[[book]] +name = "schema" +schema = "schema/schema.json" +``` + +| where | keys | +|---|---| +| top level | `reference`, `defaults`, `book` | +| `[defaults]` | `out`, `style`, `theme`, `exclude`, `reader`, `notes` | +| `[[book]]` | `name`, exactly one of `root` / `schema` / `openapi` / `har`, and optionally `reader` (`"python"` or `"code"`, only with `root`), `out`, `style`, `theme`, `exclude`, `overlay`, `notes` | + +A book's own value replaces the default for that key. Values are never merged. + +--- + +## Notes + +A note is a markdown file. docgen puts its content directly above the spot it is named +after, in both the notebook and the site. Its path is: + +``` +/_intro.md top of the whole document +/_end.md end of the whole document +//_intro.md top of that book's part +//.md above that spot in the book +``` + +`` is the book's `name` (the reader's name if none was given). + +### Spots every book has + +| spot | appears above | +|---|---| +| `larder` | "what came in": what was read | +| `pages` | the index of diagram pages | +| `overview` | the first diagram page | +| `book` | "what came out": the closing summary | +| `step-ir`, `step-view`, `step-pages`, `step-graph`, `step-index`, `step-sidebar`, `step-minimap`, `step-explore` | each intermediate step | + +### Spots that depend on the source + +- **Pages of a code tree**: named by a package's or directory's dotted path from the + root, for example `billing.invoices`. Not every directory gets a page: a level that + holds a single group is merged into it, and the top page is always `overview`. +- **Pages of a schema**: `overview`, then `tables-01`, `tables-02`, … +- **An API** (`openapi`, `usage`): `setup`, `client`, `shapes`, and one spot per call: + the method and path with every character outside letters, digits, `.`, `_` and `-` + replaced by `-`. So `GET /orders/{orderId}/lines` becomes `GET-orders-orderId-lines`. + +Page ids for a code tree depend on the actual directories. **Do not derive them yourself +unless you have the full directory listing.** Ask for the output of: + +``` +python3 -m docgen scaffold docgen.toml --list +``` + +It prints every spot that exists, one per line. + +### Writing a note + +- Plain markdown: headings, paragraphs, lists, **bold**, *italic*, `code`, fenced code + blocks, links, tables, quotes. +- No raw HTML. It is shown as text. +- Images: a path relative to the note file, e.g. `![flow](../img/flow.svg)`. They are + embedded. +- Keep it short: a few paragraphs. Start with `##` headings, because the page already + has its own title. +- A note that contains only the stub comment docgen writes renders nothing. Write below + the comment, or replace the file's contents. + +Example: `notes/code/billing.invoices.md`: + +```markdown +## Why invoices are immutable + +An invoice is never edited after it is issued. Corrections are a new credit note that +references it. This is a legal requirement, not a design preference, so do not add +update endpoints here. + +Start reading at `InvoiceIssuer`. It is the only place an invoice is created. +``` + +--- + +## When the build refuses your file + +The person running docgen pastes the errors back. Each one says what to change: + +| error contains | fix | +|---|---| +| `has unknown key 'x'` | remove the key or use a listed one; check the spelling | +| `docgen reads a schema dump and never connects` | remove `url`; use `schema = "path/to/schema.json"` | +| `no grammar for .sh` | remove those extensions from `ext` | +| `no reader named 'x'` | use one of `python`, `code`, `db`, `openapi`, `usage` | +| `no output named 'x'` | use `notebook` or `site` | +| `is inside the tree it reads` | move `output` or `build` outside the source directory | +| `needs a name when there are several` | give each `[[extract.x]]` table a different `name` | +| `must name exactly one source` | a `[[book]]` needs exactly one of `root`, `schema`, `openapi`, `har` | +| `is not a directory` / `is not a file` | the path is wrong; ask for the real layout | +| `note … belongs above 'x', which book 'y' no longer has` | that spot does not exist: rename the note to a listed spot, or delete it | +| `both [project] and [[book]]` | use one shape per file | diff --git a/soleprint/atlas2/docgen/emitters/docsite.py b/soleprint/atlas2/docgen/emitters/docsite.py new file mode 100644 index 0000000..cc902dc --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/docsite.py @@ -0,0 +1,269 @@ +""" +A document's site: an index of books, and every book as pages you click through. + + site/ + ├── index.html every book, with what came in and what came out + ├── site.css viewer.html + └── / + ├── overview.html the book's top page + ├── .html one subsystem each; its boxes open the next level + ├── .svg the same drawing, for the full-screen viewer + └── reference.html every module with its docstring — where a leaf opens + +## Inline, so the boxes are links + +The single-page site put its diagram in an ``, and an image swallows every +click. Here the SVG is inlined, so a box that stands for a subsystem opens that +subsystem's page, a box for a module opens its section of the reference page, +and a context box — something outside this subsystem — opens its own branch. +Breadcrumbs lead back up. + +## Adjustable + +A page is drawn to fit the content column, so it reads without zooming. The size +is still the reader's: `fit` scales to the column, `1:1` shows the drawing at its +natural size with scrollbars, and the choice is remembered. `full screen` opens +the pan-and-zoom viewer for anything still too dense. + +This writes HTML and nothing else. What goes on each page, the links and the +notes are prepared by `book/document.py` and handed in as data — an emitter has +never heard of a book's extractor, and this one has not either. +""" + +from html import escape +from pathlib import Path + +from .markdown import to_html +from .paged import inline +from .site import CONTENT_W, CSS, SIDEBAR_W, VIEWER, _fill, _ledger, _slots +from .site import emit as reference_page + +CONTENT_WIDTH = CONTENT_W + +SIZE_SCRIPT = """""" + + +def _note(note, cls="note") -> str: + if not note: + return "" + text, uris = note + return f'

' + to_html( + text, image=lambda alt, src: f'{escape(alt)}' + ) + "
" + + +def _missing(failed: dict | None) -> str: + """Books that were meant to be here and are not. Above the ones that are, + because a reader who does not see this has no way to know the site is short.""" + if not failed: + return "" + items = "".join(f"
  • {escape(name)} — {escape(why)}
    • " + for name, why in sorted(failed.items())) + return ('
    This document is incomplete. ' + f"{len(failed)} book(s) could not be built, so they are not below.
      {items}
    ") + + +def _tree(pages: list[dict], current: str | None) -> str: + by_parent: dict = {} + for p in pages: + by_parent.setdefault(p["parent"], []).append(p) + root = pages[0]["id"] if pages else None + + def walk(page, depth): + active = ' class="active"' if page["slug"] == current else "" + link = f'{escape(page["title"])}' + kids = by_parent.get(page["id"], []) + if not kids: + return f"
  • {link}
    • " + return f"
  • {link}
      " + "".join(walk(k, depth + 1) for k in kids) + "
    • " + + return "
      " + "".join(walk(p, 0) for p in pages if p["id"] == root) + "
    " + + +def _page(title: str, book: dict, page: dict, svg: str | None, pages: list[dict], + note, doc_title: str) -> str: + b = book["book"] + by_id = {p["id"]: p for p in pages} + trail, cur = [], page + while cur is not None: + trail.append(cur) + cur = by_id.get(cur["parent"]) if cur.get("parent") else None + crumbs = [f'{escape(doc_title)}'] + for p in reversed(trail): + if p is page: + crumbs.append(escape(p["title"])) + else: + crumbs.append(f'{escape(p["title"])}') + + from ..book.document import caption # noqa: F401 - kept out of module import time + cap = caption(page) + warn = " warn" if page.get("too_wide") else "" + if svg is not None: + figure = ( + '
    ' + '' + f'full screen' + "
    " + f'
    {inline(svg)}
    ' + ) + else: + figure = '

    Not drawn — Graphviz is not installed here.

    ' + + children = "" + if page.get("children"): + items = "".join( + f'
  • {escape(by_id[c]["title"])}
    • ' + for c in page["children"] if c in by_id + ) + children = f"

    Opens into

      {items}
    " + + return f""" + + + + +{escape(page["title"])} — {escape(b.slug)} — {escape(doc_title)} + + + +
    + +
    +
    {" › ".join(crumbs)}
    +

    {escape(page["title"])}

    + {_note(note)} + {figure} +

    {escape(cap)}

    + {children} +
    +
    +{SIZE_SCRIPT} + + +""" + + +def write(out_dir, style, *, title: str, books: list[dict], intro=None, end=None, + failed: dict | None = None) -> list[Path]: + """Write the site. Returns every file written, for the output manifest. + + `books` is a list of:: + + {"book": Book, "pages": [(page, svg)], "notes": {slug: (markdown, {ref: data uri})}, + "intro": (markdown, {}) or None} + """ + import shutil + + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + values = _slots(style) + written: list[Path] = [] + + def put(path: Path, text: str): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + written.append(path) + + put(out_dir / "site.css", + _fill(CSS.replace("__SIDEBAR__", str(SIDEBAR_W)).replace("__CONTENT__", str(CONTENT_W)), + values)) + put(out_dir / "viewer.html", _fill(VIEWER.replace("__TITLE__", escape(title)), values)) + + cards, nav = [], [] + for entry in books: + b = entry["book"] + pages = [p for p, _ in entry["pages"]] + book_dir = out_dir / b.slug + for page, svg in entry["pages"]: + put(book_dir / f"{page['slug']}.html", + _page(title, entry, page, svg, pages, entry["notes"].get(page["slug"]), title)) + if svg is not None: + put(book_dir / f"{page['slug']}.svg", svg) + + # The reference page: the single-page site as it was, one per book, where + # a module box on any page opens its own section. + ref = reference_page(b.view, style, title=f"{b.slug} — reference")["index.html"] + ref = ref.replace('href="site.css"', 'href="../site.css"') + put(book_dir / "reference.html", ref) + + explore = b.out / "explore" + if explore.is_dir(): + for f in sorted(explore.rglob("*")): + if f.is_file(): + target = book_dir / "explore" / f.relative_to(explore) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(f, target) + written.append(target) + + first = pages[0]["slug"] if pages else "reference" + too_wide = sum(1 for p in pages if p.get("too_wide")) + cards.append( + f'

    ' + f"{escape(b.slug)}

    " + + _note(entry.get("intro")) + + _ledger(b.to_dict()) + + '
    ' + + f'{len(pages)} page(s)' + + f'reference' + + (f'explore' if explore.is_dir() else "") + + (f'{too_wide} too wide' if too_wide else "") + + "
    " + ) + nav.append(f'
  • {escape(b.slug)}
    • ') + + put(out_dir / "index.html", f""" + + + + +{escape(title)} + + + +
    + +
    +

    {escape(title)}

    +

    Regenerated, not edited. Each book opens with what came in and what came out.

    + {_note((intro, {}) if intro else None)} + {_missing(failed)} +
    {"".join(cards)}
    + {_note((end, {}) if end else None)} +
    +
    + + +""") + return written diff --git a/soleprint/atlas2/docgen/emitters/dot.py b/soleprint/atlas2/docgen/emitters/dot.py index 0e4f2c7..bc4915b 100644 --- a/soleprint/atlas2/docgen/emitters/dot.py +++ b/soleprint/atlas2/docgen/emitters/dot.py @@ -59,7 +59,7 @@ def _style_words(rule: dict, *, filled: bool = True) -> str: return ",".join(words) -def _node_attrs(node, rule: dict, style) -> dict: +def _node_attrs(node, rule: dict, style, links: dict | None = None) -> dict: a = { "label": node.get("label") or node["id"], "shape": rule.get("shape", "box"), @@ -76,7 +76,16 @@ def _node_attrs(node, rule: dict, style) -> dict: "class": node["kind"], } attrs = node.get("attrs") or {} - if attrs.get("file"): + if links and node["id"] in links: + # A link the caller supplied wins over the source anchor: on a paginated + # site a box is a way into its own page, which is worth more than the + # file it came from. + a["href"] = links[node["id"]] + a["tooltip"] = attrs.get("doc") or f"open {node['id']}" + elif attrs.get("file") and links is None: + # Only when the output is not managing links itself. A relative source + # path is a dead link on a web page, so an output that passes `links` — + # even an empty one — owns every href in the drawing. line = attrs.get("line") a["href"] = f"{attrs['file']}#L{line}" if line else attrs["file"] a["tooltip"] = attrs.get("doc") or node["id"] @@ -86,8 +95,14 @@ def _node_attrs(node, rule: dict, style) -> dict: def emit(ir: dict, style, *, max_depth: int | None = None, - rankdir: str | None = None) -> str: - """IR document (a dict) + Style -> DOT text.""" + rankdir: str | None = None, links: dict | None = None) -> str: + """IR document (a dict) + Style -> DOT text. + + `links` maps node ids to URLs and becomes each box's `href` in the SVG. + Passed in rather than read from the IR, because where a box leads is a fact + about the output — a site page, a notebook anchor — and the IR is the same + for every output. Same rule that keeps colour out of it. + """ nodes = {n["id"]: n for n in ir["nodes"]} children: dict[str | None, list[str]] = {} for n in ir["nodes"]: @@ -119,7 +134,7 @@ def emit(ir: dict, style, *, max_depth: int | None = None, pad = " " * (depth + 1) if not kids or too_deep: - out.append(f"{pad}{_q(node_id)}{_attrs(_node_attrs(node, style.node(node['kind']), style))}") + out.append(f"{pad}{_q(node_id)}{_attrs(_node_attrs(node, style.node(node['kind']), style, links))}") return rule = style.group(node["kind"]) @@ -138,6 +153,8 @@ def emit(ir: dict, style, *, max_depth: int | None = None, out.append(f'{pad} labeljust=l') out.append(f'{pad} id="{_esc(node_id)}"') out.append(f'{pad} class="{node["kind"]}"') + if links and node_id in links: + out.append(f'{pad} href="{_esc(links[node_id])}"') for kid in kids: write(kid, depth + 1) out.append(f"{pad}}}") diff --git a/soleprint/atlas2/docgen/emitters/erd.py b/soleprint/atlas2/docgen/emitters/erd.py index d736c1f..86450d3 100644 --- a/soleprint/atlas2/docgen/emitters/erd.py +++ b/soleprint/atlas2/docgen/emitters/erd.py @@ -84,7 +84,19 @@ def _card_height(table: dict, columns: bool = True) -> int: return header + (len(table["fields"]) * FIELD_H if columns else 0) -def layout(tables: list[dict], edges: list[dict], columns: bool = True) -> dict[str, tuple[int, int]]: +def max_columns(width_px: int) -> int: + """How many card columns fit a width budget given in CSS pixels. + + The SVG is written in points (`width="Npt"`), and a browser draws a point as + 4/3 of a pixel — so the budget is converted before dividing, or a page sized + for 960px renders at 1280px and the notebook shrinks it after all. + """ + usable = width_px * 0.75 - 2 * PAD + COL_GAP + return max(1, int(usable // (CARD_W + COL_GAP))) + + +def layout(tables: list[dict], edges: list[dict], columns: bool = True, + max_cols: int | None = None) -> dict[str, tuple[int, int]]: """Place cards in √n columns, referenced tables first. Sorting by "is the target of a foreign key" puts the tables everything @@ -95,6 +107,10 @@ def layout(tables: list[dict], edges: list[dict], columns: bool = True) -> dict[ ordered = sorted(tables, key=lambda t: (t["id"] not in referenced, t["id"])) cols = max(2, int((len(ordered) * 1.2) ** 0.5 + 0.999)) + if max_cols is not None: + # Capped, not chosen: the page grows downward instead, which a notebook + # scrolls and does not shrink. + cols = max(1, min(cols, max_cols)) col_w = CARD_W + COL_GAP cursor = [PAD] * cols @@ -111,7 +127,7 @@ def _field_y(table: dict, index: int, top: int) -> float: return top + header + (max(index, 0) + 0.5) * FIELD_H -def emit(ir: dict, style, *, columns: bool = True) -> str: +def emit(ir: dict, style, *, columns: bool = True, width: int | None = None) -> str: """IR (a db document) + Style -> SVG text. `columns=False` draws the header of every card and none of its contents — @@ -127,7 +143,7 @@ def emit(ir: dict, style, *, columns: bool = True) -> str: ) edges = [e for e in ir["edges"] if e["kind"] in ("foreign_key", "references")] by_id = {t["id"]: t for t in tables} - pos = layout(tables, edges, columns) + pos = layout(tables, edges, columns, max_cols=max_columns(width) if width else None) s = style.slot width = max(x for x, _ in pos.values()) + CARD_W + PAD diff --git a/soleprint/atlas2/docgen/emitters/markdown.py b/soleprint/atlas2/docgen/emitters/markdown.py new file mode 100644 index 0000000..767cd78 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/markdown.py @@ -0,0 +1,145 @@ +""" +Markdown -> HTML, for notes on the site. Stdlib only, and deliberately small. + +A notebook renders markdown itself, so notes go into it as written. The site has +no renderer, and adding a dependency to turn a few paragraphs into HTML would +break the rule that the structural path is the stdlib. So this covers what notes +are actually written with: + + # headings paragraphs - bullet / 1. numbered lists + **bold** *italic* `code` ```fenced code``` + [links](url) ![images](src) | pipe | tables | + > quotes --- + +Anything it does not recognise is **escaped, not passed through** — a note is +text, and raw HTML in it is shown rather than executed. +""" + +import re +from html import escape + + +def _url(raw: str) -> str: + """An attribute-safe URL. The text around it is already escaped without + quotes, so a `"` here would close the attribute; and a `javascript:` link is + refused outright — a note is text, not a place to run code.""" + if re.match(r"^\s*(javascript|vbscript|data):", raw, re.I): + return "#" + return raw.replace('"', """) + + +INLINE = [ + (re.compile(r"!\[([^\]]*)\]\(([^)\s]+)\)"), lambda m, img: img(m.group(1), m.group(2))), + (re.compile(r"\[([^\]]+)\]\(([^)\s]+)\)"), + lambda m, img: f'{m.group(1)}'), + (re.compile(r"\*\*(.+?)\*\*"), lambda m, img: f"{m.group(1)}"), + (re.compile(r"(?{m.group(1)}"), +] + + +def _inline(text: str, image) -> str: + # Code spans first and protected, so nothing inside them is formatted. + spans: list[str] = [] + + def keep(m): + spans.append(f"{escape(m.group(1))}") + return f"\x00{len(spans) - 1}\x00" + + text = re.sub(r"`([^`]+)`", keep, text) + text = escape(text, quote=False) + for pattern, render in INLINE: + text = pattern.sub(lambda m: render(m, image), text) + return re.sub(r"\x00(\d+)\x00", lambda m: spans[int(m.group(1))], text) + + +def to_html(text: str, *, image=None) -> str: + """Render a note. `image(alt, src) -> html` lets the caller embed local images.""" + image = image or (lambda alt, src: f'{alt.replace(chr(34), ') + lines = text.replace("\r\n", "\n").split("\n") + out: list[str] = [] + i = 0 + + def para(buf): + if buf: + out.append(f"

    {_inline(' '.join(s.strip() for s in buf), image)}

    ") + + buf: list[str] = [] + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if stripped.startswith("```"): + para(buf); buf = [] + code = [] + i += 1 + while i < len(lines) and not lines[i].strip().startswith("```"): + code.append(lines[i]) + i += 1 + out.append(f"
    {escape(chr(10).join(code))}
    ") + i += 1 + continue + + if not stripped: + para(buf); buf = [] + i += 1 + continue + + m = re.match(r"^(#{1,6})\s+(.*)$", stripped) + if m: + para(buf); buf = [] + level = len(m.group(1)) + out.append(f"{_inline(m.group(2), image)}") + i += 1 + continue + + if re.match(r"^(-{3,}|\*{3,})$", stripped): + para(buf); buf = [] + out.append("
    ") + i += 1 + continue + + if stripped.startswith(">"): + para(buf); buf = [] + quote = [] + while i < len(lines) and lines[i].strip().startswith(">"): + quote.append(lines[i].strip()[1:].strip()) + i += 1 + out.append(f"
    {to_html(chr(10).join(quote), image=image)}
    ") + continue + + if re.match(r"^([-*+]|\d+[.)])\s+", stripped): + para(buf); buf = [] + ordered = bool(re.match(r"^\d", stripped)) + items = [] + while i < len(lines) and re.match(r"^\s*([-*+]|\d+[.)])\s+", lines[i]): + items.append(re.sub(r"^\s*([-*+]|\d+[.)])\s+", "", lines[i])) + i += 1 + tag = "ol" if ordered else "ul" + out.append(f"<{tag}>" + "".join(f"
  • {_inline(it, image)}
    • " for it in items) + + f"") + continue + + if stripped.startswith("|") and i + 1 < len(lines) \ + and re.match(r"^\|?\s*:?-{3,}", lines[i + 1].strip()): + para(buf); buf = [] + + def cells(row): + return [c.strip() for c in row.strip().strip("|").split("|")] + + head = cells(lines[i]) + i += 2 + rows = [] + while i < len(lines) and lines[i].strip().startswith("|"): + rows.append(cells(lines[i])) + i += 1 + html = "" + "".join(f"" for h in head) + "" + for r in rows: + html += "" + "".join(f"" for c in r) + "" + out.append(html + "
    {_inline(h, image)}
    {_inline(c, image)}
    ") + continue + + buf.append(line) + i += 1 + + para(buf) + return "\n".join(out) diff --git a/soleprint/atlas2/docgen/emitters/notebook.py b/soleprint/atlas2/docgen/emitters/notebook.py index 32df47f..964a9e4 100644 --- a/soleprint/atlas2/docgen/emitters/notebook.py +++ b/soleprint/atlas2/docgen/emitters/notebook.py @@ -40,7 +40,7 @@ METADATA = { } -def _cell(index: int, kind: str, text: str) -> dict: +def _cell(index: int, kind: str, text: str, attachments: dict | None = None) -> dict: lines = text.split("\n") source = [ln + "\n" for ln in lines[:-1]] + ([lines[-1]] if lines[-1] else []) cell = { @@ -49,6 +49,12 @@ def _cell(index: int, kind: str, text: str) -> dict: "metadata": {}, "source": source, } + if attachments and kind == "md": + # nbformat 4 lets a markdown cell carry its own images, referenced as + # `attachment:`. That is how a figure renders without the notebook + # being run: nothing is an output, so "run all" cannot clear it, and the + # same SVG gives the same bytes every build. + cell["attachments"] = {name: dict(bundle) for name, bundle in sorted(attachments.items())} if kind == "code": cell["execution_count"] = None cell["outputs"] = [] @@ -212,15 +218,50 @@ def _call_md(step: dict) -> str: return "\n".join(out) +def _figure_md(step: dict) -> str: + """A page of a paginated diagram, as markdown. + + The anchor is an explicit `` rather than the heading's own: JupyterLab + and VS Code derive heading anchors differently, and JupyterLab's sanitiser + keeps `name` where it drops `id`. The page index links to these. + """ + parts = [] + if step.get("anchor"): + parts.append(f'') + parts.append("") + if step.get("title"): + parts.append(f'{"#" * min(6, 2 + int(step.get("depth", 0)))} {step["title"]}') + parts.append("") + if step.get("svg"): + parts.append(f'![{step.get("title", "figure")}](attachment:{step["name"]})') + else: + parts.append(f'*Not drawn here — {step.get("why", "no renderer")}.*') + if step.get("caption"): + parts += ["", f'*{step["caption"]}*'] + return "\n".join(parts) + + +def _figure_attachments(step: dict) -> dict | None: + if not step.get("svg"): + return None + import base64 + + data = step["svg"].encode("utf-8") if isinstance(step["svg"], str) else step["svg"] + return {step["name"]: {"image/svg+xml": base64.b64encode(data).decode("ascii")}} + + def build(spec: dict) -> dict: """A merged notebook spec -> the notebook, as a dict.""" - blocks: list[tuple[str, str]] = [] + blocks: list[tuple] = [] for step in spec["steps"]: kind = step.get("kind", "md") before = step.get("before") if before: - blocks.append(("md", before)) + blocks.append(("md", before, step.get("before_attachments"))) + + if kind == "figure": + blocks.append(("md", _figure_md(step), _figure_attachments(step))) if kind == "md": text = [] @@ -259,7 +300,7 @@ def build(spec: dict) -> dict: blocks.append(("md", after)) return { - "cells": [_cell(i, k, t) for i, (k, t) in enumerate(blocks)], + "cells": [_cell(i, *block) for i, block in enumerate(blocks)], "metadata": METADATA, "nbformat": 4, "nbformat_minor": 5, diff --git a/soleprint/atlas2/docgen/emitters/paged.py b/soleprint/atlas2/docgen/emitters/paged.py new file mode 100644 index 0000000..92ba6f1 --- /dev/null +++ b/soleprint/atlas2/docgen/emitters/paged.py @@ -0,0 +1,118 @@ +""" +Draw pages: measure each one against a width, then render it for an output. + +`ops/pages.py` decides which pages exist and never renders anything — `ops/` +imports no emitter. This is the other half: the `fits` test it needs, and the +drawing of a laid-out page, with the links an output wants. + +## Measured, not estimated + +A layered drawing's width is its widest level, but *how* wide depends on the +labels, the font and the spacing the style sets. A count-based estimate was off +by enough to matter, so when Graphviz is present a page is rendered and its +`width="Npt"` read back. Without Graphviz the estimate is all there is, and the +page records `measured: False` so nobody mistakes one for the other. + +## Points and pixels + +Graphviz and the ERD emitter both write sizes in points, and a browser draws a +point as 4/3 of a CSS pixel. Budgets here are pixels, because the column they +have to fit — a notebook cell, a site's content column — is sized in pixels. +""" + +import re + +PT_TO_PX = 4 / 3 + +# Budgets for the outputs docgen writes. A notebook cell in JupyterLab or VS Code +# is a little under a thousand pixels at a normal window size; the site's content +# column is set by `site.CONTENT_W`. +NOTEBOOK_WIDTH = 960 + + +def svg_width_px(svg: str) -> float | None: + m = re.search(r']*\swidth="([\d.]+)(pt|px)?"', svg) + if not m: + return None + value = float(m.group(1)) + return value * PT_TO_PX if (m.group(2) or "pt") == "pt" else value + + +def _estimate_px(ir: dict, options: dict) -> float: + """Width without a renderer: widest level of boxes, or levels side by side.""" + from ..ops import shape + + sh = shape(ir) + labels = [len(n.get("label") or n["id"]) for n in ir["nodes"]] or [8] + box = 7 * (sum(labels) / len(labels)) + 40 + if options.get("rankdir") == "LR": + return sh["levels"] * (box + 58) + 30 + return sh["widest_level"] * (box + 36) + 30 + + +def fits_for(style, width: int): + """A `fits(page_ir, options) -> bool` for `ops.pages.layout`, at `width` px. + + Also returns whether it measured or estimated, so the ledger can say which. + """ + from .dot import emit, have_graphviz, render + + measured = have_graphviz() + + def fits(ir: dict, options: dict) -> bool: + if not measured: + return _estimate_px(ir, options) <= width + svg = render(emit(ir, style, rankdir=options.get("rankdir"))).decode("utf-8") + px = svg_width_px(svg) + return px is not None and px <= width + + return fits, measured + + +def links_for(pages: list[dict], page: dict, url) -> dict: + """{box id on this page: url of the page that box opens}. + + `url(target_page)` builds the address, so the site and anything else that + can follow a link decide their own scheme. Context boxes link too, when their + branch has a page of its own — that is how a reader crosses to a sibling. + """ + by_scope = {p.get("scope") or p["id"]: p for p in pages} + by_id = {p["id"]: p for p in pages} + out = {} + for n in page["ir"]["nodes"]: + target = by_id.get(n["id"]) if page.get("tables") else by_scope.get(n["id"]) + if target is not None and target is not page: + out[n["id"]] = url(target) + return out + + +def render(page: dict, style, width: int, *, links: dict | None = None) -> str | None: + """A laid-out page as SVG text, or None when it cannot be drawn here. + + Schema pages go through the ERD emitter with its columns capped to `width`; + everything else, schema overviews included, through Graphviz with the page's + layout options and the caller's links. + """ + if page.get("tables") and page["id"] != "overview": + from .erd import emit as erd + + return erd(page["ir"], style, width=width) + + from .dot import emit, have_graphviz, render as dot_render + + if not have_graphviz(): + return None + svg = dot_render(emit(page["ir"], style, rankdir=(page.get("options") or {}).get("rankdir"), + links=links)).decode("utf-8") + return svg + + +def inline(svg: str) -> str: + """SVG text ready to sit inside HTML: no XML declaration, no DOCTYPE. + + Inline rather than ``, because an image swallows every click and the + whole point of a page's boxes is that they open other pages. + """ + svg = re.sub(r"<\?xml[^>]*\?>\s*", "", svg) + svg = re.sub(r"]*>\s*", "", svg) + return re.sub(r"\s*", "", svg, flags=re.S).strip() diff --git a/soleprint/atlas2/docgen/emitters/site.py b/soleprint/atlas2/docgen/emitters/site.py index 7057d9c..e48c933 100644 --- a/soleprint/atlas2/docgen/emitters/site.py +++ b/soleprint/atlas2/docgen/emitters/site.py @@ -177,6 +177,11 @@ CSS = """/* Generated by docgen. The layout five demos converged on: a sticky si --muted: __MUTED__; --dim: __DIM__; --accent: __ACCENT__; + /* Used by the ledger's "incomplete" panel. Declared here because it was not, + and an undefined custom property falls back silently — the warning rendered + in the ordinary text colour, which is the one place it must not. */ + --artery: __ARTERY__; + --ok: __OK__; } * { margin: 0; padding: 0; box-sizing: border-box; } body { @@ -267,6 +272,44 @@ body { .gap b { color: var(--artery); } .gap ul { margin: .4rem 0 0 1.1rem; color: var(--muted); } .gap code { font-size: 11px; } + +/* -- pages ------------------------------------------------------------------ + A page is one subsystem drawn small enough to read. The diagram is inline SVG, + not an , so its boxes are links: a group opens its own page, a module + opens its section of the reference page. */ +.crumbs { color: var(--dim); font-size: 11px; margin-bottom: .6rem; } +.crumbs a { color: var(--muted); text-decoration: none; } +.crumbs a:hover { color: var(--accent); } +.note { margin: 0 0 1.2rem; } +.note h1, .note h2, .note h3 { color: var(--text); margin: 1rem 0 .4rem; font-size: 14px; } +.note p, .note li { color: var(--muted); margin-bottom: .5rem; } +.note ul, .note ol { margin: 0 0 .8rem 1.2rem; } +.note pre { background: var(--surface); border: 1px solid var(--border); border-radius: 6px; + padding: .6rem .8rem; overflow-x: auto; margin: .6rem 0; } +.note code { font-size: 11px; } +.note table { border-collapse: collapse; margin: .6rem 0; } +.note th, .note td { border-bottom: 1px solid var(--border); padding: 4px 12px 4px 0; text-align: left; } +.note blockquote { border-left: 2px solid var(--border); padding-left: .8rem; color: var(--dim); } +.note img { max-width: 100%; } +.figbar { display: flex; gap: 6px; align-items: center; margin-bottom: .4rem; font-size: 11px; color: var(--dim); } +.figbar button, .figbar a { + background: var(--surface); color: var(--muted); border: 1px solid var(--border); + border-radius: 5px; padding: 2px 8px; font-size: 11px; cursor: pointer; text-decoration: none; } +.figbar button.on { color: var(--accent); border-color: var(--accent); } +.fig { border: 1px solid var(--border); border-radius: 8px; background: var(--surface); + padding: 8px; overflow: hidden; } +.fig svg { display: block; max-width: 100%; height: auto; margin: 0 auto; } +.fig.actual { overflow: auto; } +.fig.actual svg { max-width: none; } +.fig a:hover polygon, .fig a:hover path { stroke: var(--accent); } +.caption { color: var(--dim); font-size: 11px; margin: .4rem 0 1.2rem; } +.caption.warn { color: var(--artery); } +.children { columns: 2; margin: .4rem 0 1.4rem; list-style: none; } +.children li { margin-bottom: 2px; } +.books { display: grid; gap: 1rem; margin: 1rem 0 2rem; } +.bookcard { border: 1px solid var(--border); border-radius: 8px; padding: .9rem 1rem; } +.bookcard h2 { font-size: 15px; margin-bottom: .5rem; } +.bookcard .links a { margin-right: 1rem; font-size: 12px; } """ @@ -337,6 +380,8 @@ def _slots(style) -> dict: "__MUTED__": s("text-muted"), "__DIM__": s("text-dim"), "__ACCENT__": s("accent"), + "__ARTERY__": s("error") or s("artery"), + "__OK__": s("ok") or s("atlas"), "__FONT__": '"Segoe UI", Inter, system-ui, -apple-system, Arial, sans-serif', } diff --git a/soleprint/atlas2/docgen/fixtures/notes/_intro.md b/soleprint/atlas2/docgen/fixtures/notes/_intro.md new file mode 100644 index 0000000..b92345c --- /dev/null +++ b/soleprint/atlas2/docgen/fixtures/notes/_intro.md @@ -0,0 +1,3 @@ + +The fixtures docgen tests itself against: a Python tree, a schema and an API +contract, built into one document to show the parts fitting together. diff --git a/soleprint/atlas2/docgen/fixtures/notes/schema/overview.md b/soleprint/atlas2/docgen/fixtures/notes/schema/overview.md new file mode 100644 index 0000000..bd8e658 --- /dev/null +++ b/soleprint/atlas2/docgen/fixtures/notes/schema/overview.md @@ -0,0 +1,5 @@ + +## Where to start + +`orders` is the centre: every line belongs to one, and every order to one user. +Open the page to see the keys. diff --git a/soleprint/atlas2/docgen/notebook/spec.py b/soleprint/atlas2/docgen/notebook/spec.py index cb8c7e6..76b5116 100644 --- a/soleprint/atlas2/docgen/notebook/spec.py +++ b/soleprint/atlas2/docgen/notebook/spec.py @@ -51,7 +51,7 @@ from pathlib import Path SPEC_VERSION = "1" # Step kinds. Small and closed, the same discipline the IR's `kind` follows. -KINDS = ("md", "code", "params", "call") +KINDS = ("md", "code", "params", "call", "figure") def _step(id: str, kind: str, **fields) -> dict: diff --git a/soleprint/atlas2/docgen/ops/pages.py b/soleprint/atlas2/docgen/ops/pages.py new file mode 100644 index 0000000..22dc851 --- /dev/null +++ b/soleprint/atlas2/docgen/ops/pages.py @@ -0,0 +1,382 @@ +""" +Pages: a graph too big for one picture, as a set of pictures that each fit. + + pages = paginate(view) # the structure: which pages exist + laid = layout(pages, fits) # per output: fits(page_ir, options) -> bool + +A notebook scales an image down to its cell width, so a wide graph does not +scroll — it shrinks until its labels are unreadable. The only fix that holds is +to draw less per picture. This decides *what* goes on each picture; the caller +decides whether it fits, by rendering it and measuring, because a width +estimated from node counts is wrong the moment the labels are long. + +**Which pages exist does not depend on width** — only how each one is laid out +does. So the two are separate calls: the page ids a note is filed under are the +same for a notebook column and a site column, and listing them needs no +renderer at all. + +## By structure, never by slicing + +Tiling one large image cuts boxes and edges at the seams, and a tile means +nothing on its own. A page here is a subsystem: + + overview one box per top-level group, relationships lifted + └── station one box per child of `station` + └── station.tools one box per child of `station.tools`, and so on + +Clicking a box — on the site — or following the index — in a notebook — moves +down a level. Every page is a real graph: the boxes are the scope's children, +and anything outside the scope that they touch appears as a **context** box, +collapsed to the branch it sits on, so a page still says what its subsystem +reaches without drawing the rest of the tree. + +## Groups come from the id, not from `parent` + +A directory without `__init__.py` is not a Python package, so its modules have +no `parent` — soleprint has 173 such roots — but their dotted ids still carry +the directory. `split()` groups on id segments for the same reason. A group with +no node of its own (such a directory) is synthesised on the page as a box of +the kind most of its contents are, marked `attrs.group`. + +## When a level cannot be narrowed + +A scope whose children genuinely are many peers has no smaller structure to +split into. It is redrawn left-to-right — height is free in a notebook, width is +not — and if that still does not fit, the page is kept and flagged `too_wide`. +Never cut, never dropped: a flagged page is honest, a missing one is not. + +## Schemas page differently + +A schema is peers with references, drawn as cards in columns whose count the +emitter caps to the width, so width is never the problem — count is. Tables are +paged by connected component (tables joined by foreign keys stay together), and +tables with no keys at all are chunked alphabetically. +""" + +import re + +OVERVIEW = "overview" +MAX_TABLES = 20 # past this a schema page stops being scannable + + +def slugify(page_id: str) -> str: + """A filename- and anchor-safe name for a page id. + + Ids are not safe as they stand: an endpoint id is `GET /orders/{id}`, and a + tree-sitter id can carry `#L12`. + """ + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", page_id).strip("-.") + return slug or "page" + + +def _key(node_id: str, scope: str) -> str: + """The box at `scope` that `node_id` falls into: scope + one more segment.""" + if scope == "": + return node_id.split(".", 1)[0] + if node_id == scope: + return scope + rest = node_id[len(scope) + 1:] + return f"{scope}.{rest.split('.', 1)[0]}" + + +def _under(node: dict, scope: str) -> bool: + if node["kind"] == "external": + return False # externals are context everywhere, never members + nid = node["id"] + return scope == "" or nid == scope or nid.startswith(scope + ".") + + +def _representative(node_id: str, scope: str) -> str: + """Where an outside node shows on this page: the branch it diverges on. + + The common prefix of the scope and the node, plus one more segment of the + node's id. That box is never an ancestor of the scope (it is on the node's + branch, past the point they part), so it never swallows the page's own + members — and two outside nodes on the same branch share one box. + """ + s, n = scope.split(".") if scope else [], node_id.split(".") + common = 0 + while common < len(s) and common < len(n) and s[common] == n[common]: + common += 1 + return ".".join(n[: common + 1]) + + +def page_ir(view: dict, scope: str) -> tuple[dict, dict]: + """The graph for one page, and {member key: [node ids it stands for]}. + + Built directly rather than through `_rebuild`, because a synthesised group + box and a context box have no node in the source graph to keep. + """ + by_id = {n["id"]: n for n in view["nodes"]} + members: dict[str, list] = {} + for n in view["nodes"]: + if _under(n, scope): + members.setdefault(_key(n["id"], scope), []).append(n["id"]) + + where = {nid: key for key, ids in members.items() for nid in ids} + context: dict[str, list] = {} + for e in view["edges"]: + s_in, t_in = e["source"] in where, e["target"] in where + if s_in == t_in: + continue # internal to the page (handled below) or unrelated + outside = e["target"] if s_in else e["source"] + rep = _representative(outside, scope) + if rep in members: + continue # cannot happen for a well-formed tree; stay safe + context.setdefault(rep, []).append(outside) + where.setdefault(outside, rep) + + nodes = [] + for key, ids in sorted(members.items()) + sorted(context.items()): + is_context = key in context and key not in members + own = by_id.get(key) + ids = sorted(set(ids)) + if own is not None and (ids == [key] or not is_context): + node = {"id": key, "kind": own["kind"], "label": own.get("label") or key, + "parent": None, "attrs": dict(own.get("attrs") or {})} + else: + kinds = [by_id[i]["kind"] for i in ids if i in by_id] + kind = max(set(kinds), key=lambda k: (kinds.count(k), k)) if kinds else "module" + node = {"id": key, "kind": kind, "label": key.rsplit(".", 1)[-1], + "parent": None, "attrs": {}} + # Semantic, not visual: which boxes are subsystems to open, and which are + # the outside world seen from here. How that looks is style's decision. + grouped = [i for i in ids if i != key] + if grouped or (own is None): + node["attrs"]["group"] = True + node["attrs"]["count"] = len(ids) + if is_context: + node["attrs"]["context"] = True + nodes.append(node) + + merged: dict[tuple, dict] = {} + for e in view["edges"]: + src, dst = where.get(e["source"]), where.get(e["target"]) + if src is None or dst is None or src == dst: + continue + if src in context and dst in context: + continue # two outside boxes: not this page's relationship + k = (src, dst, e["kind"]) + if k in merged: + merged[k]["attrs"]["weight"] = merged[k]["attrs"].get("weight", 1) + 1 + continue + attrs = dict(e.get("attrs") or {}) + if src != e["source"] or dst != e["target"]: + attrs = {"weight": 1} + merged[k] = {"source": src, "target": dst, "kind": e["kind"], "attrs": attrs} + + ir = {"meta": dict(view["meta"]), "nodes": nodes, + "edges": sorted(merged.values(), key=lambda e: (e["source"], e["target"], e["kind"]))} + return ir, members + + +def layout(pages: list[dict], fits) -> list[dict]: + """The same pages, each with the layout that fits: top-to-bottom, then + left-to-right, then flagged `too_wide`. Returns new page dicts. + + Schema pages are left alone — the ERD emitter caps its own columns to the + width, so they always fit — except the overview, which is a graph. + """ + out = [] + for page in pages: + page = dict(page) + if page.get("tables") and page["id"] != OVERVIEW: + page["options"], page["too_wide"] = {}, False + elif fits(page["ir"], {}): + page["options"], page["too_wide"] = {}, False + elif fits(page["ir"], {"rankdir": "LR"}): + page["options"], page["too_wide"] = {"rankdir": "LR"}, False + else: + page["options"], page["too_wide"] = {"rankdir": "LR"}, True + out.append(page) + return out + + +def paginate(view: dict, *, max_depth: int = 6) -> list[dict]: + """Pages for a graph, overview first, each drawn as a subsystem. + + Returns a list of:: + + {"id", "slug", "title", "scope", "parent", "depth", "ir", + "children": [page ids], "options": {}, "too_wide": False} + + `options` and `too_wide` are placeholders until `layout()` fills them. + + in reading order — a page is followed by its own children, depth first — so + a notebook can emit them in sequence and a site can build its tree from + `parent` and `children`. + """ + pages: list[dict] = [] + slugs: set[str] = set() + + def unique(slug: str) -> str: + base, i = slug, 2 + while slug in slugs: + slug, i = f"{base}-{i}", i + 1 + slugs.add(slug) + return slug + + def opens(key: str, scope: str, ids: list) -> bool: + # A box opens into a page when it stands for more than itself. + return key != scope and any(i != key for i in ids) + + def visit(scope: str, parent: str | None, depth: int, page_id: str | None = None): + ir, members = page_ir(view, scope) + if not ir["nodes"]: + return + # A page whose only member is one group says nothing but "open this" — + # docgen's own overview was one box reading `docgen`. Descend instead, + # keeping the id the caller expects, so the overview is still `overview`. + groups = [k for k, ids in members.items() if opens(k, scope, ids)] + if len(members) == 1 and len(groups) == 1: + return visit(groups[0], parent, depth, page_id or (scope or OVERVIEW)) + page_id = page_id or scope or OVERVIEW + page = { + "id": page_id, + "slug": unique(slugify(page_id)), + "title": scope if scope and page_id == scope else (scope or "Overview"), + "scope": scope, + "parent": parent, + "depth": depth, + "ir": ir, + "options": {}, + "children": [], + "too_wide": False, + } + pages.append(page) + if depth >= max_depth: + return + for key in sorted(members): + if opens(key, scope, members[key]): + page["children"].append(key) + visit(key, page_id, depth + 1) + + visit("", None, 0) + return pages + + +def paginate_tables(view: dict, *, max_tables: int = MAX_TABLES) -> list[dict]: + """Pages for a schema: connected components, packed, then chunked. + + Each page is the tables plus their columns, so the ERD emitter draws cards. + The overview has one box per page, which is what a reader opens from. + """ + tables = sorted(n["id"] for n in view["nodes"] if n["kind"] == "table") + links: dict[str, set] = {t: set() for t in tables} + for e in view["edges"]: + if e["source"] in links and e["target"] in links: + links[e["source"]].add(e["target"]) + links[e["target"]].add(e["source"]) + + seen, components = set(), [] + for t in tables: + if t in seen: + continue + stack, comp = [t], [] + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + comp.append(cur) + stack.extend(links[cur] - seen) + components.append(sorted(comp)) + + connected = sorted((c for c in components if len(c) > 1), key=lambda c: (-len(c), c[0])) + lonely = sorted(c[0] for c in components if len(c) == 1) + + groups: list[list[str]] = [] + for comp in connected: + # A component larger than a page is still split — by name, since a + # foreign-key cluster has no finer structure to split on — and the cut + # keys show as edges to the other page's tables in neither, which the + # overview's lifted edges make up for. + for i in range(0, len(comp), max_tables): + groups.append(comp[i:i + max_tables]) + for i in range(0, len(lonely), max_tables): + groups.append(lonely[i:i + max_tables]) + + by_id = {n["id"]: n for n in view["nodes"]} + pages: list[dict] = [] + table_page: dict[str, str] = {} + for i, group in enumerate(groups, start=1): + pid = group[0] if len(group) == 1 else f"{group[0]}…{group[-1]}" + slug = f"tables-{i:02d}" + for t in group: + table_page[t] = slug + keep = set(group) | {n["id"] for n in view["nodes"] if n.get("parent") in set(group)} + ir = {"meta": dict(view["meta"]), + "nodes": [dict(by_id[n]) for n in sorted(keep)], + "edges": [dict(e) for e in view["edges"] + if e["source"] in keep and e["target"] in keep]} + pages.append({"id": slug, "slug": slug, "title": pid, "scope": slug, + "parent": OVERVIEW, "depth": 1, "ir": ir, "options": {}, + "children": [], "too_wide": False, "tables": True}) + + over_nodes = [{"id": p["id"], "kind": "table", "label": p["title"], "parent": None, + "attrs": {"group": True, "count": sum( + 1 for n in p["ir"]["nodes"] if n["kind"] == "table")}} + for p in pages] + merged: dict[tuple, dict] = {} + for e in view["edges"]: + a, b = table_page.get(e["source"]), table_page.get(e["target"]) + if a and b and a != b: + k = (a, b, e["kind"]) + merged.setdefault(k, {"source": a, "target": b, "kind": e["kind"], + "attrs": {"weight": 0}}) + merged[k]["attrs"]["weight"] += 1 + overview = {"id": OVERVIEW, "slug": OVERVIEW, "title": "Overview", "scope": "", + "tables": True, "parent": None, + "depth": 0, "ir": {"meta": dict(view["meta"]), "nodes": over_nodes, + "edges": list(merged.values())}, + "options": {}, "children": [p["id"] for p in pages], "too_wide": False} + return [overview] + pages + + +def coverage(view: dict, pages: list[dict]) -> list[str]: + """Node ids of the view that no page accounts for. Empty means nothing lost. + + A node is accounted for when a page shows it — as itself, or inside a box + that opens onto a page that does. Externals count when their branch appears + as context somewhere. + """ + shown = set() + for p in pages: + shown |= {n["id"] for n in p["ir"]["nodes"]} + # Schema pages draw tables and their columns; an API's endpoints are the + # notebook's walkthrough, not a diagram, so they are not a page's to account for. + tables_only = bool(pages) and pages[0].get("tables") + missing = [] + for n in view["nodes"]: + if tables_only and n["kind"] not in ("table", "column"): + continue + nid = n["id"] + if nid in shown: + continue + parts = nid.split(".") + if n["kind"] == "external" and any(".".join(parts[:i]) in shown + for i in range(1, len(parts) + 1)): + continue + if n.get("parent") in shown and n["kind"] == "column": + continue + missing.append(nid) + return sorted(missing) + + +def for_view(ir: dict, view: dict, *, max_depth: int = 6) -> list[dict]: + """The pages a book gets, whatever its source. + + A schema pages by table — from the full IR, since a card needs its columns and + the default view drops them — and anything else pages by structure from the + view. The same decision `classify` makes about which emitter draws it. + """ + from .filter import classify + + if classify(view)["emitter"] == "erd": + tables = {"meta": ir["meta"], + "nodes": [n for n in ir["nodes"] if n["kind"] in ("table", "column")], + "edges": [e for e in ir["edges"] + if e["kind"] in ("foreign_key", "references")]} + if any(n["kind"] == "table" for n in tables["nodes"]): + return paginate_tables(tables) + return paginate(view, max_depth=max_depth) diff --git a/soleprint/atlas2/docgen/selftest.py b/soleprint/atlas2/docgen/selftest.py index 4d3ab2e..9575d56 100644 --- a/soleprint/atlas2/docgen/selftest.py +++ b/soleprint/atlas2/docgen/selftest.py @@ -2319,6 +2319,453 @@ def _runfiles(): _runfiles() + +# -------------------------------------------------------------------------- +print("\n19. pages — a graph too big for one picture, as pictures that each fit") + + +def _pages(): + pages_mod = __import__(f"{PKG}.ops.pages", fromlist=["*"]) + paged = __import__(f"{PKG}.emitters.paged", fromlist=["*"]) + dot = __import__(f"{PKG}.emitters.dot", fromlist=["*"]) + from_ops = __import__(f"{PKG}.ops", fromlist=["*"]) + + # ops/ decides what goes on a page and must never draw one. Paging is exactly + # where the temptation lives — measuring a width wants a renderer — so the + # measure is passed in as `fits`, and this holds the line. + reaching = [] + for path in sorted((HERE / "ops").glob("*.py")): + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, ast.ImportFrom) and node.module and node.level \ + and node.module.split(".")[0] in ("emitters", "book", "cli", "extractors"): + reaching.append(f"{path.name}: {node.module}") + check("ops/ imports no emitter, book, command line or extractor", not reaching, reaching) + + view = from_ops.overview(py_ex.extract(HERE).to_dict()) + structure = pages_mod.paginate(view) + + check( + "the first page is the overview", + structure[0]["id"] == pages_mod.OVERVIEW, + f"got {structure[0]['id']}", + ) + check( + "every page is a valid IR on its own", + not [p["id"] for p in structure if check_ir(p["ir"])], + ) + # The claim pagination is allowed to make: drawn smaller, never drawn less. + lost = pages_mod.coverage(view, structure) + check("pagination loses no node — each is on a page or inside a box that opens one", + not lost, f"{lost[:5]}") + ids = {p["id"] for p in structure} + check( + "every box that opens a page opens one that exists", + not [c for p in structure for c in p["children"] if c not in ids], + ) + check( + "slugs are unique and safe as file names", + len({p["slug"] for p in structure}) == len(structure) + and all(re.fullmatch(r"[A-Za-z0-9._-]+", p["slug"]) for p in structure), + ) + check( + "an endpoint id becomes a safe slug", + pages_mod.slugify("GET /orders/{orderId}") == "GET-orders-orderId", + pages_mod.slugify("GET /orders/{orderId}"), + ) + # docgen's own overview used to be a single box reading `docgen`: a page + # whose only content is "open this". A lone group is descended into instead. + check( + "a page is never just one box that opens another page", + not [p["id"] for p in structure + if len([n for n in p["ir"]["nodes"] if not (n.get("attrs") or {}).get("context")]) == 1 + and p["children"]], + ) + # Which pages exist must not depend on width — notes are filed by page id, + # and a note has to land in the same place in a notebook and on a site. + check( + "which pages exist does not depend on width", + [p["id"] for p in pages_mod.layout(structure, lambda ir, o: False)] + == [p["id"] for p in pages_mod.layout(structure, lambda ir, o: True)], + ) + narrow = pages_mod.layout(structure, lambda ir, o: False) + check( + "a page that fits no way is kept and flagged, never dropped", + len(narrow) == len(structure) and all(p["too_wide"] for p in narrow), + ) + + # -- measured against a real width -------------------------------------- + if not dot.have_graphviz(): + skip("pages fit their width", "Graphviz is not installed") + else: + lucid_style = Style.load("lucid") + fits, measured = paged.fits_for(lucid_style, 800) + laid = pages_mod.layout(structure, fits) + over = [] + for p in laid: + px = paged.svg_width_px(paged.render(p, lucid_style, 800)) + if px > 800 and not p["too_wide"]: + over.append(f"{p['id']} {px:.0f}px") + check("no page is wider than its budget unless it says so", not over, "; ".join(over)) + # Links are the output's, never the IR's: a site passes them, and then a + # relative source path — a dead link on a web page — must not survive. + page = next(p for p in laid if p["children"]) + links = paged.links_for(laid, page, lambda t: f"{t['slug']}.html") + svg = paged.render(page, lucid_style, 800, links=links) + check( + "a box that opens a page links to it", + all(f'href="{url}"' in svg for url in links.values()) and links, + f"{len(links)} links", + ) + check( + "an output that manages links drops the source-file hrefs", + ".py" not in "".join(re.findall(r'href="([^"]+)"', svg)), + ) + + # Schemas page by table, and a card grows downward rather than sideways. + shop = db_ex.extract(HERE / "fixtures" / "shop.json").to_dict() + tables = pages_mod.for_view(shop, from_ops.overview(shop)) + check( + "a schema pages by connected tables, cards with their columns", + tables[0]["id"] == "overview" and tables[1].get("tables") + and any(n["kind"] == "column" for n in tables[1]["ir"]["nodes"]), + ) + many = {"models": {f"t{i:02d}": {"fields": {"id": {"type": "int", "pk": True}}} + for i in range(12)}} + wide_ir = db_ex.from_schema_dict(many, root="x").to_dict() + svg = erd_mod.emit(wide_ir, lucid, width=800) + check( + "an ERD capped to a width stays inside it", + paged.svg_width_px(svg) <= 800, + f"{paged.svg_width_px(svg):.0f}px", + ) + + +_pages() + + +# -------------------------------------------------------------------------- +print("\n20. notes, figures and pages — the hand-written part, and where it lands") + + +def _documents(): + doc_mod = __import__(f"{PKG}.book.document", fromlist=["*"]) + notes_mod = __import__(f"{PKG}.book.notes", fromlist=["*"]) + build_mod = __import__(f"{PKG}.book.build", fromlist=["*"]) + checks_mod = __import__(f"{PKG}.book.checks", fromlist=["*"]) + md = __import__(f"{PKG}.emitters.markdown", fromlist=["*"]) + dot = __import__(f"{PKG}.emitters.dot", fromlist=["*"]) + import io as _io3 + import contextlib as _ctx3 + + base = Path(tmp.name) / "documents" + src = base / "src" / "shop" + for sub in ("orders", "billing"): + (src / sub).mkdir(parents=True, exist_ok=True) + (src / sub / "__init__.py").write_text(f'"""{sub}."""\n') + (src / sub / "models.py").write_text(f'"""{sub} models."""\nfrom shop.{sub} import x\n') + (src / "__init__.py").write_text('"""shop."""\n') + notes = base / "notes" + + # -- notes --------------------------------------------------------------- + stubbed = notes_mod.Notes(notes).scaffold({"app": [("overview", "page Overview")]}) + check("scaffold writes a stub per spot", (notes / "app" / "overview.md") in stubbed) + check("a stub renders nothing", notes_mod.body((notes / "app" / "overview.md").read_text()) is None) + (notes / "app" / "overview.md").write_text( + (notes / "app" / "overview.md").read_text() + "## Written by hand\n\nKept.\n") + again = notes_mod.Notes(notes).scaffold({"app": [("overview", "page Overview")]}) + check( + "scaffold never overwrites a note", + (notes / "app" / "overview.md") not in again + and "Written by hand" in (notes / "app" / "overview.md").read_text(), + ) + (notes / "img").mkdir(exist_ok=True) + (notes / "img" / "dot.svg").write_text('') + (notes / "app" / "overview.md").write_text( + (notes / "app" / "overview.md").read_text() + "\n![a dot](../img/dot.svg)\n") + (notes / "app" / "no-such-page.md").write_text("Filed under a page that is gone.\n") + + # A note is text: raw HTML is shown, not run, and a link cannot break out of + # its attribute or run script. + html = md.to_html(' [a](b"onclick=1) [c](javascript:alert(1))') + check( + "a note's HTML is escaped and its links are attribute-safe", + # The quote stays inside the attribute as " — asserted on the exact + # attribute, since stripping the entity first (as this check once did) + # makes the attribute's own closing quote look like a break-out. + "