mariano/soleprint
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
73998020b3ea712d4a48fac633d6330339a20d13
soleprint/soleprint/atlas2/docgen/book/document.py

449 lines
18 KiB
Python
Raw Blame History

"""
A document: several books, assembled into each output.
Document
├── books one per source — each keeps its own larder → steps → book
├── notes hand-written markdown, filed by page (book/notes.py)
└── outputs a notebook, a minisite, … — each one made of every book, as parts
A book is a templated part of the final output, not the output itself. A project
with an OpenAPI contract and a C# tree is one document of two books: one notebook
and one minisite, each with a part per book. A `[[book]]` run file is the same thing
with one book per document, which is what it always was.
## Built once, rendered per output
Extraction, the view and the page structure happen once per book. Rendering
does not, because what a page looks like depends on where it is going: a minisite
column is narrower than a notebook cell, a minisite's boxes are links and a
notebook's cannot be, and each output can have its own theme. So each output
lays the pages out for its own width and draws them itself.
## Both ends, at both levels
Every book part still opens with what came in and closes with what came out. A
document of several books also opens with every larder and closes with every
reconciliation, so the rule — any operation is measured at both ends — holds for
the whole as well as for each part.
"""
import json
import os
from dataclasses import dataclass, field
from pathlib import Path
from . import Book
from .notes import Notes, data_uri, images
from .larder import Larder
MANIFEST = ".docgen-output.json"
@dataclass
class Output:
emitter: str # "notebook" | "minisite"
path: Path
style: str = "lucid"
theme: str | None = None
width: int | None = None # px; each emitter has its own default
@dataclass
class Part:
"""One book to build: a source, and where its artifacts go."""
name: str
kind: str
source: Path
out: Path
exclude: tuple = ()
overlay: dict | None = None
ops: dict = field(default_factory=dict)
suffixes: tuple | None = None # code only: which extensions to read
@dataclass
class Document:
name: str
title: str
build: Path # where document.json goes
parts: list[Part]
outputs: list[Output]
notes: Path | None = None
inline: bool = False # a single [[book]]: outputs live inside its directory
books: list[Book] = field(default_factory=list)
problems: list[str] = field(default_factory=list)
failed: dict = field(default_factory=dict) # book name -> why it was not built
# -- views --------------------------------------------------------------------
OPS = ("drop_stdlib", "drop_builtins", "drop_external", "only", "drop")
def apply_ops(view: dict, ops: dict) -> dict:
"""A project's `[ops]`, on top of the default view. Unknown keys never get here."""
from ..ops import filter as F
if ops.get("drop_stdlib"):
view = F.drop_stdlib(view)
if ops.get("drop_builtins"):
view = F.drop_builtins(view)
if ops.get("drop_external"):
view = F.drop_external(view)
if ops.get("drop"):
view = F.drop_kinds(view, ops["drop"])
if ops.get("only"):
view = F.only_kinds(view, ops["only"])
return view
# -- output bookkeeping -------------------------------------------------------
def clear_output(directory: Path) -> list[str]:
"""Remove exactly what the last build wrote into an output directory.
A page that disappeared between builds would otherwise linger — still on disk,
still linked from nowhere, still found by search. The manifest lists every file
written; anything not in it, including a hand-written file someone dropped in,
is left alone.
"""
manifest = directory / MANIFEST
if not manifest.is_file():
return []
try:
written = json.loads(manifest.read_text()).get("written", [])
except (OSError, json.JSONDecodeError):
return []
removed = []
for rel in written:
target = (directory / rel)
if target.is_file() and directory in target.resolve().parents:
target.unlink()
removed.append(rel)
# Directories docgen made and has now emptied — never one with anything left.
for d in sorted({(directory / r).parent for r in written}, key=lambda p: -len(p.parts)):
if d != directory and d.is_dir() and not any(d.iterdir()):
d.rmdir()
manifest.unlink()
return removed
def write_manifest(directory: Path, paths: list[Path]) -> None:
rel = sorted({p.relative_to(directory).as_posix() for p in paths if p.is_file()})
(directory / MANIFEST).write_text(json.dumps({"written": rel}, indent=2) + "\n")
# -- pages, rendered ----------------------------------------------------------
def rendered_pages(book: Book, style, width: int, *, url=None):
"""[(page, svg or None)] laid out for `width`. `url(page)` makes boxes links."""
from ..emitters.paged import fits_for, links_for, render
from ..ops.pages import layout
fits, measured = fits_for(style, width)
laid = layout(book.pages, fits)
known = {n["id"] for n in book.ir["nodes"]}
out = []
for page in laid:
links = None
if url is not None:
links = links_for(laid, page, url)
# A box with no page of its own opens its section of the reference
# page, where its docstring and file are — when it is a real node.
for n in page["ir"]["nodes"]:
if n["id"] not in links and n["id"] in known \
and not (n.get("attrs") or {}).get("context"):
links[n["id"]] = f"reference.html#{n['id']}"
page["measured"] = measured
out.append((page, render(page, style, width, links=links)))
return out
def caption(page: dict) -> str:
nodes = page["ir"]["nodes"]
parts = [f"{len(nodes)} box{'es' if len(nodes) != 1 else ''}",
f"{len(page['ir']['edges'])} relationship{'s' if len(page['ir']['edges']) != 1 else ''}"]
context = sum(1 for n in nodes if (n.get("attrs") or {}).get("context"))
if context:
parts.append(f"{context} outside this subsystem")
if page.get("children"):
parts.append(f"opens into {len(page['children'])} page{'s' if len(page['children']) != 1 else ''}")
if (page.get("options") or {}).get("rankdir") == "LR":
parts.append("drawn left to right to fit the width")
if page.get("too_wide"):
parts.append("still wider than the page — this level has no smaller structure to split into")
return " · ".join(parts)
# -- the notebook ---------------------------------------------------------------
def _note_step(text: str, base: Path) -> tuple[str, dict]:
"""A note for a notebook cell: local images become cell attachments."""
attachments = {}
for ref, name, mime, data in images(text, base):
import base64
text = text.replace(f"]({ref})", f"](attachment:{name})")
attachments[name] = {mime: base64.b64encode(data).decode("ascii")}
return text, attachments
def _with_note(step: dict, notes: Notes, book: str | None, spot: str) -> dict:
text = notes.book(book, spot) if book else notes.document(spot)
if text is None:
return step
path = notes.path(book, spot)
text, attachments = _note_step(text, path.parent)
step = dict(step)
step["before"] = text + ("\n\n" + step["before"] if step.get("before") else "")
if attachments:
step["before_attachments"] = {**(step.get("before_attachments") or {}), **attachments}
return step
def notebook_spec(doc: Document, notes: Notes, output: Output, notebook_dir: Path) -> dict:
"""Every book's part, in order, with the document's own ends when there are several."""
from ..emitters.paged import NOTEBOOK_WIDTH
from ..notebook import spec as spec_mod
from ..ops.pages import slugify
from ..style import Style
from .build import spec_from
style = Style.load(output.style, theme=output.theme)
width = output.width or NOTEBOOK_WIDTH
several = len(doc.books) > 1
steps: list[dict] = []
missing = ""
if doc.failed:
missing = ("\n\n**This document is incomplete** — these books could not be built "
"and are not below:\n\n"
+ "\n".join(f"- `{n}` — {why}" for n, why in sorted(doc.failed.items())))
if several or doc.failed:
lines = "\n".join(f"- **{b.slug}** — {b.larder.line()}" for b in doc.books)
steps.append(spec_mod._step(
"document", "md", title=doc.title,
text=(f"Built from {len(doc.books)} book(s). Each part below opens with what "
f"came in and closes with what came out.\n\n{lines}{missing}")))
for i, book in enumerate(doc.books):
rel = os.path.relpath(book.out, notebook_dir)
base = spec_from(book, book.ir, prelude_dir=None if doc.inline else rel)
base_steps = base["steps"]
figures = []
index_lines = []
for page, svg in rendered_pages(book, style, width):
anchor = f"page-{slugify(book.slug)}-{page['slug']}"
index_lines.append(f"{' ' * page['depth']}- [{page['title']}](#{anchor})"
+ (" — *too wide*" if page.get("too_wide") else ""))
figures.append(spec_mod._step(
f"page-{page['slug']}", "figure", title=page["title"], anchor=anchor,
depth=page["depth"], svg=svg, name=f"{page['slug']}.svg",
caption=caption(page),
why="Graphviz is not installed" if svg is None else None))
index = spec_mod._step("pages", "md", title=f"{book.slug} — pages",
text="Each page is one subsystem, drawn small enough to "
"read. Follow a link to jump to it.\n\n"
+ "\n".join(index_lines))
ordered = []
for step in base_steps:
if step["id"] == "book":
ordered.extend(figures)
ordered.append(step)
if step["id"] == "larder":
ordered.append(index)
spec, problems = spec_mod.merge({"meta": {}, "steps": ordered},
doc.parts_built[i].overlay)
doc.problems += [f"{book.slug}: overlay: {p}" for p in problems]
# Notes are applied after the overlay, so an overlay that moves a step moves
# its note with it. A page's note is filed under the page's slug; every
# other step under its own id. `_with_note` prepends, so applying the book's
# `_intro` last puts it above the larder's own note.
part = []
for n, step in enumerate(spec["steps"]):
sid = step["id"]
spot = sid[len("page-"):] if step.get("kind") == "figure" else slugify(sid)
step = _with_note(step, notes, book.slug, spot)
if n == 0:
step = _with_note(step, notes, book.slug, "_intro")
if several:
step = dict(step, id=f"{book.slug}/{sid}")
part.append(step)
steps.extend(part)
if several:
lines = []
for b in doc.books:
m = b.measure()
lines.append(f"- **{b.slug}** — {m['nodes']} nodes · {m['edges']} edges · "
f"{m['external']} external")
lines += [f" - {'✓' if r['ok'] else '✗'} {r['claim']}" for r in b.compare()]
steps.append(spec_mod._step("document-end", "md", title=f"{doc.title} — what came out",
text="\n".join(lines)))
if steps:
steps[0] = _with_note(steps[0], notes, None, "_intro")
end = notes.document("_end")
if end is not None:
text, att = _note_step(end, notes.path(None, "_end").parent)
steps.append(spec_mod._step("_end", "md", text=text,
**({"before_attachments": att} if att else {})))
return {"version": spec_mod.SPEC_VERSION, "steps": steps}
# -- building -----------------------------------------------------------------
def build(doc: Document, *, quiet: bool = True) -> Document:
"""Build every book, then every output. Returns the document, ledgers written."""
from ..emitters.notebook import write as nb_write
from ..ops.pages import slugify
from ..style import Style
from .build import build_book
def say(text):
if not quiet:
print(text)
notes = Notes(doc.notes)
doc.books, doc.failed = [], {}
built_parts = []
for part in doc.parts:
# One book that cannot be built must not cost the others — the same rule a
# run of independent books follows. The outputs are assembled from what
# was built, and the ledger names what was not and why, so a partial
# document is visibly partial rather than quietly short.
try:
book = build_book(part.kind, part.source, part.out, slug=part.name,
exclude=part.exclude, ops=part.ops, suffixes=part.suffixes,
quiet=quiet)
except Exception as e: # noqa: BLE001 - recorded, not raised
reason = f"{type(e).__name__}: {str(e).splitlines()[0] if str(e) else ''}"
doc.failed[part.name] = reason
doc.problems.append(f"book {part.name!r} was not built — {reason}")
say(f" FAIL {part.name}: {reason}")
continue
doc.books.append(book)
built_parts.append(part)
doc.parts_built = built_parts
# Minisites first: in a single inline book the minisite is its last step's
# artifact, and the notebook quotes the book measure — which only counts the
# minisite once the last step exists.
sites = [o for o in doc.outputs if o.emitter == "minisite"]
notebooks = [o for o in doc.outputs if o.emitter == "notebook"]
written_outputs = []
for output in sites:
from ..emitters import minisite
site_dir = Path(output.path)
clear_output(site_dir)
style = Style.load(output.style, theme=output.theme)
width = output.width or minisite.CONTENT_WIDTH
books = []
for book in doc.books:
pages = rendered_pages(book, style, width, url=lambda p: f"{p['slug']}.html")
page_notes = {}
for page, _ in pages:
text = notes.book(book.slug, page["slug"])
if text is not None:
base = notes.path(book.slug, page["slug"]).parent
page_notes[page["slug"]] = (text, {ref: data_uri(mime, data)
for ref, _, mime, data in images(text, base)})
intro = notes.book(book.slug, "_intro")
books.append({"book": book, "pages": pages, "notes": page_notes,
"intro": (intro, {}) if intro else None})
doc_intro = notes.document("_intro")
doc_end = notes.document("_end")
files = minisite.write(site_dir, style, title=doc.title, books=books,
intro=doc_intro, end=doc_end, failed=doc.failed)
write_manifest(site_dir, files)
written_outputs.append({"emitter": "minisite", "path": str(site_dir), "width": width,
"files": len(files)})
say(f" minisite {site_dir / 'index.html'}")
for book in doc.books:
inside = [o for o in sites if doc.inline and Path(o.path).parent == book.out]
book.close(minisite=Path(inside[0].path) if inside else None)
for output in notebooks:
path = Path(output.path)
if path.suffix != ".ipynb":
clear_output(path)
path.mkdir(parents=True, exist_ok=True)
target = path / f"{slugify(doc.name)}.ipynb"
else:
path.parent.mkdir(parents=True, exist_ok=True)
target = path
spec = notebook_spec(doc, notes, output, target.parent)
nb_write(spec, target)
if path.suffix != ".ipynb":
write_manifest(path, [target])
if doc.inline and len(doc.books) == 1:
doc.books[0].notebook = os.path.relpath(target, doc.books[0].out)
written_outputs.append({"emitter": "notebook", "path": str(target),
"width": output.width or 960})
say(f" notebook {target}")
known = {b.slug: known_spots(b) for b in doc.books}
drift = notes.drift(known)
doc.problems += drift
for book in doc.books:
book.write()
ledger = {
"name": doc.name,
"title": doc.title,
"failed": doc.failed,
"books": [{"name": b.slug, "dir": str(b.out),
"pages": [{"id": p["id"], "slug": p["slug"], "parent": p["parent"]}
for p in b.pages]}
for b in doc.books],
"outputs": written_outputs,
"notes": {"root": str(doc.notes) if doc.notes else None,
"used": sorted(str(p.relative_to(doc.notes)) for p in notes.used) if doc.notes else [],
"drift": drift},
"problems": doc.problems,
}
doc.build.mkdir(parents=True, exist_ok=True)
(doc.build / "document.json").write_text(json.dumps(ledger, indent=2) + "\n")
return doc
def _spots(ir: dict, pages: list[dict]) -> list[tuple[str, str]]:
"""[(spot id, what it is)] for one book, in reading order."""
from ..notebook import spec as spec_mod
from ..ops.pages import slugify
from .build import STEP_LABELS
out = [("larder", "what came in"), ("pages", "the page index")]
out += [(p["slug"], f"page {p['title']}") for p in pages]
out += [(f"step-{sid}", f"step: {label}") for sid, label in STEP_LABELS]
# An API-shaped book also carries the endpoint walkthrough; each call is a
# spot of its own, under the slug of its id.
if any(n["kind"] in ("endpoint", "operation") for n in ir.get("nodes") or []):
out += [(slugify(s["id"]), f"{s['kind']}: {s.get('title', s['id'])}")
for s in spec_mod.from_ir(ir)["steps"] if s["id"] != "intro"]
out.append(("book", "what came out"))
return out
def known_spots(book: Book) -> set[str]:
"""Every id a note for this book may be filed under."""
return {spot for spot, _ in _spots(book.ir, book.pages)}
def plan_spots(part: Part) -> list[tuple[str, str]]:
"""The spots a book will have, without building it — extraction and pages only."""
from ..ops import overview
from ..ops.pages import for_view
from .build import extract
ir, _ = extract(part.kind, part.source, exclude=part.exclude, suffixes=part.suffixes)
view = apply_ops(overview(ir), part.ops)
return _spots(ir, for_view(ir, view))
Reference in New Issue View Git Blame Copy Permalink