383 lines
16 KiB
Python
383 lines
16 KiB
Python
"""
|
|
Pages: a graph too big for one picture, as a set of pictures that each fit.
|
|
|
|
pages = paginate(view) # the structure: which pages exist
|
|
laid = layout(pages, fits) # per output: fits(page_ir, options) -> bool
|
|
|
|
A notebook scales an image down to its cell width, so a wide graph does not
|
|
scroll — it shrinks until its labels are unreadable. The only fix that holds is
|
|
to draw less per picture. This decides *what* goes on each picture; the caller
|
|
decides whether it fits, by rendering it and measuring, because a width
|
|
estimated from node counts is wrong the moment the labels are long.
|
|
|
|
**Which pages exist does not depend on width** — only how each one is laid out
|
|
does. So the two are separate calls: the page ids a note is filed under are the
|
|
same for a notebook column and a minisite column, and listing them needs no
|
|
renderer at all.
|
|
|
|
## By structure, never by slicing
|
|
|
|
Tiling one large image cuts boxes and edges at the seams, and a tile means
|
|
nothing on its own. A page here is a subsystem:
|
|
|
|
overview one box per top-level group, relationships lifted
|
|
└── station one box per child of `station`
|
|
└── station.tools one box per child of `station.tools`, and so on
|
|
|
|
Clicking a box — on the minisite — or following the index — in a notebook — moves
|
|
down a level. Every page is a real graph: the boxes are the scope's children,
|
|
and anything outside the scope that they touch appears as a **context** box,
|
|
collapsed to the branch it sits on, so a page still says what its subsystem
|
|
reaches without drawing the rest of the tree.
|
|
|
|
## Groups come from the id, not from `parent`
|
|
|
|
A directory without `__init__.py` is not a Python package, so its modules have
|
|
no `parent` — soleprint has 173 such roots — but their dotted ids still carry
|
|
the directory. `split()` groups on id segments for the same reason. A group with
|
|
no node of its own (such a directory) is synthesised on the page as a box of
|
|
the kind most of its contents are, marked `attrs.group`.
|
|
|
|
## When a level cannot be narrowed
|
|
|
|
A scope whose children genuinely are many peers has no smaller structure to
|
|
split into. It is redrawn left-to-right — height is free in a notebook, width is
|
|
not — and if that still does not fit, the page is kept and flagged `too_wide`.
|
|
Never cut, never dropped: a flagged page is honest, a missing one is not.
|
|
|
|
## Schemas page differently
|
|
|
|
A schema is peers with references, drawn as cards in columns whose count the
|
|
emitter caps to the width, so width is never the problem — count is. Tables are
|
|
paged by connected component (tables joined by foreign keys stay together), and
|
|
tables with no keys at all are chunked alphabetically.
|
|
"""
|
|
|
|
import re
|
|
|
|
OVERVIEW = "overview"
|
|
MAX_TABLES = 20 # past this a schema page stops being scannable
|
|
|
|
|
|
def slugify(page_id: str) -> str:
|
|
"""A filename- and anchor-safe name for a page id.
|
|
|
|
Ids are not safe as they stand: an endpoint id is `GET /orders/{id}`, and a
|
|
tree-sitter id can carry `#L12`.
|
|
"""
|
|
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", page_id).strip("-.")
|
|
return slug or "page"
|
|
|
|
|
|
def _key(node_id: str, scope: str) -> str:
|
|
"""The box at `scope` that `node_id` falls into: scope + one more segment."""
|
|
if scope == "":
|
|
return node_id.split(".", 1)[0]
|
|
if node_id == scope:
|
|
return scope
|
|
rest = node_id[len(scope) + 1:]
|
|
return f"{scope}.{rest.split('.', 1)[0]}"
|
|
|
|
|
|
def _under(node: dict, scope: str) -> bool:
|
|
if node["kind"] == "external":
|
|
return False # externals are context everywhere, never members
|
|
nid = node["id"]
|
|
return scope == "" or nid == scope or nid.startswith(scope + ".")
|
|
|
|
|
|
def _representative(node_id: str, scope: str) -> str:
|
|
"""Where an outside node shows on this page: the branch it diverges on.
|
|
|
|
The common prefix of the scope and the node, plus one more segment of the
|
|
node's id. That box is never an ancestor of the scope (it is on the node's
|
|
branch, past the point they part), so it never swallows the page's own
|
|
members — and two outside nodes on the same branch share one box.
|
|
"""
|
|
s, n = scope.split(".") if scope else [], node_id.split(".")
|
|
common = 0
|
|
while common < len(s) and common < len(n) and s[common] == n[common]:
|
|
common += 1
|
|
return ".".join(n[: common + 1])
|
|
|
|
|
|
def page_ir(view: dict, scope: str) -> tuple[dict, dict]:
|
|
"""The graph for one page, and {member key: [node ids it stands for]}.
|
|
|
|
Built directly rather than through `_rebuild`, because a synthesised group
|
|
box and a context box have no node in the source graph to keep.
|
|
"""
|
|
by_id = {n["id"]: n for n in view["nodes"]}
|
|
members: dict[str, list] = {}
|
|
for n in view["nodes"]:
|
|
if _under(n, scope):
|
|
members.setdefault(_key(n["id"], scope), []).append(n["id"])
|
|
|
|
where = {nid: key for key, ids in members.items() for nid in ids}
|
|
context: dict[str, list] = {}
|
|
for e in view["edges"]:
|
|
s_in, t_in = e["source"] in where, e["target"] in where
|
|
if s_in == t_in:
|
|
continue # internal to the page (handled below) or unrelated
|
|
outside = e["target"] if s_in else e["source"]
|
|
rep = _representative(outside, scope)
|
|
if rep in members:
|
|
continue # cannot happen for a well-formed tree; stay safe
|
|
context.setdefault(rep, []).append(outside)
|
|
where.setdefault(outside, rep)
|
|
|
|
nodes = []
|
|
for key, ids in sorted(members.items()) + sorted(context.items()):
|
|
is_context = key in context and key not in members
|
|
own = by_id.get(key)
|
|
ids = sorted(set(ids))
|
|
if own is not None and (ids == [key] or not is_context):
|
|
node = {"id": key, "kind": own["kind"], "label": own.get("label") or key,
|
|
"parent": None, "attrs": dict(own.get("attrs") or {})}
|
|
else:
|
|
kinds = [by_id[i]["kind"] for i in ids if i in by_id]
|
|
kind = max(set(kinds), key=lambda k: (kinds.count(k), k)) if kinds else "module"
|
|
node = {"id": key, "kind": kind, "label": key.rsplit(".", 1)[-1],
|
|
"parent": None, "attrs": {}}
|
|
# Semantic, not visual: which boxes are subsystems to open, and which are
|
|
# the outside world seen from here. How that looks is style's decision.
|
|
grouped = [i for i in ids if i != key]
|
|
if grouped or (own is None):
|
|
node["attrs"]["group"] = True
|
|
node["attrs"]["count"] = len(ids)
|
|
if is_context:
|
|
node["attrs"]["context"] = True
|
|
nodes.append(node)
|
|
|
|
merged: dict[tuple, dict] = {}
|
|
for e in view["edges"]:
|
|
src, dst = where.get(e["source"]), where.get(e["target"])
|
|
if src is None or dst is None or src == dst:
|
|
continue
|
|
if src in context and dst in context:
|
|
continue # two outside boxes: not this page's relationship
|
|
k = (src, dst, e["kind"])
|
|
if k in merged:
|
|
merged[k]["attrs"]["weight"] = merged[k]["attrs"].get("weight", 1) + 1
|
|
continue
|
|
attrs = dict(e.get("attrs") or {})
|
|
if src != e["source"] or dst != e["target"]:
|
|
attrs = {"weight": 1}
|
|
merged[k] = {"source": src, "target": dst, "kind": e["kind"], "attrs": attrs}
|
|
|
|
ir = {"meta": dict(view["meta"]), "nodes": nodes,
|
|
"edges": sorted(merged.values(), key=lambda e: (e["source"], e["target"], e["kind"]))}
|
|
return ir, members
|
|
|
|
|
|
def layout(pages: list[dict], fits) -> list[dict]:
|
|
"""The same pages, each with the layout that fits: top-to-bottom, then
|
|
left-to-right, then flagged `too_wide`. Returns new page dicts.
|
|
|
|
Schema pages are left alone — the ERD emitter caps its own columns to the
|
|
width, so they always fit — except the overview, which is a graph.
|
|
"""
|
|
out = []
|
|
for page in pages:
|
|
page = dict(page)
|
|
if page.get("tables") and page["id"] != OVERVIEW:
|
|
page["options"], page["too_wide"] = {}, False
|
|
elif fits(page["ir"], {}):
|
|
page["options"], page["too_wide"] = {}, False
|
|
elif fits(page["ir"], {"rankdir": "LR"}):
|
|
page["options"], page["too_wide"] = {"rankdir": "LR"}, False
|
|
else:
|
|
page["options"], page["too_wide"] = {"rankdir": "LR"}, True
|
|
out.append(page)
|
|
return out
|
|
|
|
|
|
def paginate(view: dict, *, max_depth: int = 6) -> list[dict]:
|
|
"""Pages for a graph, overview first, each drawn as a subsystem.
|
|
|
|
Returns a list of::
|
|
|
|
{"id", "slug", "title", "scope", "parent", "depth", "ir",
|
|
"children": [page ids], "options": {}, "too_wide": False}
|
|
|
|
`options` and `too_wide` are placeholders until `layout()` fills them.
|
|
|
|
in reading order — a page is followed by its own children, depth first — so
|
|
a notebook can emit them in sequence and a minisite can build its tree from
|
|
`parent` and `children`.
|
|
"""
|
|
pages: list[dict] = []
|
|
slugs: set[str] = set()
|
|
|
|
def unique(slug: str) -> str:
|
|
base, i = slug, 2
|
|
while slug in slugs:
|
|
slug, i = f"{base}-{i}", i + 1
|
|
slugs.add(slug)
|
|
return slug
|
|
|
|
def opens(key: str, scope: str, ids: list) -> bool:
|
|
# A box opens into a page when it stands for more than itself.
|
|
return key != scope and any(i != key for i in ids)
|
|
|
|
def visit(scope: str, parent: str | None, depth: int, page_id: str | None = None):
|
|
ir, members = page_ir(view, scope)
|
|
if not ir["nodes"]:
|
|
return
|
|
# A page whose only member is one group says nothing but "open this" —
|
|
# docgen's own overview was one box reading `docgen`. Descend instead,
|
|
# keeping the id the caller expects, so the overview is still `overview`.
|
|
groups = [k for k, ids in members.items() if opens(k, scope, ids)]
|
|
if len(members) == 1 and len(groups) == 1:
|
|
return visit(groups[0], parent, depth, page_id or (scope or OVERVIEW))
|
|
page_id = page_id or scope or OVERVIEW
|
|
page = {
|
|
"id": page_id,
|
|
"slug": unique(slugify(page_id)),
|
|
"title": scope if scope and page_id == scope else (scope or "Overview"),
|
|
"scope": scope,
|
|
"parent": parent,
|
|
"depth": depth,
|
|
"ir": ir,
|
|
"options": {},
|
|
"children": [],
|
|
"too_wide": False,
|
|
}
|
|
pages.append(page)
|
|
if depth >= max_depth:
|
|
return
|
|
for key in sorted(members):
|
|
if opens(key, scope, members[key]):
|
|
page["children"].append(key)
|
|
visit(key, page_id, depth + 1)
|
|
|
|
visit("", None, 0)
|
|
return pages
|
|
|
|
|
|
def paginate_tables(view: dict, *, max_tables: int = MAX_TABLES) -> list[dict]:
|
|
"""Pages for a schema: connected components, packed, then chunked.
|
|
|
|
Each page is the tables plus their columns, so the ERD emitter draws cards.
|
|
The overview has one box per page, which is what a reader opens from.
|
|
"""
|
|
tables = sorted(n["id"] for n in view["nodes"] if n["kind"] == "table")
|
|
links: dict[str, set] = {t: set() for t in tables}
|
|
for e in view["edges"]:
|
|
if e["source"] in links and e["target"] in links:
|
|
links[e["source"]].add(e["target"])
|
|
links[e["target"]].add(e["source"])
|
|
|
|
seen, components = set(), []
|
|
for t in tables:
|
|
if t in seen:
|
|
continue
|
|
stack, comp = [t], []
|
|
while stack:
|
|
cur = stack.pop()
|
|
if cur in seen:
|
|
continue
|
|
seen.add(cur)
|
|
comp.append(cur)
|
|
stack.extend(links[cur] - seen)
|
|
components.append(sorted(comp))
|
|
|
|
connected = sorted((c for c in components if len(c) > 1), key=lambda c: (-len(c), c[0]))
|
|
lonely = sorted(c[0] for c in components if len(c) == 1)
|
|
|
|
groups: list[list[str]] = []
|
|
for comp in connected:
|
|
# A component larger than a page is still split — by name, since a
|
|
# foreign-key cluster has no finer structure to split on — and the cut
|
|
# keys show as edges to the other page's tables in neither, which the
|
|
# overview's lifted edges make up for.
|
|
for i in range(0, len(comp), max_tables):
|
|
groups.append(comp[i:i + max_tables])
|
|
for i in range(0, len(lonely), max_tables):
|
|
groups.append(lonely[i:i + max_tables])
|
|
|
|
by_id = {n["id"]: n for n in view["nodes"]}
|
|
pages: list[dict] = []
|
|
table_page: dict[str, str] = {}
|
|
for i, group in enumerate(groups, start=1):
|
|
pid = group[0] if len(group) == 1 else f"{group[0]}…{group[-1]}"
|
|
slug = f"tables-{i:02d}"
|
|
for t in group:
|
|
table_page[t] = slug
|
|
keep = set(group) | {n["id"] for n in view["nodes"] if n.get("parent") in set(group)}
|
|
ir = {"meta": dict(view["meta"]),
|
|
"nodes": [dict(by_id[n]) for n in sorted(keep)],
|
|
"edges": [dict(e) for e in view["edges"]
|
|
if e["source"] in keep and e["target"] in keep]}
|
|
pages.append({"id": slug, "slug": slug, "title": pid, "scope": slug,
|
|
"parent": OVERVIEW, "depth": 1, "ir": ir, "options": {},
|
|
"children": [], "too_wide": False, "tables": True})
|
|
|
|
over_nodes = [{"id": p["id"], "kind": "table", "label": p["title"], "parent": None,
|
|
"attrs": {"group": True, "count": sum(
|
|
1 for n in p["ir"]["nodes"] if n["kind"] == "table")}}
|
|
for p in pages]
|
|
merged: dict[tuple, dict] = {}
|
|
for e in view["edges"]:
|
|
a, b = table_page.get(e["source"]), table_page.get(e["target"])
|
|
if a and b and a != b:
|
|
k = (a, b, e["kind"])
|
|
merged.setdefault(k, {"source": a, "target": b, "kind": e["kind"],
|
|
"attrs": {"weight": 0}})
|
|
merged[k]["attrs"]["weight"] += 1
|
|
overview = {"id": OVERVIEW, "slug": OVERVIEW, "title": "Overview", "scope": "",
|
|
"tables": True, "parent": None,
|
|
"depth": 0, "ir": {"meta": dict(view["meta"]), "nodes": over_nodes,
|
|
"edges": list(merged.values())},
|
|
"options": {}, "children": [p["id"] for p in pages], "too_wide": False}
|
|
return [overview] + pages
|
|
|
|
|
|
def coverage(view: dict, pages: list[dict]) -> list[str]:
|
|
"""Node ids of the view that no page accounts for. Empty means nothing lost.
|
|
|
|
A node is accounted for when a page shows it — as itself, or inside a box
|
|
that opens onto a page that does. Externals count when their branch appears
|
|
as context somewhere.
|
|
"""
|
|
shown = set()
|
|
for p in pages:
|
|
shown |= {n["id"] for n in p["ir"]["nodes"]}
|
|
# Schema pages draw tables and their columns; an API's endpoints are the
|
|
# notebook's walkthrough, not a diagram, so they are not a page's to account for.
|
|
tables_only = bool(pages) and pages[0].get("tables")
|
|
missing = []
|
|
for n in view["nodes"]:
|
|
if tables_only and n["kind"] not in ("table", "column"):
|
|
continue
|
|
nid = n["id"]
|
|
if nid in shown:
|
|
continue
|
|
parts = nid.split(".")
|
|
if n["kind"] == "external" and any(".".join(parts[:i]) in shown
|
|
for i in range(1, len(parts) + 1)):
|
|
continue
|
|
if n.get("parent") in shown and n["kind"] == "column":
|
|
continue
|
|
missing.append(nid)
|
|
return sorted(missing)
|
|
|
|
|
|
def for_view(ir: dict, view: dict, *, max_depth: int = 6) -> list[dict]:
|
|
"""The pages a book gets, whatever its source.
|
|
|
|
A schema pages by table — from the full IR, since a card needs its columns and
|
|
the default view drops them — and anything else pages by structure from the
|
|
view. The same decision `classify` makes about which emitter draws it.
|
|
"""
|
|
from .filter import classify
|
|
|
|
if classify(view)["emitter"] == "erd":
|
|
tables = {"meta": ir["meta"],
|
|
"nodes": [n for n in ir["nodes"] if n["kind"] in ("table", "column")],
|
|
"edges": [e for e in ir["edges"]
|
|
if e["kind"] in ("foreign_key", "references")]}
|
|
if any(n["kind"] == "table" for n in tables["nodes"]):
|
|
return paginate_tables(tables)
|
|
return paginate(view, max_depth=max_depth)
|