328 lines
12 KiB
Python
328 lines
12 KiB
Python
"""
|
|
A run file: every book a project builds, so a rebuild is one command.
|
|
|
|
python3 -m docgen run docgen.toml
|
|
python3 -m docgen run docgen.toml --only station --only shop
|
|
python3 -m docgen run docgen.toml --list # what would run, resolved
|
|
|
|
Books are rebuilt many times — after a merge, before a release, whenever the
|
|
source moves — and each one is a source, an output directory and a handful of
|
|
options. Typed out every time, those drift: one run excludes `migrations`, the
|
|
next forgets. Written down once, the rebuild is exact.
|
|
|
|
## The shape
|
|
|
|
# docgen.toml
|
|
reference = "../soleprint" # optional; $DOCGEN_REFERENCE wins if set
|
|
|
|
[defaults]
|
|
out = "out/book" # each book goes to <out>/<name>
|
|
style = "lucid"
|
|
theme = "dark"
|
|
exclude = ["migrations", "tests"]
|
|
|
|
[[book]]
|
|
name = "station"
|
|
root = "../soleprint/station" # a tree; reader = "python" by default
|
|
|
|
[[book]]
|
|
name = "orders-api"
|
|
openapi = "specs/orders.yaml"
|
|
overlay = "overlays/orders.json"
|
|
out = "/srv/docs/orders" # overrides <defaults.out>/<name>
|
|
|
|
Each book names **exactly one** source — `root`, `schema`, `openapi` or `har` —
|
|
the same choice the `book` command makes you take.
|
|
|
|
## Three rules, each one a thing that went wrong somewhere else
|
|
|
|
**Paths are relative to the run file, not to wherever you ran the command.** A
|
|
run file is kept beside the project it describes; if its paths meant different
|
|
things from different directories, the same file would build different books.
|
|
|
|
**A book's value replaces the default, for every key.** Including `exclude`,
|
|
which is the one where merging looks tempting. One rule nobody has to remember
|
|
beats a clever one somebody has to look up.
|
|
|
|
**Unknown keys are refused, not ignored.** `exlude = [...]` silently doing
|
|
nothing is how a rebuild quietly starts reading `node_modules`. The style loader
|
|
takes the same line (requirements R30).
|
|
|
|
## Why TOML
|
|
|
|
It is read by the stdlib (`tomllib`), it allows comments — and a run file is
|
|
hand-written, so the reason a book excludes something belongs next to the
|
|
exclusion — and it is already the format of the `pyproject.toml` beside it. The
|
|
IR and style files stay JSON, because those are data that tools write; this is
|
|
configuration that people write.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import tomllib
|
|
except ModuleNotFoundError: # pragma: no cover - Python < 3.11
|
|
tomllib = None
|
|
|
|
SOURCES = {"root": "python", "schema": "db", "openapi": "openapi", "har": "usage"}
|
|
READERS = ("python", "code")
|
|
|
|
TOP_KEYS = {"reference", "defaults", "book"}
|
|
DEFAULT_KEYS = {"out", "style", "theme", "exclude", "reader"}
|
|
BOOK_KEYS = {"name", "out", "style", "theme", "exclude", "reader", "overlay", *SOURCES}
|
|
|
|
# A book name becomes a directory name and a slug, so it is held to what is safe
|
|
# as both — no separators, nothing that means something to a shell.
|
|
NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
|
|
|
|
class ConfigError(ValueError):
|
|
"""A run file that cannot be run. Carries every problem, not the first."""
|
|
|
|
def __init__(self, path, problems: list[str]):
|
|
self.path, self.problems = path, problems
|
|
super().__init__(f"{path}: {len(problems)} problem(s)\n " + "\n ".join(problems))
|
|
|
|
|
|
@dataclass
|
|
class Entry:
|
|
"""One book, fully resolved — nothing relative, nothing defaulted later."""
|
|
|
|
name: str
|
|
kind: str # python | code | db | openapi | usage
|
|
source: Path
|
|
out: Path
|
|
style: str = "lucid"
|
|
theme: str | None = None
|
|
overlay: Path | None = None
|
|
exclude: tuple = ()
|
|
|
|
def line(self) -> str:
|
|
return f"{self.name:<18} {self.kind:<8} {self.source} -> {self.out}"
|
|
|
|
|
|
@dataclass
|
|
class RunFile:
|
|
path: Path
|
|
books: list[Entry] = field(default_factory=list)
|
|
reference: Path | None = None
|
|
|
|
def select(self, names) -> list[Entry]:
|
|
"""The named books, in run-file order. An unknown name is an error."""
|
|
if not names:
|
|
return list(self.books)
|
|
known = {b.name for b in self.books}
|
|
unknown = [n for n in names if n not in known]
|
|
if unknown:
|
|
raise ConfigError(self.path, [
|
|
f"no book named {n!r} — have: {', '.join(sorted(known))}" for n in unknown
|
|
])
|
|
wanted = set(names)
|
|
return [b for b in self.books if b.name in wanted]
|
|
|
|
|
|
def _path(value, base: Path) -> Path:
|
|
# Normalised lexically, so `--list` shows `atlas2/out` rather than
|
|
# `atlas2/docgen/../out`. Not resolved: a symlinked project should be named
|
|
# the way its owner names it.
|
|
p = Path(str(value)).expanduser()
|
|
return Path(os.path.normpath(p if p.is_absolute() else base / p))
|
|
|
|
|
|
def _within(inner: Path, outer: Path) -> bool:
|
|
inner, outer = inner.resolve(), outer.resolve()
|
|
return inner == outer or outer in inner.parents
|
|
|
|
|
|
def load(path) -> RunFile:
|
|
"""Read and resolve a run file. Raises ConfigError listing every problem."""
|
|
path = Path(path)
|
|
if tomllib is None:
|
|
raise ConfigError(path, ["run files need Python 3.11+ (tomllib)"])
|
|
try:
|
|
data = tomllib.loads(path.read_text())
|
|
except OSError as e:
|
|
raise ConfigError(path, [f"cannot read: {e}"]) from None
|
|
except tomllib.TOMLDecodeError as e:
|
|
raise ConfigError(path, [f"not valid TOML: {e}"]) from None
|
|
|
|
base = path.resolve().parent
|
|
problems: list[str] = []
|
|
|
|
for key in sorted(set(data) - TOP_KEYS):
|
|
problems.append(f"unknown top-level key {key!r} — have: {', '.join(sorted(TOP_KEYS))}")
|
|
|
|
defaults = data.get("defaults") or {}
|
|
if not isinstance(defaults, dict):
|
|
problems.append("[defaults] must be a table")
|
|
defaults = {}
|
|
for key in sorted(set(defaults) - DEFAULT_KEYS):
|
|
problems.append(f"[defaults] has unknown key {key!r} — have: "
|
|
f"{', '.join(sorted(DEFAULT_KEYS))}")
|
|
|
|
reference = None
|
|
if "reference" in data:
|
|
reference = _path(data["reference"], base)
|
|
if not reference.is_dir():
|
|
problems.append(f"reference {reference} is not a directory")
|
|
|
|
raw_books = data.get("book") or []
|
|
if not isinstance(raw_books, list) or not raw_books:
|
|
problems.append("no books — add at least one [[book]] table")
|
|
raw_books = []
|
|
|
|
books: list[Entry] = []
|
|
for i, raw in enumerate(raw_books):
|
|
where = f"book[{i}]"
|
|
if not isinstance(raw, dict):
|
|
problems.append(f"{where} must be a table")
|
|
continue
|
|
name = raw.get("name")
|
|
if isinstance(name, str):
|
|
where = f"book {name!r}"
|
|
if not isinstance(name, str) or not NAME.match(name):
|
|
problems.append(f"{where} needs a name of letters, digits, '.', '_' or '-'")
|
|
continue
|
|
|
|
for key in sorted(set(raw) - BOOK_KEYS):
|
|
problems.append(f"{where} has unknown key {key!r} — have: "
|
|
f"{', '.join(sorted(BOOK_KEYS))}")
|
|
|
|
named = [k for k in SOURCES if k in raw]
|
|
if len(named) != 1:
|
|
problems.append(
|
|
f"{where} must name exactly one source ({', '.join(SOURCES)}); "
|
|
f"it names {', '.join(named) if named else 'none'}"
|
|
)
|
|
continue
|
|
source_key = named[0]
|
|
source = _path(raw[source_key], base)
|
|
|
|
def pick(key, default=None):
|
|
# A book's value replaces the default, for every key.
|
|
return raw[key] if key in raw else defaults.get(key, default)
|
|
|
|
reader = pick("reader", "python")
|
|
if source_key == "root":
|
|
if reader not in READERS:
|
|
problems.append(f"{where}: reader must be one of {READERS}, got {reader!r}")
|
|
continue
|
|
kind = reader
|
|
if not source.is_dir():
|
|
problems.append(f"{where}: root {source} is not a directory")
|
|
else:
|
|
if "reader" in raw:
|
|
problems.append(f"{where}: reader only applies to root, not {source_key}")
|
|
kind = SOURCES[source_key]
|
|
if not source.is_file():
|
|
problems.append(f"{where}: {source_key} {source} is not a file")
|
|
|
|
if "out" in raw:
|
|
out = _path(raw["out"], base)
|
|
else:
|
|
out = _path(defaults.get("out", "out"), base) / name
|
|
|
|
overlay = _path(raw["overlay"], base) if "overlay" in raw else None
|
|
if overlay is not None and not overlay.is_file():
|
|
problems.append(f"{where}: overlay {overlay} is not a file")
|
|
|
|
exclude = pick("exclude", [])
|
|
if not isinstance(exclude, list) or not all(isinstance(x, str) for x in exclude):
|
|
problems.append(f"{where}: exclude must be a list of directory names")
|
|
exclude = []
|
|
elif source_key != "root":
|
|
# Only an exclude the book sets itself is a mistake. One inherited
|
|
# from [defaults] is meant for the trees and simply does not apply —
|
|
# refusing it would make a shared default impossible to write.
|
|
if "exclude" in raw:
|
|
problems.append(f"{where}: exclude only applies to a root, not {source_key}")
|
|
exclude = []
|
|
|
|
# Writing a book inside the tree it reads means the next run reads the
|
|
# last run's output. That is a rebuild that changes on every rebuild.
|
|
if source_key == "root" and source.is_dir() and _within(out, source):
|
|
problems.append(f"{where}: out {out} is inside the tree it reads ({source})")
|
|
|
|
books.append(Entry(
|
|
name=name, kind=kind, source=source, out=out,
|
|
style=pick("style", "lucid"), theme=pick("theme"),
|
|
overlay=overlay, exclude=tuple(exclude),
|
|
))
|
|
|
|
seen_names, seen_outs = {}, {}
|
|
for b in books:
|
|
if b.name in seen_names:
|
|
problems.append(f"book {b.name!r} is listed twice")
|
|
seen_names[b.name] = b
|
|
key = b.out.resolve()
|
|
if key in seen_outs:
|
|
# Two books into one directory: the second clears the first on every
|
|
# run, and nothing says so.
|
|
problems.append(f"books {seen_outs[key]!r} and {b.name!r} both write to {b.out}")
|
|
seen_outs[key] = b.name
|
|
|
|
if problems:
|
|
raise ConfigError(path, problems)
|
|
return RunFile(path=path, books=books, reference=reference)
|
|
|
|
|
|
def apply_reference(runfile: RunFile) -> str | None:
|
|
"""Point the one seam at the run file's reference — unless the caller's
|
|
environment already does. The caller's env beats the file, which is the
|
|
precedence rig uses for every setting it has.
|
|
"""
|
|
from .. import reference as ref
|
|
|
|
if runfile.reference is None or os.environ.get(ref.ENV_VAR):
|
|
return None
|
|
os.environ[ref.ENV_VAR] = str(runfile.reference)
|
|
return str(runfile.reference)
|
|
|
|
|
|
def run(runfile: RunFile, names=None, *, check: bool = False, quiet: bool = True):
|
|
"""Build every selected book, in order. One failure never stops the rest.
|
|
|
|
Returns one result per book:
|
|
{"name", "out", "ok", "error", "lost": [claims], "checks_failed": [names]}
|
|
"""
|
|
from . import checks as checks_mod
|
|
from .build import run as build
|
|
|
|
apply_reference(runfile)
|
|
results = []
|
|
for entry in runfile.select(names):
|
|
result = {"name": entry.name, "out": str(entry.out), "ok": False,
|
|
"error": None, "lost": [], "checks_failed": []}
|
|
try:
|
|
overlay = None
|
|
if entry.overlay is not None:
|
|
from ..notebook import spec as spec_mod
|
|
overlay = spec_mod.load(entry.overlay)
|
|
book = build(entry.kind, entry.source, entry.out, slug=entry.name,
|
|
style=entry.style, theme=entry.theme, exclude=entry.exclude,
|
|
overlay=overlay, quiet=quiet)
|
|
result["lost"] = [r["claim"] for r in book.compare() if not r["ok"]]
|
|
except Exception as e: # noqa: BLE001 - one bad book must not cost the run
|
|
result["error"] = f"{type(e).__name__}: {e}"
|
|
results.append(result)
|
|
continue
|
|
|
|
if check:
|
|
import contextlib
|
|
import io
|
|
|
|
report = checks_mod.Report()
|
|
loaded = checks_mod.Loaded(entry.out)
|
|
sink = io.StringIO() if quiet else None
|
|
with contextlib.redirect_stdout(sink) if sink else contextlib.nullcontext():
|
|
checks_mod.generated(loaded, report)
|
|
checks_mod.custom(loaded, report)
|
|
result["checks_failed"] = list(report.failed)
|
|
|
|
result["ok"] = not result["lost"] and not result["checks_failed"]
|
|
results.append(result)
|
|
return results
|