soleprint/soleprint/atlas2/docgen/book/build.py

"""
Run a book: larder, the steps, the web output, the book measure.

    python3 -m docgen book --root ../station -o out/book/station

Composes functions that already exist. Nothing here parses, lays out or styles
anything — `ops.overview`, `emitters.auto.draw`, `emitters.site` and `notebook.spec`
do all of it, and this decides the order and writes the ledger. If this file
ever starts doing the work, the seam has moved to the wrong place.

## The notebook is the sequence, not an item in it

So it is not a step. Its **first cell is the larder measure and its last cell is
the book measure**, which is what makes the two ends part of the document rather
than part of the tooling. Between them, one pair of cells per step: what the
step did, and a code cell that loads that step's artifact on its own.

That last part is the "usable by themselves" property made executable. Each cell
reads one artifact and prints one fact, so a reader can start anywhere in the
sequence, and `selftest.py` runs them.
"""

import json
from pathlib import Path

from . import Book
from .larder import Larder

# Loader for the notebook's step cells. Guarded on purpose: a notebook is
# opened from wherever somebody happens to open it, and a traceback on cell 2
# is a worse answer than a sentence saying which directory to run it from.
PRELUDE = '''from pathlib import Path
import json

BOOK = Path.cwd()          # the book directory — change if you opened this elsewhere


def load(rel):
    """One step's artifact, on its own. Returns None when it is not here.

    A step's artifact can be a directory — the explorer is one — so this returns
    a file list for those rather than trying to read a directory as text.
    """
    p = BOOK / rel
    if not p.exists():
        print(f"{rel} is not here — run: make book OUT={BOOK}")
        return None
    if p.is_dir():
        return sorted(f.relative_to(p).as_posix() for f in p.rglob("*") if f.is_file())
    return json.loads(p.read_text()) if p.suffix == ".json" else p.read_text()
'''


# Every extractor a book can run, as data rather than as an if/elif chain.
#
# This is the shape that gives the extension contract teeth. `selftest.py` loops
# over this dict and asserts each entry reports a larder measure — so adding an
# extractor makes the test start asking about it without anyone remembering to
# go and add a case. rig's `CONFIG_OVERRIDABLE` loop is the precedent, and it
# found two real bugs that way.
#
#   kind -> (module, function, what the source is)
EXTRACTORS = {
    "python": (".extractors.python", "extract", "tree"),
    "code": (".extractors.code", "extract", "tree"),
    "db": (".extractors.db", "extract", "file"),
    "openapi": (".extractors.openapi", "extract", "file"),
    "usage": (".extractors.usage", "extract", "file"),
}

# Extractors that walk a directory take an exclude list; the ones that read a
# single document have nothing to exclude.
TAKES_EXCLUDE = {"python", "code"}


def extract(kind: str, source, *, exclude=(), identity=None, suffixes=None):
    """(IR dict, Larder) for one source type, dispatched through EXTRACTORS."""
    from importlib import import_module

    if kind not in EXTRACTORS:
        raise ValueError(
            f"no extractor named {kind!r} — have {', '.join(sorted(EXTRACTORS))}"
        )
    module_name, fn_name, _ = EXTRACTORS[kind]
    # The package name is derived, not written: the Makefile takes PKG from the
    # directory name so the folder can be copied anywhere and renamed, and a
    # literal "docgen" here would quietly undo that.
    root_pkg = __package__.rsplit(".", 1)[0]
    fn = getattr(import_module(module_name, package=root_pkg), fn_name)

    kwargs = {"identity": identity or str(source)}
    if kind in TAKES_EXCLUDE:
        kwargs["exclude"] = exclude
    if kind == "code" and suffixes:
        kwargs["suffixes"] = suffixes
    g = fn(source, **kwargs)

    ir = g.to_dict()
    measured = ir["meta"].get("larder")
    if measured is None:
        # Not fatal. An extractor that cannot count its input still produces a
        # book; what it does not get to do is pretend it measured one.
        return ir, None
    return ir, Larder.from_dict(measured)


def spec_from(book: Book, ir: dict, *, prelude_dir: str | None = None) -> dict:
    """The book's ledger -> a notebook spec, with the two measures at the ends.

    Reuses `notebook.spec`'s step vocabulary rather than inventing one, so
    `merge()` keeps working and a hand-written overlay can annotate a spine step
    the same way it annotates any other.
    """
    from ..notebook import spec as spec_mod

    steps = [
        spec_mod._step(
            "larder", "md", title=f"{book.slug} — what came in",
            text=(
                f"`{book.larder.identity}`\n\n**{book.larder.line()}**\n\n"
                "This is the first step of the book and the only measure of the "
                "input. Everything below is derived from it, so a number here "
                "that looks wrong makes everything below it suspect."
                + (
                    "\n\nCould not be read:\n\n"
                    + "\n".join(f"- `{f['name']}` — {f['error']}"
                                for f in book.larder.failed)
                    if book.larder.failed else ""
                )
            ),
        ),
        spec_mod._step("prelude", "code", title="Reading a step on its own",
                       # In its own book directory the notebook reads steps from
                       # where it is opened; elsewhere it points back at the book.
                       code=PRELUDE if prelude_dir is None else PRELUDE.replace(
                           "BOOK = Path.cwd()", f"BOOK = Path.cwd() / {prelude_dir!r}", 1)),
    ]

    for s in book.steps:
        if s.id in ("larder", "book") or not s.artifact:
            continue
        steps.append(spec_mod._step(
            f"step-{s.id}", "md", title=s.label,
            text=f"`{s.artifact}` — {s.bytes:,} bytes" + (f"\n\n{s.note}" if s.note else ""),
        ))
        steps.append(spec_mod._step(
            f"load-{s.id}", "code", code=_load_cell(s.id, s.artifact),
        ))

    # Where the larder is API-shaped, the generated endpoint walkthrough slots in
    # as further steps. For a tree of source there are no endpoints and this adds
    # nothing, which is the correct amount for it to add.
    if any(n["kind"] in ("endpoint", "operation") for n in ir.get("nodes") or []):
        generated = spec_mod.from_ir(ir)
        steps.extend(s for s in generated["steps"] if s["id"] not in ("intro",))

    m = book.measure()
    # `external` is reported on its own below, so it is dropped here rather than
    # appearing twice in one line — which is how it read before.
    summary = " · ".join(f"{v} {k}" for k, v in
                         sorted(m["by_kind"].items(), key=lambda kv: -kv[1])
                         if k != "external")
    reconciled = "\n".join(
        f"- {'✓' if r['ok'] else '✗'} {r['claim']} — {r['why']}"
        for r in book.compare()
    )
    steps.append(spec_mod._step(
        "book", "md", title=f"{book.slug} — what came out",
        text=(
            f"**{summary} · {m['edges']} edges · {m['external']} external**\n\n"
            f"{len(m['artifacts'])} artifact(s), {m['bytes']:,} bytes.\n\n"
            f"Reconciled against what came in:\n\n{reconciled}\n\n"
            "The web output is `site/index.html`."
        ),
    ))

    return {"version": spec_mod.SPEC_VERSION, "steps": steps}


def _load_cell(step_id: str, artifact: str) -> str:
    """A cell that reads one artifact and prints one fact about it.

    The fact has to suit the artifact. An earlier version printed "nodes, edges"
    for every JSON file, so `sidebar.json` — which has neither — reported
    "0 nodes, 0 edges", which is a true sentence about the wrong thing and worse
    than saying nothing.
    """
    if not artifact.endswith((".json", ".svg", ".md")):
        # A directory, e.g. the explorer.
        return (
            f'files = load("{artifact}")\n'
            'if files is not None:\n'
            '    print(f"{len(files)} file(s)")\n'
            '    print("\\n".join(files[:5]))'
        )
    if artifact.endswith(".json"):
        return (
            f'data = load("{artifact}")\n'
            'if isinstance(data, list):\n'
            '    print(f"{len(data)} item(s)")\n'
            'elif data:\n'
            '    if "nodes" in data:\n'
            '        print(f\'{len(data["nodes"])} nodes, {len(data.get("edges", []))} edges\')\n'
            '    else:\n'
            '        print(", ".join(f"{k}: {len(v) if isinstance(v, (list, dict)) else v}"\n'
            '                        for k, v in data.items()))'
        )
    if artifact.endswith(".svg"):
        # No IPython import, guarded or otherwise. A generated notebook is a
        # build artifact and has to run wherever it is opened; requiring a
        # kernel package in order to *load a file* would make the cell fail on
        # the machine that produced it, which is where this was found.
        return (
            f'svg = load("{artifact}")\n'
            'if svg:\n'
            '    print(f"{len(svg):,} bytes of SVG")\n'
            '    # In Jupyter: from IPython.display import SVG; SVG(svg)'
        )
    if artifact.endswith(".md"):
        return (
            f'text = load("{artifact}")\n'
            'if text:\n'
            '    print(text[:400])'
        )
    return f'print(load("{artifact}") is not None)'


# What a build writes into a book directory, and therefore what a rebuild may
# remove first. Named, never globbed: a book directory also holds the hand-written
# `checks.py` and `overlay.json`, and a misconfigured output path could point at
# somebody's source tree — so nothing outside this list is ever deleted.
GENERATED = ("book.json", "notebook.ipynb", "steps", "site", "explore", "document.json")

# The steps a book runs, by id — fixed, so a note can be filed under one before
# the book has ever been built, and a skipped step does not make its note drift.
STEP_LABELS = (
    ("ir", "the graph, extracted"),
    ("view", "the default view for this source"),
    ("pages", "the diagram, split into pages that fit"),
    ("graph", "the whole view in one drawing"),
    ("index", "readable without a diagram"),
    ("sidebar", "navigation, for whatever renders it"),
    ("minimap", "what is where, read from the colours"),
    ("explore", "navigate on one side, explore on the other"),
)


def clear(out) -> list[str]:
    """Remove a previous build's outputs from `out`. Returns what was removed.

    Without this, re-running a book leaves the last run's artifacts beside the new
    ledger: a graph that is now `graph.md` keeps its old `graph.svg`, and the site
    goes on showing it. The ledger would not list the stale file, so nothing would
    say it was stale — which is the failure the book measure exists to prevent.

    Only acts on a directory that already holds a `book.json`, i.e. one this code
    wrote. Anything else is left exactly as found.
    """
    import shutil

    out = Path(out)
    if not (out / "book.json").is_file():
        return []
    removed = []
    for name in GENERATED:
        target = out / name
        if target.is_dir():
            shutil.rmtree(target)
            removed.append(name)
        elif target.is_file():
            target.unlink()
            removed.append(name)
    return removed


def build_book(kind: str, source, out, *, slug: str | None = None, exclude=(),
               ops: dict | None = None, suffixes=None, quiet: bool = False) -> Book:
    """One book's steps: larder, IR, view, pages, drawings. No site, no notebook.

    Those are a document's outputs and are assembled by `book/document.py`, which
    may put several books into one of each. The book is returned open — its last
    step is added once the outputs it belongs to exist.
    """
    from ..ir import check
    from ..ops import classify, overview
    from ..ops.pages import for_view
    from ..style import Style
    from .document import apply_ops

    out = Path(out)
    slug = slug or Path(str(source)).name or "book"
    clear(out)
    steps_dir = out / "steps"
    steps_dir.mkdir(parents=True, exist_ok=True)

    def say(text):
        if not quiet:
            print(text)

    ir, larder = extract(kind, source, exclude=exclude, suffixes=suffixes)
    if larder is None:
        larder = Larder(kind=kind, identity=str(source), unit="document", seen=0)
    book = Book(slug=slug, larder=larder, out=out)
    book.ir = ir
    say(f"  larder   {larder.line()}")

    problems = check(ir)
    if problems:
        # Recorded as a step rather than raised: a book that cannot be trusted
        # should exist and say so, because the alternative is that nobody can
        # see what went wrong.
        book.step("validate", "the IR did not validate", note="; ".join(problems[:3]))
        say(f"  WARNING  IR has {len(problems)} problem(s)")

    p = steps_dir / "ir.json"
    p.write_text(json.dumps(ir, indent=2) + "\n")
    book.step("ir", "the graph, extracted", p, note=f"{len(ir['nodes'])} nodes")

    view = apply_ops(overview(ir), ops or {})
    book.view = view
    p = steps_dir / "view.json"
    p.write_text(json.dumps(view, indent=2) + "\n")
    verdict = classify(view)
    book.step("view", "the default view for this source", p,
              note=f"{verdict['kind']} — {verdict['why']}")

    # Which pages exist — width-free, so it is decided once. Each output lays
    # them out for its own column when it is assembled.
    book.pages = for_view(ir, view)
    p = steps_dir / "pages.json"
    p.write_text(json.dumps(
        [{k: v for k, v in page.items() if k != "ir"} for page in book.pages],
        indent=2) + "\n")
    book.step("pages", "the diagram, split into pages that fit", p,
              note=f"{len(book.pages)} page(s)")

    style_obj = Style.load("lucid")

    # The whole view in one drawing — still worth having as a step on its own,
    # and the same `draw()` the `emit auto` command uses.
    from ..emitters.auto import draw

    drawn = draw(view, style_obj)
    labels = {"erd": "drawn as an ERD", "dot": f"drawn as a {verdict['kind']}",
              "index": "not a diagram — written as a list"}
    if drawn["content"] is None:
        book.step("graph", "not drawn", skipped=drawn["why"])
    else:
        p = steps_dir / f"graph{drawn['suffix']}"
        if isinstance(drawn["content"], bytes):
            p.write_bytes(drawn["content"])
        else:
            p.write_text(drawn["content"])
        book.step("graph", labels[drawn["emitter"]], p, note=drawn["why"])

    from ..emitters.index import to_markdown, to_sidebar
    p = steps_dir / "index.md"
    p.write_text(to_markdown(ir))
    book.step("index", "readable without a diagram", p)
    p = steps_dir / "sidebar.json"
    p.write_text(json.dumps(to_sidebar(ir), indent=2) + "\n")
    book.step("sidebar", "navigation, for whatever renders it", p)

    try:
        from ..emitters.minimap import emit as mm_emit
        p = steps_dir / "minimap.svg"
        p.write_text(mm_emit(ir, style_obj))
        book.step("minimap", "what is where, read from the colours", p)
    except Exception as e:  # noqa: BLE001
        book.step("minimap", "minimap not drawn", skipped=f"{type(e).__name__}: {e}")

    try:
        from ..emitters.explore import write as exp_write
        exp_write(ir, style_obj, out / "explore")
        book.step("explore", "navigate on one side, explore on the other",
                  out / "explore")
    except Exception as e:  # noqa: BLE001
        book.step("explore", "explorer not built", skipped=f"{type(e).__name__}: {e}")

    return book


def run(kind: str, source, out, *, slug: str | None = None, style: str = "lucid",
        theme: str | None = None, exclude=(), overlay=None, notes=None,
        quiet: bool = False) -> Book:
    """One book, as a one-book document whose outputs live in its own directory.

    `book.json`, `steps/`, `site/` and `notebook.ipynb` land where they always did;
    the site and notebook are now assembled the same way a many-book document's are.
    """
    from .document import Document, Output, Part, build

    out = Path(out)
    slug = slug or Path(str(source)).name or "book"
    doc = Document(
        name=slug, title=slug, build=out, inline=True,
        notes=Path(notes) if notes else None,
        parts=[Part(name=slug, kind=kind, source=Path(source), out=out,
                    exclude=tuple(exclude), overlay=overlay)],
        outputs=[Output("site", out / "site", style=style, theme=theme),
                 Output("notebook", out / "notebook.ipynb", style=style, theme=theme)],
    )
    build(doc, quiet=quiet)
    book = doc.books[0]

    for pr in doc.problems:
        if not quiet:
            print(f"  note     {pr}")
    m = book.measure()
    if not quiet:
        print(f"  book     {m['nodes']} nodes · {m['edges']} edges · "
              f"{m['external']} external · {len(m['artifacts'])} artifacts")
        for r in book.compare():
            print(f"  {'ok  ' if r['ok'] else 'LOST'}     {r['claim']}")
        print(f"  open     {out / 'site' / 'index.html'}")
        print(f"           {out / 'book.json'}")
    return book