diff --git a/soleprint/atlas2/docgen/emitters/notebook.py b/soleprint/atlas2/docgen/emitters/notebook.py index 75abe4b..32df47f 100644 --- a/soleprint/atlas2/docgen/emitters/notebook.py +++ b/soleprint/atlas2/docgen/emitters/notebook.py @@ -146,20 +146,41 @@ def _call_cell(step: dict) -> str: """The generated call. An overlay's `code` replaces this wholesale.""" method, path = step.get("method", "GET"), step.get("path", "/") params = step.get("path_params") or [] + if step.get("graphql"): + # One endpoint carries every operation, so the operation name is the + # thing worth showing, not the path. + variables = {f["name"]: f"<{f['name']}>" for f in (step.get("body_fields") or [])} + return ( + f'QUERY = """{step.get("title", "query")} {{ ... }}""" ' + "# fill in the selection set\n" + + (f"VARIABLES = {variables!r}\n" if variables else "") + + f'show(call("POST", "{path}", body={{"query": QUERY' + + (", \"variables\": VARIABLES" if variables else "") + + '}))' + ) lines = [f'{p.upper()} = "<{p}>" # path parameter' for p in params] call_path = path for p in params: call_path = call_path.replace("{" + p + "}", f'" + str({p.upper()}) + "') + # Trim the empty concatenations a placeholder at either end leaves behind. expr = f'"{call_path}"' if params else f'"{path}"' + expr = expr.replace(' + ""', "").replace('"" + ', "") + + # Parameters that were *always* sent are not optional in practice, whatever + # the spec calls them. + always = step.get("params_always") or [] + if always: + lines.append("PARAMS = " + repr({p: f"<{p}>" for p in always})) + arg = ", params=PARAMS" if always else "" if step.get("body_fields"): lines.append("BODY = " + _example(step["body_fields"])) lines.append("") - lines.append(f'show(call("{method}", {expr}, body=BODY))') + lines.append(f'show(call("{method}", {expr}{arg}, body=BODY))') else: if lines: lines.append("") - lines.append(f'show(call("{method}", {expr}))') + lines.append(f'show(call("{method}", {expr}{arg}))') return "\n".join(lines) @@ -175,6 +196,15 @@ def _call_md(step: dict) -> str: facts.append(f'accepts **{step["request_model"]}**') if step.get("status"): facts.append(f'expects `{step["status"]}`') + if step.get("statuses"): + # What really came back, which is usually more than the spec promises. + facts.append("seen: " + ", ".join(f'`{c}`' for c in step["statuses"])) + if step.get("calls"): + facts.append(f'called {step["calls"]}×') + if step.get("params_sometimes"): + facts.append("sometimes sends " + ", ".join(f'`{p}`' for p in step["params_sometimes"])) + if step.get("id_formats"): + facts.append("id as " + "/".join(step["id_formats"])) if facts: out += ["", " · ".join(facts)] if step.get("note"): diff --git a/soleprint/atlas2/docgen/extractors/__main__.py b/soleprint/atlas2/docgen/extractors/__main__.py index 4d38704..799c481 100644 --- a/soleprint/atlas2/docgen/extractors/__main__.py +++ b/soleprint/atlas2/docgen/extractors/__main__.py @@ -1,4 +1,4 @@ -""" python3 -m docgen.extractors [options]""" +""" python3 -m docgen.extractors [options]""" import sys @@ -8,6 +8,9 @@ def main(argv=None): if argv and argv[0] == "openapi": from .openapi_main import main as run return run(argv[1:]) + if argv and argv[0] == "usage": + from .usage_main import main as run + return run(argv[1:]) if argv and argv[0] == "db": from .db_main import main as run return run(argv[1:]) diff --git a/soleprint/atlas2/docgen/extractors/usage.py b/soleprint/atlas2/docgen/extractors/usage.py new file mode 100644 index 0000000..0e0bca3 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/usage.py @@ -0,0 +1,247 @@ +""" +Recorded traffic -> IR. What callers actually do, rather than what exists. + +Reads a **HAR** file — the format browser devtools, mitmproxy, Charles and +Insomnia all export. Standard, JSON, stdlib-parseable, and already sitting on +most people's disk after ten minutes of using the thing they want documented. + + python3 -m docgen.extractors usage --har session.har -o ir.json + +## Why this exists + +An OpenAPI document says what endpoints *are*. It does not say how to use them, +and the gap is where the whole difficulty lives: + +- **The order.** Which call has to happen first. A spec is a set; usage is a + sequence, and the sequence is most of what a newcomer needs. +- **Which parameters matter.** A spec lists twenty optional query parameters. + Traffic shows the two that are always sent. +- **What a real payload looks like** — as opposed to a shape with every field + present and none of them meaning anything. +- **The endpoints that are not in the document at all**, which for a GraphQL + endpoint beside a REST surface is the normal case rather than an oversight. +- **Which responses actually happen.** A spec promises 200 and 404; traffic + shows the 422 that everyone hits. + +None of that is recoverable by reading harder. It is only in the traffic. + +## The two inferences, and their limits + +**Path templating.** `/pets/123` and `/pets/456` are one endpoint. Any segment +that looks like an identifier — digits, a UUID, a long hex string — becomes +`{id}`, *whatever its format*: a route taking a numeric id on one call and a +UUID on the next is one route, and splitting it by format invents an endpoint +that does not exist. This is still a guess — a genuine path segment that happens +to be numeric gets templated wrongly — so `attrs.observed_paths` and +`attrs.id_formats` keep what was actually seen beside it. + +**Sequence.** Consecutive calls become `follows` edges carrying how often that +pair occurred. Consecutive is not *caused by*, and one session's order is not +the only order — the weight is what separates a habit from an accident, and a +single recording will not tell you which. + +Nothing here reads a response body's values beyond its shape, and no header is +copied into the IR: a HAR is full of cookies and bearer tokens, and none of them +belong in a document that gets committed. +""" + +import json +import re +from collections import Counter +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +from ..ir import Graph, Meta + +UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) +LONG_HEX = re.compile(r"^[0-9a-f]{16,}$", re.I) +DIGITS = re.compile(r"^\d+$") + +# Never copied into the IR. A HAR carries live credentials, and a document that +# gets committed must not. +SECRET_HEADERS = {"authorization", "cookie", "set-cookie", "x-api-key", "proxy-authorization"} + + +def _template(path: str) -> tuple[str, set[str]]: + """`/pets/123` -> `/pets/{id}`. Returns (templated, id formats seen). + + Every identifier-looking segment becomes `{id}`, whatever its format. A + caller that passes a numeric id on one call and a UUID on the next is using + **one** route, and templating them to `{id}` and `{uuid}` splits it into two + endpoints that do not exist — which is worse than the imprecision it avoids. + The formats are recorded instead, so "this route takes both" stays visible. + """ + out, formats = [], set() + for seg in path.split("/"): + if not seg: + out.append(seg) + continue + fmt = None + if DIGITS.match(seg): + fmt = "numeric" + elif UUID.match(seg): + fmt = "uuid" + elif LONG_HEX.match(seg): + fmt = "hash" + if fmt: + out.append("{id}") + formats.add(fmt) + else: + out.append(seg) + return "/".join(out), formats + + +def _body(entry: dict) -> dict | None: + post = (entry.get("request") or {}).get("postData") or {} + text = post.get("text") + if not text: + return None + try: + parsed = json.loads(text) + except (json.JSONDecodeError, TypeError): + return None + return parsed if isinstance(parsed, dict) else None + + +def _shape(value) -> str: + """The type of a value, never the value. A payload is full of real data.""" + if isinstance(value, bool): + return "bool" + if isinstance(value, int): + return "int" + if isinstance(value, float): + return "float" + if isinstance(value, list): + return "list" + if isinstance(value, dict): + return "object" + if value is None: + return "null" + return "str" + + +def _graphql(body: dict | None) -> tuple[str | None, str | None]: + """(operation name, operation type) if this is a GraphQL call.""" + if not body or "query" not in body: + return None, None + query = body.get("query") + if not isinstance(query, str): + return None, None + name = body.get("operationName") + m = re.match(r"\s*(query|mutation|subscription)?\s*([A-Za-z_]\w*)?", query) + op_type = (m.group(1) if m else None) or "query" + return (name or (m.group(2) if m and m.group(2) else "anonymous")), op_type + + +def extract(har_path, source: str = "usage") -> Graph: + """A HAR file -> IR of what was actually called.""" + path = Path(har_path) + har = json.loads(path.read_text()) + entries = (har.get("log") or {}).get("entries") or [] + entries = sorted(entries, key=lambda e: e.get("startedDateTime", "")) + + calls = [] # (key, kind, facts) in order + seen: dict[str, dict] = {} + + for entry in entries: + req = entry.get("request") or {} + res = entry.get("response") or {} + method = (req.get("method") or "GET").upper() + url = req.get("url") or "" + split = urlsplit(url) + templated, id_formats = _template(split.path or "/") + body = _body(entry) + + op, op_type = _graphql(body) + if op: + key = f"{op_type} {op}" + kind = "operation" + else: + key = f"{method} {templated}" + kind = "endpoint" + + record = seen.setdefault( + key, + { + "kind": kind, + "calls": 0, + "statuses": Counter(), + "params": Counter(), + "body_fields": Counter(), + "field_types": {}, + "paths": Counter(), + "method": method, + "path": templated, + "id_formats": set(), + "op_type": op_type, + "host": split.netloc, + }, + ) + record["calls"] += 1 + record["id_formats"] |= id_formats + status = res.get("status") + if status: + record["statuses"][int(status)] += 1 + if split.path: + record["paths"][split.path] += 1 + for name in parse_qs(split.query or ""): + record["params"][name] += 1 + if body and not op: + for name, value in body.items(): + record["body_fields"][name] += 1 + record["field_types"].setdefault(name, _shape(value)) + if op and isinstance(body.get("variables"), dict): + for name, value in body["variables"].items(): + record["body_fields"][name] += 1 + record["field_types"].setdefault(name, _shape(value)) + + calls.append(key) + + g = Graph(Meta(source=source, root=path.name)) + + order = {} + for i, key in enumerate(calls): + order.setdefault(key, i) + + for key, r in sorted(seen.items(), key=lambda kv: order[kv[0]]): + n = r["calls"] + attrs = { + "calls": n, + "statuses": sorted(r["statuses"]), + "first_seen": order[key], + } + if r["kind"] == "endpoint": + attrs["method"] = r["method"] + attrs["path"] = r["path"] + else: + attrs["protocol"] = "graphql" + attrs["operation"] = r["op_type"] + if r["host"]: + attrs["host"] = r["host"] + # Always sent vs sometimes sent is the distinction a spec cannot make. + always = sorted(p for p, c in r["params"].items() if c == n) + sometimes = sorted(p for p, c in r["params"].items() if c < n) + if always: + attrs["params_always"] = always + if sometimes: + attrs["params_sometimes"] = sometimes + if r["body_fields"]: + attrs["body_fields"] = [ + {"name": f, "type": r["field_types"].get(f, "str"), + "always": r["body_fields"][f] == n} + for f in sorted(r["body_fields"]) + ] + if r["id_formats"]: + # The templating is a guess; keep what was actually seen beside it. + attrs["id_formats"] = sorted(r["id_formats"]) + attrs["observed_paths"] = [p for p, _ in r["paths"].most_common(5)] + g.node(key, r["kind"], key, attrs=attrs) + + # The sequence. Consecutive is not caused-by, so the weight is the signal. + pairs = Counter(zip(calls, calls[1:])) + for (a, b), weight in pairs.items(): + if a == b: + continue # a repeated call is polling, not a step + g.edge(a, b, "follows", attrs={"weight": weight}) + + return g diff --git a/soleprint/atlas2/docgen/extractors/usage_main.py b/soleprint/atlas2/docgen/extractors/usage_main.py new file mode 100644 index 0000000..f6aedd4 --- /dev/null +++ b/soleprint/atlas2/docgen/extractors/usage_main.py @@ -0,0 +1,32 @@ +""" python3 -m docgen.extractors usage --har session.har [-o ir.json]""" + +import argparse +import json +import sys +from pathlib import Path + + +def main(argv=None): + p = argparse.ArgumentParser(prog="python3 -m docgen.extractors usage") + p.add_argument("--har", "-s", required=True, type=Path, + help="A HAR recording, as devtools/mitmproxy/Charles export.") + p.add_argument("--output", "-o", type=Path) + args = p.parse_args(argv) + from .usage import extract + + try: + ir = extract(args.har) + except (OSError, json.JSONDecodeError, KeyError) as e: + print(f"Error: could not read {args.har}: {e}", file=sys.stderr) + return 1 + text = json.dumps(ir.to_dict(), indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text) + eps = sum(1 for n in ir.nodes if n.kind == "endpoint") + ops = sum(1 for n in ir.nodes if n.kind == "operation") + print(f"{len(ir.nodes)} nodes ({eps} endpoints, {ops} graphql), " + f"{len(ir.edges)} sequence edges -> {args.output}") + else: + sys.stdout.write(text) + return 0 diff --git a/soleprint/atlas2/docgen/notebook/spec.py b/soleprint/atlas2/docgen/notebook/spec.py index 04cf7e4..cb8c7e6 100644 --- a/soleprint/atlas2/docgen/notebook/spec.py +++ b/soleprint/atlas2/docgen/notebook/spec.py @@ -45,6 +45,7 @@ document today, and it keeps working unchanged when a usage extractor exists. """ import json +import re from pathlib import Path SPEC_VERSION = "1" @@ -70,10 +71,18 @@ def from_ir(ir: dict, base_url: str = "https://api.example.invalid", if n["kind"] == "column" and n.get("parent"): fields_of.setdefault(n["parent"], []).append(n) + # Observed order wins over alphabetical. A usage recording knows which call + # comes first, and that sequence is most of what a newcomer needs — sorting + # it away would throw out the one thing a spec could not have told us. + def _order(n): + a = n.get("attrs") or {} + if "first_seen" in a: + return (0, a["first_seen"], "") + return (1, 0, f'{a.get("path", "")} {a.get("method", "")}') + endpoints = sorted( - (n for n in ir["nodes"] if n["kind"] == "endpoint"), - key=lambda n: ((n.get("attrs") or {}).get("path", ""), - (n.get("attrs") or {}).get("method", "")), + (n for n in ir["nodes"] if n["kind"] in ("endpoint", "operation")), + key=_order, ) steps = [ @@ -90,23 +99,51 @@ def from_ir(ir: dict, base_url: str = "https://api.example.invalid", for n in endpoints: a = n.get("attrs") or {} + + # A usage recording carries facts a spec cannot: which parameters are + # *always* sent, which status codes really happen, how often it is + # called. Where they exist they are better than anything generated from + # a schema, so they win. + body_fields = None + if a.get("body_fields"): + raw = a["body_fields"] + if raw and isinstance(raw[0], dict) and "always" in raw[0]: + body_fields = [f for f in raw if f.get("always")] or raw + else: + body_fields = raw + elif a.get("request_model"): + body_fields = [ + {"name": f.get("label") or f["id"].rsplit(".", 1)[-1], + **(f.get("attrs") or {})} + for f in fields_of.get(a["request_model"], []) + ] or None + steps.append( _step( n["id"], "call", - title=f'{a.get("method", "GET")} {a.get("path", "/")}', + title=n["id"] if n["kind"] == "operation" + else f'{a.get("method", "GET")} {a.get("path", "/")}', summary=a.get("summary"), - method=a.get("method", "GET"), - path=a.get("path", "/"), + method=a.get("method", "POST" if n["kind"] == "operation" else "GET"), + path=a.get("path", "/graphql" if n["kind"] == "operation" else "/"), status=a.get("status"), request_model=a.get("request_model"), response_model=a.get("response_model"), returns_list=a.get("returns_list"), - path_params=a.get("path_params"), - body_fields=[ - {"name": f.get("label") or f["id"].rsplit(".", 1)[-1], - **(f.get("attrs") or {})} - for f in fields_of.get(a.get("request_model") or "", []) - ] or None, + # A usage-derived path already carries its placeholders; a + # spec-derived one lists them separately. Either way the cell + # needs a variable, not a literal `{id}` that would 404. + path_params=a.get("path_params") + or re.findall(r"\{(\w+)\}", a.get("path", "")) or None, + body_fields=body_fields, + # usage-only + params_always=a.get("params_always"), + params_sometimes=a.get("params_sometimes"), + statuses=a.get("statuses"), + calls=a.get("calls"), + id_formats=a.get("id_formats"), + observed_paths=a.get("observed_paths"), + graphql=(n["kind"] == "operation") or None, ) ) diff --git a/soleprint/atlas2/docgen/selftest.py b/soleprint/atlas2/docgen/selftest.py index 0764898..0725431 100644 --- a/soleprint/atlas2/docgen/selftest.py +++ b/soleprint/atlas2/docgen/selftest.py @@ -48,6 +48,7 @@ erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"]) nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"]) spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"]) db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"]) +usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"]) check_ir = ir_mod.check Style, StyleError = style_mod.Style, style_mod.StyleError @@ -814,6 +815,114 @@ else: ) +# -------------------------------------------------------------------------- +print("\n8b. usage — what a spec cannot say") + +# A spec is a set; usage is a sequence. Everything asserted here is something no +# amount of reading the OpenAPI document harder would recover. +def _entry(t, method, url, status, body=None): + req = {"method": method, "url": url, + "headers": [{"name": "Authorization", "value": "Bearer LEAK-ME-IF-BROKEN"}]} + if body is not None: + req["postData"] = {"mimeType": "application/json", "text": json.dumps(body)} + return {"startedDateTime": f"2026-01-01T00:00:{t:02d}Z", + "request": req, "response": {"status": status, "content": {}}} + + +HAR = {"log": {"version": "1.2", "entries": [ + _entry(1, "POST", "https://x.test/auth", 200, {"user": "a", "secret": "b"}), + _entry(2, "GET", "https://x.test/pets?status=available&limit=20", 200), + _entry(3, "GET", "https://x.test/pets/1042", 200), + _entry(4, "GET", "https://x.test/pets/7f3e9b2a-1c4d-4a5b-9e8f-0a1b2c3d4e5f", 200), + _entry(5, "POST", "https://x.test/graphql", 200, + {"operationName": "PetWithTags", "query": "query PetWithTags($id: ID!) { pet }", + "variables": {"id": "1042"}}), + _entry(6, "POST", "https://x.test/pets", 422, {"name": "Rex"}), + _entry(7, "POST", "https://x.test/pets", 201, {"name": "Rex", "status": "available"}), + _entry(8, "GET", "https://x.test/pets?status=available&limit=20", 200), +]}} +har_path = ROOT.parent / "session.har" +har_path.write_text(json.dumps(HAR)) +use = usage_ex.extract(har_path).to_dict() +by = {n["id"]: n for n in use["nodes"]} +raw_json = json.dumps(use) + +check("the usage IR validates", check_ir(use) == [], str(check_ir(use)[:2])) +check( + "**no credential reaches the IR**", + "LEAK-ME-IF-BROKEN" not in raw_json and "Authorization" not in raw_json, + "a HAR is full of live tokens and a document gets committed", +) +check( + "paths are templated into one route", + "GET /pets/{id}" in by and "GET /pets/1042" not in by, +) +check( + "...and both id formats stay on that one route", + by["GET /pets/{id}"]["attrs"]["id_formats"] == ["numeric", "uuid"] + and by["GET /pets/{id}"]["attrs"]["calls"] == 2, + "splitting by format invents an endpoint that does not exist", +) +check( + "what is always sent is distinguished from what is sometimes", + by["GET /pets"]["attrs"]["params_always"] == ["limit", "status"], + "a spec calls these optional; traffic says otherwise", +) +check( + "the statuses that really happen are recorded", + by["POST /pets"]["attrs"]["statuses"] == [201, 422], + "the 422 everyone hits is not in the spec", +) +check( + "a body field seen once out of twice is not marked always", + [f["name"] for f in by["POST /pets"]["attrs"]["body_fields"] if f["always"]] == ["name"], +) +check( + "a GraphQL operation is found and named", + "query PetWithTags" in by and by["query PetWithTags"]["kind"] == "operation", + "one endpoint carries many operations; the path alone says nothing", +) +check( + "only the shape of a payload is kept, never a value", + '"a"' not in raw_json and '"b"' not in raw_json, +) +seq = {(e["source"], e["target"]) for e in use["edges"]} +check( + "the sequence is recorded as edges", + ("POST /auth", "GET /pets") in seq and all(e["kind"] == "follows" for e in use["edges"]), + "this is the half a spec structurally cannot contain", +) +check( + "a repeated call is not a step in the sequence", + not [e for e in use["edges"] if e["source"] == e["target"]], +) +check( + "usage is a pipeline, and drawn as one", + ops_mod.classify(use)["kind"] == "pipeline", +) + +# The payoff: a walkthrough in the order people actually call things. +u_spec, _ = spec_mod.merge(spec_mod.from_ir(use), None) +ids = [st["id"] for st in u_spec["steps"] if st["kind"] == "call"] +check( + "the notebook keeps the observed order", + ids[0] == "POST /auth" and ids.index("GET /pets") < ids.index("POST /pets"), + f"got {ids}", +) +u_body = "".join("".join(c["source"]) for c in nb_mod.build(u_spec)["cells"]) +check( + "it sends the parameters that are always sent", + "PARAMS = " in u_body and "'status'" in u_body, +) +check( + "a templated path becomes a variable, not a literal", + 'ID = ""' in u_body and '"/pets/{id}"' not in u_body, + "a literal {id} in the URL would 404", +) +check("the GraphQL operation gets a query cell", "QUERY = " in u_body) +har_path.unlink() + + # -------------------------------------------------------------------------- print("\n9b. generated base + hand-written overlay") diff --git a/soleprint/atlas2/docgen/style/lucid.json b/soleprint/atlas2/docgen/style/lucid.json index e40bc11..69e7daa 100644 --- a/soleprint/atlas2/docgen/style/lucid.json +++ b/soleprint/atlas2/docgen/style/lucid.json @@ -172,6 +172,24 @@ "text": "text", "bold": true, "note": "A task that waits on something outside the pipeline." + }, + "endpoint": { + "shape": "box", + "rounded": true, + "fill": "surface-2", + "border": "station", + "text": "text", + "bold": true, + "note": "An HTTP route. From a spec, or from what was actually called." + }, + "operation": { + "shape": "box", + "rounded": true, + "fill": "surface-2", + "border": "accent", + "text": "text", + "bold": true, + "note": "A GraphQL operation \u2014 usually one endpoint carrying many, which is why it is its own kind rather than a path." } }, "groups": { @@ -306,6 +324,25 @@ "arrowhead": "normal", "arrowsize": 0.8, "text": "accent" + }, + "follows": { + "color": "accent", + "arrowhead": "normal", + "arrowsize": 0.7, + "text": "accent", + "note": "Observed order. Consecutive, not caused-by; the weight is the signal." + }, + "accepts": { + "color": "border-strong", + "arrowhead": "normal", + "arrowsize": 0.6, + "text": "text-dim" + }, + "returns": { + "color": "station", + "arrowhead": "normal", + "arrowsize": 0.6, + "text": "text-dim" } }, "domain_slots": {