add cases for code, dbs, and outline notebook generation
This commit is contained in:
@@ -146,20 +146,41 @@ def _call_cell(step: dict) -> str:
|
|||||||
"""The generated call. An overlay's `code` replaces this wholesale."""
|
"""The generated call. An overlay's `code` replaces this wholesale."""
|
||||||
method, path = step.get("method", "GET"), step.get("path", "/")
|
method, path = step.get("method", "GET"), step.get("path", "/")
|
||||||
params = step.get("path_params") or []
|
params = step.get("path_params") or []
|
||||||
|
if step.get("graphql"):
|
||||||
|
# One endpoint carries every operation, so the operation name is the
|
||||||
|
# thing worth showing, not the path.
|
||||||
|
variables = {f["name"]: f"<{f['name']}>" for f in (step.get("body_fields") or [])}
|
||||||
|
return (
|
||||||
|
f'QUERY = """{step.get("title", "query")} {{ ... }}""" '
|
||||||
|
"# fill in the selection set\n"
|
||||||
|
+ (f"VARIABLES = {variables!r}\n" if variables else "")
|
||||||
|
+ f'show(call("POST", "{path}", body={{"query": QUERY'
|
||||||
|
+ (", \"variables\": VARIABLES" if variables else "")
|
||||||
|
+ '}))'
|
||||||
|
)
|
||||||
lines = [f'{p.upper()} = "<{p}>" # path parameter' for p in params]
|
lines = [f'{p.upper()} = "<{p}>" # path parameter' for p in params]
|
||||||
call_path = path
|
call_path = path
|
||||||
for p in params:
|
for p in params:
|
||||||
call_path = call_path.replace("{" + p + "}", f'" + str({p.upper()}) + "')
|
call_path = call_path.replace("{" + p + "}", f'" + str({p.upper()}) + "')
|
||||||
|
# Trim the empty concatenations a placeholder at either end leaves behind.
|
||||||
expr = f'"{call_path}"' if params else f'"{path}"'
|
expr = f'"{call_path}"' if params else f'"{path}"'
|
||||||
|
expr = expr.replace(' + ""', "").replace('"" + ', "")
|
||||||
|
|
||||||
|
# Parameters that were *always* sent are not optional in practice, whatever
|
||||||
|
# the spec calls them.
|
||||||
|
always = step.get("params_always") or []
|
||||||
|
if always:
|
||||||
|
lines.append("PARAMS = " + repr({p: f"<{p}>" for p in always}))
|
||||||
|
arg = ", params=PARAMS" if always else ""
|
||||||
|
|
||||||
if step.get("body_fields"):
|
if step.get("body_fields"):
|
||||||
lines.append("BODY = " + _example(step["body_fields"]))
|
lines.append("BODY = " + _example(step["body_fields"]))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(f'show(call("{method}", {expr}, body=BODY))')
|
lines.append(f'show(call("{method}", {expr}{arg}, body=BODY))')
|
||||||
else:
|
else:
|
||||||
if lines:
|
if lines:
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(f'show(call("{method}", {expr}))')
|
lines.append(f'show(call("{method}", {expr}{arg}))')
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
@@ -175,6 +196,15 @@ def _call_md(step: dict) -> str:
|
|||||||
facts.append(f'accepts **{step["request_model"]}**')
|
facts.append(f'accepts **{step["request_model"]}**')
|
||||||
if step.get("status"):
|
if step.get("status"):
|
||||||
facts.append(f'expects `{step["status"]}`')
|
facts.append(f'expects `{step["status"]}`')
|
||||||
|
if step.get("statuses"):
|
||||||
|
# What really came back, which is usually more than the spec promises.
|
||||||
|
facts.append("seen: " + ", ".join(f'`{c}`' for c in step["statuses"]))
|
||||||
|
if step.get("calls"):
|
||||||
|
facts.append(f'called {step["calls"]}×')
|
||||||
|
if step.get("params_sometimes"):
|
||||||
|
facts.append("sometimes sends " + ", ".join(f'`{p}`' for p in step["params_sometimes"]))
|
||||||
|
if step.get("id_formats"):
|
||||||
|
facts.append("id as " + "/".join(step["id_formats"]))
|
||||||
if facts:
|
if facts:
|
||||||
out += ["", " · ".join(facts)]
|
out += ["", " · ".join(facts)]
|
||||||
if step.get("note"):
|
if step.get("note"):
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
""" python3 -m docgen.extractors <db|openapi> [options]"""
|
""" python3 -m docgen.extractors <db|openapi|usage> [options]"""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -8,6 +8,9 @@ def main(argv=None):
|
|||||||
if argv and argv[0] == "openapi":
|
if argv and argv[0] == "openapi":
|
||||||
from .openapi_main import main as run
|
from .openapi_main import main as run
|
||||||
return run(argv[1:])
|
return run(argv[1:])
|
||||||
|
if argv and argv[0] == "usage":
|
||||||
|
from .usage_main import main as run
|
||||||
|
return run(argv[1:])
|
||||||
if argv and argv[0] == "db":
|
if argv and argv[0] == "db":
|
||||||
from .db_main import main as run
|
from .db_main import main as run
|
||||||
return run(argv[1:])
|
return run(argv[1:])
|
||||||
|
|||||||
247
soleprint/atlas2/docgen/extractors/usage.py
Normal file
247
soleprint/atlas2/docgen/extractors/usage.py
Normal file
@@ -0,0 +1,247 @@
|
|||||||
|
"""
|
||||||
|
Recorded traffic -> IR. What callers actually do, rather than what exists.
|
||||||
|
|
||||||
|
Reads a **HAR** file — the format browser devtools, mitmproxy, Charles and
|
||||||
|
Insomnia all export. Standard, JSON, stdlib-parseable, and already sitting on
|
||||||
|
most people's disk after ten minutes of using the thing they want documented.
|
||||||
|
|
||||||
|
python3 -m docgen.extractors usage --har session.har -o ir.json
|
||||||
|
|
||||||
|
## Why this exists
|
||||||
|
|
||||||
|
An OpenAPI document says what endpoints *are*. It does not say how to use them,
|
||||||
|
and the gap is where the whole difficulty lives:
|
||||||
|
|
||||||
|
- **The order.** Which call has to happen first. A spec is a set; usage is a
|
||||||
|
sequence, and the sequence is most of what a newcomer needs.
|
||||||
|
- **Which parameters matter.** A spec lists twenty optional query parameters.
|
||||||
|
Traffic shows the two that are always sent.
|
||||||
|
- **What a real payload looks like** — as opposed to a shape with every field
|
||||||
|
present and none of them meaning anything.
|
||||||
|
- **The endpoints that are not in the document at all**, which for a GraphQL
|
||||||
|
endpoint beside a REST surface is the normal case rather than an oversight.
|
||||||
|
- **Which responses actually happen.** A spec promises 200 and 404; traffic
|
||||||
|
shows the 422 that everyone hits.
|
||||||
|
|
||||||
|
None of that is recoverable by reading harder. It is only in the traffic.
|
||||||
|
|
||||||
|
## The two inferences, and their limits
|
||||||
|
|
||||||
|
**Path templating.** `/pets/123` and `/pets/456` are one endpoint. Any segment
|
||||||
|
that looks like an identifier — digits, a UUID, a long hex string — becomes
|
||||||
|
`{id}`, *whatever its format*: a route taking a numeric id on one call and a
|
||||||
|
UUID on the next is one route, and splitting it by format invents an endpoint
|
||||||
|
that does not exist. This is still a guess — a genuine path segment that happens
|
||||||
|
to be numeric gets templated wrongly — so `attrs.observed_paths` and
|
||||||
|
`attrs.id_formats` keep what was actually seen beside it.
|
||||||
|
|
||||||
|
**Sequence.** Consecutive calls become `follows` edges carrying how often that
|
||||||
|
pair occurred. Consecutive is not *caused by*, and one session's order is not
|
||||||
|
the only order — the weight is what separates a habit from an accident, and a
|
||||||
|
single recording will not tell you which.
|
||||||
|
|
||||||
|
Nothing here reads a response body's values beyond its shape, and no header is
|
||||||
|
copied into the IR: a HAR is full of cookies and bearer tokens, and none of them
|
||||||
|
belong in a document that gets committed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from ..ir import Graph, Meta
|
||||||
|
|
||||||
|
UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
|
||||||
|
LONG_HEX = re.compile(r"^[0-9a-f]{16,}$", re.I)
|
||||||
|
DIGITS = re.compile(r"^\d+$")
|
||||||
|
|
||||||
|
# Never copied into the IR. A HAR carries live credentials, and a document that
|
||||||
|
# gets committed must not.
|
||||||
|
SECRET_HEADERS = {"authorization", "cookie", "set-cookie", "x-api-key", "proxy-authorization"}
|
||||||
|
|
||||||
|
|
||||||
|
def _template(path: str) -> tuple[str, set[str]]:
|
||||||
|
"""`/pets/123` -> `/pets/{id}`. Returns (templated, id formats seen).
|
||||||
|
|
||||||
|
Every identifier-looking segment becomes `{id}`, whatever its format. A
|
||||||
|
caller that passes a numeric id on one call and a UUID on the next is using
|
||||||
|
**one** route, and templating them to `{id}` and `{uuid}` splits it into two
|
||||||
|
endpoints that do not exist — which is worse than the imprecision it avoids.
|
||||||
|
The formats are recorded instead, so "this route takes both" stays visible.
|
||||||
|
"""
|
||||||
|
out, formats = [], set()
|
||||||
|
for seg in path.split("/"):
|
||||||
|
if not seg:
|
||||||
|
out.append(seg)
|
||||||
|
continue
|
||||||
|
fmt = None
|
||||||
|
if DIGITS.match(seg):
|
||||||
|
fmt = "numeric"
|
||||||
|
elif UUID.match(seg):
|
||||||
|
fmt = "uuid"
|
||||||
|
elif LONG_HEX.match(seg):
|
||||||
|
fmt = "hash"
|
||||||
|
if fmt:
|
||||||
|
out.append("{id}")
|
||||||
|
formats.add(fmt)
|
||||||
|
else:
|
||||||
|
out.append(seg)
|
||||||
|
return "/".join(out), formats
|
||||||
|
|
||||||
|
|
||||||
|
def _body(entry: dict) -> dict | None:
|
||||||
|
post = (entry.get("request") or {}).get("postData") or {}
|
||||||
|
text = post.get("text")
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = json.loads(text)
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
return None
|
||||||
|
return parsed if isinstance(parsed, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _shape(value) -> str:
|
||||||
|
"""The type of a value, never the value. A payload is full of real data."""
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "bool"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return "int"
|
||||||
|
if isinstance(value, float):
|
||||||
|
return "float"
|
||||||
|
if isinstance(value, list):
|
||||||
|
return "list"
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return "object"
|
||||||
|
if value is None:
|
||||||
|
return "null"
|
||||||
|
return "str"
|
||||||
|
|
||||||
|
|
||||||
|
def _graphql(body: dict | None) -> tuple[str | None, str | None]:
|
||||||
|
"""(operation name, operation type) if this is a GraphQL call."""
|
||||||
|
if not body or "query" not in body:
|
||||||
|
return None, None
|
||||||
|
query = body.get("query")
|
||||||
|
if not isinstance(query, str):
|
||||||
|
return None, None
|
||||||
|
name = body.get("operationName")
|
||||||
|
m = re.match(r"\s*(query|mutation|subscription)?\s*([A-Za-z_]\w*)?", query)
|
||||||
|
op_type = (m.group(1) if m else None) or "query"
|
||||||
|
return (name or (m.group(2) if m and m.group(2) else "anonymous")), op_type
|
||||||
|
|
||||||
|
|
||||||
|
def extract(har_path, source: str = "usage") -> Graph:
|
||||||
|
"""A HAR file -> IR of what was actually called."""
|
||||||
|
path = Path(har_path)
|
||||||
|
har = json.loads(path.read_text())
|
||||||
|
entries = (har.get("log") or {}).get("entries") or []
|
||||||
|
entries = sorted(entries, key=lambda e: e.get("startedDateTime", ""))
|
||||||
|
|
||||||
|
calls = [] # (key, kind, facts) in order
|
||||||
|
seen: dict[str, dict] = {}
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
|
req = entry.get("request") or {}
|
||||||
|
res = entry.get("response") or {}
|
||||||
|
method = (req.get("method") or "GET").upper()
|
||||||
|
url = req.get("url") or ""
|
||||||
|
split = urlsplit(url)
|
||||||
|
templated, id_formats = _template(split.path or "/")
|
||||||
|
body = _body(entry)
|
||||||
|
|
||||||
|
op, op_type = _graphql(body)
|
||||||
|
if op:
|
||||||
|
key = f"{op_type} {op}"
|
||||||
|
kind = "operation"
|
||||||
|
else:
|
||||||
|
key = f"{method} {templated}"
|
||||||
|
kind = "endpoint"
|
||||||
|
|
||||||
|
record = seen.setdefault(
|
||||||
|
key,
|
||||||
|
{
|
||||||
|
"kind": kind,
|
||||||
|
"calls": 0,
|
||||||
|
"statuses": Counter(),
|
||||||
|
"params": Counter(),
|
||||||
|
"body_fields": Counter(),
|
||||||
|
"field_types": {},
|
||||||
|
"paths": Counter(),
|
||||||
|
"method": method,
|
||||||
|
"path": templated,
|
||||||
|
"id_formats": set(),
|
||||||
|
"op_type": op_type,
|
||||||
|
"host": split.netloc,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
record["calls"] += 1
|
||||||
|
record["id_formats"] |= id_formats
|
||||||
|
status = res.get("status")
|
||||||
|
if status:
|
||||||
|
record["statuses"][int(status)] += 1
|
||||||
|
if split.path:
|
||||||
|
record["paths"][split.path] += 1
|
||||||
|
for name in parse_qs(split.query or ""):
|
||||||
|
record["params"][name] += 1
|
||||||
|
if body and not op:
|
||||||
|
for name, value in body.items():
|
||||||
|
record["body_fields"][name] += 1
|
||||||
|
record["field_types"].setdefault(name, _shape(value))
|
||||||
|
if op and isinstance(body.get("variables"), dict):
|
||||||
|
for name, value in body["variables"].items():
|
||||||
|
record["body_fields"][name] += 1
|
||||||
|
record["field_types"].setdefault(name, _shape(value))
|
||||||
|
|
||||||
|
calls.append(key)
|
||||||
|
|
||||||
|
g = Graph(Meta(source=source, root=path.name))
|
||||||
|
|
||||||
|
order = {}
|
||||||
|
for i, key in enumerate(calls):
|
||||||
|
order.setdefault(key, i)
|
||||||
|
|
||||||
|
for key, r in sorted(seen.items(), key=lambda kv: order[kv[0]]):
|
||||||
|
n = r["calls"]
|
||||||
|
attrs = {
|
||||||
|
"calls": n,
|
||||||
|
"statuses": sorted(r["statuses"]),
|
||||||
|
"first_seen": order[key],
|
||||||
|
}
|
||||||
|
if r["kind"] == "endpoint":
|
||||||
|
attrs["method"] = r["method"]
|
||||||
|
attrs["path"] = r["path"]
|
||||||
|
else:
|
||||||
|
attrs["protocol"] = "graphql"
|
||||||
|
attrs["operation"] = r["op_type"]
|
||||||
|
if r["host"]:
|
||||||
|
attrs["host"] = r["host"]
|
||||||
|
# Always sent vs sometimes sent is the distinction a spec cannot make.
|
||||||
|
always = sorted(p for p, c in r["params"].items() if c == n)
|
||||||
|
sometimes = sorted(p for p, c in r["params"].items() if c < n)
|
||||||
|
if always:
|
||||||
|
attrs["params_always"] = always
|
||||||
|
if sometimes:
|
||||||
|
attrs["params_sometimes"] = sometimes
|
||||||
|
if r["body_fields"]:
|
||||||
|
attrs["body_fields"] = [
|
||||||
|
{"name": f, "type": r["field_types"].get(f, "str"),
|
||||||
|
"always": r["body_fields"][f] == n}
|
||||||
|
for f in sorted(r["body_fields"])
|
||||||
|
]
|
||||||
|
if r["id_formats"]:
|
||||||
|
# The templating is a guess; keep what was actually seen beside it.
|
||||||
|
attrs["id_formats"] = sorted(r["id_formats"])
|
||||||
|
attrs["observed_paths"] = [p for p, _ in r["paths"].most_common(5)]
|
||||||
|
g.node(key, r["kind"], key, attrs=attrs)
|
||||||
|
|
||||||
|
# The sequence. Consecutive is not caused-by, so the weight is the signal.
|
||||||
|
pairs = Counter(zip(calls, calls[1:]))
|
||||||
|
for (a, b), weight in pairs.items():
|
||||||
|
if a == b:
|
||||||
|
continue # a repeated call is polling, not a step
|
||||||
|
g.edge(a, b, "follows", attrs={"weight": weight})
|
||||||
|
|
||||||
|
return g
|
||||||
32
soleprint/atlas2/docgen/extractors/usage_main.py
Normal file
32
soleprint/atlas2/docgen/extractors/usage_main.py
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
""" python3 -m docgen.extractors usage --har session.har [-o ir.json]"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
p = argparse.ArgumentParser(prog="python3 -m docgen.extractors usage")
|
||||||
|
p.add_argument("--har", "-s", required=True, type=Path,
|
||||||
|
help="A HAR recording, as devtools/mitmproxy/Charles export.")
|
||||||
|
p.add_argument("--output", "-o", type=Path)
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
from .usage import extract
|
||||||
|
|
||||||
|
try:
|
||||||
|
ir = extract(args.har)
|
||||||
|
except (OSError, json.JSONDecodeError, KeyError) as e:
|
||||||
|
print(f"Error: could not read {args.har}: {e}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
text = json.dumps(ir.to_dict(), indent=2) + "\n"
|
||||||
|
if args.output:
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(text)
|
||||||
|
eps = sum(1 for n in ir.nodes if n.kind == "endpoint")
|
||||||
|
ops = sum(1 for n in ir.nodes if n.kind == "operation")
|
||||||
|
print(f"{len(ir.nodes)} nodes ({eps} endpoints, {ops} graphql), "
|
||||||
|
f"{len(ir.edges)} sequence edges -> {args.output}")
|
||||||
|
else:
|
||||||
|
sys.stdout.write(text)
|
||||||
|
return 0
|
||||||
@@ -45,6 +45,7 @@ document today, and it keeps working unchanged when a usage extractor exists.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
SPEC_VERSION = "1"
|
SPEC_VERSION = "1"
|
||||||
@@ -70,10 +71,18 @@ def from_ir(ir: dict, base_url: str = "https://api.example.invalid",
|
|||||||
if n["kind"] == "column" and n.get("parent"):
|
if n["kind"] == "column" and n.get("parent"):
|
||||||
fields_of.setdefault(n["parent"], []).append(n)
|
fields_of.setdefault(n["parent"], []).append(n)
|
||||||
|
|
||||||
|
# Observed order wins over alphabetical. A usage recording knows which call
|
||||||
|
# comes first, and that sequence is most of what a newcomer needs — sorting
|
||||||
|
# it away would throw out the one thing a spec could not have told us.
|
||||||
|
def _order(n):
|
||||||
|
a = n.get("attrs") or {}
|
||||||
|
if "first_seen" in a:
|
||||||
|
return (0, a["first_seen"], "")
|
||||||
|
return (1, 0, f'{a.get("path", "")} {a.get("method", "")}')
|
||||||
|
|
||||||
endpoints = sorted(
|
endpoints = sorted(
|
||||||
(n for n in ir["nodes"] if n["kind"] == "endpoint"),
|
(n for n in ir["nodes"] if n["kind"] in ("endpoint", "operation")),
|
||||||
key=lambda n: ((n.get("attrs") or {}).get("path", ""),
|
key=_order,
|
||||||
(n.get("attrs") or {}).get("method", "")),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
steps = [
|
steps = [
|
||||||
@@ -90,23 +99,51 @@ def from_ir(ir: dict, base_url: str = "https://api.example.invalid",
|
|||||||
|
|
||||||
for n in endpoints:
|
for n in endpoints:
|
||||||
a = n.get("attrs") or {}
|
a = n.get("attrs") or {}
|
||||||
|
|
||||||
|
# A usage recording carries facts a spec cannot: which parameters are
|
||||||
|
# *always* sent, which status codes really happen, how often it is
|
||||||
|
# called. Where they exist they are better than anything generated from
|
||||||
|
# a schema, so they win.
|
||||||
|
body_fields = None
|
||||||
|
if a.get("body_fields"):
|
||||||
|
raw = a["body_fields"]
|
||||||
|
if raw and isinstance(raw[0], dict) and "always" in raw[0]:
|
||||||
|
body_fields = [f for f in raw if f.get("always")] or raw
|
||||||
|
else:
|
||||||
|
body_fields = raw
|
||||||
|
elif a.get("request_model"):
|
||||||
|
body_fields = [
|
||||||
|
{"name": f.get("label") or f["id"].rsplit(".", 1)[-1],
|
||||||
|
**(f.get("attrs") or {})}
|
||||||
|
for f in fields_of.get(a["request_model"], [])
|
||||||
|
] or None
|
||||||
|
|
||||||
steps.append(
|
steps.append(
|
||||||
_step(
|
_step(
|
||||||
n["id"], "call",
|
n["id"], "call",
|
||||||
title=f'{a.get("method", "GET")} {a.get("path", "/")}',
|
title=n["id"] if n["kind"] == "operation"
|
||||||
|
else f'{a.get("method", "GET")} {a.get("path", "/")}',
|
||||||
summary=a.get("summary"),
|
summary=a.get("summary"),
|
||||||
method=a.get("method", "GET"),
|
method=a.get("method", "POST" if n["kind"] == "operation" else "GET"),
|
||||||
path=a.get("path", "/"),
|
path=a.get("path", "/graphql" if n["kind"] == "operation" else "/"),
|
||||||
status=a.get("status"),
|
status=a.get("status"),
|
||||||
request_model=a.get("request_model"),
|
request_model=a.get("request_model"),
|
||||||
response_model=a.get("response_model"),
|
response_model=a.get("response_model"),
|
||||||
returns_list=a.get("returns_list"),
|
returns_list=a.get("returns_list"),
|
||||||
path_params=a.get("path_params"),
|
# A usage-derived path already carries its placeholders; a
|
||||||
body_fields=[
|
# spec-derived one lists them separately. Either way the cell
|
||||||
{"name": f.get("label") or f["id"].rsplit(".", 1)[-1],
|
# needs a variable, not a literal `{id}` that would 404.
|
||||||
**(f.get("attrs") or {})}
|
path_params=a.get("path_params")
|
||||||
for f in fields_of.get(a.get("request_model") or "", [])
|
or re.findall(r"\{(\w+)\}", a.get("path", "")) or None,
|
||||||
] or None,
|
body_fields=body_fields,
|
||||||
|
# usage-only
|
||||||
|
params_always=a.get("params_always"),
|
||||||
|
params_sometimes=a.get("params_sometimes"),
|
||||||
|
statuses=a.get("statuses"),
|
||||||
|
calls=a.get("calls"),
|
||||||
|
id_formats=a.get("id_formats"),
|
||||||
|
observed_paths=a.get("observed_paths"),
|
||||||
|
graphql=(n["kind"] == "operation") or None,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ erd_mod = __import__(f"{PKG}.emitters.erd", fromlist=["*"])
|
|||||||
nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"])
|
nb_mod = __import__(f"{PKG}.emitters.notebook", fromlist=["*"])
|
||||||
spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"])
|
spec_mod = __import__(f"{PKG}.notebook", fromlist=["*"])
|
||||||
db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"])
|
db_ex = __import__(f"{PKG}.extractors.db", fromlist=["*"])
|
||||||
|
usage_ex = __import__(f"{PKG}.extractors.usage", fromlist=["*"])
|
||||||
|
|
||||||
check_ir = ir_mod.check
|
check_ir = ir_mod.check
|
||||||
Style, StyleError = style_mod.Style, style_mod.StyleError
|
Style, StyleError = style_mod.Style, style_mod.StyleError
|
||||||
@@ -814,6 +815,114 @@ else:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
print("\n8b. usage — what a spec cannot say")
|
||||||
|
|
||||||
|
# A spec is a set; usage is a sequence. Everything asserted here is something no
|
||||||
|
# amount of reading the OpenAPI document harder would recover.
|
||||||
|
def _entry(t, method, url, status, body=None):
|
||||||
|
req = {"method": method, "url": url,
|
||||||
|
"headers": [{"name": "Authorization", "value": "Bearer LEAK-ME-IF-BROKEN"}]}
|
||||||
|
if body is not None:
|
||||||
|
req["postData"] = {"mimeType": "application/json", "text": json.dumps(body)}
|
||||||
|
return {"startedDateTime": f"2026-01-01T00:00:{t:02d}Z",
|
||||||
|
"request": req, "response": {"status": status, "content": {}}}
|
||||||
|
|
||||||
|
|
||||||
|
HAR = {"log": {"version": "1.2", "entries": [
|
||||||
|
_entry(1, "POST", "https://x.test/auth", 200, {"user": "a", "secret": "b"}),
|
||||||
|
_entry(2, "GET", "https://x.test/pets?status=available&limit=20", 200),
|
||||||
|
_entry(3, "GET", "https://x.test/pets/1042", 200),
|
||||||
|
_entry(4, "GET", "https://x.test/pets/7f3e9b2a-1c4d-4a5b-9e8f-0a1b2c3d4e5f", 200),
|
||||||
|
_entry(5, "POST", "https://x.test/graphql", 200,
|
||||||
|
{"operationName": "PetWithTags", "query": "query PetWithTags($id: ID!) { pet }",
|
||||||
|
"variables": {"id": "1042"}}),
|
||||||
|
_entry(6, "POST", "https://x.test/pets", 422, {"name": "Rex"}),
|
||||||
|
_entry(7, "POST", "https://x.test/pets", 201, {"name": "Rex", "status": "available"}),
|
||||||
|
_entry(8, "GET", "https://x.test/pets?status=available&limit=20", 200),
|
||||||
|
]}}
|
||||||
|
har_path = ROOT.parent / "session.har"
|
||||||
|
har_path.write_text(json.dumps(HAR))
|
||||||
|
use = usage_ex.extract(har_path).to_dict()
|
||||||
|
by = {n["id"]: n for n in use["nodes"]}
|
||||||
|
raw_json = json.dumps(use)
|
||||||
|
|
||||||
|
check("the usage IR validates", check_ir(use) == [], str(check_ir(use)[:2]))
|
||||||
|
check(
|
||||||
|
"**no credential reaches the IR**",
|
||||||
|
"LEAK-ME-IF-BROKEN" not in raw_json and "Authorization" not in raw_json,
|
||||||
|
"a HAR is full of live tokens and a document gets committed",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"paths are templated into one route",
|
||||||
|
"GET /pets/{id}" in by and "GET /pets/1042" not in by,
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"...and both id formats stay on that one route",
|
||||||
|
by["GET /pets/{id}"]["attrs"]["id_formats"] == ["numeric", "uuid"]
|
||||||
|
and by["GET /pets/{id}"]["attrs"]["calls"] == 2,
|
||||||
|
"splitting by format invents an endpoint that does not exist",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"what is always sent is distinguished from what is sometimes",
|
||||||
|
by["GET /pets"]["attrs"]["params_always"] == ["limit", "status"],
|
||||||
|
"a spec calls these optional; traffic says otherwise",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"the statuses that really happen are recorded",
|
||||||
|
by["POST /pets"]["attrs"]["statuses"] == [201, 422],
|
||||||
|
"the 422 everyone hits is not in the spec",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"a body field seen once out of twice is not marked always",
|
||||||
|
[f["name"] for f in by["POST /pets"]["attrs"]["body_fields"] if f["always"]] == ["name"],
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"a GraphQL operation is found and named",
|
||||||
|
"query PetWithTags" in by and by["query PetWithTags"]["kind"] == "operation",
|
||||||
|
"one endpoint carries many operations; the path alone says nothing",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"only the shape of a payload is kept, never a value",
|
||||||
|
'"a"' not in raw_json and '"b"' not in raw_json,
|
||||||
|
)
|
||||||
|
seq = {(e["source"], e["target"]) for e in use["edges"]}
|
||||||
|
check(
|
||||||
|
"the sequence is recorded as edges",
|
||||||
|
("POST /auth", "GET /pets") in seq and all(e["kind"] == "follows" for e in use["edges"]),
|
||||||
|
"this is the half a spec structurally cannot contain",
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"a repeated call is not a step in the sequence",
|
||||||
|
not [e for e in use["edges"] if e["source"] == e["target"]],
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"usage is a pipeline, and drawn as one",
|
||||||
|
ops_mod.classify(use)["kind"] == "pipeline",
|
||||||
|
)
|
||||||
|
|
||||||
|
# The payoff: a walkthrough in the order people actually call things.
|
||||||
|
u_spec, _ = spec_mod.merge(spec_mod.from_ir(use), None)
|
||||||
|
ids = [st["id"] for st in u_spec["steps"] if st["kind"] == "call"]
|
||||||
|
check(
|
||||||
|
"the notebook keeps the observed order",
|
||||||
|
ids[0] == "POST /auth" and ids.index("GET /pets") < ids.index("POST /pets"),
|
||||||
|
f"got {ids}",
|
||||||
|
)
|
||||||
|
u_body = "".join("".join(c["source"]) for c in nb_mod.build(u_spec)["cells"])
|
||||||
|
check(
|
||||||
|
"it sends the parameters that are always sent",
|
||||||
|
"PARAMS = " in u_body and "'status'" in u_body,
|
||||||
|
)
|
||||||
|
check(
|
||||||
|
"a templated path becomes a variable, not a literal",
|
||||||
|
'ID = "<id>"' in u_body and '"/pets/{id}"' not in u_body,
|
||||||
|
"a literal {id} in the URL would 404",
|
||||||
|
)
|
||||||
|
check("the GraphQL operation gets a query cell", "QUERY = " in u_body)
|
||||||
|
har_path.unlink()
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
print("\n9b. generated base + hand-written overlay")
|
print("\n9b. generated base + hand-written overlay")
|
||||||
|
|
||||||
|
|||||||
@@ -172,6 +172,24 @@
|
|||||||
"text": "text",
|
"text": "text",
|
||||||
"bold": true,
|
"bold": true,
|
||||||
"note": "A task that waits on something outside the pipeline."
|
"note": "A task that waits on something outside the pipeline."
|
||||||
|
},
|
||||||
|
"endpoint": {
|
||||||
|
"shape": "box",
|
||||||
|
"rounded": true,
|
||||||
|
"fill": "surface-2",
|
||||||
|
"border": "station",
|
||||||
|
"text": "text",
|
||||||
|
"bold": true,
|
||||||
|
"note": "An HTTP route. From a spec, or from what was actually called."
|
||||||
|
},
|
||||||
|
"operation": {
|
||||||
|
"shape": "box",
|
||||||
|
"rounded": true,
|
||||||
|
"fill": "surface-2",
|
||||||
|
"border": "accent",
|
||||||
|
"text": "text",
|
||||||
|
"bold": true,
|
||||||
|
"note": "A GraphQL operation \u2014 usually one endpoint carrying many, which is why it is its own kind rather than a path."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"groups": {
|
"groups": {
|
||||||
@@ -306,6 +324,25 @@
|
|||||||
"arrowhead": "normal",
|
"arrowhead": "normal",
|
||||||
"arrowsize": 0.8,
|
"arrowsize": 0.8,
|
||||||
"text": "accent"
|
"text": "accent"
|
||||||
|
},
|
||||||
|
"follows": {
|
||||||
|
"color": "accent",
|
||||||
|
"arrowhead": "normal",
|
||||||
|
"arrowsize": 0.7,
|
||||||
|
"text": "accent",
|
||||||
|
"note": "Observed order. Consecutive, not caused-by; the weight is the signal."
|
||||||
|
},
|
||||||
|
"accepts": {
|
||||||
|
"color": "border-strong",
|
||||||
|
"arrowhead": "normal",
|
||||||
|
"arrowsize": 0.6,
|
||||||
|
"text": "text-dim"
|
||||||
|
},
|
||||||
|
"returns": {
|
||||||
|
"color": "station",
|
||||||
|
"arrowhead": "normal",
|
||||||
|
"arrowsize": 0.6,
|
||||||
|
"text": "text-dim"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"domain_slots": {
|
"domain_slots": {
|
||||||
|
|||||||
Reference in New Issue
Block a user