updates 33.1 84
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -19,6 +19,10 @@ dist/
|
|||||||
# Generated runnable instance (entirely gitignored - regenerate with build.py)
|
# Generated runnable instance (entirely gitignored - regenerate with build.py)
|
||||||
gen/
|
gen/
|
||||||
|
|
||||||
|
# Specs and sheets uploaded through shuntgen's UI. Inputs someone dropped in a
|
||||||
|
# browser, not source — and often a client's real data.
|
||||||
|
soleprint/station/tools/shuntgen/uploads/
|
||||||
|
|
||||||
# Room configurations (separate repo - contains credentials and room-specific data)
|
# Room configurations (separate repo - contains credentials and room-specific data)
|
||||||
# Keep cfg/standalone/ and cfg/sample/ as templates, ignore actual rooms
|
# Keep cfg/standalone/ and cfg/sample/ as templates, ignore actual rooms
|
||||||
cfg/amar/
|
cfg/amar/
|
||||||
|
|||||||
220
build.py
220
build.py
@@ -317,6 +317,221 @@ def copy_cfg(output_dir: Path, room: str):
|
|||||||
copy_path(item, output_dir / item.name)
|
copy_path(item, output_dir / item.name)
|
||||||
|
|
||||||
|
|
||||||
|
def load_cabinets(room: str) -> list[dict]:
|
||||||
|
"""The dependency containers a room asked for, in the order it listed them.
|
||||||
|
|
||||||
|
Read from cfg/<room>/data/cabinets.json — the same shape and the same place
|
||||||
|
as its sibling data/*.json files, so nothing new has to know about it.
|
||||||
|
Entries are {"name": "postgres"} and may carry an "env" override.
|
||||||
|
"""
|
||||||
|
path = SPR_ROOT / "cfg" / room / "data" / "cabinets.json"
|
||||||
|
if not path.exists():
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
entries = json.loads(path.read_text())
|
||||||
|
except ValueError as e:
|
||||||
|
log.warning(f" cabinets.json is not valid JSON, ignoring: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
out = []
|
||||||
|
for entry in entries if isinstance(entries, list) else []:
|
||||||
|
if isinstance(entry, str):
|
||||||
|
entry = {"name": entry}
|
||||||
|
if isinstance(entry, dict) and entry.get("name"):
|
||||||
|
out.append(entry)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_cabinets(requested: list[dict]) -> list[dict]:
|
||||||
|
"""Expand each request into its definition, pulling in what it depends on.
|
||||||
|
|
||||||
|
Airflow without postgres is a container that exits on boot, so a cabinet's
|
||||||
|
depends_on is added for you rather than left as something to remember.
|
||||||
|
"""
|
||||||
|
cabinets_dir = SPR_ROOT / "soleprint" / "station" / "cabinets"
|
||||||
|
resolved: dict[str, dict] = {}
|
||||||
|
|
||||||
|
def add(name: str, overrides: dict) -> None:
|
||||||
|
if name in resolved:
|
||||||
|
# Already pulled in as somebody's dependency. The room asking for it
|
||||||
|
# by name is the more specific statement, so its env still applies —
|
||||||
|
# otherwise declaring airflow before postgres would silently drop
|
||||||
|
# postgres's settings.
|
||||||
|
if overrides.get("env"):
|
||||||
|
resolved[name]["env"] = {
|
||||||
|
**resolved[name].get("env", {}),
|
||||||
|
**overrides["env"],
|
||||||
|
}
|
||||||
|
return
|
||||||
|
definition_path = cabinets_dir / name / "cabinet.json"
|
||||||
|
if not definition_path.exists():
|
||||||
|
available = sorted(
|
||||||
|
p.name for p in cabinets_dir.iterdir() if p.is_dir()
|
||||||
|
) if cabinets_dir.exists() else []
|
||||||
|
log.warning(f" no such cabinet: {name} (available: {', '.join(available) or 'none'})")
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
definition = json.loads(definition_path.read_text())
|
||||||
|
except ValueError as e:
|
||||||
|
log.warning(f" cabinet {name} has invalid cabinet.json: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Mark it claimed before recursing, so a dependency cycle terminates.
|
||||||
|
resolved[name] = definition
|
||||||
|
for dependency in definition.get("depends_on", []) or []:
|
||||||
|
add(dependency, {})
|
||||||
|
|
||||||
|
definition["env"] = {**definition.get("env", {}), **overrides.get("env", {})}
|
||||||
|
|
||||||
|
for entry in requested:
|
||||||
|
add(entry["name"], entry)
|
||||||
|
|
||||||
|
# Dependencies first, so compose reads in the order things start.
|
||||||
|
ordered: list[dict] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
def emit(name: str) -> None:
|
||||||
|
if name in seen or name not in resolved:
|
||||||
|
return
|
||||||
|
seen.add(name)
|
||||||
|
for dependency in resolved[name].get("depends_on", []) or []:
|
||||||
|
emit(dependency)
|
||||||
|
ordered.append(resolved[name])
|
||||||
|
|
||||||
|
for name in resolved:
|
||||||
|
emit(name)
|
||||||
|
return ordered
|
||||||
|
|
||||||
|
|
||||||
|
def compose_cabinets(output_dir: Path, room: str):
|
||||||
|
"""Merge the room's cabinets into its docker-compose.yml and .env.example.
|
||||||
|
|
||||||
|
This is the compile step for dependencies: a room declares postgres, and the
|
||||||
|
built instance comes out with postgres in its compose file rather than with
|
||||||
|
instructions for adding it.
|
||||||
|
"""
|
||||||
|
requested = load_cabinets(room)
|
||||||
|
if not requested:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml
|
||||||
|
except ImportError:
|
||||||
|
log.warning(
|
||||||
|
" cabinets need PyYAML to merge into docker-compose.yml "
|
||||||
|
"(pip install pyyaml) — skipping"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
cabinets = resolve_cabinets(requested)
|
||||||
|
if not cabinets:
|
||||||
|
return
|
||||||
|
|
||||||
|
compose_path = output_dir / "docker-compose.yml"
|
||||||
|
if not compose_path.exists():
|
||||||
|
log.warning(
|
||||||
|
f" no docker-compose.yml in {output_dir.name}, "
|
||||||
|
f"so there is nothing to merge {len(cabinets)} cabinet(s) into"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
original = compose_path.read_text()
|
||||||
|
# A YAML round-trip drops every comment, and the room's compose file leads
|
||||||
|
# with the one that says how to run it. Keep the header block; the rest is
|
||||||
|
# generated anyway.
|
||||||
|
header = []
|
||||||
|
for line in original.splitlines():
|
||||||
|
if line.startswith("#") or not line.strip():
|
||||||
|
header.append(line)
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
while header and not header[-1].strip():
|
||||||
|
header.pop()
|
||||||
|
|
||||||
|
compose = yaml.safe_load(original) or {}
|
||||||
|
services = compose.setdefault("services", {})
|
||||||
|
volumes = compose.setdefault("volumes", {}) or {}
|
||||||
|
cabinets_dir = SPR_ROOT / "soleprint" / "station" / "cabinets"
|
||||||
|
|
||||||
|
added, skipped = [], []
|
||||||
|
for cabinet in cabinets:
|
||||||
|
name = cabinet["name"]
|
||||||
|
service_name = cabinet.get("service", name)
|
||||||
|
|
||||||
|
# The room's own compose file is the authority. A room that already
|
||||||
|
# declares `db` has arranged it deliberately, and silently replacing it
|
||||||
|
# would be the worst possible outcome of switching a cabinet on.
|
||||||
|
if service_name in services:
|
||||||
|
skipped.append(service_name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
fragment_path = cabinets_dir / name / "service.yml"
|
||||||
|
if not fragment_path.exists():
|
||||||
|
log.warning(f" cabinet {name} has no service.yml")
|
||||||
|
continue
|
||||||
|
|
||||||
|
fragment = yaml.safe_load(fragment_path.read_text()) or {}
|
||||||
|
for key, value in fragment.items():
|
||||||
|
if key in services:
|
||||||
|
skipped.append(key)
|
||||||
|
continue
|
||||||
|
services[key] = value
|
||||||
|
added.append(key)
|
||||||
|
|
||||||
|
for volume in cabinet.get("volumes", []) or []:
|
||||||
|
volumes.setdefault(volume, None)
|
||||||
|
|
||||||
|
if volumes:
|
||||||
|
compose["volumes"] = volumes
|
||||||
|
|
||||||
|
rendered = yaml.safe_dump(compose, sort_keys=False, default_flow_style=False)
|
||||||
|
banner = f"# Cabinets merged in by build.py: {', '.join(c['name'] for c in cabinets)}.\n"
|
||||||
|
preamble = ("\n".join(header) + "\n" + banner + "\n") if header else banner + "\n"
|
||||||
|
compose_path.write_text(preamble + rendered)
|
||||||
|
|
||||||
|
if added:
|
||||||
|
log.info(f" cabinets: {', '.join(added)}")
|
||||||
|
if skipped:
|
||||||
|
log.info(f" cabinets already declared by the room, left alone: {', '.join(skipped)}")
|
||||||
|
|
||||||
|
_append_cabinet_env(output_dir, cabinets)
|
||||||
|
|
||||||
|
|
||||||
|
def _append_cabinet_env(output_dir: Path, cabinets: list[dict]):
|
||||||
|
"""Add each cabinet's settings to .env.example, without touching .env."""
|
||||||
|
example = output_dir / ".env.example"
|
||||||
|
existing = example.read_text() if example.exists() else ""
|
||||||
|
# Match whole settings, not substrings: `POSTGRES_DB=` appears inside
|
||||||
|
# `MY_POSTGRES_DB=`, and a substring test would decide the setting was
|
||||||
|
# already there and skip it.
|
||||||
|
declared = {
|
||||||
|
line.split("=", 1)[0].strip()
|
||||||
|
for line in existing.splitlines()
|
||||||
|
if "=" in line and not line.lstrip().startswith("#")
|
||||||
|
}
|
||||||
|
|
||||||
|
lines = []
|
||||||
|
for cabinet in cabinets:
|
||||||
|
env = cabinet.get("env", {})
|
||||||
|
if not env:
|
||||||
|
continue
|
||||||
|
block = [f"\n# ── {cabinet.get('title', cabinet['name'])} (cabinet) ──"]
|
||||||
|
for note in cabinet.get("notes", []) or []:
|
||||||
|
block.append(f"# {note}")
|
||||||
|
wrote = False
|
||||||
|
for key, value in env.items():
|
||||||
|
if key in declared:
|
||||||
|
continue
|
||||||
|
block.append(f"{key}={value}")
|
||||||
|
declared.add(key)
|
||||||
|
wrote = True
|
||||||
|
if wrote:
|
||||||
|
lines.extend(block)
|
||||||
|
|
||||||
|
if lines:
|
||||||
|
example.write_text(existing.rstrip("\n") + "\n" + "\n".join(lines) + "\n")
|
||||||
|
|
||||||
|
|
||||||
def build_soleprint(output_dir: Path, room: str):
|
def build_soleprint(output_dir: Path, room: str):
|
||||||
"""Build soleprint folder with core + room config merged."""
|
"""Build soleprint folder with core + room config merged."""
|
||||||
soleprint = SPR_ROOT / "soleprint"
|
soleprint = SPR_ROOT / "soleprint"
|
||||||
@@ -348,6 +563,11 @@ def build_soleprint(output_dir: Path, room: str):
|
|||||||
# Room config (includes merging room-specific artery/atlas/station)
|
# Room config (includes merging room-specific artery/atlas/station)
|
||||||
copy_cfg(output_dir, room)
|
copy_cfg(output_dir, room)
|
||||||
|
|
||||||
|
# Dependency containers the room asked for, merged into its compose file.
|
||||||
|
# After copy_cfg, because the compose file being merged into is the room's.
|
||||||
|
log.info("Composing cabinets...")
|
||||||
|
compose_cabinets(output_dir, room)
|
||||||
|
|
||||||
# Generate models
|
# Generate models
|
||||||
log.info("Generating models...")
|
log.info("Generating models...")
|
||||||
if not generate_models(output_dir, room):
|
if not generate_models(output_dir, room):
|
||||||
|
|||||||
124
docs/data/en/export.md
Normal file
124
docs/data/en/export.md
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
# Export / Compile
|
||||||
|
|
||||||
|
Soleprint's source tree is not what runs. `build.py` compiles the framework plus
|
||||||
|
a room's configuration into a self-contained instance under `gen/<room>/`, and
|
||||||
|
that directory is what a container boots, what `deploy.sh` rsyncs, and what the
|
||||||
|
cluster manifests point at.
|
||||||
|
|
||||||
|
Everything below is a `make` target, and every target is one script in `ctrl/`.
|
||||||
|
The logic lives in the scripts, never in the Makefile.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make build # cfg/standalone -> gen/standalone
|
||||||
|
make build sample # cfg/sample -> gen/sample
|
||||||
|
make start # run it
|
||||||
|
```
|
||||||
|
|
||||||
|
## The targets
|
||||||
|
|
||||||
|
| Command | Runs | Does |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `make build [<room>\|all\|models]` | `ctrl/build.sh` | compile a room into `gen/` |
|
||||||
|
| `make start [<room>] [-d]` | `ctrl/start.sh` | run a built room's compose stack |
|
||||||
|
| `make stop [<room>]` | `ctrl/stop.sh` | stop it |
|
||||||
|
| `make cluster [up\|down\|status]` | `ctrl/cluster.sh` | the shared kind cluster |
|
||||||
|
| `make component [list\|publish\|sync\|watch\|diff]` | `ctrl/spr.py` | publish a distributable component |
|
||||||
|
| `make deploy` | `ctrl/deploy.sh` | rsync `gen/standalone` to the server and restart |
|
||||||
|
|
||||||
|
Bare words pass straight through, so `make build sample` becomes
|
||||||
|
`ctrl/build.sh sample`. Anything starting with a dash would be eaten by make
|
||||||
|
itself, so those go through `ARGS`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make deploy ARGS="--build"
|
||||||
|
make component ARGS="publish soleprint-ui /tmp/out --dist"
|
||||||
|
```
|
||||||
|
|
||||||
|
## What a build does
|
||||||
|
|
||||||
|
`python build.py --cfg <room>` runs these in order:
|
||||||
|
|
||||||
|
1. **Clean** `gen/<room>/`. A build is not incremental — a stale file left
|
||||||
|
behind is worse than a slow build.
|
||||||
|
2. **Copy the framework.** `main.py`, `run.py`, `index.html`, `Dockerfile`,
|
||||||
|
`requirements.txt`, `dataloader/`, `common/`, and the three systems
|
||||||
|
(`artery/`, `atlas/`, `station/`).
|
||||||
|
3. **Merge the room** (`copy_cfg`). `cfg/<room>/config.json` lands in `cfg/`,
|
||||||
|
`data/*.json` in `data/`, and anything under `cfg/<room>/soleprint/artery|atlas|station/`
|
||||||
|
is merged *over* the framework copy — which is how a room adds its own vein,
|
||||||
|
shunt, tool or generator without forking the tree.
|
||||||
|
4. **Compose cabinets.** The dependency containers the room declared in
|
||||||
|
`data/cabinets.json` are merged into its `docker-compose.yml`. See
|
||||||
|
[Cabinets](#station-cabinets).
|
||||||
|
5. **Generate models.** modelgen reads the room's `config.json` and writes
|
||||||
|
`models/pydantic/__init__.py`.
|
||||||
|
6. **Render k8s** (optional). When the room's config enables it,
|
||||||
|
`soleprint/ctrl/k8s/` writes manifests and lifecycle scripts.
|
||||||
|
|
||||||
|
## What comes out
|
||||||
|
|
||||||
|
A standalone room is a flat instance:
|
||||||
|
|
||||||
|
```
|
||||||
|
gen/standalone/
|
||||||
|
run.py main.py Dockerfile docker-compose.yml
|
||||||
|
artery/ atlas/ station/ common/
|
||||||
|
cfg/config.json
|
||||||
|
data/*.json
|
||||||
|
models/pydantic/
|
||||||
|
```
|
||||||
|
|
||||||
|
A **managed** room — one that wraps an existing application — is three folders
|
||||||
|
instead, because soleprint sits beside the app rather than containing it:
|
||||||
|
|
||||||
|
```
|
||||||
|
gen/<room>/
|
||||||
|
<app>/ the application's repos, plus its ctrl scripts
|
||||||
|
link/ bridge code between the two
|
||||||
|
soleprint/ the instance, exactly as above
|
||||||
|
```
|
||||||
|
|
||||||
|
`build.py` picks between them on whether the room's `config.json` has a
|
||||||
|
`managed` block. `gen/` is gitignored in full: it is an artifact, and the way to
|
||||||
|
change it is to change `cfg/<room>/` and rebuild.
|
||||||
|
|
||||||
|
## Distributing components
|
||||||
|
|
||||||
|
Rooms are compiled; *components* are published. `registry.json` lists what can
|
||||||
|
be shipped out of this repo on its own:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make component # list
|
||||||
|
make component ARGS="publish soleprint-ui /tmp/out --dist"
|
||||||
|
make component ARGS="watch soleprint-ui ../unt/ui/framework"
|
||||||
|
```
|
||||||
|
|
||||||
|
`--dist` copies only the built bundle — `dist/**` plus `package.json`,
|
||||||
|
`README.md` and `LICENSE` — rather than the source. It refuses to publish an
|
||||||
|
empty `dist/`, because a component whose bundle was never built is the failure
|
||||||
|
that shows up later as a container that starts and renders nothing:
|
||||||
|
|
||||||
|
```
|
||||||
|
no build at soleprint/common/ui
|
||||||
|
build it first: cd soleprint/common/ui && pnpm build
|
||||||
|
```
|
||||||
|
|
||||||
|
Each publish leaves a `.spr` stamp in the destination recording name, version,
|
||||||
|
type, source and mode, so a copy can say where it came from.
|
||||||
|
|
||||||
|
## Deploying
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make deploy ARGS="--build" # rebuild, sync, restart
|
||||||
|
make deploy ARGS="--sync-only" # sync, leave it running
|
||||||
|
```
|
||||||
|
|
||||||
|
`deploy.sh` rsyncs `gen/standalone/` and runs `docker compose up -d --build` on
|
||||||
|
the far side. `.env` is excluded, so server secrets stay on the server.
|
||||||
|
|
||||||
|
## Running without building
|
||||||
|
|
||||||
|
`python run.py` from `soleprint/` serves every subsystem on one port (12000 by
|
||||||
|
default) straight from the source tree. It is for developing the framework
|
||||||
|
itself; a room's `cfg/config.json` does not exist there, so the landing pages
|
||||||
|
fall back to their defaults. Rooms use docker.
|
||||||
107
docs/data/en/station-cabinets.md
Normal file
107
docs/data/en/station-cabinets.md
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
# Cabinets
|
||||||
|
|
||||||
|
A cabinet is a **dependency container** a room can switch on: postgres, redis,
|
||||||
|
airflow. The vocabulary already had the word — `execution.container` in every
|
||||||
|
room's `config.json` is *"Cabinet — tool container"* — and until now nothing
|
||||||
|
stood behind it.
|
||||||
|
|
||||||
|
The problem it solves: a generated artifact knows what it needs and had no way
|
||||||
|
to say so. A shunt built from a client's spreadsheets holds its rows in memory
|
||||||
|
happily, but the moment you want them to survive a restart you need postgres,
|
||||||
|
and wiring postgres in meant hand-editing a room's `docker-compose.yml` and then
|
||||||
|
hand-editing the cluster too. A cabinet is that declaration, made once and read
|
||||||
|
by both paths.
|
||||||
|
|
||||||
|
## Switching one on
|
||||||
|
|
||||||
|
Add `cfg/<room>/data/cabinets.json`, the same shape as its sibling `data/*.json`
|
||||||
|
files:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{ "name": "postgres" },
|
||||||
|
{ "name": "redis" },
|
||||||
|
{ "name": "airflow", "env": { "AIRFLOW_ADMIN_PASSWORD": "change-me" } }
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Then build. The compose merge is a step in [Export / Compile](#export):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python build.py --cfg sample
|
||||||
|
cd gen/sample && docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
`build.py` merges each cabinet's compose fragment into the room's
|
||||||
|
`docker-compose.yml`, declares its named volumes, and appends its settings to
|
||||||
|
`.env.example` — never to `.env`.
|
||||||
|
|
||||||
|
**A service the room already declares wins.** `cfg/amar/docker-compose.yml`
|
||||||
|
ships its own `db`; switching the postgres cabinet on will not replace it. The
|
||||||
|
build says so when it skips one:
|
||||||
|
|
||||||
|
```
|
||||||
|
Composing cabinets...
|
||||||
|
cabinets: redis, airflow
|
||||||
|
cabinets already declared by the room, left alone: postgres
|
||||||
|
```
|
||||||
|
|
||||||
|
Dependencies come along automatically. Airflow without a metadata database is a
|
||||||
|
container that exits on boot, so asking for `airflow` brings `postgres` and
|
||||||
|
`redis` with it, ordered so compose reads them before the thing that needs them.
|
||||||
|
|
||||||
|
## On a cluster
|
||||||
|
|
||||||
|
Every cabinet names a `rig_addon`. Where a room runs on kind rather than
|
||||||
|
compose, the same dependency installs as a rig addon of that name:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd rig
|
||||||
|
PROFILE=data make cluster up
|
||||||
|
PROFILE=data make addons install
|
||||||
|
|
||||||
|
kubectl -n data port-forward svc/postgres 5432:5432
|
||||||
|
kubectl -n data port-forward svc/airflow 8080:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
The two paths are deliberately separate — compose for a laptop, manifests for a
|
||||||
|
cluster — and `rig_addon` is the thread between them, so the room declares the
|
||||||
|
dependency once either way. The addons generate their own passwords on first
|
||||||
|
install and keep them across re-runs, so re-running never rotates a credential
|
||||||
|
out from under something already connected.
|
||||||
|
|
||||||
|
## What ships
|
||||||
|
|
||||||
|
| Cabinet | Image | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `postgres` | `postgres:16-alpine` | healthcheck wired, so `depends_on: service_healthy` works |
|
||||||
|
| `redis` | `redis:7-alpine` | cache, and the broker for anything queue-shaped |
|
||||||
|
| `airflow` | `apache/airflow:2.10.4` | one container on `standalone`; needs postgres and redis |
|
||||||
|
|
||||||
|
## Writing one
|
||||||
|
|
||||||
|
```
|
||||||
|
soleprint/station/cabinets/<name>/
|
||||||
|
cabinet.json what it is, what it needs, what it exports
|
||||||
|
service.yml the compose service, verbatim
|
||||||
|
```
|
||||||
|
|
||||||
|
`cabinet.json`:
|
||||||
|
|
||||||
|
| Key | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `name` | must match the directory |
|
||||||
|
| `title`, `description` | shown on the station index |
|
||||||
|
| `service` | the key to merge under in `services:` (defaults to `name`) |
|
||||||
|
| `env` | settings and defaults, written to `.env.example` |
|
||||||
|
| `volumes` | named volumes to declare at the top level |
|
||||||
|
| `depends_on` | other cabinets that must come with it |
|
||||||
|
| `rig_addon` | the matching `rig/ctrl/addons/<name>.sh`, if there is one |
|
||||||
|
| `notes` | lines written into `.env.example` as comments |
|
||||||
|
|
||||||
|
`service.yml` is a plain compose fragment with one top-level key — the service
|
||||||
|
name. It stays YAML rather than being generated from JSON so it reads like the
|
||||||
|
file it becomes, and so anything compose supports is available without this tool
|
||||||
|
learning about it first.
|
||||||
|
|
||||||
|
Adding a cabinet is adding a directory. Nothing dispatches on the name.
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
# Datagen
|
# Datagen
|
||||||
|
|
||||||
Test data generator using faker. Produces realistic, domain-specific data for testing and development.
|
Test data generator. Produces realistic, domain-specific records for testing and
|
||||||
|
development, from generators a room writes or that
|
||||||
|
[modelgen](#station-modelgen) writes for it.
|
||||||
|
|
||||||
**Status:** live
|
**Status:** live
|
||||||
|
|
||||||
@@ -8,50 +10,95 @@ Test data generator using faker. Produces realistic, domain-specific data for te
|
|||||||
|
|
||||||
## What It Does
|
## What It Does
|
||||||
|
|
||||||
Datagen generates fake but realistic data. Names, emails, addresses, transactions -- whatever your domain needs. It uses Python's faker library under the hood.
|
Datagen hands out instances of a room's models. Core ships the base class, the
|
||||||
|
discovery, the HTTP API and the browser UI; the generators themselves belong to
|
||||||
|
a room, because what counts as realistic is a property of the domain.
|
||||||
|
|
||||||
Core datagen is a placeholder. The real work happens in room-specific generators.
|
Generation is stdlib `random`, `uuid` and `datetime` — **not** faker, which is
|
||||||
|
not a dependency of this repo.
|
||||||
|
|
||||||
## Structure
|
## Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
soleprint/station/tools/datagen/ # Core (base classes, placeholder)
|
soleprint/station/tools/datagen/ # base class, api, UI
|
||||||
cfg/<room>/soleprint/station/tools/datagen/ # Room-specific generators
|
cfg/<room>/soleprint/station/tools/datagen/ # the room's generators
|
||||||
```
|
```
|
||||||
|
|
||||||
After build, both merge into `gen/<room>/station/tools/datagen/`.
|
After build, both merge into `gen/<room>/station/tools/datagen/`.
|
||||||
|
|
||||||
## Pattern
|
## The contract
|
||||||
|
|
||||||
Rooms subclass a base generator and provide domain-specific data factories:
|
A generator subclasses `BaseDataGenerator` and defines **one method per model,
|
||||||
|
named after it**. There is no registration step: the method name *is* the model
|
||||||
|
name.
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from station.tools.datagen.base import BaseGenerator
|
from station.tools.datagen.base import BaseDataGenerator
|
||||||
|
|
||||||
class RoomDataGenerator(BaseGenerator):
|
class RoomDataGenerator(BaseDataGenerator):
|
||||||
def generate_customers(self, count=10):
|
def customer(self, **kwargs):
|
||||||
return [self.fake_customer() for _ in range(count)]
|
return {"id": str(uuid4()), "name": ..., "email": ..., **kwargs}
|
||||||
|
|
||||||
def fake_customer(self):
|
def invoice(self, customer_id=None, **kwargs):
|
||||||
return {
|
return {"id": str(uuid4()), "customer_id": customer_id, **kwargs}
|
||||||
"name": self.faker.name(),
|
```
|
||||||
"email": self.faker.email(),
|
|
||||||
"phone": self.faker.phone_number(),
|
The base class provides:
|
||||||
}
|
|
||||||
|
| Method | Does |
|
||||||
|
| --- | --- |
|
||||||
|
| `generate(model, count=1, **kwargs)` | calls the matching method `count` times; `kwargs` pass through to every call |
|
||||||
|
| `available_models()` | the method names, which are the model names |
|
||||||
|
| `schema()` | override to return a graphgen-compatible schema |
|
||||||
|
|
||||||
|
Discovery is by convention too: any `*.py` in the datagen directory whose first
|
||||||
|
class ends in `Generator` is loaded and keyed by its filename.
|
||||||
|
|
||||||
|
## Generating a generator
|
||||||
|
|
||||||
|
Writing one by hand is optional. modelgen's `datagen` target emits the whole
|
||||||
|
class from a schema — and when the schema came from spreadsheets, the generated
|
||||||
|
class **samples the real rows** rather than inventing values:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t datagen
|
||||||
|
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t datagen
|
||||||
```
|
```
|
||||||
|
|
||||||
Each room defines what data it needs. Core provides the faker instance and base class. Rooms provide the factories.
|
This is how [shuntgen](#station-shuntgen) fills a generated shunt.
|
||||||
|
|
||||||
|
## HTTP API
|
||||||
|
|
||||||
|
Mounted under `/station/tools/datagen/`:
|
||||||
|
|
||||||
|
| Route | Returns |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /` | the browser UI |
|
||||||
|
| `GET /api/generators` | loaded generator files and their models |
|
||||||
|
| `GET /api/models?generator=` | model names |
|
||||||
|
| `POST /api/generate` | `{model, count, generator?, kwargs?}` → the records |
|
||||||
|
| `GET /api/schema?generator=` | the generator's schema, if it exposes one |
|
||||||
|
|
||||||
## Room Configuration
|
## Feeding graphgen
|
||||||
|
|
||||||
Room generators live in `cfg/<room>/soleprint/station/tools/datagen/`. They are fully self-contained -- they define their own models, factories, and output formats.
|
A generator that overrides `schema()` is surfaced at `/api/schema` in the format
|
||||||
|
[graphgen](#station-graphgen) reads, so the same definition draws the diagram:
|
||||||
|
|
||||||
The core module provides:
|
```python
|
||||||
- Base generator class with faker instance
|
def schema(self):
|
||||||
- CLI entry point
|
return {
|
||||||
- Output formatting (JSON, CSV)
|
"models": {
|
||||||
|
"Invoice": {
|
||||||
|
"doc": "A billed order.",
|
||||||
|
"fields": {
|
||||||
|
"id": {"type": "UUID", "pk": True},
|
||||||
|
"customer_id": {"type": "FK:Customer"},
|
||||||
|
"total": {"type": "float"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
Rooms provide:
|
`FK:<Model>` and `M2M:<Model>` are how relations are written. modelgen's
|
||||||
- Domain-specific generator subclasses
|
`datagen` target emits this method for you.
|
||||||
- Field definitions and relationships
|
|
||||||
- Volume and distribution configuration
|
|
||||||
|
|||||||
@@ -1,54 +1,111 @@
|
|||||||
# Modelgen
|
# Modelgen
|
||||||
|
|
||||||
Generates platform-specific models from JSON Schema. Reads schema once, writes models for multiple targets.
|
Multi-source, multi-target model generator. Reads a schema from wherever it
|
||||||
|
already lives, and writes it out for every stack that needs it.
|
||||||
|
|
||||||
**Status:** dev
|
**Status:** live
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## What It Does
|
## What It Does
|
||||||
|
|
||||||
Modelgen takes a JSON Schema definition and produces model code for different platforms:
|
Everything passes through one intermediate representation — `ModelDefinition`,
|
||||||
|
`FieldDefinition`, `EnumDefinition`. **Loaders** fill it, **generators** emit
|
||||||
|
from it, and the two sides do not know about each other. Adding an input means
|
||||||
|
one extractor and every output comes with it; adding an output means one
|
||||||
|
generator and every input already feeds it.
|
||||||
|
|
||||||
- **Pydantic** -- Python data validation models
|
```
|
||||||
- **Django ORM** -- Django model classes
|
dataclasses ─┐ ┌─ pydantic
|
||||||
- **Prisma** -- Prisma schema definitions
|
Django │ ├─ django
|
||||||
|
SQLAlchemy ├──▶ ModelDefinition ──▶├─ sqlmodel
|
||||||
|
a live DB │ FieldDefinition ├─ typescript
|
||||||
|
OpenAPI │ EnumDefinition ├─ protobuf
|
||||||
|
CSV/ODS ─┘ ├─ prisma
|
||||||
|
├─ strawberry
|
||||||
|
├─ schema (graphgen)
|
||||||
|
└─ datagen
|
||||||
|
```
|
||||||
|
|
||||||
One schema, multiple outputs.
|
Core is **pure standard library**. It is published as `soleprint-modelgen` and
|
||||||
|
installs with no dependencies; live-database extraction is an extra
|
||||||
|
(`pip install "soleprint-modelgen[db]"`), and YAML specs need PyYAML.
|
||||||
|
|
||||||
## Extractors
|
## Sources
|
||||||
|
|
||||||
Modelgen also works in reverse. Extractors read existing codebases and produce a normalized schema representation:
|
| Command | Reads |
|
||||||
|
| --- | --- |
|
||||||
|
| `from-schema` | Python dataclasses in a `schema/` folder |
|
||||||
|
| `from-config` | a room's `config.json` |
|
||||||
|
| `extract` | a Django or SQLAlchemy codebase (`--framework auto` detects) |
|
||||||
|
| `from-db` | a live database, any SQLAlchemy dialect |
|
||||||
|
| `from-openapi` | an OpenAPI 3.x / Swagger 2.0 document |
|
||||||
|
| `from-tabular` | a directory of `.csv` / `.tsv` / `.ods` spreadsheets |
|
||||||
|
|
||||||
- **Django extractor** -- reads Django model files
|
```bash
|
||||||
- **SQLAlchemy extractor** -- reads SQLAlchemy model files
|
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript,schema
|
||||||
- **Prisma extractor** -- reads Prisma schema files
|
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t pydantic,datagen
|
||||||
|
python -m station.tools.modelgen extract -s /path/to/django -o out/ -t prisma
|
||||||
|
python -m station.tools.modelgen from-db -u postgresql://… -o out/ -t typescript
|
||||||
|
python -m station.tools.modelgen list-formats
|
||||||
|
```
|
||||||
|
|
||||||
Extractors feed into graphgen for visualization.
|
### From a spec
|
||||||
|
|
||||||
## Output
|
`components.schemas` (or Swagger's `definitions`) become models. `$ref` chains
|
||||||
|
and `allOf` are resolved, enums are materialised as real `Enum` classes so every
|
||||||
|
target names them properly, and a referenced object becomes a relation rather
|
||||||
|
than a nested type — the same call the database extractor makes, and what keeps
|
||||||
|
the generated code valid for every target.
|
||||||
|
|
||||||
Generated models are written to `gen/<room>/models/`.
|
The parse also yields the *operations*, which is what
|
||||||
|
[shuntgen](#station-shuntgen) turns into routes.
|
||||||
|
|
||||||
```
|
### From spreadsheets
|
||||||
gen/<room>/models/
|
|
||||||
├── pydantic/
|
|
||||||
├── django/
|
|
||||||
└── prisma/
|
|
||||||
```
|
|
||||||
|
|
||||||
## CLI
|
One model per CSV file, one per sheet in an ODS workbook. Column types are
|
||||||
|
inferred from the values actually present, and a blank cell makes the column
|
||||||
|
optional. Keys and relations are inferred by name and then confirmed against the
|
||||||
|
data: an `id` column that is not unique is not treated as a key, and
|
||||||
|
`customer_id` is only a foreign key if a `customers` sheet came with it.
|
||||||
|
|
||||||
```bash
|
The rows are kept, not just the shape — which is what lets the `datagen` target
|
||||||
python -m modelgen
|
sample real values instead of inventing them.
|
||||||
```
|
|
||||||
|
|
||||||
Reads from `schema.json` (the project source of truth) and writes to the configured output directory.
|
ODS is read with `zipfile` and `ElementTree`. No odfpy, no pandas: the
|
||||||
|
dependency-free promise is what makes this package publishable on its own.
|
||||||
|
|
||||||
## Shared Distribution
|
## Targets
|
||||||
|
|
||||||
Modelgen is also distributed as a shared component via `ctrl/spr.py`. This allows other projects to use model generation without running full soleprint.
|
`pydantic`, `django`, `sqlmodel`, `typescript` (`ts`), `protobuf` (`proto`),
|
||||||
|
`prisma`, `strawberry`, `schema` (`jsonschema`), `datagen`.
|
||||||
|
|
||||||
## Schema Source
|
Two are worth calling out:
|
||||||
|
|
||||||
|
- **`schema`** writes the graphgen-compatible `schema.json` — the portable
|
||||||
|
artifact [graphgen](#station-graphgen) and databrowse read directly.
|
||||||
|
Relations come out as `FK:<Model>` and `M2M:<Model>`.
|
||||||
|
- **`datagen`** writes a `BaseDataGenerator` subclass for
|
||||||
|
[datagen](#station-datagen), including its `schema()` override. Given
|
||||||
|
spreadsheet rows it samples them; otherwise it synthesises from the types.
|
||||||
|
|
||||||
|
Multiple targets in one run get one file each, named `models_<target><ext>`.
|
||||||
|
|
||||||
|
## In a build
|
||||||
|
|
||||||
|
`build.py` calls modelgen during every room build, writing
|
||||||
|
`gen/<room>/models/pydantic/__init__.py` from the room's `config.json`. See
|
||||||
|
[Export / Compile](#export).
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd soleprint/station/tools
|
||||||
|
python -m unittest modelgen.tests.test_extractors
|
||||||
|
```
|
||||||
|
|
||||||
The source of truth is `schema.json` at the project root. All model generation starts from this file. Room-specific schema extensions live in `cfg/<room>/models/`.
|
stdlib `unittest`, no pytest, and every input is built in a temp directory — the
|
||||||
|
tests have to pass with nothing installed and nothing else in the tree. Run them
|
||||||
|
from `station/tools/`, not from inside `modelgen/`: the package ships a
|
||||||
|
`types.py`, and putting its own directory on `sys.path` shadows the standard
|
||||||
|
library module of that name.
|
||||||
|
|||||||
129
docs/data/en/station-shuntgen.md
Normal file
129
docs/data/en/station-shuntgen.md
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
# Shuntgen
|
||||||
|
|
||||||
|
Generates runnable [shunts](#artery-shunts) from the two things people actually
|
||||||
|
have: a service contract, or a folder of spreadsheets.
|
||||||
|
|
||||||
|
Writing a shunt by hand means copying `artery/shunts/example/` and filling in
|
||||||
|
`responses.json` entry by entry. That is fine for three endpoints and untenable
|
||||||
|
for eighty — and it is the wrong work anyway, because the endpoints are already
|
||||||
|
described in the spec somebody handed you.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# a spec you were handed
|
||||||
|
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
|
||||||
|
|
||||||
|
# sheets a client sent
|
||||||
|
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
|
||||||
|
|
||||||
|
python -m station.tools.shuntgen list
|
||||||
|
```
|
||||||
|
|
||||||
|
Run from `soleprint/`. Also in the browser at `/station/tools/shuntgen/`, where
|
||||||
|
you can upload a spec, preview the routes it would serve, and generate.
|
||||||
|
|
||||||
|
## What comes out
|
||||||
|
|
||||||
|
```
|
||||||
|
artery/shunts/<name>/
|
||||||
|
main.py builds the app from the spec
|
||||||
|
run.py uvicorn entry point (PORT, or depot/config.json)
|
||||||
|
shunt_runtime.py vendored runtime — no soleprint import
|
||||||
|
models.py pydantic, via modelgen
|
||||||
|
datagen_<name>.py BaseDataGenerator subclass, via modelgen
|
||||||
|
depot/spec.json routes, collections and schema
|
||||||
|
depot/responses.json pinned overrides — yours, never overwritten
|
||||||
|
depot/config.json delays, error rate, prefill — yours, never overwritten
|
||||||
|
depot/data.json imported rows
|
||||||
|
templates/index.html config UI
|
||||||
|
README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd artery/shunts/books && python run.py
|
||||||
|
curl localhost:8098/customers
|
||||||
|
```
|
||||||
|
|
||||||
|
The routes are built at startup from `spec.json` rather than written out as
|
||||||
|
source. That keeps the generated code short enough to read, and puts the
|
||||||
|
behaviour in one reviewable place: fixing `runtime.py` fixes every shunt, and
|
||||||
|
regenerating is a copy.
|
||||||
|
|
||||||
|
## Where a response comes from
|
||||||
|
|
||||||
|
First hit wins:
|
||||||
|
|
||||||
|
1. `depot/responses.json` — a pinned override, keyed `"METHOD /path"`
|
||||||
|
2. the store — rows imported from sheets, plus anything POSTed since
|
||||||
|
3. the spec's `example`, if the source document carried one
|
||||||
|
4. `datagen_<name>.py`, synthesising from the schema
|
||||||
|
5. `{}`
|
||||||
|
|
||||||
|
The store is what makes it behave like a service rather than a random-value
|
||||||
|
faucet: POST something and GET it back, ask for `/pets/7` and get the pet whose
|
||||||
|
id is 7. Collections that arrived with no rows are prefilled with generated
|
||||||
|
ones, so the first call answers with something.
|
||||||
|
|
||||||
|
## Two sources, one pipeline
|
||||||
|
|
||||||
|
Both inputs are [modelgen](#station-modelgen) extractors, so the same shapes
|
||||||
|
also generate pydantic, TypeScript, prisma and a
|
||||||
|
[graphgen](#station-graphgen) schema:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript
|
||||||
|
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t schema,datagen
|
||||||
|
```
|
||||||
|
|
||||||
|
| Source | Becomes | Routes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| OpenAPI 3.x / Swagger 2.0 | one model per schema; enums become real Enums, `$ref` becomes a relation | the operations the document declares |
|
||||||
|
| `.csv` / `.tsv` / `.ods` | one model per file or sheet, types inferred per column | five CRUD routes per table |
|
||||||
|
|
||||||
|
Keys and relations are inferred by name and then **checked against the data**:
|
||||||
|
an `id` column that is not unique is not treated as a key, and `customer_id` is
|
||||||
|
only a foreign key if a `customers` sheet came with it.
|
||||||
|
|
||||||
|
ODS is read with `zipfile` and `ElementTree` — no odfpy, no pandas — which is
|
||||||
|
what lets modelgen stay dependency-free and publishable on its own.
|
||||||
|
|
||||||
|
## Control endpoints
|
||||||
|
|
||||||
|
Every generated shunt serves these:
|
||||||
|
|
||||||
|
| Endpoint | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /health` | liveness |
|
||||||
|
| `GET /mock/spec` | the routes it was built from |
|
||||||
|
| `GET /mock/stats` | call counts and row counts |
|
||||||
|
| `POST /mock/reset` | restore the imported rows, clear counters |
|
||||||
|
| `GET,POST /mock/config` | delays, error rate, `unknown_id`, page size |
|
||||||
|
| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# make it slow and flaky, the way the real thing is
|
||||||
|
curl -X POST localhost:8098/mock/config \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"enable_random_delays": true, "error_rate": 0.2}'
|
||||||
|
|
||||||
|
# make one call answer something specific
|
||||||
|
curl -X POST localhost:8098/mock/responses \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"GET /customers/1": {"id": 1, "name": "PINNED"}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
`unknown_id` decides what an unknown key does: `generate` (the default) invents
|
||||||
|
a record wearing the id that was asked for; `404` refuses it. Generate by
|
||||||
|
default, because a client pointed at a fresh shunt should just work — flip it
|
||||||
|
when the error path is what you are testing.
|
||||||
|
|
||||||
|
## Dependency containers
|
||||||
|
|
||||||
|
`--cabinet postgres,redis` writes a `cabinet.json` declaring what the shunt
|
||||||
|
expects. `build.py` composes those services into the room's compose file, and on
|
||||||
|
a cluster they install as rig addons of the same name. See
|
||||||
|
[Cabinets](#station-cabinets).
|
||||||
|
|
||||||
|
## Regenerating
|
||||||
|
|
||||||
|
Everything is overwritten except `depot/responses.json` and `depot/config.json`.
|
||||||
|
Those two are yours.
|
||||||
@@ -22,8 +22,11 @@
|
|||||||
{"id": "station-datagen", "title": {"en": "↳ Datagen"}, "sub": true},
|
{"id": "station-datagen", "title": {"en": "↳ Datagen"}, "sub": true},
|
||||||
{"id": "station-modelgen", "title": {"en": "↳ Modelgen"}, "sub": true},
|
{"id": "station-modelgen", "title": {"en": "↳ Modelgen"}, "sub": true},
|
||||||
{"id": "station-graphgen", "title": {"en": "↳ Graphgen"}, "sub": true},
|
{"id": "station-graphgen", "title": {"en": "↳ Graphgen"}, "sub": true},
|
||||||
|
{"id": "station-shuntgen", "title": {"en": "↳ Shuntgen"}, "sub": true},
|
||||||
{"id": "station-databrowse", "title": {"en": "↳ Databrowse"}, "sub": true},
|
{"id": "station-databrowse", "title": {"en": "↳ Databrowse"}, "sub": true},
|
||||||
|
{"id": "station-cabinets", "title": {"en": "↳ Cabinets"}, "sub": true},
|
||||||
|
|
||||||
{"id": "components", "title": {"en": "Shared Components"}},
|
{"id": "components", "title": {"en": "Shared Components"}},
|
||||||
|
{"id": "export", "title": {"en": "Export / Compile"}},
|
||||||
{"id": "deployment", "title": {"en": "Deployment"}}
|
{"id": "deployment", "title": {"en": "Deployment"}}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<!doctype html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
html {
|
html {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
}
|
}
|
||||||
body {
|
body {
|
||||||
font-family:
|
font-family:
|
||||||
@@ -25,8 +25,8 @@
|
|||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 2rem 1rem;
|
padding: 2rem 1rem;
|
||||||
line-height: 1.6;
|
line-height: 1.6;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
background: #b91c1c;
|
background: var(--system-accent);
|
||||||
}
|
}
|
||||||
header {
|
header {
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -51,7 +51,7 @@
|
|||||||
padding-bottom: 2rem;
|
padding-bottom: 2rem;
|
||||||
}
|
}
|
||||||
section {
|
section {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
padding: 1.5rem;
|
padding: 1.5rem;
|
||||||
margin: 1.5rem 0;
|
margin: 1.5rem 0;
|
||||||
border-radius: 12px;
|
border-radius: 12px;
|
||||||
@@ -59,23 +59,23 @@
|
|||||||
section h2 {
|
section h2 {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 1.2rem;
|
font-size: 1.2rem;
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition {
|
.composition {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 2px solid #b91c1c;
|
border: 2px solid var(--system-accent);
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
border-radius: 12px;
|
border-radius: 12px;
|
||||||
}
|
}
|
||||||
.composition h3 {
|
.composition h3 {
|
||||||
margin: 0 0 0.75rem 0;
|
margin: 0 0 0.75rem 0;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition > p {
|
.composition > p {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.components {
|
.components {
|
||||||
display: grid;
|
display: grid;
|
||||||
@@ -83,20 +83,20 @@
|
|||||||
gap: 0.75rem;
|
gap: 0.75rem;
|
||||||
}
|
}
|
||||||
.component {
|
.component {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
padding: 0.75rem;
|
padding: 0.75rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
.component h4 {
|
.component h4 {
|
||||||
margin: 0 0 0.25rem 0;
|
margin: 0 0 0.25rem 0;
|
||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.component p {
|
.component p {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.veins {
|
.veins {
|
||||||
display: grid;
|
display: grid;
|
||||||
@@ -104,8 +104,8 @@
|
|||||||
gap: 1rem;
|
gap: 1rem;
|
||||||
}
|
}
|
||||||
.vein {
|
.vein {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
text-align: center;
|
text-align: center;
|
||||||
@@ -113,16 +113,16 @@
|
|||||||
transition: all 0.15s;
|
transition: all 0.15s;
|
||||||
}
|
}
|
||||||
.vein:hover {
|
.vein:hover {
|
||||||
background: #2a2a2a;
|
background: var(--border);
|
||||||
}
|
}
|
||||||
.vein.selected {
|
.vein.selected {
|
||||||
border-color: #b91c1c;
|
border-color: var(--system-accent);
|
||||||
border-width: 2px;
|
border-width: 2px;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
}
|
}
|
||||||
.vein.active {
|
.vein.active {
|
||||||
background: #b91c1c;
|
background: var(--system-accent);
|
||||||
border-color: #b91c1c;
|
border-color: var(--system-accent);
|
||||||
}
|
}
|
||||||
.vein.active h3 {
|
.vein.active h3 {
|
||||||
color: white;
|
color: white;
|
||||||
@@ -139,12 +139,12 @@
|
|||||||
cursor: not-allowed;
|
cursor: not-allowed;
|
||||||
}
|
}
|
||||||
.vein.disabled:hover {
|
.vein.disabled:hover {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
}
|
}
|
||||||
.vein h3 {
|
.vein h3 {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-size: 1rem;
|
font-size: 1rem;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
}
|
}
|
||||||
.endpoints {
|
.endpoints {
|
||||||
list-style: none;
|
list-style: none;
|
||||||
@@ -153,7 +153,7 @@
|
|||||||
}
|
}
|
||||||
.endpoints li {
|
.endpoints li {
|
||||||
padding: 0.75rem 0;
|
padding: 0.75rem 0;
|
||||||
border-bottom: 1px solid #3f3f3f;
|
border-bottom: 1px solid var(--border-strong);
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
@@ -164,18 +164,18 @@
|
|||||||
}
|
}
|
||||||
.endpoints code {
|
.endpoints code {
|
||||||
font-family: monospace;
|
font-family: monospace;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
padding: 0.25rem 0.5rem;
|
padding: 0.25rem 0.5rem;
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.endpoints .desc {
|
.endpoints .desc {
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
}
|
}
|
||||||
code {
|
code {
|
||||||
background: #2a2a2a;
|
background: var(--border);
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
padding: 0.1rem 0.3rem;
|
padding: 0.1rem 0.3rem;
|
||||||
border-radius: 3px;
|
border-radius: 3px;
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
@@ -210,41 +210,41 @@
|
|||||||
display: block;
|
display: block;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
margin-bottom: 0.5rem;
|
margin-bottom: 0.5rem;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
}
|
}
|
||||||
.api-form input[type="text"] {
|
.api-form input[type="text"] {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
padding: 0.75rem;
|
padding: 0.75rem;
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
font-family: monospace;
|
font-family: monospace;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
}
|
}
|
||||||
.api-form input[type="text"]:focus,
|
.api-form input[type="text"]:focus,
|
||||||
.api-form select:focus {
|
.api-form select:focus {
|
||||||
outline: none;
|
outline: none;
|
||||||
border-color: #b91c1c;
|
border-color: var(--system-accent);
|
||||||
}
|
}
|
||||||
.api-form select {
|
.api-form select {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
padding: 0.75rem;
|
padding: 0.75rem;
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
.api-form select:disabled {
|
.api-form select:disabled {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
color: #666;
|
color: var(--dim);
|
||||||
cursor: not-allowed;
|
cursor: not-allowed;
|
||||||
}
|
}
|
||||||
.api-form input:disabled {
|
.api-form input:disabled {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
color: #666;
|
color: var(--dim);
|
||||||
cursor: not-allowed;
|
cursor: not-allowed;
|
||||||
}
|
}
|
||||||
.api-controls {
|
.api-controls {
|
||||||
@@ -254,7 +254,7 @@
|
|||||||
margin-top: 1rem;
|
margin-top: 1rem;
|
||||||
}
|
}
|
||||||
.api-controls button {
|
.api-controls button {
|
||||||
background: #b91c1c;
|
background: var(--system-accent);
|
||||||
color: white;
|
color: white;
|
||||||
border: none;
|
border: none;
|
||||||
padding: 0.75rem 1.5rem;
|
padding: 0.75rem 1.5rem;
|
||||||
@@ -271,17 +271,17 @@
|
|||||||
cursor: not-allowed;
|
cursor: not-allowed;
|
||||||
}
|
}
|
||||||
.tab-button {
|
.tab-button {
|
||||||
background: #1a1a1a !important;
|
background: var(--surface) !important;
|
||||||
border: 1px solid #3f3f3f !important;
|
border: 1px solid var(--border-strong) !important;
|
||||||
color: #e5e5e5 !important;
|
color: var(--text) !important;
|
||||||
}
|
}
|
||||||
.tab-button:hover {
|
.tab-button:hover {
|
||||||
background: #2a2a2a !important;
|
background: var(--border) !important;
|
||||||
}
|
}
|
||||||
.tab-button.active {
|
.tab-button.active {
|
||||||
border-color: white !important;
|
border-color: white !important;
|
||||||
border-width: 2px !important;
|
border-width: 2px !important;
|
||||||
background: #b91c1c !important;
|
background: var(--system-accent) !important;
|
||||||
color: white !important;
|
color: white !important;
|
||||||
}
|
}
|
||||||
.tab-button.active:hover {
|
.tab-button.active:hover {
|
||||||
@@ -291,19 +291,19 @@
|
|||||||
font-size: 2rem;
|
font-size: 2rem;
|
||||||
font-weight: bold;
|
font-weight: bold;
|
||||||
padding: 2rem;
|
padding: 2rem;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
animation: pulse 2s ease-in-out infinite;
|
animation: pulse 2s ease-in-out infinite;
|
||||||
}
|
}
|
||||||
.epic-status.error {
|
.epic-status.error {
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
@keyframes pulse {
|
@keyframes pulse {
|
||||||
0%,
|
0%,
|
||||||
100% {
|
100% {
|
||||||
color: #0a0a0a;
|
color: var(--bg);
|
||||||
}
|
}
|
||||||
50% {
|
50% {
|
||||||
color: #b91c1c;
|
color: var(--system-accent);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.api-controls label {
|
.api-controls label {
|
||||||
@@ -312,7 +312,7 @@
|
|||||||
gap: 0.5rem;
|
gap: 0.5rem;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
}
|
}
|
||||||
.output-container {
|
.output-container {
|
||||||
position: relative;
|
position: relative;
|
||||||
@@ -324,8 +324,8 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.output-area {
|
.output-area {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
padding-top: 2.5rem;
|
padding-top: 2.5rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
@@ -337,7 +337,7 @@
|
|||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
.output-area.error {
|
.output-area.error {
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.output-area.scrollable {
|
.output-area.scrollable {
|
||||||
max-height: 1000px;
|
max-height: 1000px;
|
||||||
@@ -353,7 +353,7 @@
|
|||||||
margin-top: 1.5rem;
|
margin-top: 1.5rem;
|
||||||
}
|
}
|
||||||
.attachments-container h3 {
|
.attachments-container h3 {
|
||||||
color: #fca5a5;
|
color: var(--system-accent-text);
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 1rem;
|
font-size: 1rem;
|
||||||
}
|
}
|
||||||
@@ -370,11 +370,16 @@
|
|||||||
}
|
}
|
||||||
.attachment-label {
|
.attachment-label {
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
margin-bottom: 0.5rem;
|
margin-bottom: 0.5rem;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
<style>
|
||||||
|
/* Artery keeps its own colour under every theme. */
|
||||||
|
:root { --system-accent: #b91c1c; --system-accent-text: #fca5a5; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header style="position: relative">
|
<header style="position: relative">
|
||||||
<!-- Flux capacitor -->
|
<!-- Flux capacitor -->
|
||||||
@@ -1824,5 +1829,6 @@
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
</body>
|
<script src="/theme.js" defer></script>
|
||||||
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -7,15 +7,8 @@ from fastapi.responses import JSONResponse
|
|||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
# Import datagen from ward/tools
|
from core.config import settings
|
||||||
import sys
|
from datagen import MercadoPagoDataGenerator
|
||||||
from pathlib import Path
|
|
||||||
ward_tools_path = Path(__file__).parent.parent.parent.parent.parent / "ward" / "tools"
|
|
||||||
sys.path.insert(0, str(ward_tools_path))
|
|
||||||
|
|
||||||
from datagen.mercadopago import MercadoPagoDataGenerator
|
|
||||||
|
|
||||||
from ..core.config import settings
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|||||||
182
soleprint/artery/shunts/mercadopago/datagen.py
Normal file
182
soleprint/artery/shunts/mercadopago/datagen.py
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
"""
|
||||||
|
MercadoPago response shapes.
|
||||||
|
|
||||||
|
This lived under ward/tools/datagen/ before that tree was renamed to station/,
|
||||||
|
and the module was lost in the move — api/routes.py has been importing a path
|
||||||
|
that does not exist since. It belongs here rather than back in station/tools:
|
||||||
|
a shunt runs as its own process, and its payload shapes are part of it.
|
||||||
|
|
||||||
|
Amounts are in whole currency units, matching what the real API returns for
|
||||||
|
ARS. Every method is static; the routes hold the state, this holds the shapes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import random
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
CURRENCY = "ARS"
|
||||||
|
SITE = "MLA" # MercadoPago Argentina
|
||||||
|
|
||||||
|
PAYMENT_METHODS = ["visa", "master", "amex", "account_money", "rapipago"]
|
||||||
|
PAYMENT_TYPES = ["credit_card", "debit_card", "account_money", "ticket"]
|
||||||
|
|
||||||
|
STATUS_DETAIL = {
|
||||||
|
"approved": "accredited",
|
||||||
|
"pending": "pending_contingency",
|
||||||
|
"in_process": "pending_review_manual",
|
||||||
|
"rejected": "cc_rejected_insufficient_amount",
|
||||||
|
"cancelled": "expired",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _later(minutes: int) -> str:
|
||||||
|
return (datetime.now(timezone.utc) + timedelta(minutes=minutes)).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _numeric_id() -> int:
|
||||||
|
"""A MercadoPago-sized numeric id — the routes look these up as ints."""
|
||||||
|
return random.randint(1_000_000_000, 9_999_999_999)
|
||||||
|
|
||||||
|
|
||||||
|
class MercadoPagoDataGenerator:
|
||||||
|
"""Builds MercadoPago-shaped payloads for the shunt to hand back."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def preference(
|
||||||
|
description: str = "Payment",
|
||||||
|
total: float = 0.0,
|
||||||
|
external_reference: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
preference_id = f"{random.randint(100000000, 999999999)}-{uuid.uuid4()}"
|
||||||
|
return {
|
||||||
|
"id": preference_id,
|
||||||
|
"client_id": str(random.randint(1_000_000_000_000_000, 9_999_999_999_999_999)),
|
||||||
|
"collector_id": _numeric_id(),
|
||||||
|
"date_created": _now(),
|
||||||
|
"expires": False,
|
||||||
|
"external_reference": external_reference or "",
|
||||||
|
"init_point": f"https://www.mercadopago.com.ar/checkout/v1/redirect?pref_id={preference_id}",
|
||||||
|
"sandbox_init_point": (
|
||||||
|
f"https://sandbox.mercadopago.com.ar/checkout/v1/redirect?pref_id={preference_id}"
|
||||||
|
),
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": str(uuid.uuid4()),
|
||||||
|
"title": description,
|
||||||
|
"description": description,
|
||||||
|
"quantity": 1,
|
||||||
|
"unit_price": total,
|
||||||
|
"currency_id": CURRENCY,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"marketplace": "NONE",
|
||||||
|
"marketplace_fee": 0,
|
||||||
|
"operation_type": "regular_payment",
|
||||||
|
"site_id": SITE,
|
||||||
|
"total_amount": total,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def payment(
|
||||||
|
transaction_amount: float = 0.0,
|
||||||
|
description: str = "Payment",
|
||||||
|
status: str = "approved",
|
||||||
|
application_fee: float | None = None,
|
||||||
|
) -> dict:
|
||||||
|
method = random.choice(PAYMENT_METHODS)
|
||||||
|
fee = round(transaction_amount * 0.0579, 2) if transaction_amount else 0.0
|
||||||
|
return {
|
||||||
|
"id": _numeric_id(),
|
||||||
|
"date_created": _now(),
|
||||||
|
"date_approved": _now() if status == "approved" else None,
|
||||||
|
"date_last_updated": _now(),
|
||||||
|
"money_release_date": _later(60 * 24 * 14) if status == "approved" else None,
|
||||||
|
"operation_type": "regular_payment",
|
||||||
|
"payment_method_id": method,
|
||||||
|
"payment_type_id": random.choice(PAYMENT_TYPES),
|
||||||
|
"status": status,
|
||||||
|
"status_detail": STATUS_DETAIL.get(status, "accredited"),
|
||||||
|
"currency_id": CURRENCY,
|
||||||
|
"description": description,
|
||||||
|
"live_mode": False,
|
||||||
|
"collector_id": _numeric_id(),
|
||||||
|
"payer": {
|
||||||
|
"id": str(_numeric_id()),
|
||||||
|
"email": f"test_user_{random.randint(1000, 99999)}@testuser.com",
|
||||||
|
"identification": {"type": "DNI", "number": str(random.randint(10_000_000, 45_000_000))},
|
||||||
|
"type": "customer",
|
||||||
|
},
|
||||||
|
"transaction_amount": transaction_amount,
|
||||||
|
"transaction_amount_refunded": 0,
|
||||||
|
"installments": 1,
|
||||||
|
"transaction_details": {
|
||||||
|
"net_received_amount": (
|
||||||
|
round(transaction_amount - fee, 2) if status == "approved" else 0
|
||||||
|
),
|
||||||
|
"total_paid_amount": transaction_amount,
|
||||||
|
"overpaid_amount": 0,
|
||||||
|
"installment_amount": transaction_amount,
|
||||||
|
},
|
||||||
|
"fee_details": (
|
||||||
|
[{"type": "mercadopago_fee", "amount": fee, "fee_payer": "collector"}]
|
||||||
|
if status == "approved"
|
||||||
|
else []
|
||||||
|
),
|
||||||
|
"application_fee": application_fee,
|
||||||
|
"captured": status == "approved",
|
||||||
|
"external_reference": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def merchant_order(
|
||||||
|
preference_id: str = "",
|
||||||
|
total: float = 0.0,
|
||||||
|
paid_amount: float = 0.0,
|
||||||
|
) -> dict:
|
||||||
|
return {
|
||||||
|
"id": _numeric_id(),
|
||||||
|
"status": "closed" if paid_amount >= total and total else "opened",
|
||||||
|
"external_reference": "",
|
||||||
|
"preference_id": preference_id,
|
||||||
|
"payments": [],
|
||||||
|
"shipments": [],
|
||||||
|
"date_created": _now(),
|
||||||
|
"last_updated": _now(),
|
||||||
|
"site_id": SITE,
|
||||||
|
"total_amount": total,
|
||||||
|
"paid_amount": paid_amount,
|
||||||
|
"refunded_amount": 0,
|
||||||
|
"order_status": "paid" if paid_amount >= total and total else "payment_required",
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def oauth_token() -> dict:
|
||||||
|
return {
|
||||||
|
"access_token": f"APP_USR-{uuid.uuid4().hex}",
|
||||||
|
"token_type": "Bearer",
|
||||||
|
"expires_in": 15552000,
|
||||||
|
"scope": "offline_access read write",
|
||||||
|
"user_id": _numeric_id(),
|
||||||
|
"refresh_token": f"TG-{uuid.uuid4().hex}",
|
||||||
|
"public_key": f"APP_USR-{uuid.uuid4()}",
|
||||||
|
"live_mode": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def webhook_notification(topic: str = "payment", resource_id: str = "") -> dict:
|
||||||
|
return {
|
||||||
|
"id": _numeric_id(),
|
||||||
|
"live_mode": False,
|
||||||
|
"type": topic,
|
||||||
|
"date_created": _now(),
|
||||||
|
"application_id": _numeric_id(),
|
||||||
|
"user_id": _numeric_id(),
|
||||||
|
"version": 1,
|
||||||
|
"api_version": "v1",
|
||||||
|
"action": f"{topic}.updated",
|
||||||
|
"data": {"id": str(resource_id)},
|
||||||
|
}
|
||||||
@@ -5,8 +5,10 @@ from fastapi import FastAPI, Request
|
|||||||
from fastapi.responses import HTMLResponse
|
from fastapi.responses import HTMLResponse
|
||||||
from fastapi.templating import Jinja2Templates
|
from fastapi.templating import Jinja2Templates
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from .api.routes import router
|
# Absolute imports: a shunt is started from its own directory (`python run.py`),
|
||||||
from .core.config import settings
|
# not imported as a package, so relative imports have no parent to resolve.
|
||||||
|
from api.routes import router
|
||||||
|
from core.config import settings
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="MercadoPago (MOCK)",
|
title="MercadoPago (MOCK)",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<!doctype html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
html {
|
html {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
}
|
}
|
||||||
body {
|
body {
|
||||||
font-family:
|
font-family:
|
||||||
@@ -25,7 +25,7 @@
|
|||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 2rem 1rem;
|
padding: 2rem 1rem;
|
||||||
line-height: 1.6;
|
line-height: 1.6;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
background: #163528;
|
background: #163528;
|
||||||
}
|
}
|
||||||
header {
|
header {
|
||||||
@@ -59,7 +59,7 @@
|
|||||||
section h2 {
|
section h2 {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 1.2rem;
|
font-size: 1.2rem;
|
||||||
color: #86efac;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition {
|
.composition {
|
||||||
background: #000000;
|
background: #000000;
|
||||||
@@ -70,12 +70,12 @@
|
|||||||
.composition h3 {
|
.composition h3 {
|
||||||
margin: 0 0 0.75rem 0;
|
margin: 0 0 0.75rem 0;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
color: #86efac;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition > p {
|
.composition > p {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.components {
|
.components {
|
||||||
display: grid;
|
display: grid;
|
||||||
@@ -83,7 +83,7 @@
|
|||||||
gap: 0.75rem;
|
gap: 0.75rem;
|
||||||
}
|
}
|
||||||
.component {
|
.component {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
border: 1px solid #6b665e;
|
border: 1px solid #6b665e;
|
||||||
padding: 0.75rem;
|
padding: 0.75rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
@@ -91,12 +91,12 @@
|
|||||||
.component h4 {
|
.component h4 {
|
||||||
margin: 0 0 0.25rem 0;
|
margin: 0 0 0.25rem 0;
|
||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
color: #86efac;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.component p {
|
.component p {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.books {
|
.books {
|
||||||
list-style: none;
|
list-style: none;
|
||||||
@@ -114,7 +114,7 @@
|
|||||||
border-bottom: none;
|
border-bottom: none;
|
||||||
}
|
}
|
||||||
.books a {
|
.books a {
|
||||||
color: #86efac;
|
color: var(--system-accent-text);
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
}
|
}
|
||||||
@@ -138,7 +138,12 @@
|
|||||||
opacity: 0.5;
|
opacity: 0.5;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
<style>
|
||||||
|
/* Atlas keeps its own colour under every theme. */
|
||||||
|
:root { --system-accent: #15803d; --system-accent-text: #86efac; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header style="position: relative">
|
<header style="position: relative">
|
||||||
<!-- Open book -->
|
<!-- Open book -->
|
||||||
@@ -265,5 +270,6 @@
|
|||||||
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
|
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
|
||||||
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
|
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
|
||||||
</footer>
|
</footer>
|
||||||
</body>
|
<script src="/theme.js" defer></script>
|
||||||
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
118
soleprint/common/theme/theme.js
Normal file
118
soleprint/common/theme/theme.js
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
/* Theme selection, and the toggle that drives it.
|
||||||
|
*
|
||||||
|
* Served by run.py at /theme.js, next to /theme.css. Include both and a page
|
||||||
|
* is themed; there is nothing else to wire.
|
||||||
|
*
|
||||||
|
* <link rel="stylesheet" href="/theme.css">
|
||||||
|
* <script src="/theme.js" defer></script>
|
||||||
|
*
|
||||||
|
* Resolution order, strongest first:
|
||||||
|
* 1. ?theme=mcrn — a link that carries its own theme
|
||||||
|
* 2. localStorage — what this browser last chose
|
||||||
|
* 3. <html data-theme> — what the page was served with
|
||||||
|
* 4. the server default — framework.theme in cfg/config.json
|
||||||
|
*
|
||||||
|
* The attribute is set before first paint when this script is loaded in the
|
||||||
|
* head; with `defer` the page renders once in the served theme and then
|
||||||
|
* switches, which is why run.py stamps data-theme into the served HTML.
|
||||||
|
*/
|
||||||
|
(function () {
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
var THEMES = ["soleprint", "mcrn"];
|
||||||
|
var KEY = "spr-theme";
|
||||||
|
var root = document.documentElement;
|
||||||
|
|
||||||
|
function fromQuery() {
|
||||||
|
var match = /[?&]theme=([^&#]+)/.exec(window.location.search);
|
||||||
|
return match ? decodeURIComponent(match[1]) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stored() {
|
||||||
|
try {
|
||||||
|
return window.localStorage.getItem(KEY);
|
||||||
|
} catch (e) {
|
||||||
|
// Private mode and file:// origins throw on access rather than
|
||||||
|
// returning null, and a theme is not worth breaking a page over.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolve() {
|
||||||
|
var candidates = [fromQuery(), stored(), root.getAttribute("data-theme")];
|
||||||
|
for (var i = 0; i < candidates.length; i++) {
|
||||||
|
if (candidates[i] && THEMES.indexOf(candidates[i]) !== -1) {
|
||||||
|
return candidates[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return THEMES[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
function apply(theme, persist) {
|
||||||
|
root.setAttribute("data-theme", theme);
|
||||||
|
if (persist) {
|
||||||
|
try {
|
||||||
|
window.localStorage.setItem(KEY, theme);
|
||||||
|
} catch (e) {
|
||||||
|
/* see stored() */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var buttons = document.querySelectorAll("#spr-theme-toggle button");
|
||||||
|
for (var i = 0; i < buttons.length; i++) {
|
||||||
|
buttons[i].setAttribute(
|
||||||
|
"aria-pressed",
|
||||||
|
buttons[i].dataset.theme === theme ? "true" : "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
window.dispatchEvent(new CustomEvent("spr:theme", { detail: theme }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildToggle() {
|
||||||
|
if (document.getElementById("spr-theme-toggle")) return;
|
||||||
|
// Opt out with <body data-theme-toggle="off"> — the shunt config UIs
|
||||||
|
// and any embedded view want the theme without the chrome.
|
||||||
|
if (document.body.dataset.themeToggle === "off") return;
|
||||||
|
|
||||||
|
var box = document.createElement("div");
|
||||||
|
box.id = "spr-theme-toggle";
|
||||||
|
box.setAttribute("role", "group");
|
||||||
|
box.setAttribute("aria-label", "Theme");
|
||||||
|
|
||||||
|
THEMES.forEach(function (theme) {
|
||||||
|
var button = document.createElement("button");
|
||||||
|
button.type = "button";
|
||||||
|
button.dataset.theme = theme;
|
||||||
|
button.textContent = theme === "mcrn" ? "MCRN" : "SPR";
|
||||||
|
button.title = "Switch to the " + theme + " theme";
|
||||||
|
button.addEventListener("click", function () {
|
||||||
|
apply(theme, true);
|
||||||
|
});
|
||||||
|
box.appendChild(button);
|
||||||
|
});
|
||||||
|
|
||||||
|
document.body.appendChild(box);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set the attribute immediately; the toggle needs a body to attach to.
|
||||||
|
apply(resolve(), false);
|
||||||
|
|
||||||
|
if (document.readyState === "loading") {
|
||||||
|
document.addEventListener("DOMContentLoaded", function () {
|
||||||
|
buildToggle();
|
||||||
|
apply(root.getAttribute("data-theme"), false);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
buildToggle();
|
||||||
|
apply(root.getAttribute("data-theme"), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
window.sprTheme = {
|
||||||
|
get: function () {
|
||||||
|
return root.getAttribute("data-theme");
|
||||||
|
},
|
||||||
|
set: function (theme) {
|
||||||
|
if (THEMES.indexOf(theme) !== -1) apply(theme, true);
|
||||||
|
},
|
||||||
|
themes: THEMES.slice(),
|
||||||
|
};
|
||||||
|
})();
|
||||||
110
soleprint/common/theme/themes/mcrn.css
Normal file
110
soleprint/common/theme/themes/mcrn.css
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
/* MCRN — the terminal read of the Expanse aesthetic, as built at mariano.mcrn.ar.
|
||||||
|
*
|
||||||
|
* The defining choices, all of them load-bearing:
|
||||||
|
* - zero corner radius, everywhere, including inputs and buttons
|
||||||
|
* - one hairline border colour and one accent; nothing in between
|
||||||
|
* - monospace for everything, not just code
|
||||||
|
* - headings are uppercase, letterspaced, unbolded, and ruled underneath
|
||||||
|
* - the only hover is a burnt-orange border and a glow behind it
|
||||||
|
*
|
||||||
|
* Values are lifted from ~/wdir/mcrn.ar/css/mcrn.css rather than approximated,
|
||||||
|
* so the two sites read as one system.
|
||||||
|
*/
|
||||||
|
|
||||||
|
[data-theme="mcrn"] {
|
||||||
|
--bg: #0a0a0a;
|
||||||
|
--bg-2: #141414;
|
||||||
|
--surface: #1a1a1a;
|
||||||
|
--surface-raised: #202020;
|
||||||
|
--border: #2a2a2a;
|
||||||
|
--border-strong: #3d3d3d;
|
||||||
|
|
||||||
|
--text: #e0e0e0;
|
||||||
|
--muted: #888;
|
||||||
|
--dim: #555;
|
||||||
|
|
||||||
|
--accent: #d35400;
|
||||||
|
--accent-dim: #c0392b;
|
||||||
|
--accent-text: #d35400;
|
||||||
|
--glow: rgba(211, 84, 0, 0.3);
|
||||||
|
|
||||||
|
--status-ok: #2ecc71;
|
||||||
|
--status-info: #5dade2;
|
||||||
|
--status-warn: #f39c12;
|
||||||
|
--status-error: #e74c3c;
|
||||||
|
--status-idle: #555;
|
||||||
|
|
||||||
|
/* Square. This is the single most recognisable thing about the theme, so
|
||||||
|
* it applies to the small radii too — a 4px input in a 0px page reads as a
|
||||||
|
* mistake rather than a detail. */
|
||||||
|
--radius-sm: 0;
|
||||||
|
--radius: 0;
|
||||||
|
--radius-lg: 0;
|
||||||
|
--radius-xl: 0;
|
||||||
|
|
||||||
|
--font-ui: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
|
||||||
|
--font-mono: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
|
||||||
|
--font-heading: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
|
||||||
|
--heading-transform: uppercase;
|
||||||
|
--heading-spacing: 0.1em;
|
||||||
|
--heading-weight: 400;
|
||||||
|
--label-spacing: 0.05em;
|
||||||
|
|
||||||
|
--speed-fast: 0.15s;
|
||||||
|
--speed: 0.2s;
|
||||||
|
|
||||||
|
/* Glow instead of lift: this theme never moves anything on hover. */
|
||||||
|
--hover-shadow: 0 0 20px var(--glow);
|
||||||
|
--hover-lift: none;
|
||||||
|
--focus-shadow: 0 0 10px var(--glow);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Section headings carry a rule, and the rule is part of the type. */
|
||||||
|
[data-theme="mcrn"] h2 {
|
||||||
|
color: var(--system-accent-text, var(--accent-text));
|
||||||
|
padding-bottom: var(--space-2);
|
||||||
|
border-bottom: var(--hairline) solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Prompt bullets. Lists in this theme are terminal output, not prose. */
|
||||||
|
[data-theme="mcrn"] ul:not([class*="reset"]) > li::marker {
|
||||||
|
content: "";
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="mcrn"] .prompt-list {
|
||||||
|
list-style: none;
|
||||||
|
padding-left: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="mcrn"] .prompt-list > li {
|
||||||
|
position: relative;
|
||||||
|
padding-left: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="mcrn"] .prompt-list > li::before {
|
||||||
|
content: ">";
|
||||||
|
position: absolute;
|
||||||
|
left: 0;
|
||||||
|
color: var(--accent);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The active state is a solid fill with the page colour punched out of it —
|
||||||
|
* the segmented-control move from the portfolio's language toggle. */
|
||||||
|
[data-theme="mcrn"] button[aria-pressed="true"],
|
||||||
|
[data-theme="mcrn"] .active,
|
||||||
|
[data-theme="mcrn"] .selected {
|
||||||
|
background: var(--accent);
|
||||||
|
color: var(--bg);
|
||||||
|
border-color: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="mcrn"] .card:hover,
|
||||||
|
[data-theme="mcrn"] .panel:hover,
|
||||||
|
[data-theme="mcrn"] .system-card:hover,
|
||||||
|
[data-theme="mcrn"] .tool-card:hover,
|
||||||
|
[data-theme="mcrn"] .model-card:hover {
|
||||||
|
border-color: var(--system-accent, var(--accent));
|
||||||
|
box-shadow: 0 0 20px var(--glow);
|
||||||
|
transform: none;
|
||||||
|
}
|
||||||
113
soleprint/common/theme/themes/soleprint.css
Normal file
113
soleprint/common/theme/themes/soleprint.css
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
/* Soleprint — minimal, rounded, neon.
|
||||||
|
*
|
||||||
|
* The surface ramp and text colours come from common/ui/src/tokens.css, which
|
||||||
|
* is the most considered palette in the repo: near-black with a violet cast
|
||||||
|
* (#0d0d0f -> #26262f) rather than flat grey. The accent is the amber the brand
|
||||||
|
* page has always used (#d4a574) but which the token file never carried, so
|
||||||
|
* this is where the two finally agree.
|
||||||
|
*
|
||||||
|
* "Neon" here means light around an edge, not saturated fills: a 1px halo in
|
||||||
|
* the accent, translucent badge fills at 18/66 alpha, and a small lift on
|
||||||
|
* hover. Restrained enough to sit under dense tool UIs for an hour.
|
||||||
|
*/
|
||||||
|
|
||||||
|
[data-theme="soleprint"] {
|
||||||
|
--bg: #0d0d0f;
|
||||||
|
--bg-2: #16161a;
|
||||||
|
--surface: #16161a;
|
||||||
|
--surface-raised: #1e1e24;
|
||||||
|
--border: #2e2e38;
|
||||||
|
--border-strong: #3d3d4a;
|
||||||
|
|
||||||
|
--text: #e8e8f0;
|
||||||
|
--muted: #8888a0;
|
||||||
|
--dim: #555568;
|
||||||
|
|
||||||
|
--accent: #d4a574;
|
||||||
|
--accent-dim: #b8956a;
|
||||||
|
--accent-text: #e0b98d; /* lifted off the fill colour so small text holds up */
|
||||||
|
--glow: rgba(212, 165, 116, 0.32);
|
||||||
|
|
||||||
|
--status-ok: #3ecf8e;
|
||||||
|
--status-info: #4f9cf9;
|
||||||
|
--status-warn: #f5a623;
|
||||||
|
--status-error: #f06565;
|
||||||
|
--status-idle: #555568;
|
||||||
|
|
||||||
|
/* Rounded but restrained — nothing above 12px, so panels read as soft
|
||||||
|
* rather than as pills. */
|
||||||
|
--radius-sm: 4px;
|
||||||
|
--radius: 6px;
|
||||||
|
--radius-lg: 8px;
|
||||||
|
--radius-xl: 12px;
|
||||||
|
|
||||||
|
--font-ui: "Inter", system-ui, -apple-system, sans-serif;
|
||||||
|
--font-mono: "JetBrains Mono", "Fira Code", monospace;
|
||||||
|
--font-heading: "Inter", system-ui, sans-serif;
|
||||||
|
--heading-transform: none;
|
||||||
|
--heading-spacing: 0.02em;
|
||||||
|
--heading-weight: 600;
|
||||||
|
--label-spacing: 0.04em;
|
||||||
|
|
||||||
|
--speed-fast: 0.12s;
|
||||||
|
--speed: 0.2s;
|
||||||
|
|
||||||
|
--hover-shadow: 0 4px 14px rgba(0, 0, 0, 0.45), 0 0 0 1px var(--accent);
|
||||||
|
--hover-lift: translateY(-2px);
|
||||||
|
--focus-shadow: 0 0 0 2px var(--glow);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="soleprint"] .card:hover,
|
||||||
|
[data-theme="soleprint"] .panel:hover,
|
||||||
|
[data-theme="soleprint"] .system-card:hover,
|
||||||
|
[data-theme="soleprint"] .tool-card:hover,
|
||||||
|
[data-theme="soleprint"] .model-card:hover {
|
||||||
|
border-color: var(--system-accent, var(--accent));
|
||||||
|
box-shadow: 0 4px 14px rgba(0, 0, 0, 0.45),
|
||||||
|
0 0 0 1px var(--system-accent, var(--accent));
|
||||||
|
transform: translateY(-2px);
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="soleprint"] button[aria-pressed="true"],
|
||||||
|
[data-theme="soleprint"] .active,
|
||||||
|
[data-theme="soleprint"] .selected {
|
||||||
|
border-color: var(--accent);
|
||||||
|
color: var(--bg);
|
||||||
|
background: linear-gradient(135deg, var(--accent), var(--accent-dim));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Uppercase micro-labels — the recurring motif across the station tools. */
|
||||||
|
[data-theme="soleprint"] .label,
|
||||||
|
[data-theme="soleprint"] .panel-title {
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
font-weight: 600;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: var(--label-spacing);
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Translucent badges, the <hex>18 fill / <hex>66 border pattern graphgen
|
||||||
|
* already uses for pk/fk/m2m chips. */
|
||||||
|
[data-theme="soleprint"] .badge {
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
padding: 1px 6px;
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: 10px;
|
||||||
|
letter-spacing: 0.03em;
|
||||||
|
background: #d4a57418;
|
||||||
|
border: 1px solid #d4a57466;
|
||||||
|
color: var(--accent-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The one animated state, carried over from common/ui's tokens.css. */
|
||||||
|
@keyframes spr-waiting-glow {
|
||||||
|
0% { box-shadow: 0 0 3px 1px var(--status-info); }
|
||||||
|
33% { box-shadow: 0 0 3px 1px var(--status-ok); }
|
||||||
|
66% { box-shadow: 0 0 3px 1px var(--status-warn); }
|
||||||
|
100% { box-shadow: 0 0 3px 1px var(--status-info); }
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="soleprint"] .waiting {
|
||||||
|
animation: spr-waiting-glow 2s linear infinite;
|
||||||
|
outline: 1px solid transparent;
|
||||||
|
}
|
||||||
217
soleprint/common/theme/tokens.css
Normal file
217
soleprint/common/theme/tokens.css
Normal file
@@ -0,0 +1,217 @@
|
|||||||
|
/* Soleprint theme contract — the variable names every page may rely on.
|
||||||
|
*
|
||||||
|
* This file declares the vocabulary and a neutral default. It sets no colours
|
||||||
|
* of its own worth looking at: themes/*.css supply those, scoped to
|
||||||
|
* [data-theme="..."] on <html>, and theme.js decides which one is active.
|
||||||
|
*
|
||||||
|
* Two naming families existed before this file and both are still in use, so
|
||||||
|
* both are answered here rather than renamed across a dozen templates:
|
||||||
|
*
|
||||||
|
* --bg / --surface / --border / --text / --muted / --amber
|
||||||
|
* the station tool templates and the docs site
|
||||||
|
* --surface-0..3 / --text-primary / --text-secondary / --panel-radius
|
||||||
|
* common/ui's Vue component library
|
||||||
|
*
|
||||||
|
* The theme files set the first family; the aliases at the bottom derive the
|
||||||
|
* second from it. A page that uses either name gets the same colour, and a
|
||||||
|
* theme author has one set of values to fill in.
|
||||||
|
*
|
||||||
|
* Served by run.py at /theme.css together with the theme files — see the
|
||||||
|
* handler beside /sidebar.css.
|
||||||
|
*/
|
||||||
|
|
||||||
|
@import url("https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;600&display=swap");
|
||||||
|
|
||||||
|
:root {
|
||||||
|
/* ── surfaces ─────────────────────────────────────────────────────── */
|
||||||
|
--bg: #0a0a0a;
|
||||||
|
--bg-2: #141414;
|
||||||
|
--surface: #1a1a1a;
|
||||||
|
--surface-raised: #242424;
|
||||||
|
--border: #333;
|
||||||
|
--border-strong: #4a4a4a;
|
||||||
|
|
||||||
|
/* ── text ─────────────────────────────────────────────────────────── */
|
||||||
|
--text: #e5e5e5;
|
||||||
|
--muted: #a3a3a3;
|
||||||
|
--dim: #666;
|
||||||
|
|
||||||
|
/* ── accent ───────────────────────────────────────────────────────── */
|
||||||
|
--accent: #d4a574;
|
||||||
|
--accent-dim: #b8956a;
|
||||||
|
--accent-text: #d4a574; /* accent legible on --bg, where the fill is not */
|
||||||
|
--glow: rgba(212, 165, 116, 0.3);
|
||||||
|
|
||||||
|
/* Each subsystem keeps its own colour under every theme, so a page still
|
||||||
|
* announces which of artery / atlas / station you are looking at. Pages set
|
||||||
|
* --system-accent; this is the fallback for those that do not. */
|
||||||
|
--system-accent: var(--accent);
|
||||||
|
--system-accent-text: var(--accent-text);
|
||||||
|
|
||||||
|
/* ── status ───────────────────────────────────────────────────────── */
|
||||||
|
--status-ok: #3ecf8e;
|
||||||
|
--status-info: #4f9cf9;
|
||||||
|
--status-warn: #f5a623;
|
||||||
|
--status-error: #f06565;
|
||||||
|
--status-idle: #555568;
|
||||||
|
|
||||||
|
/* ── shape ────────────────────────────────────────────────────────── */
|
||||||
|
--radius-sm: 4px;
|
||||||
|
--radius: 6px;
|
||||||
|
--radius-lg: 8px;
|
||||||
|
--radius-xl: 12px;
|
||||||
|
--hairline: 1px;
|
||||||
|
|
||||||
|
/* ── type ─────────────────────────────────────────────────────────── */
|
||||||
|
--font-ui: "Inter", system-ui, -apple-system, sans-serif;
|
||||||
|
--font-mono: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
|
||||||
|
--font-heading: var(--font-ui);
|
||||||
|
--font-size-sm: 11px;
|
||||||
|
--font-size-base: 13px;
|
||||||
|
--font-size-lg: 15px;
|
||||||
|
--heading-transform: none;
|
||||||
|
--heading-spacing: 0;
|
||||||
|
--heading-weight: 600;
|
||||||
|
--label-spacing: 0.04em;
|
||||||
|
|
||||||
|
/* ── space ────────────────────────────────────────────────────────── */
|
||||||
|
--space-1: 4px;
|
||||||
|
--space-2: 8px;
|
||||||
|
--space-3: 12px;
|
||||||
|
--space-4: 16px;
|
||||||
|
--space-6: 24px;
|
||||||
|
--space-8: 32px;
|
||||||
|
|
||||||
|
/* ── motion ───────────────────────────────────────────────────────── */
|
||||||
|
--speed-fast: 0.12s;
|
||||||
|
--speed: 0.2s;
|
||||||
|
--ease: ease;
|
||||||
|
|
||||||
|
/* Themes differ most in how a surface reacts, not in what colour it is.
|
||||||
|
* Both hover treatments are declared here so a page writes one rule and
|
||||||
|
* the theme decides whether it lifts, glows, or both. */
|
||||||
|
--hover-shadow: 0 4px 12px var(--glow);
|
||||||
|
--hover-lift: translateY(-2px);
|
||||||
|
--focus-shadow: 0 0 0 1px var(--accent);
|
||||||
|
|
||||||
|
/* ── aliases for names already in use ─────────────────────────────── */
|
||||||
|
|
||||||
|
/* The station tool templates and the docs site call the accent "amber". */
|
||||||
|
--amber: var(--accent);
|
||||||
|
--amber-dim: var(--accent-dim);
|
||||||
|
|
||||||
|
/* ── aliases for common/ui's token names ──────────────────────────── */
|
||||||
|
--surface-0: var(--bg);
|
||||||
|
--surface-1: var(--surface);
|
||||||
|
--surface-2: var(--surface-raised);
|
||||||
|
--surface-3: var(--border);
|
||||||
|
--text-primary: var(--text);
|
||||||
|
--text-secondary: var(--muted);
|
||||||
|
--text-dim: var(--dim);
|
||||||
|
--panel-radius: var(--radius);
|
||||||
|
--panel-border: var(--hairline) solid var(--border);
|
||||||
|
--panel-header-height: 36px;
|
||||||
|
--status-live: var(--status-ok);
|
||||||
|
--status-processing: var(--status-info);
|
||||||
|
--status-escalating: var(--status-warn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── element defaults ─────────────────────────────────────────────────── */
|
||||||
|
|
||||||
|
body {
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font-family: var(--font-ui);
|
||||||
|
line-height: 1.6;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1, h2, h3, h4, h5, h6 {
|
||||||
|
font-family: var(--font-heading);
|
||||||
|
font-weight: var(--heading-weight);
|
||||||
|
text-transform: var(--heading-transform);
|
||||||
|
letter-spacing: var(--heading-spacing);
|
||||||
|
}
|
||||||
|
|
||||||
|
code, pre, kbd, samp {
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
color: var(--accent-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
background: var(--surface-raised);
|
||||||
|
color: var(--text);
|
||||||
|
border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
font-family: inherit;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: border-color var(--speed-fast) var(--ease),
|
||||||
|
color var(--speed-fast) var(--ease),
|
||||||
|
background var(--speed-fast) var(--ease);
|
||||||
|
}
|
||||||
|
|
||||||
|
button:hover:not(:disabled) {
|
||||||
|
border-color: var(--accent);
|
||||||
|
color: var(--accent-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
button:disabled {
|
||||||
|
opacity: 0.4;
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
|
input, select, textarea {
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
font-family: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
|
input:focus, select:focus, textarea:focus {
|
||||||
|
outline: none;
|
||||||
|
border-color: var(--accent);
|
||||||
|
box-shadow: var(--focus-shadow);
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar { width: 10px; height: 10px; }
|
||||||
|
::-webkit-scrollbar-thumb { background: var(--border); border-radius: 5px; }
|
||||||
|
::-webkit-scrollbar-track { background: transparent; }
|
||||||
|
|
||||||
|
/* The theme toggle theme.js injects. Fixed rather than placed, so it needs no
|
||||||
|
* cooperation from the page it lands on. */
|
||||||
|
#spr-theme-toggle {
|
||||||
|
position: fixed;
|
||||||
|
right: var(--space-3);
|
||||||
|
bottom: var(--space-3);
|
||||||
|
z-index: 99998;
|
||||||
|
display: flex;
|
||||||
|
gap: 0;
|
||||||
|
border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
overflow: hidden;
|
||||||
|
background: var(--surface);
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#spr-theme-toggle button {
|
||||||
|
border: none;
|
||||||
|
border-radius: 0;
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--dim);
|
||||||
|
padding: var(--space-1) var(--space-2);
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: var(--label-spacing);
|
||||||
|
}
|
||||||
|
|
||||||
|
#spr-theme-toggle button + button {
|
||||||
|
border-left: var(--hairline) solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
#spr-theme-toggle button[aria-pressed="true"] {
|
||||||
|
background: var(--accent);
|
||||||
|
color: var(--bg);
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<!doctype html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
html {
|
html {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
}
|
}
|
||||||
body {
|
body {
|
||||||
font-family:
|
font-family:
|
||||||
@@ -25,8 +25,8 @@
|
|||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 2rem 1rem;
|
padding: 2rem 1rem;
|
||||||
line-height: 1.6;
|
line-height: 1.6;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
}
|
}
|
||||||
/* Sidebar styles */
|
/* Sidebar styles */
|
||||||
.sidebar {
|
.sidebar {
|
||||||
@@ -35,8 +35,8 @@
|
|||||||
left: 0;
|
left: 0;
|
||||||
width: 60px;
|
width: 60px;
|
||||||
height: 100vh;
|
height: 100vh;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border-right: 1px solid #333;
|
border-right: 1px solid var(--border);
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
@@ -52,16 +52,16 @@
|
|||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
margin-bottom: 0.5rem;
|
margin-bottom: 0.5rem;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
transition: all 0.2s;
|
transition: all 0.2s;
|
||||||
}
|
}
|
||||||
a.sidebar-item:hover {
|
a.sidebar-item:hover {
|
||||||
background: #333;
|
background: var(--border);
|
||||||
color: white;
|
color: white;
|
||||||
}
|
}
|
||||||
.sidebar-item.active {
|
.sidebar-item.active {
|
||||||
background: #d4a574;
|
background: var(--accent);
|
||||||
color: #0a0a0a;
|
color: var(--bg);
|
||||||
}
|
}
|
||||||
.sidebar-item svg {
|
.sidebar-item svg {
|
||||||
width: 24px;
|
width: 24px;
|
||||||
@@ -70,7 +70,7 @@
|
|||||||
.sidebar-divider {
|
.sidebar-divider {
|
||||||
width: 32px;
|
width: 32px;
|
||||||
height: 1px;
|
height: 1px;
|
||||||
background: #333;
|
background: var(--border);
|
||||||
margin: 0.5rem 0;
|
margin: 0.5rem 0;
|
||||||
}
|
}
|
||||||
.sidebar-icon {
|
.sidebar-icon {
|
||||||
@@ -89,7 +89,7 @@
|
|||||||
.sidebar-item .tooltip {
|
.sidebar-item .tooltip {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
left: 70px;
|
left: 70px;
|
||||||
background: #333;
|
background: var(--border);
|
||||||
color: white;
|
color: white;
|
||||||
padding: 0.5rem 0.75rem;
|
padding: 0.5rem 0.75rem;
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
@@ -121,18 +121,18 @@
|
|||||||
color: white;
|
color: white;
|
||||||
}
|
}
|
||||||
.tagline {
|
.tagline {
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
margin-bottom: 2rem;
|
margin-bottom: 2rem;
|
||||||
border-bottom: 1px solid #333;
|
border-bottom: 1px solid var(--border);
|
||||||
padding-bottom: 2rem;
|
padding-bottom: 2rem;
|
||||||
}
|
}
|
||||||
.mission {
|
.mission {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border-left: 3px solid #d4a574;
|
border-left: 3px solid var(--accent);
|
||||||
padding: 1rem 1.5rem;
|
padding: 1rem 1.5rem;
|
||||||
margin: 2rem 0;
|
margin: 2rem 0;
|
||||||
border-radius: 0 8px 8px 0;
|
border-radius: 0 8px 8px 0;
|
||||||
color: #d4a574;
|
color: var(--accent);
|
||||||
}
|
}
|
||||||
.systems {
|
.systems {
|
||||||
display: grid;
|
display: grid;
|
||||||
@@ -171,11 +171,11 @@
|
|||||||
.system-info p {
|
.system-info p {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
|
|
||||||
.artery {
|
.artery {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 1px solid #b91c1c;
|
border: 1px solid #b91c1c;
|
||||||
}
|
}
|
||||||
.artery h2 {
|
.artery h2 {
|
||||||
@@ -186,7 +186,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.atlas {
|
.atlas {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 1px solid #15803d;
|
border: 1px solid #15803d;
|
||||||
}
|
}
|
||||||
.atlas h2 {
|
.atlas h2 {
|
||||||
@@ -197,7 +197,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.station {
|
.station {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 1px solid #1d4ed8;
|
border: 1px solid #1d4ed8;
|
||||||
}
|
}
|
||||||
.station h2 {
|
.station h2 {
|
||||||
@@ -210,9 +210,9 @@
|
|||||||
footer {
|
footer {
|
||||||
margin-top: 3rem;
|
margin-top: 3rem;
|
||||||
padding-top: 1.5rem;
|
padding-top: 1.5rem;
|
||||||
border-top: 1px solid #333;
|
border-top: 1px solid var(--border);
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #666;
|
color: var(--dim);
|
||||||
}
|
}
|
||||||
|
|
||||||
.showcase-container {
|
.showcase-container {
|
||||||
@@ -224,8 +224,8 @@
|
|||||||
}
|
}
|
||||||
.showcase-link {
|
.showcase-link {
|
||||||
display: inline-block;
|
display: inline-block;
|
||||||
background: linear-gradient(135deg, #d4a574, #b8956a);
|
background: linear-gradient(135deg, var(--accent), var(--accent-dim));
|
||||||
color: #0a0a0a;
|
color: var(--bg);
|
||||||
padding: 0.75rem 1.5rem;
|
padding: 0.75rem 1.5rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
@@ -237,19 +237,20 @@
|
|||||||
box-shadow: 0 4px 12px rgba(212, 165, 116, 0.3);
|
box-shadow: 0 4px 12px rgba(212, 165, 116, 0.3);
|
||||||
}
|
}
|
||||||
.showcase-hint {
|
.showcase-hint {
|
||||||
color: #666;
|
color: var(--dim);
|
||||||
font-size: 0.8rem;
|
font-size: 0.8rem;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
}
|
}
|
||||||
.showcase-hint:hover {
|
.showcase-hint:hover {
|
||||||
color: #d4a574;
|
color: var(--accent);
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
{% if managed %}
|
{% if managed %}
|
||||||
<link rel="stylesheet" href="/sidebar.css">
|
<link rel="stylesheet" href="/sidebar.css">
|
||||||
<script src="/sidebar.js"></script>
|
<script src="/sidebar.js"></script>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</head>
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
</head>
|
||||||
<body{% if managed %} class="has-sidebar"{% endif %}>
|
<body{% if managed %} class="has-sidebar"{% endif %}>
|
||||||
|
|
||||||
<header>
|
<header>
|
||||||
@@ -428,5 +429,6 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<footer>soleprint</footer>
|
<footer>soleprint</footer>
|
||||||
</body>
|
<script src="/theme.js" defer></script>
|
||||||
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -5,6 +5,12 @@ pydantic>=2.5.0
|
|||||||
pydantic-settings>=2.0.0
|
pydantic-settings>=2.0.0
|
||||||
httpx>=0.25.0
|
httpx>=0.25.0
|
||||||
jinja2>=3.1.0
|
jinja2>=3.1.0
|
||||||
|
# YAML: OpenAPI specs for shuntgen, and compose fragments for cabinets.
|
||||||
|
# modelgen keeps this optional (it imports it lazily) so the pip package stays
|
||||||
|
# dependency-free; the server always has it.
|
||||||
|
pyyaml>=6.0
|
||||||
|
# Multipart form parsing — file uploads in shuntgen's UI.
|
||||||
|
python-multipart>=0.0.9
|
||||||
|
|
||||||
# Database (databrowse)
|
# Database (databrowse)
|
||||||
sqlalchemy>=2.0.0
|
sqlalchemy>=2.0.0
|
||||||
|
|||||||
156
soleprint/run.py
156
soleprint/run.py
@@ -249,19 +249,41 @@ def load_config() -> dict:
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
class SafeDict(dict):
|
||||||
|
"""A dict whose missing keys read as empty rather than exploding a template.
|
||||||
|
|
||||||
|
The landing pages ask for two levels at once — `components.composed.title`.
|
||||||
|
Jinja tolerates one missing level (it returns Undefined, and `or 'Desk'`
|
||||||
|
catches it) but not two: attribute access on Undefined raises, and the
|
||||||
|
whole page 500s.
|
||||||
|
|
||||||
|
That is not hypothetical. cfg/config.json is written into a room by
|
||||||
|
build.py, so it does not exist in the source tree at all, and running
|
||||||
|
`python run.py` from soleprint/ — the documented way to develop — took
|
||||||
|
/station/ down every time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __getattr__(self, name: str):
|
||||||
|
# The dict's own contents win; only genuinely absent keys become empty.
|
||||||
|
if name in self:
|
||||||
|
value = self[name]
|
||||||
|
return SafeDict(value) if isinstance(value, dict) else value
|
||||||
|
return SafeDict()
|
||||||
|
|
||||||
|
|
||||||
def get_system_config(system_key: str) -> dict:
|
def get_system_config(system_key: str) -> dict:
|
||||||
"""Get system configuration by key (data_flow, documentation, execution)."""
|
"""Get system configuration by key (data_flow, documentation, execution)."""
|
||||||
config = load_config()
|
config = load_config()
|
||||||
for system in config.get("systems", []):
|
for system in config.get("systems", []):
|
||||||
if system.get("key") == system_key:
|
if system.get("key") == system_key:
|
||||||
return system
|
return SafeDict(system)
|
||||||
return {}
|
return SafeDict()
|
||||||
|
|
||||||
|
|
||||||
def get_components(system_key: str) -> dict:
|
def get_components(system_key: str) -> dict:
|
||||||
"""Get component definitions for a system."""
|
"""Get component definitions for a system."""
|
||||||
config = load_config()
|
config = load_config()
|
||||||
return config.get("components", {}).get(system_key, {})
|
return SafeDict(config.get("components", {}).get(system_key, {}))
|
||||||
|
|
||||||
|
|
||||||
def load_data(filename: str) -> list[dict]:
|
def load_data(filename: str) -> list[dict]:
|
||||||
@@ -461,6 +483,41 @@ def atlas_route(path: str):
|
|||||||
# === Station ===
|
# === Station ===
|
||||||
|
|
||||||
|
|
||||||
|
def load_station_cabinets() -> list[dict]:
|
||||||
|
"""The dependency containers this room switched on.
|
||||||
|
|
||||||
|
Same two-step as artery's shunts: what the room declared, else what is on
|
||||||
|
disk. The room's list is the honest answer — the catalog holds every
|
||||||
|
cabinet that could be used, not the ones that were.
|
||||||
|
"""
|
||||||
|
declared = load_data("cabinets.json")
|
||||||
|
catalog = SPR_ROOT / "station" / "cabinets"
|
||||||
|
|
||||||
|
if declared:
|
||||||
|
cabinets = declared
|
||||||
|
else:
|
||||||
|
cabinets = [
|
||||||
|
{"name": path.name}
|
||||||
|
for path in sorted(catalog.iterdir())
|
||||||
|
if path.is_dir() and not path.name.startswith(("_", "."))
|
||||||
|
] if catalog.exists() else []
|
||||||
|
|
||||||
|
for cabinet in cabinets:
|
||||||
|
name = cabinet.get("name", "")
|
||||||
|
definition_path = catalog / name / "cabinet.json"
|
||||||
|
if definition_path.exists():
|
||||||
|
try:
|
||||||
|
definition = json.loads(definition_path.read_text())
|
||||||
|
for key in ("title", "description", "image", "rig_addon"):
|
||||||
|
cabinet.setdefault(key, definition.get(key))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
cabinet.setdefault("slug", name)
|
||||||
|
cabinet.setdefault("title", name.replace("-", " ").title())
|
||||||
|
cabinet.setdefault("status", "declared" if declared else "available")
|
||||||
|
return cabinets
|
||||||
|
|
||||||
|
|
||||||
@app.get("/station", response_class=HTMLResponse)
|
@app.get("/station", response_class=HTMLResponse)
|
||||||
@app.get("/station/", response_class=HTMLResponse)
|
@app.get("/station/", response_class=HTMLResponse)
|
||||||
def station_index(request: Request):
|
def station_index(request: Request):
|
||||||
@@ -486,6 +543,7 @@ def station_index(request: Request):
|
|||||||
d["slug"] = d["name"]
|
d["slug"] = d["name"]
|
||||||
d["title"] = d["name"].replace("-", " ").title()
|
d["title"] = d["name"].replace("-", " ").title()
|
||||||
d["status"] = "ready"
|
d["status"] = "ready"
|
||||||
|
cabinets = load_station_cabinets()
|
||||||
from jinja2 import Template
|
from jinja2 import Template
|
||||||
|
|
||||||
template = Template(html_path.read_text())
|
template = Template(html_path.read_text())
|
||||||
@@ -497,6 +555,7 @@ def station_index(request: Request):
|
|||||||
tools=tools,
|
tools=tools,
|
||||||
monitors=monitors,
|
monitors=monitors,
|
||||||
desks=desks,
|
desks=desks,
|
||||||
|
cabinets=cabinets,
|
||||||
soleprint_url="/",
|
soleprint_url="/",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -512,24 +571,18 @@ def station_index(request: Request):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Mount station tool routers
|
# Mount station tool routers.
|
||||||
try:
|
#
|
||||||
from station.tools.tester.api import router as tester_router
|
# Broad except on purpose: a tool that cannot load should cost you that tool,
|
||||||
app.include_router(tester_router, prefix="/station")
|
# not the server. Route registration raises more than ImportError — FastAPI
|
||||||
except ImportError as e:
|
# turns a missing optional dependency into a RuntimeError at decoration time —
|
||||||
print(f"Warning: Could not load tester router: {e}")
|
# and catching only ImportError meant one such tool took the whole app down.
|
||||||
|
for _tool in ("tester", "graphgen", "datagen", "shuntgen"):
|
||||||
try:
|
try:
|
||||||
from station.tools.graphgen.api import router as graphgen_router
|
_module = importlib.import_module(f"station.tools.{_tool}.api")
|
||||||
app.include_router(graphgen_router, prefix="/station")
|
app.include_router(_module.router, prefix="/station")
|
||||||
except ImportError as e:
|
except Exception as e:
|
||||||
print(f"Warning: Could not load graphgen router: {e}")
|
print(f"Warning: Could not load {_tool} router: {e}")
|
||||||
|
|
||||||
try:
|
|
||||||
from station.tools.datagen.api import router as datagen_router
|
|
||||||
app.include_router(datagen_router, prefix="/station")
|
|
||||||
except ImportError as e:
|
|
||||||
print(f"Warning: Could not load datagen router: {e}")
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/station/{path:path}")
|
@app.get("/station/{path:path}")
|
||||||
@@ -541,6 +594,67 @@ def station_route(path: str):
|
|||||||
# === Sidebar Wrapper (served at /spr/* when proxied) ===
|
# === Sidebar Wrapper (served at /spr/* when proxied) ===
|
||||||
|
|
||||||
|
|
||||||
|
# === Theme ===
|
||||||
|
#
|
||||||
|
# One stylesheet for every page, served the same way the sidebar is. tokens.css
|
||||||
|
# declares the variables and both theme files ship in the same response, so a
|
||||||
|
# page can switch themes without a second request and without FOUC.
|
||||||
|
|
||||||
|
|
||||||
|
def get_default_theme() -> str:
|
||||||
|
"""The theme a page is served in, before the browser has an opinion."""
|
||||||
|
framework = load_config().get("framework", {})
|
||||||
|
theme = framework.get("theme", "soleprint")
|
||||||
|
return theme if theme in ("soleprint", "mcrn") else "soleprint"
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/theme.css")
|
||||||
|
def theme_css():
|
||||||
|
"""Serve the theme contract plus every theme, concatenated."""
|
||||||
|
from fastapi.responses import Response
|
||||||
|
|
||||||
|
theme_dir = SPR_ROOT / "common" / "theme"
|
||||||
|
parts = []
|
||||||
|
|
||||||
|
tokens = theme_dir / "tokens.css"
|
||||||
|
if tokens.exists():
|
||||||
|
parts.append(tokens.read_text())
|
||||||
|
|
||||||
|
# Sorted so the response is byte-stable and cacheable; the theme files are
|
||||||
|
# scoped to [data-theme] selectors, so their order carries no meaning.
|
||||||
|
for sheet in sorted((theme_dir / "themes").glob("*.css")):
|
||||||
|
parts.append(f"\n/* ── {sheet.stem} ── */\n")
|
||||||
|
parts.append(sheet.read_text())
|
||||||
|
|
||||||
|
if not parts:
|
||||||
|
return Response(
|
||||||
|
content="/* theme not found — is common/theme/ present? */",
|
||||||
|
media_type="text/css",
|
||||||
|
)
|
||||||
|
return Response(content="".join(parts), media_type="text/css")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/theme.js")
|
||||||
|
def theme_js():
|
||||||
|
"""Serve the theme switcher."""
|
||||||
|
from fastapi.responses import Response
|
||||||
|
|
||||||
|
js_path = SPR_ROOT / "common" / "theme" / "theme.js"
|
||||||
|
if js_path.exists():
|
||||||
|
return Response(
|
||||||
|
content=js_path.read_text(), media_type="application/javascript"
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
content="/* theme.js not found */", media_type="application/javascript"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/theme")
|
||||||
|
def theme_config():
|
||||||
|
"""The server-side default, for pages that render their own <html> tag."""
|
||||||
|
return {"theme": get_default_theme(), "themes": ["soleprint", "mcrn"]}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/sidebar.css")
|
@app.get("/sidebar.css")
|
||||||
def sidebar_css():
|
def sidebar_css():
|
||||||
"""Serve sidebar CSS for injection."""
|
"""Serve sidebar CSS for injection."""
|
||||||
|
|||||||
91
soleprint/station/cabinets/README.md
Normal file
91
soleprint/station/cabinets/README.md
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
# Cabinets
|
||||||
|
|
||||||
|
A cabinet is a **dependency container** a room can switch on: postgres, redis,
|
||||||
|
airflow. The vocabulary already had the word — `execution.container` in every
|
||||||
|
room's `config.json` is *"Cabinet — tool container"* — and until now nothing
|
||||||
|
stood behind it.
|
||||||
|
|
||||||
|
The problem it solves is that a generated artifact knows what it needs and had
|
||||||
|
no way to say so. A shunt built from a client's spreadsheets can hold its rows
|
||||||
|
in memory, but the moment you want them to survive a restart you need postgres,
|
||||||
|
and wiring postgres in meant hand-editing a room's `docker-compose.yml` and then
|
||||||
|
hand-editing the cluster too. A cabinet is that declaration, made once and read
|
||||||
|
by both paths.
|
||||||
|
|
||||||
|
```
|
||||||
|
soleprint/station/cabinets/<name>/
|
||||||
|
cabinet.json what it is, what it needs, what it exports
|
||||||
|
service.yml the compose service, verbatim
|
||||||
|
```
|
||||||
|
|
||||||
|
## Turning one on
|
||||||
|
|
||||||
|
Add `cfg/<room>/data/cabinets.json` — the same shape as its sibling `data/*.json`
|
||||||
|
files:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{ "name": "postgres" },
|
||||||
|
{ "name": "redis" },
|
||||||
|
{ "name": "airflow", "env": { "AIRFLOW_ADMIN_PASSWORD": "change-me" } }
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Then build. `python build.py --cfg <room>` merges each cabinet's `service.yml`
|
||||||
|
into the room's `docker-compose.yml` and appends its settings to `.env.example`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python build.py --cfg sample
|
||||||
|
cd gen/sample && docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
**A service the room already declares wins.** `cfg/amar/docker-compose.yml`
|
||||||
|
ships its own `db`; switching on the postgres cabinet will not overwrite it.
|
||||||
|
Build says so when it skips one.
|
||||||
|
|
||||||
|
## On a cluster
|
||||||
|
|
||||||
|
`cabinet.json` names a `rig_addon`. Where the room runs on kind rather than
|
||||||
|
compose, the same dependency installs as a rig addon of that name:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd rig
|
||||||
|
PROFILE=data make cluster up
|
||||||
|
PROFILE=data make addons install
|
||||||
|
```
|
||||||
|
|
||||||
|
The two paths are deliberately separate — compose for a laptop, helm for a
|
||||||
|
cluster — and `rig_addon` is the thread between them, so a room declares the
|
||||||
|
dependency once either way.
|
||||||
|
|
||||||
|
## Writing one
|
||||||
|
|
||||||
|
`cabinet.json`:
|
||||||
|
|
||||||
|
| Key | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `name` | must match the directory |
|
||||||
|
| `title`, `description` | shown on the station index |
|
||||||
|
| `image` | for the record; `service.yml` is what runs |
|
||||||
|
| `service` | the key to merge under in `services:` (defaults to `name`) |
|
||||||
|
| `env` | settings and defaults, written to `.env.example` |
|
||||||
|
| `volumes` | named volumes to declare at the top level |
|
||||||
|
| `depends_on` | other cabinets that must come with it |
|
||||||
|
| `rig_addon` | the matching `rig/ctrl/addons/<name>.sh`, if there is one |
|
||||||
|
| `ports` | host ports it wants, for the collision note in the docs |
|
||||||
|
|
||||||
|
`service.yml` is a plain compose fragment — one top-level key, the service name:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-soleprint}
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
Kept as YAML rather than generated from JSON so it reads like the compose file it
|
||||||
|
becomes, and so anything compose supports is available without this tool
|
||||||
|
learning about it first.
|
||||||
|
|
||||||
|
Adding a cabinet is adding a directory. Nothing dispatches on the name.
|
||||||
22
soleprint/station/cabinets/airflow/cabinet.json
Normal file
22
soleprint/station/cabinets/airflow/cabinet.json
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"name": "airflow",
|
||||||
|
"title": "Apache Airflow",
|
||||||
|
"description": "Scheduled pipelines. DAGs live in the room, under dags/.",
|
||||||
|
"image": "apache/airflow:2.10.4",
|
||||||
|
"service": "airflow",
|
||||||
|
"rig_addon": "airflow",
|
||||||
|
"ports": [8080],
|
||||||
|
"volumes": ["airflow_logs"],
|
||||||
|
"depends_on": ["postgres", "redis"],
|
||||||
|
"env": {
|
||||||
|
"AIRFLOW_PORT": "8080",
|
||||||
|
"AIRFLOW_ADMIN_USER": "admin",
|
||||||
|
"AIRFLOW_ADMIN_PASSWORD": "change-me",
|
||||||
|
"AIRFLOW_DAGS_DIR": "./dags"
|
||||||
|
},
|
||||||
|
"notes": [
|
||||||
|
"Brings postgres and redis with it — Airflow needs a metadata database and a broker, and will not start without both.",
|
||||||
|
"LocalExecutor by default: one container, no separate worker. Switch to CeleryExecutor in service.yml when the room outgrows it.",
|
||||||
|
"8080 collides with almost everything. Set AIRFLOW_PORT in the room's .env."
|
||||||
|
]
|
||||||
|
}
|
||||||
37
soleprint/station/cabinets/airflow/service.yml
Normal file
37
soleprint/station/cabinets/airflow/service.yml
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# Airflow — scheduler and webserver in one container, on LocalExecutor.
|
||||||
|
#
|
||||||
|
# One container rather than the five the official compose file ships, because a
|
||||||
|
# room switching this on wants pipelines, not a distributed deployment. The
|
||||||
|
# metadata database is the postgres cabinet, so the two arrive together; moving
|
||||||
|
# to CeleryExecutor is changing the executor here and adding a worker service.
|
||||||
|
airflow:
|
||||||
|
image: apache/airflow:2.10.4
|
||||||
|
container_name: ${DEPLOYMENT_NAME:-soleprint}_airflow
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||||
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: >-
|
||||||
|
postgresql+psycopg2://${POSTGRES_USER:-soleprint}:${POSTGRES_PASSWORD:-change-me}@postgres:5432/${POSTGRES_DB:-soleprint}
|
||||||
|
AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0
|
||||||
|
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
||||||
|
# Without a fixed key, every restart invalidates stored connections.
|
||||||
|
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-}
|
||||||
|
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "true"
|
||||||
|
_AIRFLOW_DB_MIGRATE: "true"
|
||||||
|
_AIRFLOW_WWW_USER_CREATE: "true"
|
||||||
|
_AIRFLOW_WWW_USER_USERNAME: ${AIRFLOW_ADMIN_USER:-admin}
|
||||||
|
_AIRFLOW_WWW_USER_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-change-me}
|
||||||
|
volumes:
|
||||||
|
- ${AIRFLOW_DAGS_DIR:-./dags}:/opt/airflow/dags
|
||||||
|
- airflow_logs:/opt/airflow/logs
|
||||||
|
ports:
|
||||||
|
- "${AIRFLOW_PORT:-8080}:8080"
|
||||||
|
# `standalone` runs the migration, creates the admin user, and starts both
|
||||||
|
# the scheduler and the webserver — the whole first-boot sequence that the
|
||||||
|
# official compose file spreads across an init container and four services.
|
||||||
|
command: standalone
|
||||||
|
restart: unless-stopped
|
||||||
20
soleprint/station/cabinets/postgres/cabinet.json
Normal file
20
soleprint/station/cabinets/postgres/cabinet.json
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"name": "postgres",
|
||||||
|
"title": "PostgreSQL",
|
||||||
|
"description": "Relational database. Backs rooms that need their data to outlive a restart.",
|
||||||
|
"image": "postgres:16-alpine",
|
||||||
|
"service": "postgres",
|
||||||
|
"rig_addon": "postgres",
|
||||||
|
"ports": [5432],
|
||||||
|
"volumes": ["pgdata"],
|
||||||
|
"env": {
|
||||||
|
"POSTGRES_DB": "soleprint",
|
||||||
|
"POSTGRES_USER": "soleprint",
|
||||||
|
"POSTGRES_PASSWORD": "change-me",
|
||||||
|
"POSTGRES_PORT": "5432"
|
||||||
|
},
|
||||||
|
"notes": [
|
||||||
|
"POSTGRES_PASSWORD is a placeholder. Set the real one in the room's .env, which is gitignored.",
|
||||||
|
"The connection string other services want is postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}"
|
||||||
|
]
|
||||||
|
}
|
||||||
24
soleprint/station/cabinets/postgres/service.yml
Normal file
24
soleprint/station/cabinets/postgres/service.yml
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
# PostgreSQL — merged into a room's docker-compose.yml when the room asks for it.
|
||||||
|
#
|
||||||
|
# The healthcheck is not decoration: anything with `depends_on: condition:
|
||||||
|
# service_healthy` waits on it, and without one a backend races the database on
|
||||||
|
# every cold start and fails its first migration.
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: ${DEPLOYMENT_NAME:-soleprint}_postgres
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-soleprint}
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER:-soleprint}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-change-me}
|
||||||
|
volumes:
|
||||||
|
- pgdata:/var/lib/postgresql/data
|
||||||
|
ports:
|
||||||
|
# Bound on the host so psql and databrowse can reach it from outside the
|
||||||
|
# compose network. Override POSTGRES_PORT when 5432 is already taken.
|
||||||
|
- "${POSTGRES_PORT:-5432}:5432"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-soleprint} -d ${POSTGRES_DB:-soleprint}"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
restart: unless-stopped
|
||||||
17
soleprint/station/cabinets/redis/cabinet.json
Normal file
17
soleprint/station/cabinets/redis/cabinet.json
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"name": "redis",
|
||||||
|
"title": "Redis",
|
||||||
|
"description": "In-memory store. Cache, and the broker Celery and Airflow run their queues on.",
|
||||||
|
"image": "redis:7-alpine",
|
||||||
|
"service": "redis",
|
||||||
|
"rig_addon": "redis",
|
||||||
|
"ports": [6379],
|
||||||
|
"volumes": ["redisdata"],
|
||||||
|
"env": {
|
||||||
|
"REDIS_PORT": "6379"
|
||||||
|
},
|
||||||
|
"notes": [
|
||||||
|
"The URL other services want is redis://redis:6379/0",
|
||||||
|
"Airflow depends on this one; switching airflow on brings it along."
|
||||||
|
]
|
||||||
|
}
|
||||||
14
soleprint/station/cabinets/redis/service.yml
Normal file
14
soleprint/station/cabinets/redis/service.yml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# Redis — cache, and the broker for anything queue-shaped in the room.
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
container_name: ${DEPLOYMENT_NAME:-soleprint}_redis
|
||||||
|
volumes:
|
||||||
|
- redisdata:/data
|
||||||
|
ports:
|
||||||
|
- "${REDIS_PORT:-6379}:6379"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
restart: unless-stopped
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<!doctype html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
html {
|
html {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
}
|
}
|
||||||
body {
|
body {
|
||||||
font-family:
|
font-family:
|
||||||
@@ -25,8 +25,8 @@
|
|||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 2rem 1rem;
|
padding: 2rem 1rem;
|
||||||
line-height: 1.6;
|
line-height: 1.6;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
background: #1d4ed8;
|
background: var(--system-accent);
|
||||||
}
|
}
|
||||||
header {
|
header {
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -51,7 +51,7 @@
|
|||||||
padding-bottom: 2rem;
|
padding-bottom: 2rem;
|
||||||
}
|
}
|
||||||
section {
|
section {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
padding: 1.5rem;
|
padding: 1.5rem;
|
||||||
margin: 1.5rem 0;
|
margin: 1.5rem 0;
|
||||||
border-radius: 12px;
|
border-radius: 12px;
|
||||||
@@ -59,23 +59,23 @@
|
|||||||
section h2 {
|
section h2 {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 1.2rem;
|
font-size: 1.2rem;
|
||||||
color: #93c5fd;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition {
|
.composition {
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 2px solid #1d4ed8;
|
border: 2px solid var(--system-accent);
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
border-radius: 12px;
|
border-radius: 12px;
|
||||||
}
|
}
|
||||||
.composition h3 {
|
.composition h3 {
|
||||||
margin: 0 0 0.75rem 0;
|
margin: 0 0 0.75rem 0;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
color: #93c5fd;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.composition > p {
|
.composition > p {
|
||||||
margin: 0 0 1rem 0;
|
margin: 0 0 1rem 0;
|
||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.components {
|
.components {
|
||||||
display: grid;
|
display: grid;
|
||||||
@@ -83,20 +83,20 @@
|
|||||||
gap: 0.75rem;
|
gap: 0.75rem;
|
||||||
}
|
}
|
||||||
.component {
|
.component {
|
||||||
background: #0a0a0a;
|
background: var(--bg);
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
padding: 0.75rem;
|
padding: 0.75rem;
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
.component h4 {
|
.component h4 {
|
||||||
margin: 0 0 0.25rem 0;
|
margin: 0 0 0.25rem 0;
|
||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
color: #93c5fd;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.component p {
|
.component p {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.tables {
|
.tables {
|
||||||
list-style: none;
|
list-style: none;
|
||||||
@@ -105,7 +105,7 @@
|
|||||||
}
|
}
|
||||||
.tables li {
|
.tables li {
|
||||||
padding: 0.75rem 0;
|
padding: 0.75rem 0;
|
||||||
border-bottom: 1px solid #3f3f3f;
|
border-bottom: 1px solid var(--border-strong);
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
@@ -116,32 +116,32 @@
|
|||||||
.tables .name {
|
.tables .name {
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
color: #e5e5e5;
|
color: var(--text);
|
||||||
}
|
}
|
||||||
.tables a.name:hover {
|
.tables a.name:hover {
|
||||||
color: #93c5fd;
|
color: var(--system-accent-text);
|
||||||
}
|
}
|
||||||
.status {
|
.status {
|
||||||
font-size: 0.75rem;
|
font-size: 0.75rem;
|
||||||
padding: 0.2rem 0.5rem;
|
padding: 0.2rem 0.5rem;
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
background: #2a2a2a;
|
background: var(--border);
|
||||||
color: #a3a3a3;
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
.health {
|
.health {
|
||||||
display: inline-block;
|
display: inline-block;
|
||||||
margin-top: 1rem;
|
margin-top: 1rem;
|
||||||
padding: 0.5rem 1rem;
|
padding: 0.5rem 1rem;
|
||||||
background: #1a1a1a;
|
background: var(--surface);
|
||||||
border: 1px solid #3f3f3f;
|
border: 1px solid var(--border-strong);
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
font-family: monospace;
|
font-family: monospace;
|
||||||
color: #93c5fd;
|
color: var(--system-accent-text);
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
}
|
}
|
||||||
.health:hover {
|
.health:hover {
|
||||||
background: #2a2a2a;
|
background: var(--border);
|
||||||
}
|
}
|
||||||
footer {
|
footer {
|
||||||
margin-top: 3rem;
|
margin-top: 3rem;
|
||||||
@@ -157,7 +157,12 @@
|
|||||||
opacity: 0.5;
|
opacity: 0.5;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
<style>
|
||||||
|
/* Station keeps its own colour under every theme. */
|
||||||
|
:root { --system-accent: #1d4ed8; --system-accent-text: #93c5fd; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header style="position: relative">
|
<header style="position: relative">
|
||||||
<!-- Control station / monitor -->
|
<!-- Control station / monitor -->
|
||||||
@@ -217,6 +222,20 @@
|
|||||||
</ul>
|
</ul>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h2>{{ (components.container.plural or 'cabinets')|title }}</h2>
|
||||||
|
<ul class="tables">
|
||||||
|
{% for cabinet in cabinets %}
|
||||||
|
<li>
|
||||||
|
<span class="name">{{ cabinet.title }}</span
|
||||||
|
><span class="status">{{ cabinet.status }}</span>
|
||||||
|
</li>
|
||||||
|
{% else %}
|
||||||
|
<li><span class="name">--</span></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
|
||||||
<section>
|
<section>
|
||||||
<h2>{{ (components.watcher.plural or 'monitors')|title }}</h2>
|
<h2>{{ (components.watcher.plural or 'monitors')|title }}</h2>
|
||||||
<ul class="tables">
|
<ul class="tables">
|
||||||
@@ -263,5 +282,6 @@
|
|||||||
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
|
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
|
||||||
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
|
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
|
||||||
</footer>
|
</footer>
|
||||||
</body>
|
<script src="/theme.js" defer></script>
|
||||||
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -1,27 +1,19 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>datagen — Test Data Generator</title>
|
<title>datagen — Test Data Generator</title>
|
||||||
|
<!-- Palette, fonts and the theme switcher. The :root block that used to sit
|
||||||
|
here was one of eight copies that had already drifted apart. -->
|
||||||
|
<link rel="stylesheet" href="/theme.css">
|
||||||
<style>
|
<style>
|
||||||
:root {
|
|
||||||
--bg: #0a0a0a;
|
|
||||||
--surface: #1a1a1a;
|
|
||||||
--border: #333;
|
|
||||||
--text: #e5e5e5;
|
|
||||||
--muted: #a3a3a3;
|
|
||||||
--dim: #666;
|
|
||||||
--amber: #d4a574;
|
|
||||||
--amber-dim: #b8956a;
|
|
||||||
}
|
|
||||||
|
|
||||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||||
|
|
||||||
body {
|
body {
|
||||||
background: var(--bg);
|
background: var(--bg);
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
font-family: system-ui, -apple-system, sans-serif;
|
font-family: var(--font-ui);
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
height: 100vh;
|
height: 100vh;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -493,5 +485,6 @@ document.addEventListener('keydown', e => {
|
|||||||
|
|
||||||
init();
|
init();
|
||||||
</script>
|
</script>
|
||||||
|
<script src="/theme.js" defer></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -1,27 +1,18 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="soleprint">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>graphgen — Schema Explorer</title>
|
<title>graphgen — Schema Explorer</title>
|
||||||
|
<!-- Palette, fonts and the theme switcher — see common/theme/. -->
|
||||||
|
<link rel="stylesheet" href="/theme.css">
|
||||||
<style>
|
<style>
|
||||||
:root {
|
|
||||||
--bg: #0a0a0a;
|
|
||||||
--surface: #1a1a1a;
|
|
||||||
--border: #333;
|
|
||||||
--text: #e5e5e5;
|
|
||||||
--muted: #a3a3a3;
|
|
||||||
--dim: #666;
|
|
||||||
--amber: #d4a574;
|
|
||||||
--amber-dim: #b8956a;
|
|
||||||
}
|
|
||||||
|
|
||||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||||
|
|
||||||
body {
|
body {
|
||||||
background: var(--bg);
|
background: var(--bg);
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
font-family: system-ui, -apple-system, sans-serif;
|
font-family: var(--font-ui);
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
height: 100vh;
|
height: 100vh;
|
||||||
@@ -725,5 +716,6 @@ function svgEl(tag) {
|
|||||||
applyViewport();
|
applyViewport();
|
||||||
init();
|
init();
|
||||||
</script>
|
</script>
|
||||||
|
<script src="/theme.js" defer></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -4,9 +4,12 @@ Modelgen - Generic Model Generation Tool
|
|||||||
Generates typed models from various sources to various formats.
|
Generates typed models from various sources to various formats.
|
||||||
|
|
||||||
Input sources:
|
Input sources:
|
||||||
- from-config: Configuration files (soleprint config.json style)
|
- from-config: Configuration files (soleprint config.json style)
|
||||||
- from-schema: Python dataclasses in schema/ folder
|
- from-schema: Python dataclasses in schema/ folder
|
||||||
- extract: Existing codebases (Django, SQLAlchemy, Prisma)
|
- extract: Existing codebases (Django, SQLAlchemy)
|
||||||
|
- from-db: A live database, any SQLAlchemy dialect
|
||||||
|
- from-openapi: An OpenAPI 3.x / Swagger 2.0 document
|
||||||
|
- from-tabular: A directory of CSV/TSV/ODS spreadsheets
|
||||||
|
|
||||||
Output formats:
|
Output formats:
|
||||||
- pydantic: Pydantic BaseModel classes
|
- pydantic: Pydantic BaseModel classes
|
||||||
@@ -14,12 +17,16 @@ Output formats:
|
|||||||
- typescript: TypeScript interfaces
|
- typescript: TypeScript interfaces
|
||||||
- protobuf: Protocol Buffer definitions
|
- protobuf: Protocol Buffer definitions
|
||||||
- prisma: Prisma schema
|
- prisma: Prisma schema
|
||||||
|
- schema: graphgen-compatible schema.json
|
||||||
|
- datagen: BaseDataGenerator subclass for station's datagen tool
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python -m soleprint.station.tools.modelgen --help
|
python -m soleprint.station.tools.modelgen --help
|
||||||
python -m soleprint.station.tools.modelgen from-config -c config.json -o models.py
|
python -m soleprint.station.tools.modelgen from-config -c config.json -o models.py
|
||||||
python -m soleprint.station.tools.modelgen from-schema -o models/ --targets pydantic,typescript
|
python -m soleprint.station.tools.modelgen from-schema -o models/ --targets pydantic,typescript
|
||||||
python -m soleprint.station.tools.modelgen extract --source /path/to/django --targets pydantic
|
python -m soleprint.station.tools.modelgen extract --source /path/to/django --targets pydantic
|
||||||
|
python -m soleprint.station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,schema
|
||||||
|
python -m soleprint.station.tools.modelgen from-tabular -s ./sheets -o out/ -t pydantic,datagen
|
||||||
python -m soleprint.station.tools.modelgen generate --config schema/modelgen.json
|
python -m soleprint.station.tools.modelgen generate --config schema/modelgen.json
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -225,6 +232,88 @@ def cmd_from_db(args):
|
|||||||
print("Done!")
|
print("Done!")
|
||||||
|
|
||||||
|
|
||||||
|
def _emit(payload, targets_arg: str, output: str) -> None:
|
||||||
|
"""Run one extraction result through every requested target.
|
||||||
|
|
||||||
|
The three older commands each carry their own copy of this loop; the two
|
||||||
|
below share it, because they also have to pass datasets through and a
|
||||||
|
fourth copy would be a fourth place to keep in step.
|
||||||
|
"""
|
||||||
|
targets = [t.strip() for t in targets_arg.split(",") if t.strip()]
|
||||||
|
output_dir = Path(output)
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
if target not in GENERATORS:
|
||||||
|
print(f"Warning: Unknown target '{target}', skipping", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
generator = GENERATORS[target]()
|
||||||
|
ext = generator.file_extension()
|
||||||
|
|
||||||
|
# Determine output filename (use target name to avoid overwrites)
|
||||||
|
if len(targets) == 1 and output.endswith(ext):
|
||||||
|
output_file = output_dir
|
||||||
|
else:
|
||||||
|
output_file = output_dir / f"models_{target}{ext}"
|
||||||
|
|
||||||
|
print(f"Generating {target} to: {output_file}")
|
||||||
|
generator.generate(payload, output_file)
|
||||||
|
|
||||||
|
print("Done!")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_from_openapi(args):
|
||||||
|
"""Generate models from an OpenAPI 3.x / Swagger 2.0 document."""
|
||||||
|
from .loader.extract.openapi import OpenAPIExtractor
|
||||||
|
|
||||||
|
spec_path = Path(args.spec)
|
||||||
|
if not spec_path.exists():
|
||||||
|
print(f"Error: Spec not found: {spec_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
extractor = OpenAPIExtractor(spec_path)
|
||||||
|
print(f"Reading spec: {spec_path}")
|
||||||
|
try:
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
except (RuntimeError, ValueError) as e:
|
||||||
|
print(f"Error: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
endpoints = extractor.endpoints()
|
||||||
|
print(
|
||||||
|
f"Extracted {len(models)} models, {len(enums)} enums, "
|
||||||
|
f"{len(endpoints)} endpoints"
|
||||||
|
)
|
||||||
|
|
||||||
|
_emit((models, enums), args.targets, args.output)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_from_tabular(args):
|
||||||
|
"""Generate models from a directory of CSV/TSV/ODS spreadsheets."""
|
||||||
|
from .loader.extract.tabular import TabularExtractor
|
||||||
|
|
||||||
|
source_path = Path(args.source)
|
||||||
|
if not source_path.exists():
|
||||||
|
print(f"Error: Source not found: {source_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
extractor = TabularExtractor(source_path)
|
||||||
|
print(f"Reading sheets: {source_path}")
|
||||||
|
try:
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
except ValueError as e:
|
||||||
|
print(f"Error: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
datasets = extractor.datasets()
|
||||||
|
rows = sum(len(d.rows) for d in datasets)
|
||||||
|
print(f"Extracted {len(models)} models, {rows} rows")
|
||||||
|
|
||||||
|
# Datasets ride along as a third element: the datagen target seeds from
|
||||||
|
# them, and every other target ignores the extra slot.
|
||||||
|
_emit((models, enums, datasets), args.targets, args.output)
|
||||||
|
|
||||||
|
|
||||||
def cmd_generate(args):
|
def cmd_generate(args):
|
||||||
"""Generate all targets from a JSON config file."""
|
"""Generate all targets from a JSON config file."""
|
||||||
import json
|
import json
|
||||||
@@ -430,6 +519,63 @@ def main():
|
|||||||
)
|
)
|
||||||
db_parser.set_defaults(func=cmd_from_db)
|
db_parser.set_defaults(func=cmd_from_db)
|
||||||
|
|
||||||
|
# from-openapi command (service contract -> models)
|
||||||
|
openapi_parser = subparsers.add_parser(
|
||||||
|
"from-openapi",
|
||||||
|
help="Generate models from an OpenAPI 3.x / Swagger 2.0 document",
|
||||||
|
)
|
||||||
|
openapi_parser.add_argument(
|
||||||
|
"--spec",
|
||||||
|
"-s",
|
||||||
|
type=str,
|
||||||
|
required=True,
|
||||||
|
help="Path to the spec (.json, .yaml or .yml; YAML needs PyYAML)",
|
||||||
|
)
|
||||||
|
openapi_parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
"-o",
|
||||||
|
type=str,
|
||||||
|
required=True,
|
||||||
|
help="Output path (file or directory)",
|
||||||
|
)
|
||||||
|
openapi_parser.add_argument(
|
||||||
|
"--targets",
|
||||||
|
"-t",
|
||||||
|
type=str,
|
||||||
|
default="pydantic",
|
||||||
|
help=f"Comma-separated output targets ({formats_str})",
|
||||||
|
)
|
||||||
|
openapi_parser.set_defaults(func=cmd_from_openapi)
|
||||||
|
|
||||||
|
# from-tabular command (spreadsheets -> models + rows)
|
||||||
|
tabular_parser = subparsers.add_parser(
|
||||||
|
"from-tabular",
|
||||||
|
help="Generate models from a directory of CSV/TSV/ODS spreadsheets",
|
||||||
|
)
|
||||||
|
tabular_parser.add_argument(
|
||||||
|
"--source",
|
||||||
|
"-s",
|
||||||
|
type=str,
|
||||||
|
required=True,
|
||||||
|
help="Directory of sheets (or a single .csv/.tsv/.ods file)",
|
||||||
|
)
|
||||||
|
tabular_parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
"-o",
|
||||||
|
type=str,
|
||||||
|
required=True,
|
||||||
|
help="Output path (file or directory)",
|
||||||
|
)
|
||||||
|
tabular_parser.add_argument(
|
||||||
|
"--targets",
|
||||||
|
"-t",
|
||||||
|
type=str,
|
||||||
|
default="pydantic",
|
||||||
|
help=f"Comma-separated output targets ({formats_str}). "
|
||||||
|
"The datagen target seeds from the imported rows.",
|
||||||
|
)
|
||||||
|
tabular_parser.set_defaults(func=cmd_from_tabular)
|
||||||
|
|
||||||
# generate command (config-driven multi-target)
|
# generate command (config-driven multi-target)
|
||||||
gen_parser = subparsers.add_parser(
|
gen_parser = subparsers.add_parser(
|
||||||
"generate",
|
"generate",
|
||||||
|
|||||||
@@ -8,11 +8,13 @@ Supported generators:
|
|||||||
- ProtobufGenerator: Protocol Buffer definitions
|
- ProtobufGenerator: Protocol Buffer definitions
|
||||||
- PrismaGenerator: Prisma schema
|
- PrismaGenerator: Prisma schema
|
||||||
- StrawberryGenerator: Strawberry type/input/enum classes
|
- StrawberryGenerator: Strawberry type/input/enum classes
|
||||||
|
- DatagenGenerator: BaseDataGenerator subclass for station's datagen tool
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Dict, Type
|
from typing import Dict, Type
|
||||||
|
|
||||||
from .base import BaseGenerator
|
from .base import BaseGenerator
|
||||||
|
from .datagen import DatagenGenerator
|
||||||
from .django import DjangoGenerator
|
from .django import DjangoGenerator
|
||||||
from .jsonschema import JsonSchemaGenerator
|
from .jsonschema import JsonSchemaGenerator
|
||||||
from .prisma import PrismaGenerator
|
from .prisma import PrismaGenerator
|
||||||
@@ -35,10 +37,12 @@ GENERATORS: Dict[str, Type[BaseGenerator]] = {
|
|||||||
"strawberry": StrawberryGenerator,
|
"strawberry": StrawberryGenerator,
|
||||||
"schema": JsonSchemaGenerator,
|
"schema": JsonSchemaGenerator,
|
||||||
"jsonschema": JsonSchemaGenerator, # Alias
|
"jsonschema": JsonSchemaGenerator, # Alias
|
||||||
|
"datagen": DatagenGenerator,
|
||||||
}
|
}
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BaseGenerator",
|
"BaseGenerator",
|
||||||
|
"DatagenGenerator",
|
||||||
"PydanticGenerator",
|
"PydanticGenerator",
|
||||||
"DjangoGenerator",
|
"DjangoGenerator",
|
||||||
"StrawberryGenerator",
|
"StrawberryGenerator",
|
||||||
|
|||||||
399
soleprint/station/tools/modelgen/generator/datagen.py
Normal file
399
soleprint/station/tools/modelgen/generator/datagen.py
Normal file
@@ -0,0 +1,399 @@
|
|||||||
|
"""
|
||||||
|
Datagen Generator
|
||||||
|
|
||||||
|
Emits a BaseDataGenerator subclass — the bridge between modelgen (which knows
|
||||||
|
the shapes) and datagen (which hands out instances of them).
|
||||||
|
|
||||||
|
The contract it targets is datagen/base.py: a method per model, named the way
|
||||||
|
`generate()` looks it up (`model.lower()`), plus a `schema()` override in the
|
||||||
|
graphgen-compatible format so the result also renders in graphgen with no extra
|
||||||
|
work.
|
||||||
|
|
||||||
|
Two modes, chosen by what the loader had:
|
||||||
|
|
||||||
|
- No rows → every field is synthesised from its type.
|
||||||
|
- Rows present → the generated class samples depot/data.json, so a shunt
|
||||||
|
built from a spreadsheet answers with the real values and
|
||||||
|
only invents where it must.
|
||||||
|
|
||||||
|
Synthesis uses random/uuid/datetime from the standard library rather than
|
||||||
|
faker, which is not a dependency of this repo despite what datagen's README
|
||||||
|
examples suggest.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from pprint import pformat
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from ..helpers import unwrap_optional
|
||||||
|
from ..loader.schema import ModelDefinition
|
||||||
|
from .base import BaseGenerator
|
||||||
|
|
||||||
|
# Field-name heuristics. A column called `email` deserves something that looks
|
||||||
|
# like an email — a mock whose every string is "string_4" is hard to read and
|
||||||
|
# hard to demo.
|
||||||
|
_NAME_HINTS = (
|
||||||
|
("email", '"user{n}@example.com".format(n=random.randint(1, 999))'),
|
||||||
|
("phone", '"+1-555-{n:04d}".format(n=random.randint(0, 9999))'),
|
||||||
|
("url", '"https://example.com/{n}".format(n=random.randint(1, 999))'),
|
||||||
|
("slug", 'random.choice(_WORDS) + "-" + str(random.randint(1, 99))'),
|
||||||
|
("first_name", "random.choice(_FIRST_NAMES)"),
|
||||||
|
("last_name", "random.choice(_LAST_NAMES)"),
|
||||||
|
("name", 'random.choice(_FIRST_NAMES) + " " + random.choice(_LAST_NAMES)'),
|
||||||
|
("title", 'random.choice(_WORDS).title() + " " + random.choice(_WORDS)'),
|
||||||
|
("description", '" ".join(random.choices(_WORDS, k=8))'),
|
||||||
|
("address", '"{n} ".format(n=random.randint(1, 9999)) + random.choice(_WORDS).title() + " St"'),
|
||||||
|
("city", "random.choice(_CITIES)"),
|
||||||
|
("country", "random.choice(_COUNTRIES)"),
|
||||||
|
("currency", 'random.choice(["USD", "EUR", "ARS", "GBP"])'),
|
||||||
|
("status", 'random.choice(["active", "pending", "closed"])'),
|
||||||
|
("code", '"".join(random.choices("ABCDEFGHJKLMNPQRSTUVWXYZ0123456789", k=6))'),
|
||||||
|
("token", "uuid.uuid4().hex"),
|
||||||
|
("color", '"#{n:06x}".format(n=random.randint(0, 0xFFFFFF))'),
|
||||||
|
)
|
||||||
|
|
||||||
|
_HEADER_HELPERS = '''
|
||||||
|
_WORDS = [
|
||||||
|
"alpha", "bravo", "cobalt", "delta", "ember", "falcon", "granite", "harbor",
|
||||||
|
"indigo", "juniper", "kestrel", "lumen", "meridian", "nimbus", "onyx",
|
||||||
|
]
|
||||||
|
_FIRST_NAMES = ["Ada", "Bruno", "Camila", "Diego", "Elena", "Facundo", "Gabriela", "Hugo"]
|
||||||
|
_LAST_NAMES = ["Alvarez", "Bianchi", "Castro", "Duarte", "Esposito", "Ferrari", "Gomez"]
|
||||||
|
_CITIES = ["Buenos Aires", "Rosario", "Cordoba", "Montevideo", "Santiago", "Lisbon"]
|
||||||
|
_COUNTRIES = ["AR", "UY", "CL", "BR", "PT", "ES"]
|
||||||
|
'''
|
||||||
|
|
||||||
|
_FALLBACK_BASE = '''
|
||||||
|
# Standalone fallback: a generated generator has to keep working inside a
|
||||||
|
# shunt, which runs as its own process with no soleprint on the path. This
|
||||||
|
# mirrors station/tools/datagen/base.py.
|
||||||
|
class BaseDataGenerator: # type: ignore[no-redef]
|
||||||
|
"""Minimal stand-in for station.tools.datagen.base.BaseDataGenerator."""
|
||||||
|
|
||||||
|
_RESERVED = frozenset({"generate", "available_models", "schema"})
|
||||||
|
|
||||||
|
def generate(self, model: str, count: int = 1, **kwargs) -> list:
|
||||||
|
method = getattr(self, model.lower(), None)
|
||||||
|
if method is None or not callable(method):
|
||||||
|
raise ValueError(
|
||||||
|
f"No generator for '{model}'. Available: {self.available_models()}"
|
||||||
|
)
|
||||||
|
return [method(**kwargs) for _ in range(count)]
|
||||||
|
|
||||||
|
def available_models(self) -> list:
|
||||||
|
return sorted(
|
||||||
|
name for name in dir(self)
|
||||||
|
if not name.startswith("_")
|
||||||
|
and name not in self._RESERVED
|
||||||
|
and callable(getattr(self, name))
|
||||||
|
)
|
||||||
|
|
||||||
|
def schema(self):
|
||||||
|
return None
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
class DatagenGenerator(BaseGenerator):
|
||||||
|
"""Generates a BaseDataGenerator subclass from model definitions."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
name_map: Optional[Dict[str, str]] = None,
|
||||||
|
class_name: Optional[str] = None,
|
||||||
|
depot: str = "depot/data.json",
|
||||||
|
):
|
||||||
|
super().__init__(name_map)
|
||||||
|
self.class_name = class_name
|
||||||
|
# Relative to the generated file, so the shunt directory stays movable.
|
||||||
|
self.depot = depot
|
||||||
|
|
||||||
|
def file_extension(self) -> str:
|
||||||
|
return ".py"
|
||||||
|
|
||||||
|
def generate(self, models, output_path: Path) -> None:
|
||||||
|
model_defs, _enums, datasets = self._unpack(models)
|
||||||
|
|
||||||
|
output_path = Path(output_path)
|
||||||
|
if output_path.suffix != ".py":
|
||||||
|
output_path = output_path / "generated_datagen.py"
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
class_name = self.class_name or self._derive_class_name(output_path)
|
||||||
|
seeded = {d.model for d in datasets if d.rows}
|
||||||
|
model_names = {self.map_name(m.name) for m in model_defs}
|
||||||
|
|
||||||
|
lines = self._header(class_name, bool(seeded))
|
||||||
|
for model_def in model_defs:
|
||||||
|
lines.extend(self._model_method(model_def, model_names, seeded))
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
lines.extend(self._schema_method(model_defs, model_names))
|
||||||
|
lines.append("")
|
||||||
|
lines.extend(self._helpers(bool(seeded)))
|
||||||
|
|
||||||
|
output_path.write_text("\n".join(lines))
|
||||||
|
|
||||||
|
# ── input handling ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _unpack(models) -> tuple:
|
||||||
|
"""Accept the shapes the other generators accept, plus datasets."""
|
||||||
|
if isinstance(models, tuple):
|
||||||
|
model_defs = list(models[0])
|
||||||
|
enum_defs = list(models[1]) if len(models) > 1 else []
|
||||||
|
datasets = list(models[2]) if len(models) > 2 else []
|
||||||
|
return model_defs, enum_defs, datasets
|
||||||
|
if hasattr(models, "models"):
|
||||||
|
# SchemaLoader
|
||||||
|
model_defs = list(models.models) + list(getattr(models, "api_models", []))
|
||||||
|
return model_defs, list(getattr(models, "enums", [])), []
|
||||||
|
if isinstance(models, list):
|
||||||
|
return list(models), [], []
|
||||||
|
raise ValueError(f"Unsupported input type: {type(models)}")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _derive_class_name(output_path: Path) -> str:
|
||||||
|
stem = output_path.stem.replace("datagen_", "").replace("_datagen", "")
|
||||||
|
parts = [p for p in stem.replace("-", "_").split("_") if p]
|
||||||
|
base = "".join(p[:1].upper() + p[1:] for p in parts) or "Generated"
|
||||||
|
return f"{base}Generator"
|
||||||
|
|
||||||
|
# ── emission ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _header(self, class_name: str, seeded: bool) -> List[str]:
|
||||||
|
lines = [
|
||||||
|
'"""',
|
||||||
|
"Data generator - GENERATED FILE",
|
||||||
|
"",
|
||||||
|
"Do not edit directly. Regenerate using modelgen (target: datagen).",
|
||||||
|
'"""',
|
||||||
|
"",
|
||||||
|
"import json",
|
||||||
|
"import random",
|
||||||
|
"import uuid",
|
||||||
|
"from datetime import datetime, timedelta, timezone",
|
||||||
|
"from pathlib import Path",
|
||||||
|
"",
|
||||||
|
"try:",
|
||||||
|
" from station.tools.datagen.base import BaseDataGenerator",
|
||||||
|
"except ImportError: # pragma: no cover - standalone use",
|
||||||
|
_FALLBACK_BASE.strip("\n"),
|
||||||
|
"",
|
||||||
|
_HEADER_HELPERS.strip("\n"),
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
f"class {class_name}(BaseDataGenerator):",
|
||||||
|
' """Generated from a modelgen schema."""',
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
if seeded:
|
||||||
|
lines.extend([
|
||||||
|
f' _DEPOT = Path(__file__).parent / "{self.depot}"',
|
||||||
|
"",
|
||||||
|
" def __init__(self):",
|
||||||
|
" self._seed = {}",
|
||||||
|
" if self._DEPOT.exists():",
|
||||||
|
" try:",
|
||||||
|
" self._seed = json.loads(self._DEPOT.read_text())",
|
||||||
|
" except (OSError, ValueError):",
|
||||||
|
" self._seed = {}",
|
||||||
|
"",
|
||||||
|
])
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _model_method(
|
||||||
|
self, model_def: ModelDefinition, model_names: set, seeded: set
|
||||||
|
) -> List[str]:
|
||||||
|
mapped = self.map_name(model_def.name)
|
||||||
|
doc = (model_def.docstring or mapped).strip().splitlines()[0]
|
||||||
|
|
||||||
|
lines = [
|
||||||
|
f" def {mapped.lower()}(self, **kwargs) -> dict:",
|
||||||
|
f' """{doc}"""',
|
||||||
|
]
|
||||||
|
|
||||||
|
if model_def.name in seeded:
|
||||||
|
# Real rows first; synthesis is the fallback for when the depot is
|
||||||
|
# missing or a caller asked for more rows than were imported.
|
||||||
|
lines.append(f' record = self._sample("{model_def.name}")')
|
||||||
|
lines.append(" if record is None:")
|
||||||
|
indent = " "
|
||||||
|
else:
|
||||||
|
indent = " "
|
||||||
|
|
||||||
|
if model_def.fields:
|
||||||
|
lines.append(f"{indent}record = {{")
|
||||||
|
for field in model_def.fields:
|
||||||
|
value = self._value(field, model_names)
|
||||||
|
lines.append(f'{indent} "{field.name}": {value},')
|
||||||
|
lines.append(f"{indent}}}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{indent}record = {{}}")
|
||||||
|
|
||||||
|
lines.append(" record.update(kwargs)")
|
||||||
|
lines.append(" return record")
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def _value(self, field: Any, model_names: set) -> str:
|
||||||
|
base, _ = unwrap_optional(field.type_hint)
|
||||||
|
name = field.name.lower()
|
||||||
|
|
||||||
|
if getattr(field, "primary_key", False):
|
||||||
|
if base in (int, "bigint"):
|
||||||
|
return "self._next_id()"
|
||||||
|
return "str(uuid.uuid4())"
|
||||||
|
|
||||||
|
fk = getattr(field, "foreign_key", None)
|
||||||
|
if fk:
|
||||||
|
target = self.map_name(fk)
|
||||||
|
# How a relation is carried depends on the shape it was found in.
|
||||||
|
# A `customer_id` column holds a key; a `category` property in a
|
||||||
|
# spec holds the object itself, and answering that one with a key
|
||||||
|
# would be the wrong shape, not merely a dull value.
|
||||||
|
if base == "dict":
|
||||||
|
return f'self._nested("{target}")'
|
||||||
|
if base == "list":
|
||||||
|
return f'self._nested_list("{target}")'
|
||||||
|
if base in (int, "bigint"):
|
||||||
|
# A plausible existing key, not a fresh one: a foreign key that
|
||||||
|
# never matches anything makes the mock useless for joins.
|
||||||
|
return "random.randint(1, 100)"
|
||||||
|
return "str(uuid.uuid4())"
|
||||||
|
|
||||||
|
if isinstance(base, type) and issubclass(base, Enum):
|
||||||
|
values = json.dumps([m.value for m in base])
|
||||||
|
return f"random.choice({values})"
|
||||||
|
|
||||||
|
if base is bool:
|
||||||
|
return "random.choice([True, False])"
|
||||||
|
if base in (int, "bigint"):
|
||||||
|
return "random.randint(1, 10000)"
|
||||||
|
if base is float:
|
||||||
|
return "round(random.uniform(1, 10000), 2)"
|
||||||
|
if base == "UUID":
|
||||||
|
return "str(uuid.uuid4())"
|
||||||
|
if base == "datetime":
|
||||||
|
return "self._recent()"
|
||||||
|
if base == "dict":
|
||||||
|
return "{}"
|
||||||
|
if base == "list":
|
||||||
|
return "[]"
|
||||||
|
if base == "bytes":
|
||||||
|
return '""'
|
||||||
|
|
||||||
|
for hint, expression in _NAME_HINTS:
|
||||||
|
if hint in name:
|
||||||
|
return expression
|
||||||
|
|
||||||
|
return 'random.choice(_WORDS) + "-" + str(random.randint(1, 999))'
|
||||||
|
|
||||||
|
def _schema_method(
|
||||||
|
self, model_defs: List[ModelDefinition], model_names: set
|
||||||
|
) -> List[str]:
|
||||||
|
schema: Dict[str, Any] = {"models": {}}
|
||||||
|
for model_def in model_defs:
|
||||||
|
mapped = self.map_name(model_def.name)
|
||||||
|
entry: Dict[str, Any] = {}
|
||||||
|
if model_def.docstring:
|
||||||
|
entry["doc"] = model_def.docstring.strip().splitlines()[0]
|
||||||
|
fields: Dict[str, Any] = {}
|
||||||
|
for field in model_def.fields:
|
||||||
|
fields[field.name] = self._schema_field(field, model_names)
|
||||||
|
entry["fields"] = fields
|
||||||
|
schema["models"][mapped] = entry
|
||||||
|
|
||||||
|
# pformat, not json.dumps: the result is spliced into a source file, so
|
||||||
|
# it has to be a Python literal. Post-processing JSON with string
|
||||||
|
# replacement would rewrite "true" inside a docstring too.
|
||||||
|
body = pformat(schema, indent=4, width=88, sort_dicts=False)
|
||||||
|
indented = "\n".join(f" {line}" for line in body.splitlines())
|
||||||
|
|
||||||
|
return [
|
||||||
|
" def schema(self) -> dict:",
|
||||||
|
' """Graphgen-compatible schema — surfaced at /tools/datagen/api/schema."""',
|
||||||
|
f" return {indented.lstrip()}",
|
||||||
|
]
|
||||||
|
|
||||||
|
def _schema_field(self, field: Any, model_names: set) -> Dict[str, Any]:
|
||||||
|
base, is_optional = unwrap_optional(field.type_hint)
|
||||||
|
fk = getattr(field, "foreign_key", None)
|
||||||
|
|
||||||
|
if fk:
|
||||||
|
relation = "M2M" if getattr(field, "many", False) else "FK"
|
||||||
|
type_value = f"{relation}:{self.map_name(fk)}"
|
||||||
|
elif isinstance(base, type) and issubclass(base, Enum):
|
||||||
|
type_value = base.__name__
|
||||||
|
elif isinstance(base, str):
|
||||||
|
type_value = base
|
||||||
|
elif hasattr(base, "__name__"):
|
||||||
|
type_value = base.__name__
|
||||||
|
else:
|
||||||
|
type_value = "Any"
|
||||||
|
|
||||||
|
out: Dict[str, Any] = {
|
||||||
|
"type": type_value,
|
||||||
|
"nullable": bool(getattr(field, "optional", False) or is_optional),
|
||||||
|
}
|
||||||
|
if getattr(field, "primary_key", False):
|
||||||
|
out["pk"] = True
|
||||||
|
if getattr(field, "unique", False):
|
||||||
|
out["unique"] = True
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _helpers(seeded: bool) -> List[str]:
|
||||||
|
lines = [
|
||||||
|
" # ── helpers ────────────────────────────────────────────────",
|
||||||
|
"",
|
||||||
|
" _counter = 0",
|
||||||
|
"",
|
||||||
|
" def _next_id(self) -> int:",
|
||||||
|
' """Monotonic ids, so generated relations can be joined."""',
|
||||||
|
" type(self)._counter += 1",
|
||||||
|
" return type(self)._counter",
|
||||||
|
"",
|
||||||
|
" @staticmethod",
|
||||||
|
" def _recent() -> str:",
|
||||||
|
' """An ISO timestamp within the last 90 days."""',
|
||||||
|
" moment = datetime.now(timezone.utc) - timedelta(",
|
||||||
|
" days=random.randint(0, 90), seconds=random.randint(0, 86399)",
|
||||||
|
" )",
|
||||||
|
" return moment.isoformat()",
|
||||||
|
"",
|
||||||
|
" # Nesting is capped rather than followed: schemas refer to each",
|
||||||
|
" # other in cycles, and an uncapped expansion recurses forever.",
|
||||||
|
" _depth = 0",
|
||||||
|
" _MAX_DEPTH = 2",
|
||||||
|
"",
|
||||||
|
" def _nested(self, model: str):",
|
||||||
|
' """One related object, or None once the depth cap is reached."""',
|
||||||
|
" if type(self)._depth >= self._MAX_DEPTH:",
|
||||||
|
" return None",
|
||||||
|
" method = getattr(self, model.lower(), None)",
|
||||||
|
" if not callable(method):",
|
||||||
|
" return None",
|
||||||
|
" type(self)._depth += 1",
|
||||||
|
" try:",
|
||||||
|
" return method()",
|
||||||
|
" finally:",
|
||||||
|
" type(self)._depth -= 1",
|
||||||
|
"",
|
||||||
|
" def _nested_list(self, model: str, count: int = 2) -> list:",
|
||||||
|
' """A short list of related objects."""',
|
||||||
|
" items = [self._nested(model) for _ in range(count)]",
|
||||||
|
" return [item for item in items if item is not None]",
|
||||||
|
]
|
||||||
|
if seeded:
|
||||||
|
lines.extend([
|
||||||
|
"",
|
||||||
|
" def _sample(self, model: str):",
|
||||||
|
' """A copy of a real imported row, or None if there are none."""',
|
||||||
|
" rows = self._seed.get(model) or []",
|
||||||
|
" if not rows:",
|
||||||
|
" return None",
|
||||||
|
" return dict(random.choice(rows))",
|
||||||
|
])
|
||||||
|
lines.append("")
|
||||||
|
return lines
|
||||||
@@ -84,7 +84,8 @@ class JsonSchemaGenerator(BaseGenerator):
|
|||||||
|
|
||||||
# Resolve the relationship-aware type string graphgen expects.
|
# Resolve the relationship-aware type string graphgen expects.
|
||||||
if fk_target:
|
if fk_target:
|
||||||
type_value = f"FK:{self.map_name(fk_target)}"
|
relation = "M2M" if getattr(field, "many", False) else "FK"
|
||||||
|
type_value = f"{relation}:{self.map_name(fk_target)}"
|
||||||
elif type_str in model_names:
|
elif type_str in model_names:
|
||||||
type_value = f"FK:{type_str}"
|
type_value = f"FK:{type_str}"
|
||||||
elif type_str == "M2M":
|
elif type_str == "M2M":
|
||||||
|
|||||||
@@ -27,8 +27,9 @@ class ProtobufGenerator(BaseGenerator):
|
|||||||
if hasattr(models, "grpc_messages"):
|
if hasattr(models, "grpc_messages"):
|
||||||
# SchemaLoader with gRPC definitions
|
# SchemaLoader with gRPC definitions
|
||||||
content = self._generate_from_loader(models)
|
content = self._generate_from_loader(models)
|
||||||
elif isinstance(models, tuple) and len(models) >= 3:
|
elif isinstance(models, tuple) and len(models) >= 2:
|
||||||
# (messages, service_def) tuple
|
# (models, enums, ...) tuple — same first two slots as every other
|
||||||
|
# generator; extra slots (datasets) belong to targets that use them.
|
||||||
content = self._generate_from_definitions(models[0], models[1])
|
content = self._generate_from_definitions(models[0], models[1])
|
||||||
elif isinstance(models, list):
|
elif isinstance(models, list):
|
||||||
# List of dataclasses (MPR style)
|
# List of dataclasses (MPR style)
|
||||||
|
|||||||
@@ -8,8 +8,16 @@ Supported loaders:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from .config import ConfigLoader, load_config
|
from .config import ConfigLoader, load_config
|
||||||
from .extract import EXTRACTORS, BaseExtractor, DjangoExtractor
|
from .extract import (
|
||||||
|
EXTRACTORS,
|
||||||
|
BaseExtractor,
|
||||||
|
DjangoExtractor,
|
||||||
|
OpenAPIExtractor,
|
||||||
|
TabularExtractor,
|
||||||
|
)
|
||||||
from .schema import (
|
from .schema import (
|
||||||
|
DatasetDefinition,
|
||||||
|
EndpointDefinition,
|
||||||
EnumDefinition,
|
EnumDefinition,
|
||||||
FieldDefinition,
|
FieldDefinition,
|
||||||
GrpcServiceDefinition,
|
GrpcServiceDefinition,
|
||||||
@@ -30,8 +38,12 @@ __all__ = [
|
|||||||
"FieldDefinition",
|
"FieldDefinition",
|
||||||
"EnumDefinition",
|
"EnumDefinition",
|
||||||
"GrpcServiceDefinition",
|
"GrpcServiceDefinition",
|
||||||
|
"EndpointDefinition",
|
||||||
|
"DatasetDefinition",
|
||||||
# Extractors
|
# Extractors
|
||||||
"BaseExtractor",
|
"BaseExtractor",
|
||||||
"DjangoExtractor",
|
"DjangoExtractor",
|
||||||
|
"OpenAPIExtractor",
|
||||||
|
"TabularExtractor",
|
||||||
"EXTRACTORS",
|
"EXTRACTORS",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,29 +1,41 @@
|
|||||||
"""
|
"""
|
||||||
Extractors - Extract model definitions from existing codebases.
|
Extractors - Extract model definitions from existing codebases.
|
||||||
|
|
||||||
Supported frameworks:
|
Supported sources:
|
||||||
- Django: Extract from Django ORM models
|
- Django: Extract from Django ORM models
|
||||||
- SQLAlchemy: Extract from SQLAlchemy models (planned)
|
- SQLAlchemy: Extract from SQLAlchemy models
|
||||||
- Prisma: Extract from Prisma schema (planned)
|
- OpenAPI: Extract from an OpenAPI 3.x / Swagger 2.0 document
|
||||||
|
- Tabular: Extract from a directory of CSV/TSV/ODS spreadsheets
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Dict, Type
|
from typing import Dict, Type
|
||||||
|
|
||||||
from .base import BaseExtractor
|
from .base import BaseExtractor
|
||||||
from .django import DjangoExtractor
|
from .django import DjangoExtractor
|
||||||
|
from .openapi import OpenAPIExtractor
|
||||||
from .sqlalchemy_models import SqlAlchemyExtractor
|
from .sqlalchemy_models import SqlAlchemyExtractor
|
||||||
|
from .tabular import TabularExtractor
|
||||||
|
|
||||||
# Registry of code-source extractors (auto-detectable via detect()).
|
# Registry of source extractors (auto-detectable via detect()).
|
||||||
|
#
|
||||||
|
# Ordering matters for `--framework auto`: detection runs in insertion order and
|
||||||
|
# stops at the first match, so the two that inspect a *file* come after the two
|
||||||
|
# that inspect a *source tree* and cannot be confused with them.
|
||||||
|
#
|
||||||
# Note: live-database introspection lives in database.py (DatabaseExtractor),
|
# Note: live-database introspection lives in database.py (DatabaseExtractor),
|
||||||
# invoked explicitly via the `from-db` command since it takes a URL, not a path.
|
# invoked explicitly via the `from-db` command since it takes a URL, not a path.
|
||||||
EXTRACTORS: Dict[str, Type[BaseExtractor]] = {
|
EXTRACTORS: Dict[str, Type[BaseExtractor]] = {
|
||||||
"django": DjangoExtractor,
|
"django": DjangoExtractor,
|
||||||
"sqlalchemy": SqlAlchemyExtractor,
|
"sqlalchemy": SqlAlchemyExtractor,
|
||||||
|
"openapi": OpenAPIExtractor,
|
||||||
|
"tabular": TabularExtractor,
|
||||||
}
|
}
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BaseExtractor",
|
"BaseExtractor",
|
||||||
"DjangoExtractor",
|
"DjangoExtractor",
|
||||||
"SqlAlchemyExtractor",
|
"SqlAlchemyExtractor",
|
||||||
|
"OpenAPIExtractor",
|
||||||
|
"TabularExtractor",
|
||||||
"EXTRACTORS",
|
"EXTRACTORS",
|
||||||
]
|
]
|
||||||
|
|||||||
440
soleprint/station/tools/modelgen/loader/extract/openapi.py
Normal file
440
soleprint/station/tools/modelgen/loader/extract/openapi.py
Normal file
@@ -0,0 +1,440 @@
|
|||||||
|
"""
|
||||||
|
OpenAPI Extractor
|
||||||
|
|
||||||
|
Reads an OpenAPI 3.x or Swagger 2.0 document and produces modelgen's IR.
|
||||||
|
|
||||||
|
One parse yields two products:
|
||||||
|
|
||||||
|
extractor = OpenAPIExtractor("api.yaml")
|
||||||
|
models, enums = extractor.extract() # the shapes (BaseExtractor contract)
|
||||||
|
endpoints = extractor.endpoints() # the routes (what shuntgen consumes)
|
||||||
|
|
||||||
|
Only the first is part of the BaseExtractor contract; a spec is the one input
|
||||||
|
that describes calls as well as shapes, so endpoints() is an addition rather
|
||||||
|
than a widening of the ABC.
|
||||||
|
|
||||||
|
Referenced schemas become foreign_key metadata rather than nested types, which
|
||||||
|
is the same call DatabaseExtractor makes (see database.py) and for the same
|
||||||
|
reason: it keeps every generator emitting valid code, and graphgen still draws
|
||||||
|
the edge. Enums are materialised as real Enum classes so the existing
|
||||||
|
PYDANTIC_RESOLVERS/TS_RESOLVERS "enum" branch resolves them by name.
|
||||||
|
|
||||||
|
YAML is optional. JSON specs parse with the stdlib alone — which is what keeps
|
||||||
|
modelgen installable with no dependencies — and a .yaml spec needs PyYAML.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from ..schema import EndpointDefinition, EnumDefinition, FieldDefinition, ModelDefinition
|
||||||
|
from .base import BaseExtractor
|
||||||
|
|
||||||
|
_YAML_HINT = (
|
||||||
|
"Reading a YAML spec requires PyYAML. Install it with:\n"
|
||||||
|
" pip install pyyaml\n"
|
||||||
|
"(JSON specs need nothing beyond the standard library.)"
|
||||||
|
)
|
||||||
|
|
||||||
|
# OpenAPI `format` is more specific than `type`, so it wins where both are set.
|
||||||
|
_FORMAT_HINTS: Dict[str, Any] = {
|
||||||
|
"date-time": "datetime",
|
||||||
|
"date": "datetime",
|
||||||
|
"uuid": "UUID",
|
||||||
|
"binary": "bytes",
|
||||||
|
"byte": "bytes",
|
||||||
|
"int64": "bigint",
|
||||||
|
}
|
||||||
|
|
||||||
|
_TYPE_HINTS: Dict[str, Any] = {
|
||||||
|
"string": str,
|
||||||
|
"integer": int,
|
||||||
|
"number": float,
|
||||||
|
"boolean": bool,
|
||||||
|
"array": "list",
|
||||||
|
"object": "dict",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Response codes worth reading a body off, best first. "default" is last
|
||||||
|
# because a spec that defines it usually means the error case.
|
||||||
|
_STATUS_PREFERENCE = ("200", "201", "2XX", "202", "203", "204", "default")
|
||||||
|
|
||||||
|
|
||||||
|
def _to_model_name(name: str) -> str:
|
||||||
|
"""Convert a schema name to PascalCase (pet_category -> PetCategory)."""
|
||||||
|
parts = [p for p in re.split(r"[^0-9a-zA-Z]+", name) if p]
|
||||||
|
return "".join(p[:1].upper() + p[1:] for p in parts) or name
|
||||||
|
|
||||||
|
|
||||||
|
def _to_member_name(value: Any) -> str:
|
||||||
|
"""Convert an enum value to a legal Python identifier."""
|
||||||
|
name = re.sub(r"[^0-9a-zA-Z]+", "_", str(value)).strip("_").upper()
|
||||||
|
if not name:
|
||||||
|
return "EMPTY"
|
||||||
|
if name[0].isdigit():
|
||||||
|
return f"V_{name}"
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def _load_document(path: Path) -> dict:
|
||||||
|
"""Parse a spec file. JSON via stdlib; YAML only if PyYAML is installed."""
|
||||||
|
text = path.read_text()
|
||||||
|
|
||||||
|
if path.suffix.lower() in (".yaml", ".yml"):
|
||||||
|
try:
|
||||||
|
import yaml
|
||||||
|
except ImportError as e: # pragma: no cover - exercised only without the extra
|
||||||
|
raise RuntimeError(_YAML_HINT) from e
|
||||||
|
return yaml.safe_load(text) or {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
return json.loads(text)
|
||||||
|
except ValueError:
|
||||||
|
# A spec is often handed over with no extension or a wrong one, so fall
|
||||||
|
# back to YAML rather than failing on what is really a naming mistake.
|
||||||
|
try:
|
||||||
|
import yaml
|
||||||
|
except ImportError as e:
|
||||||
|
raise ValueError(
|
||||||
|
f"{path} is not valid JSON, and YAML support is unavailable.\n{_YAML_HINT}"
|
||||||
|
) from e
|
||||||
|
return yaml.safe_load(text) or {}
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAPIExtractor(BaseExtractor):
|
||||||
|
"""Extracts modelgen IR from an OpenAPI 3.x / Swagger 2.0 document."""
|
||||||
|
|
||||||
|
def __init__(self, source_path):
|
||||||
|
super().__init__(source_path)
|
||||||
|
self._doc: Optional[dict] = None
|
||||||
|
self._enums: Dict[str, EnumDefinition] = {}
|
||||||
|
self._enum_types: Dict[str, type] = {}
|
||||||
|
|
||||||
|
# ── document access ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@property
|
||||||
|
def doc(self) -> dict:
|
||||||
|
if self._doc is None:
|
||||||
|
self._doc = _load_document(self.source_path)
|
||||||
|
return self._doc
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_swagger2(self) -> bool:
|
||||||
|
return "swagger" in self.doc and "openapi" not in self.doc
|
||||||
|
|
||||||
|
def _schemas(self) -> Dict[str, dict]:
|
||||||
|
"""Named schemas, wherever this spec version keeps them."""
|
||||||
|
if self.is_swagger2:
|
||||||
|
return self.doc.get("definitions", {}) or {}
|
||||||
|
return (self.doc.get("components", {}) or {}).get("schemas", {}) or {}
|
||||||
|
|
||||||
|
def detect(self) -> bool:
|
||||||
|
"""True when the source is a parseable document declaring a spec version."""
|
||||||
|
if not self.source_path.is_file():
|
||||||
|
return False
|
||||||
|
if self.source_path.suffix.lower() not in (".json", ".yaml", ".yml"):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
doc = self.doc
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return isinstance(doc, dict) and ("openapi" in doc or "swagger" in doc)
|
||||||
|
|
||||||
|
# ── $ref plumbing ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ref_name(ref: str) -> str:
|
||||||
|
return _to_model_name(ref.rsplit("/", 1)[-1])
|
||||||
|
|
||||||
|
def _deref(self, schema: Any, _seen: Optional[set] = None) -> dict:
|
||||||
|
"""Follow $ref chains to the schema they land on."""
|
||||||
|
if not isinstance(schema, dict):
|
||||||
|
return {}
|
||||||
|
seen = _seen or set()
|
||||||
|
ref = schema.get("$ref")
|
||||||
|
if not ref or ref in seen:
|
||||||
|
return schema
|
||||||
|
seen.add(ref)
|
||||||
|
target = self.doc
|
||||||
|
for part in ref.lstrip("#/").split("/"):
|
||||||
|
if not isinstance(target, dict):
|
||||||
|
return {}
|
||||||
|
target = target.get(part, {})
|
||||||
|
return self._deref(target, seen)
|
||||||
|
|
||||||
|
def _flatten(self, schema: dict) -> Tuple[Dict[str, dict], List[str]]:
|
||||||
|
"""Resolve allOf/$ref into one property map plus the required names."""
|
||||||
|
schema = self._deref(schema)
|
||||||
|
props: Dict[str, dict] = {}
|
||||||
|
required: List[str] = []
|
||||||
|
|
||||||
|
for sub in schema.get("allOf", []) or []:
|
||||||
|
sub_props, sub_required = self._flatten(sub)
|
||||||
|
props.update(sub_props)
|
||||||
|
required.extend(sub_required)
|
||||||
|
|
||||||
|
# anyOf/oneOf describe alternatives, not a single shape. Taking the
|
||||||
|
# first branch's properties beats emitting an empty model — a shunt
|
||||||
|
# answering with one valid variant is more useful than one answering {}.
|
||||||
|
for key in ("oneOf", "anyOf"):
|
||||||
|
for sub in (schema.get(key) or [])[:1]:
|
||||||
|
sub_props, sub_required = self._flatten(sub)
|
||||||
|
props.update(sub_props)
|
||||||
|
required.extend(sub_required)
|
||||||
|
|
||||||
|
props.update(schema.get("properties", {}) or {})
|
||||||
|
required.extend(schema.get("required", []) or [])
|
||||||
|
return props, required
|
||||||
|
|
||||||
|
# ── type resolution ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _register_enum(self, owner: str, field_name: str, values: List[Any]) -> type:
|
||||||
|
"""Materialise a spec enum as a real Enum class, deduped by value set."""
|
||||||
|
name = f"{_to_model_name(owner)}{_to_model_name(field_name)}"
|
||||||
|
pairs = [(_to_member_name(v), str(v)) for v in values]
|
||||||
|
|
||||||
|
# Two fields can legitimately declare the same enum name with different
|
||||||
|
# members; suffix rather than let the second silently win.
|
||||||
|
existing = self._enums.get(name)
|
||||||
|
if existing and existing.values != pairs:
|
||||||
|
suffix = 2
|
||||||
|
while f"{name}{suffix}" in self._enums and (
|
||||||
|
self._enums[f"{name}{suffix}"].values != pairs
|
||||||
|
):
|
||||||
|
suffix += 1
|
||||||
|
name = f"{name}{suffix}"
|
||||||
|
|
||||||
|
if name not in self._enums:
|
||||||
|
self._enums[name] = EnumDefinition(name=name, values=pairs)
|
||||||
|
self._enum_types[name] = Enum(name, pairs)
|
||||||
|
return self._enum_types[name]
|
||||||
|
|
||||||
|
def _resolve(
|
||||||
|
self, owner: str, field_name: str, prop: dict
|
||||||
|
) -> Tuple[Any, Optional[str], bool]:
|
||||||
|
"""Map a property schema to (type_hint, foreign_key, many)."""
|
||||||
|
ref = prop.get("$ref")
|
||||||
|
if ref:
|
||||||
|
target = self._ref_name(ref)
|
||||||
|
resolved = self._deref(prop)
|
||||||
|
# A $ref to an enum or a plain scalar is not a relationship — only
|
||||||
|
# a ref to an object shape is.
|
||||||
|
if resolved.get("enum"):
|
||||||
|
return self._register_enum(owner, field_name, resolved["enum"]), None, False
|
||||||
|
if resolved.get("type") in _TYPE_HINTS and resolved.get("type") != "object":
|
||||||
|
return self._resolve(owner, field_name, resolved)
|
||||||
|
return "dict", target, False
|
||||||
|
|
||||||
|
if prop.get("enum") and prop.get("type", "string") == "string":
|
||||||
|
return self._register_enum(owner, field_name, prop["enum"]), None, False
|
||||||
|
|
||||||
|
if prop.get("allOf") or prop.get("oneOf") or prop.get("anyOf"):
|
||||||
|
branch = (
|
||||||
|
(prop.get("allOf") or prop.get("oneOf") or prop.get("anyOf")) or [{}]
|
||||||
|
)[0]
|
||||||
|
if branch.get("$ref"):
|
||||||
|
return "dict", self._ref_name(branch["$ref"]), False
|
||||||
|
return self._resolve(owner, field_name, self._deref(branch))
|
||||||
|
|
||||||
|
prop_type = prop.get("type")
|
||||||
|
|
||||||
|
if prop_type == "array":
|
||||||
|
items = prop.get("items", {}) or {}
|
||||||
|
if items.get("$ref"):
|
||||||
|
inner = self._deref(items)
|
||||||
|
if inner.get("type", "object") == "object":
|
||||||
|
return "list", self._ref_name(items["$ref"]), True
|
||||||
|
return "list", None, False
|
||||||
|
|
||||||
|
fmt = prop.get("format")
|
||||||
|
if fmt in _FORMAT_HINTS:
|
||||||
|
return _FORMAT_HINTS[fmt], None, False
|
||||||
|
|
||||||
|
return _TYPE_HINTS.get(prop_type, str), None, False
|
||||||
|
|
||||||
|
# ── extraction ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def extract(self) -> Tuple[List[ModelDefinition], List[EnumDefinition]]:
|
||||||
|
self._enums = {}
|
||||||
|
self._enum_types = {}
|
||||||
|
|
||||||
|
models: List[ModelDefinition] = []
|
||||||
|
for raw_name, schema in self._schemas().items():
|
||||||
|
model_name = _to_model_name(raw_name)
|
||||||
|
schema = schema or {}
|
||||||
|
|
||||||
|
# A top-level enum is a type, not a shape — register it and skip.
|
||||||
|
if schema.get("enum") and not schema.get("properties"):
|
||||||
|
self._register_enum(model_name, "", schema["enum"])
|
||||||
|
continue
|
||||||
|
|
||||||
|
props, required = self._flatten(schema)
|
||||||
|
required_set = set(required)
|
||||||
|
|
||||||
|
fields: List[FieldDefinition] = []
|
||||||
|
for prop_name, prop in props.items():
|
||||||
|
prop = prop or {}
|
||||||
|
type_hint, fk, many = self._resolve(model_name, prop_name, prop)
|
||||||
|
is_pk = prop_name == "id" or prop_name == f"{raw_name.lower()}_id"
|
||||||
|
fields.append(
|
||||||
|
FieldDefinition(
|
||||||
|
name=prop_name,
|
||||||
|
type_hint=type_hint,
|
||||||
|
default=prop.get("default"),
|
||||||
|
optional=prop_name not in required_set and not is_pk,
|
||||||
|
primary_key=is_pk,
|
||||||
|
foreign_key=fk,
|
||||||
|
many=many,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
models.append(
|
||||||
|
ModelDefinition(
|
||||||
|
name=model_name,
|
||||||
|
fields=fields,
|
||||||
|
docstring=schema.get("description") or schema.get("title"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return models, list(self._enums.values())
|
||||||
|
|
||||||
|
# ── endpoints ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def endpoints(self) -> List[EndpointDefinition]:
|
||||||
|
"""Every operation in the document, normalised for routing."""
|
||||||
|
out: List[EndpointDefinition] = []
|
||||||
|
base = self.doc.get("basePath", "") if self.is_swagger2 else ""
|
||||||
|
|
||||||
|
for path, item in (self.doc.get("paths", {}) or {}).items():
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
shared_params = item.get("parameters", []) or []
|
||||||
|
|
||||||
|
for method, op in item.items():
|
||||||
|
if method.lower() not in (
|
||||||
|
"get", "post", "put", "patch", "delete", "head", "options"
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if not isinstance(op, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
status, response_schema, example = self._response(op)
|
||||||
|
request_model = self._request_model(op, shared_params)
|
||||||
|
response_model, is_list, envelope = self._schema_target(response_schema)
|
||||||
|
|
||||||
|
full_path = f"{base.rstrip('/')}{path}" if base else path
|
||||||
|
model = response_model or request_model
|
||||||
|
|
||||||
|
if is_list:
|
||||||
|
kind = "collection"
|
||||||
|
elif re.search(r"\{[^}]+\}$", full_path):
|
||||||
|
kind = "item"
|
||||||
|
elif method.lower() == "post" and model:
|
||||||
|
kind = "collection"
|
||||||
|
else:
|
||||||
|
kind = "action"
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
EndpointDefinition(
|
||||||
|
method=method.upper(),
|
||||||
|
path=full_path,
|
||||||
|
operation_id=op.get("operationId"),
|
||||||
|
summary=op.get("summary") or op.get("description"),
|
||||||
|
kind=kind,
|
||||||
|
model=model,
|
||||||
|
request_model=request_model,
|
||||||
|
response_model=response_model,
|
||||||
|
response_is_list=is_list,
|
||||||
|
envelope_key=envelope,
|
||||||
|
status=status,
|
||||||
|
path_params=re.findall(r"\{([^}]+)\}", full_path),
|
||||||
|
example=example,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _response(self, op: dict) -> Tuple[int, dict, Any]:
|
||||||
|
"""Pick the success response and return (status, schema, example)."""
|
||||||
|
responses = op.get("responses", {}) or {}
|
||||||
|
codes = [str(c) for c in responses]
|
||||||
|
|
||||||
|
chosen = next((c for c in _STATUS_PREFERENCE if c in codes), None)
|
||||||
|
if chosen is None:
|
||||||
|
chosen = next((c for c in sorted(codes) if c.startswith("2")), None)
|
||||||
|
if chosen is None:
|
||||||
|
return 200, {}, None
|
||||||
|
|
||||||
|
body = responses.get(chosen) or responses.get(int(chosen), {}) or {}
|
||||||
|
status = 200 if chosen in ("default", "2XX") else int(chosen)
|
||||||
|
|
||||||
|
if self.is_swagger2:
|
||||||
|
return status, body.get("schema", {}) or {}, body.get("examples")
|
||||||
|
|
||||||
|
content = body.get("content", {}) or {}
|
||||||
|
media = content.get("application/json") or next(
|
||||||
|
(v for k, v in content.items() if "json" in k), {}
|
||||||
|
)
|
||||||
|
example = media.get("example")
|
||||||
|
if example is None:
|
||||||
|
examples = media.get("examples") or {}
|
||||||
|
first = next(iter(examples.values()), None)
|
||||||
|
if isinstance(first, dict):
|
||||||
|
example = first.get("value")
|
||||||
|
return status, media.get("schema", {}) or {}, example
|
||||||
|
|
||||||
|
def _request_model(self, op: dict, shared_params: list) -> Optional[str]:
|
||||||
|
if self.is_swagger2:
|
||||||
|
params = list(shared_params) + list(op.get("parameters", []) or [])
|
||||||
|
for param in params:
|
||||||
|
if isinstance(param, dict) and param.get("in") == "body":
|
||||||
|
name, _, _ = self._schema_target(param.get("schema", {}) or {})
|
||||||
|
return name
|
||||||
|
return None
|
||||||
|
|
||||||
|
body = op.get("requestBody") or {}
|
||||||
|
content = self._deref(body).get("content", {}) or {}
|
||||||
|
media = content.get("application/json") or next(
|
||||||
|
(v for k, v in content.items() if "json" in k), {}
|
||||||
|
)
|
||||||
|
name, _, _ = self._schema_target(media.get("schema", {}) or {})
|
||||||
|
return name
|
||||||
|
|
||||||
|
def _schema_target(
|
||||||
|
self, schema: dict
|
||||||
|
) -> Tuple[Optional[str], bool, Optional[str]]:
|
||||||
|
"""Return (model name, is_list, envelope key) for a request/response schema."""
|
||||||
|
if not isinstance(schema, dict) or not schema:
|
||||||
|
return None, False, None
|
||||||
|
|
||||||
|
if schema.get("$ref"):
|
||||||
|
resolved = self._deref(schema)
|
||||||
|
# A named wrapper (PetPage) is still a wrapper; look through it.
|
||||||
|
if resolved.get("type") == "array" or resolved.get("properties"):
|
||||||
|
inner = self._schema_target(resolved)
|
||||||
|
if inner[0]:
|
||||||
|
return inner
|
||||||
|
return self._ref_name(schema["$ref"]), False, None
|
||||||
|
|
||||||
|
if schema.get("type") == "array":
|
||||||
|
items = schema.get("items", {}) or {}
|
||||||
|
if items.get("$ref"):
|
||||||
|
return self._ref_name(items["$ref"]), True, None
|
||||||
|
return None, True, None
|
||||||
|
|
||||||
|
# A wrapped collection — {"items": [...], "total": n} and friends.
|
||||||
|
for prop_name, prop in (schema.get("properties", {}) or {}).items():
|
||||||
|
if prop_name in ("items", "results", "data") and (prop or {}).get(
|
||||||
|
"type"
|
||||||
|
) == "array":
|
||||||
|
inner = (prop.get("items") or {}).get("$ref")
|
||||||
|
if inner:
|
||||||
|
return self._ref_name(inner), True, prop_name
|
||||||
|
|
||||||
|
for key in ("allOf", "oneOf", "anyOf"):
|
||||||
|
for sub in (schema.get(key) or [])[:1]:
|
||||||
|
return self._schema_target(sub)
|
||||||
|
|
||||||
|
return None, False, None
|
||||||
450
soleprint/station/tools/modelgen/loader/extract/tabular.py
Normal file
450
soleprint/station/tools/modelgen/loader/extract/tabular.py
Normal file
@@ -0,0 +1,450 @@
|
|||||||
|
"""
|
||||||
|
Tabular Extractor
|
||||||
|
|
||||||
|
Turns a directory of spreadsheets into modelgen's IR — one model per CSV file,
|
||||||
|
one per sheet inside an ODS workbook — and keeps the rows.
|
||||||
|
|
||||||
|
extractor = TabularExtractor("./sheets")
|
||||||
|
models, enums = extractor.extract() # the shapes (BaseExtractor contract)
|
||||||
|
datasets = extractor.datasets() # the rows (what shuntgen seeds with)
|
||||||
|
|
||||||
|
ODS is read with zipfile + ElementTree rather than odfpy or pandas. An .ods is
|
||||||
|
a zip with an XML part in it, and reading it directly is what lets modelgen keep
|
||||||
|
its "no dependencies" promise — the same promise that makes it publishable as a
|
||||||
|
standalone pip package.
|
||||||
|
|
||||||
|
Types are inferred per column from the values actually present, and blanks make
|
||||||
|
a column optional. Keys are inferred by name and confirmed by the data: an `id`
|
||||||
|
column that is neither unique nor complete is not treated as a primary key, and
|
||||||
|
a `customer_id` column is only a foreign key if a matching table exists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import re
|
||||||
|
import zipfile
|
||||||
|
from datetime import date, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
from uuid import UUID
|
||||||
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
|
from ..schema import (
|
||||||
|
DatasetDefinition,
|
||||||
|
EnumDefinition,
|
||||||
|
FieldDefinition,
|
||||||
|
ModelDefinition,
|
||||||
|
)
|
||||||
|
from .base import BaseExtractor
|
||||||
|
|
||||||
|
SUFFIXES = (".csv", ".tsv", ".ods")
|
||||||
|
|
||||||
|
_TABLE = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}"
|
||||||
|
_OFFICE = "{urn:oasis:names:tc:opendocument:xmlns:office:1.0}"
|
||||||
|
|
||||||
|
# ODS pads every sheet out to the full grid with repeat counts in the thousands.
|
||||||
|
# Expanding those verbatim would turn a 5-column sheet into a 1024-column one,
|
||||||
|
# so repeats are honoured only up to a width a real sheet could plausibly have.
|
||||||
|
_MAX_COLS = 512
|
||||||
|
_MAX_ROWS = 100_000
|
||||||
|
|
||||||
|
_TRUE = {"true", "yes", "y", "t"}
|
||||||
|
_FALSE = {"false", "no", "n", "f"}
|
||||||
|
|
||||||
|
# Ordered widest-last: the first format that parses every value in the column
|
||||||
|
# wins, so a stricter pattern must be offered before a looser one.
|
||||||
|
_DATE_FORMATS = ("%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%m/%d")
|
||||||
|
_DATETIME_FORMATS = (
|
||||||
|
"%Y-%m-%dT%H:%M:%S",
|
||||||
|
"%Y-%m-%dT%H:%M",
|
||||||
|
"%Y-%m-%d %H:%M:%S",
|
||||||
|
"%Y-%m-%d %H:%M",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_model_name(name: str) -> str:
|
||||||
|
"""Convert a file or sheet name to PascalCase (line_items -> LineItems)."""
|
||||||
|
parts = [p for p in re.split(r"[^0-9a-zA-Z]+", name) if p]
|
||||||
|
return "".join(p[:1].upper() + p[1:] for p in parts) or name
|
||||||
|
|
||||||
|
|
||||||
|
def _to_slug(name: str) -> str:
|
||||||
|
"""Convert a file or sheet name to a url-safe collection name."""
|
||||||
|
slug = re.sub(r"[^0-9a-zA-Z]+", "-", name).strip("-").lower()
|
||||||
|
return slug or "items"
|
||||||
|
|
||||||
|
|
||||||
|
def _singular(word: str) -> str:
|
||||||
|
"""Crude singulariser — enough to match a `customer_id` to a `customers` sheet."""
|
||||||
|
word = word.lower()
|
||||||
|
if word.endswith("ies") and len(word) > 3:
|
||||||
|
return word[:-3] + "y"
|
||||||
|
for ending in ("ches", "shes", "sses", "xes", "zes"):
|
||||||
|
if word.endswith(ending):
|
||||||
|
return word[: -len(ending) + 1]
|
||||||
|
if word.endswith("s") and not word.endswith("ss"):
|
||||||
|
return word[:-1]
|
||||||
|
return word
|
||||||
|
|
||||||
|
|
||||||
|
# ── readers ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _read_delimited(path: Path) -> List[List[str]]:
|
||||||
|
"""Read a CSV/TSV into a grid of raw strings."""
|
||||||
|
delimiter = "\t" if path.suffix.lower() == ".tsv" else ","
|
||||||
|
with path.open(newline="", encoding="utf-8-sig") as fh:
|
||||||
|
return [row for row in csv.reader(fh, delimiter=delimiter)]
|
||||||
|
|
||||||
|
|
||||||
|
def _cell_text(cell: ElementTree.Element) -> str:
|
||||||
|
"""The displayed text of an ODS cell, typed value preferred over its label."""
|
||||||
|
value_type = cell.get(f"{_OFFICE}value-type")
|
||||||
|
for attr in ("value", "date-value", "time-value", "boolean-value"):
|
||||||
|
raw = cell.get(f"{_OFFICE}{attr}")
|
||||||
|
if raw is not None:
|
||||||
|
return raw
|
||||||
|
if value_type == "string" or value_type is None:
|
||||||
|
return "".join(cell.itertext()).strip()
|
||||||
|
return "".join(cell.itertext()).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ods(path: Path) -> Dict[str, List[List[str]]]:
|
||||||
|
"""Read an ODS workbook into {sheet name: grid of raw strings}."""
|
||||||
|
with zipfile.ZipFile(path) as archive:
|
||||||
|
try:
|
||||||
|
content = archive.read("content.xml")
|
||||||
|
except KeyError as e:
|
||||||
|
raise ValueError(f"{path} is not a readable ODS file (no content.xml)") from e
|
||||||
|
|
||||||
|
root = ElementTree.fromstring(content)
|
||||||
|
sheets: Dict[str, List[List[str]]] = {}
|
||||||
|
|
||||||
|
for table in root.iter(f"{_TABLE}table"):
|
||||||
|
name = table.get(f"{_TABLE}name") or f"sheet{len(sheets) + 1}"
|
||||||
|
grid: List[List[str]] = []
|
||||||
|
|
||||||
|
for row in table.iter(f"{_TABLE}table-row"):
|
||||||
|
cells: List[str] = []
|
||||||
|
for cell in row:
|
||||||
|
if cell.tag not in (
|
||||||
|
f"{_TABLE}table-cell",
|
||||||
|
f"{_TABLE}covered-table-cell",
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
repeat = int(cell.get(f"{_TABLE}number-columns-repeated", 1) or 1)
|
||||||
|
text = "" if cell.tag.endswith("covered-table-cell") else _cell_text(cell)
|
||||||
|
# Padding is always empty and always repeated; a real repeated
|
||||||
|
# value is worth expanding, an empty run at the end is not.
|
||||||
|
if not text and repeat > 1 and len(cells) + repeat > _MAX_COLS:
|
||||||
|
break
|
||||||
|
cells.extend([text] * min(repeat, _MAX_COLS - len(cells)))
|
||||||
|
if len(cells) >= _MAX_COLS:
|
||||||
|
break
|
||||||
|
|
||||||
|
while cells and not cells[-1].strip():
|
||||||
|
cells.pop()
|
||||||
|
|
||||||
|
row_repeat = int(row.get(f"{_TABLE}number-rows-repeated", 1) or 1)
|
||||||
|
if not cells:
|
||||||
|
# A repeated blank row is padding; a single one may be a gap.
|
||||||
|
if row_repeat == 1 and grid:
|
||||||
|
grid.append([])
|
||||||
|
continue
|
||||||
|
for _ in range(min(row_repeat, _MAX_ROWS - len(grid))):
|
||||||
|
grid.append(list(cells))
|
||||||
|
|
||||||
|
if grid:
|
||||||
|
sheets[name] = grid
|
||||||
|
|
||||||
|
return sheets
|
||||||
|
|
||||||
|
|
||||||
|
# ── inference ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_bool(value: str) -> Optional[bool]:
|
||||||
|
lowered = value.strip().lower()
|
||||||
|
if lowered in _TRUE:
|
||||||
|
return True
|
||||||
|
if lowered in _FALSE:
|
||||||
|
return False
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_int(value: str) -> Optional[int]:
|
||||||
|
text = value.strip()
|
||||||
|
# A float-shaped string must not read as an int, or a column of prices
|
||||||
|
# silently truncates.
|
||||||
|
if not re.fullmatch(r"[+-]?\d+", text):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(text)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_float(value: str) -> Optional[float]:
|
||||||
|
try:
|
||||||
|
return float(value.strip())
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_uuid(value: str) -> Optional[str]:
|
||||||
|
try:
|
||||||
|
return str(UUID(value.strip()))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_temporal(value: str) -> Optional[str]:
|
||||||
|
"""Parse a date/datetime into an ISO string, or None."""
|
||||||
|
text = value.strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
for fmt in _DATETIME_FORMATS:
|
||||||
|
try:
|
||||||
|
return datetime.strptime(text, fmt).isoformat()
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
for fmt in _DATE_FORMATS:
|
||||||
|
try:
|
||||||
|
return datetime.strptime(text, fmt).date().isoformat()
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
# ODS writes ISO-8601 with a timezone or fractional seconds; let the
|
||||||
|
# stdlib parser have the ones the explicit formats miss.
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(text.replace("Z", "+00:00")).isoformat()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Each entry is (type hint, parser). Order is the precedence: the first type
|
||||||
|
# every value in the column satisfies wins, so narrow types come first.
|
||||||
|
_CANDIDATES: List[Tuple[Any, Any]] = [
|
||||||
|
(bool, _parse_bool),
|
||||||
|
(int, _parse_int),
|
||||||
|
(float, _parse_float),
|
||||||
|
("UUID", _parse_uuid),
|
||||||
|
("datetime", _parse_temporal),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _infer_column(values: List[str]) -> Tuple[Any, Any]:
|
||||||
|
"""Return (type_hint, coercer) for a column, from its non-blank values."""
|
||||||
|
present = [v for v in values if v is not None and str(v).strip() != ""]
|
||||||
|
if not present:
|
||||||
|
return str, lambda v: v
|
||||||
|
|
||||||
|
for type_hint, parser in _CANDIDATES:
|
||||||
|
if all(parser(v) is not None for v in present):
|
||||||
|
return type_hint, parser
|
||||||
|
|
||||||
|
return str, lambda v: v.strip() if isinstance(v, str) else v
|
||||||
|
|
||||||
|
|
||||||
|
# ── extractor ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TabularExtractor(BaseExtractor):
|
||||||
|
"""Extracts modelgen IR and seed rows from a directory of spreadsheets."""
|
||||||
|
|
||||||
|
def __init__(self, source_path):
|
||||||
|
super().__init__(source_path)
|
||||||
|
self._datasets: List[DatasetDefinition] = []
|
||||||
|
|
||||||
|
def detect(self) -> bool:
|
||||||
|
"""True when the source is a directory holding at least one sheet."""
|
||||||
|
if self.source_path.is_file():
|
||||||
|
return self.source_path.suffix.lower() in SUFFIXES
|
||||||
|
if not self.source_path.is_dir():
|
||||||
|
return False
|
||||||
|
return any(self._sources())
|
||||||
|
|
||||||
|
def _sources(self) -> List[Path]:
|
||||||
|
if self.source_path.is_file():
|
||||||
|
return [self.source_path]
|
||||||
|
return sorted(
|
||||||
|
p
|
||||||
|
for p in self.source_path.iterdir()
|
||||||
|
if p.is_file() and p.suffix.lower() in SUFFIXES and not p.name.startswith(".")
|
||||||
|
)
|
||||||
|
|
||||||
|
def datasets(self) -> List[DatasetDefinition]:
|
||||||
|
"""Seed rows harvested by the last extract() call."""
|
||||||
|
if not self._datasets:
|
||||||
|
self.extract()
|
||||||
|
return self._datasets
|
||||||
|
|
||||||
|
def extract(self) -> Tuple[List[ModelDefinition], List[EnumDefinition]]:
|
||||||
|
grids: List[Tuple[str, str, List[List[str]]]] = [] # (label, source, grid)
|
||||||
|
|
||||||
|
for path in self._sources():
|
||||||
|
if path.suffix.lower() == ".ods":
|
||||||
|
for sheet_name, grid in _read_ods(path).items():
|
||||||
|
grids.append((sheet_name, f"{path.name}#{sheet_name}", grid))
|
||||||
|
else:
|
||||||
|
grids.append((path.stem, path.name, _read_delimited(path)))
|
||||||
|
|
||||||
|
if not grids:
|
||||||
|
raise ValueError(f"No .csv/.tsv/.ods files found in {self.source_path}")
|
||||||
|
|
||||||
|
# Names have to be known before fields, because a foreign key is only a
|
||||||
|
# foreign key when the table it points at is one of the others.
|
||||||
|
known: Dict[str, str] = {}
|
||||||
|
for label, _, _ in grids:
|
||||||
|
model_name = _to_model_name(label)
|
||||||
|
known[_singular(label)] = model_name
|
||||||
|
known[label.lower()] = model_name
|
||||||
|
|
||||||
|
models: List[ModelDefinition] = []
|
||||||
|
self._datasets = []
|
||||||
|
|
||||||
|
for label, source, grid in grids:
|
||||||
|
parsed = self._build(label, source, grid, known)
|
||||||
|
if parsed:
|
||||||
|
model, dataset = parsed
|
||||||
|
models.append(model)
|
||||||
|
self._datasets.append(dataset)
|
||||||
|
|
||||||
|
return models, []
|
||||||
|
|
||||||
|
def _build(
|
||||||
|
self,
|
||||||
|
label: str,
|
||||||
|
source: str,
|
||||||
|
grid: List[List[str]],
|
||||||
|
known: Dict[str, str],
|
||||||
|
) -> Optional[Tuple[ModelDefinition, DatasetDefinition]]:
|
||||||
|
rows = [r for r in grid if any(str(c).strip() for c in r)]
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
|
||||||
|
headers = self._headers(rows[0])
|
||||||
|
if not headers:
|
||||||
|
return None
|
||||||
|
|
||||||
|
body = rows[1:]
|
||||||
|
columns: Dict[str, List[str]] = {h: [] for h in headers}
|
||||||
|
for row in body:
|
||||||
|
for index, header in enumerate(headers):
|
||||||
|
columns[header].append(row[index] if index < len(row) else "")
|
||||||
|
|
||||||
|
model_name = _to_model_name(label)
|
||||||
|
pk = self._primary_key(label, model_name, headers, columns, len(body), known)
|
||||||
|
|
||||||
|
fields: List[FieldDefinition] = []
|
||||||
|
coercers: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
for header in headers:
|
||||||
|
values = columns[header]
|
||||||
|
type_hint, coercer = _infer_column(values)
|
||||||
|
coercers[header] = coercer
|
||||||
|
|
||||||
|
is_pk = header == pk
|
||||||
|
fk = None if is_pk else self._foreign_key(header, model_name, known)
|
||||||
|
has_blank = any(str(v).strip() == "" for v in values)
|
||||||
|
|
||||||
|
fields.append(
|
||||||
|
FieldDefinition(
|
||||||
|
name=header,
|
||||||
|
type_hint=type_hint,
|
||||||
|
default=None,
|
||||||
|
optional=has_blank and not is_pk,
|
||||||
|
primary_key=is_pk,
|
||||||
|
foreign_key=fk,
|
||||||
|
unique=is_pk,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
typed_rows: List[Dict[str, Any]] = []
|
||||||
|
for row in body:
|
||||||
|
record: Dict[str, Any] = {}
|
||||||
|
for index, header in enumerate(headers):
|
||||||
|
raw = row[index] if index < len(row) else ""
|
||||||
|
if str(raw).strip() == "":
|
||||||
|
record[header] = None
|
||||||
|
continue
|
||||||
|
parsed = coercers[header](raw)
|
||||||
|
record[header] = raw.strip() if parsed is None else parsed
|
||||||
|
typed_rows.append(record)
|
||||||
|
|
||||||
|
model = ModelDefinition(
|
||||||
|
name=model_name,
|
||||||
|
fields=fields,
|
||||||
|
docstring=f"Imported from {source} ({len(typed_rows)} rows).",
|
||||||
|
)
|
||||||
|
dataset = DatasetDefinition(
|
||||||
|
model=model_name,
|
||||||
|
rows=typed_rows,
|
||||||
|
source=source,
|
||||||
|
collection=_to_slug(label),
|
||||||
|
)
|
||||||
|
return model, dataset
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _headers(row: List[str]) -> List[str]:
|
||||||
|
"""Normalise the header row, filling blanks and de-duplicating."""
|
||||||
|
headers: List[str] = []
|
||||||
|
seen: Dict[str, int] = {}
|
||||||
|
for index, raw in enumerate(row):
|
||||||
|
name = re.sub(r"[^0-9a-zA-Z]+", "_", str(raw).strip()).strip("_").lower()
|
||||||
|
if not name:
|
||||||
|
name = f"column_{index + 1}"
|
||||||
|
if name[0].isdigit():
|
||||||
|
name = f"c_{name}"
|
||||||
|
if name in seen:
|
||||||
|
seen[name] += 1
|
||||||
|
name = f"{name}_{seen[name]}"
|
||||||
|
else:
|
||||||
|
seen[name] = 1
|
||||||
|
headers.append(name)
|
||||||
|
return headers
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _primary_key(
|
||||||
|
cls,
|
||||||
|
label: str,
|
||||||
|
model_name: str,
|
||||||
|
headers: List[str],
|
||||||
|
columns: Dict[str, List[str]],
|
||||||
|
row_count: int,
|
||||||
|
known: Dict[str, str],
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Pick the key column — named like one, and unique and complete in fact."""
|
||||||
|
|
||||||
|
def holds(header: str) -> bool:
|
||||||
|
values = [str(v).strip() for v in columns[header]]
|
||||||
|
return bool(row_count) and all(values) and len(set(values)) == row_count
|
||||||
|
|
||||||
|
stem = _singular(label)
|
||||||
|
for candidate in ("id", f"{stem}_id", f"{label.lower()}_id"):
|
||||||
|
if candidate in headers and holds(candidate):
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
# A sheet may name its key after the row rather than the table —
|
||||||
|
# `line_id` in a `line_items` sheet. Accept the leading column when it
|
||||||
|
# is shaped like a key, holds like one, and does not point elsewhere.
|
||||||
|
if headers:
|
||||||
|
first = headers[0]
|
||||||
|
if (
|
||||||
|
(first == "id" or first.endswith("_id"))
|
||||||
|
and cls._foreign_key(first, model_name, known) is None
|
||||||
|
and holds(first)
|
||||||
|
):
|
||||||
|
return first
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _foreign_key(
|
||||||
|
header: str, model_name: str, known: Dict[str, str]
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""A `<thing>_id` column pointing at another sheet in the same import."""
|
||||||
|
if not header.endswith("_id"):
|
||||||
|
return None
|
||||||
|
stem = header[:-3]
|
||||||
|
target = known.get(_singular(stem)) or known.get(stem)
|
||||||
|
if target and target != model_name:
|
||||||
|
return target
|
||||||
|
return None
|
||||||
@@ -13,7 +13,7 @@ Expects the folder to have an __init__.py that exports:
|
|||||||
import dataclasses as dc
|
import dataclasses as dc
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import sys
|
import sys
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Type, get_type_hints
|
from typing import Any, Dict, List, Optional, Type, get_type_hints
|
||||||
@@ -32,6 +32,10 @@ class FieldDefinition:
|
|||||||
primary_key: bool = False
|
primary_key: bool = False
|
||||||
foreign_key: Optional[str] = None # target model name
|
foreign_key: Optional[str] = None # target model name
|
||||||
unique: bool = False
|
unique: bool = False
|
||||||
|
# True when foreign_key points at many rows rather than one — an array of
|
||||||
|
# $ref, a m2m table. graphgen renders the two differently, so losing the
|
||||||
|
# distinction would draw every collection as a single edge.
|
||||||
|
many: bool = False
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -60,6 +64,51 @@ class GrpcServiceDefinition:
|
|||||||
methods: List[Dict[str, Any]]
|
methods: List[Dict[str, Any]]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class EndpointDefinition:
|
||||||
|
"""Represents one HTTP operation on a service.
|
||||||
|
|
||||||
|
Models describe the shapes a service passes around; endpoints describe how
|
||||||
|
it is called. Loaders that read a service contract (OpenAPI) or infer one
|
||||||
|
(tabular) emit these alongside the models, and shuntgen turns them into
|
||||||
|
routes. Loaders that only see shapes — dataclasses, a Django app — emit
|
||||||
|
none, which is why this is not part of the BaseExtractor contract.
|
||||||
|
"""
|
||||||
|
|
||||||
|
method: str # GET, POST, ...
|
||||||
|
path: str # /pets/{petId}
|
||||||
|
operation_id: Optional[str] = None
|
||||||
|
summary: Optional[str] = None
|
||||||
|
# "collection" returns/accepts many, "item" one, "action" neither.
|
||||||
|
kind: str = "item"
|
||||||
|
model: Optional[str] = None # what this operation is about
|
||||||
|
request_model: Optional[str] = None
|
||||||
|
response_model: Optional[str] = None
|
||||||
|
response_is_list: bool = False
|
||||||
|
# Set when the collection arrives wrapped — {"items": [...], "total": n}
|
||||||
|
# rather than a bare array. Answering a wrapped endpoint with an array is
|
||||||
|
# the kind of mismatch a client only discovers at parse time.
|
||||||
|
envelope_key: Optional[str] = None
|
||||||
|
status: int = 200
|
||||||
|
path_params: List[str] = field(default_factory=list)
|
||||||
|
example: Any = None # response example carried through from the source
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DatasetDefinition:
|
||||||
|
"""Concrete rows harvested next to a model.
|
||||||
|
|
||||||
|
Importing a spreadsheet yields both a shape and the data that shaped it.
|
||||||
|
Throwing the rows away would mean generating a service that answers with
|
||||||
|
invented values when the real ones were right there.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model: str
|
||||||
|
rows: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
source: Optional[str] = None # file the rows came from
|
||||||
|
collection: Optional[str] = None # url-facing name, e.g. "customers"
|
||||||
|
|
||||||
|
|
||||||
class SchemaLoader:
|
class SchemaLoader:
|
||||||
"""Loads model definitions from Python dataclasses in schema/ folder."""
|
"""Loads model definitions from Python dataclasses in schema/ folder."""
|
||||||
|
|
||||||
|
|||||||
18
soleprint/station/tools/modelgen/tests/__init__.py
Normal file
18
soleprint/station/tools/modelgen/tests/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
"""
|
||||||
|
Tests for modelgen.
|
||||||
|
|
||||||
|
Run from station/tools/, not from inside modelgen/ — the package ships a
|
||||||
|
types.py, and having its own directory on sys.path shadows the standard
|
||||||
|
library module of that name:
|
||||||
|
|
||||||
|
cd soleprint/station/tools
|
||||||
|
python -m unittest modelgen.tests.test_extractors
|
||||||
|
|
||||||
|
stdlib unittest only, and every input is built in a temp directory. modelgen is
|
||||||
|
published as a standalone pip package, so its tests have to pass with nothing
|
||||||
|
installed and nothing else in the tree.
|
||||||
|
|
||||||
|
These are unit tests of the loaders and generators. They are not contract
|
||||||
|
tests — those talk HTTP, belong to a room, and are never committed to core
|
||||||
|
(see station/tools/tester/tests/README.md).
|
||||||
|
"""
|
||||||
406
soleprint/station/tools/modelgen/tests/test_extractors.py
Normal file
406
soleprint/station/tools/modelgen/tests/test_extractors.py
Normal file
@@ -0,0 +1,406 @@
|
|||||||
|
"""
|
||||||
|
Tests for the OpenAPI and tabular extractors and the datagen target.
|
||||||
|
|
||||||
|
stdlib unittest and nothing else, and every input is built in a temp directory
|
||||||
|
rather than read from a fixture file — modelgen is published as a standalone
|
||||||
|
package, so its tests have to pass with nothing installed and nothing else in
|
||||||
|
the tree.
|
||||||
|
|
||||||
|
cd soleprint/station/tools && python -m unittest modelgen.tests.test_extractors
|
||||||
|
|
||||||
|
Run it from station/tools/, not from inside modelgen/: modelgen ships a
|
||||||
|
types.py, and putting the package's own directory on sys.path shadows the
|
||||||
|
standard library module of that name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1].parent))
|
||||||
|
|
||||||
|
from modelgen.generator import GENERATORS, DatagenGenerator # noqa: E402
|
||||||
|
from modelgen.loader.extract.openapi import OpenAPIExtractor # noqa: E402
|
||||||
|
from modelgen.loader.extract.tabular import TabularExtractor # noqa: E402
|
||||||
|
|
||||||
|
SPEC = {
|
||||||
|
"openapi": "3.0.3",
|
||||||
|
"info": {"title": "Zoo", "version": "1.0.0"},
|
||||||
|
"paths": {
|
||||||
|
"/pets": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "listPets",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {"$ref": "#/components/schemas/PetPage"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"post": {
|
||||||
|
"operationId": "createPet",
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {"$ref": "#/components/schemas/Pet"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"responses": {
|
||||||
|
"201": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {"$ref": "#/components/schemas/Pet"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"/pets/{petId}": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "getPet",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {"$ref": "#/components/schemas/Pet"},
|
||||||
|
"example": {"id": 7, "name": "Rocinante"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"components": {
|
||||||
|
"schemas": {
|
||||||
|
"Pet": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "An animal.",
|
||||||
|
"required": ["id", "name"],
|
||||||
|
"properties": {
|
||||||
|
"id": {"type": "integer", "format": "int64"},
|
||||||
|
"name": {"type": "string"},
|
||||||
|
"status": {"type": "string", "enum": ["available", "sold"]},
|
||||||
|
"weight": {"type": "number"},
|
||||||
|
"born_on": {"type": "string", "format": "date"},
|
||||||
|
"category": {"$ref": "#/components/schemas/Category"},
|
||||||
|
"tags": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"$ref": "#/components/schemas/Category"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"Category": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["id"],
|
||||||
|
"properties": {
|
||||||
|
"id": {"type": "integer"},
|
||||||
|
"name": {"type": "string"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"PetPage": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"items": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"$ref": "#/components/schemas/Pet"},
|
||||||
|
},
|
||||||
|
"total": {"type": "integer"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
CUSTOMERS_CSV = """id,name,email,active,joined,balance
|
||||||
|
1,Ada,ada@example.com,true,2024-03-11,150.5
|
||||||
|
2,Bruno,bruno@example.com,false,2024-05-02,80
|
||||||
|
3,Camila,camila@example.com,true,2023-11-27,
|
||||||
|
"""
|
||||||
|
|
||||||
|
ORDERS_CSV = """order_id,customer_id,total,placed
|
||||||
|
1001,1,42.00,2025-02-03
|
||||||
|
1002,2,15.25,2025-03-14
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def write_ods(path: Path, sheet: str, rows: list[list[str]]) -> None:
|
||||||
|
"""A minimal ODS, padded the way a real one is."""
|
||||||
|
ns = (
|
||||||
|
'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
|
||||||
|
'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" '
|
||||||
|
'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"'
|
||||||
|
)
|
||||||
|
|
||||||
|
def cell(value: str) -> str:
|
||||||
|
if value == "":
|
||||||
|
return "<table:table-cell/>"
|
||||||
|
return (
|
||||||
|
'<table:table-cell office:value-type="string">'
|
||||||
|
f"<text:p>{value}</text:p></table:table-cell>"
|
||||||
|
)
|
||||||
|
|
||||||
|
body = "".join(
|
||||||
|
"<table:table-row>"
|
||||||
|
+ "".join(cell(c) for c in row)
|
||||||
|
+ '<table:table-cell table:number-columns-repeated="1018"/>'
|
||||||
|
"</table:table-row>"
|
||||||
|
for row in rows
|
||||||
|
)
|
||||||
|
content = (
|
||||||
|
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||||
|
f"<office:document-content {ns}>"
|
||||||
|
"<office:body><office:spreadsheet>"
|
||||||
|
f'<table:table table:name="{sheet}">{body}'
|
||||||
|
'<table:table-row table:number-rows-repeated="1048570">'
|
||||||
|
'<table:table-cell table:number-columns-repeated="1024"/></table:table-row>'
|
||||||
|
"</table:table></office:spreadsheet></office:body></office:document-content>"
|
||||||
|
)
|
||||||
|
with zipfile.ZipFile(path, "w") as archive:
|
||||||
|
archive.writestr("content.xml", content)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAPIExtractorTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-openapi-"))
|
||||||
|
self.spec = self.dir / "zoo.json"
|
||||||
|
self.spec.write_text(json.dumps(SPEC))
|
||||||
|
self.extractor = OpenAPIExtractor(self.spec)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
shutil.rmtree(self.dir, ignore_errors=True)
|
||||||
|
|
||||||
|
def test_detects_a_spec(self):
|
||||||
|
self.assertTrue(self.extractor.detect())
|
||||||
|
|
||||||
|
def test_does_not_detect_arbitrary_json(self):
|
||||||
|
other = self.dir / "not-a-spec.json"
|
||||||
|
other.write_text('{"hello": "world"}')
|
||||||
|
self.assertFalse(OpenAPIExtractor(other).detect())
|
||||||
|
|
||||||
|
def test_extracts_every_object_schema(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
self.assertEqual({m.name for m in models}, {"Pet", "Category", "PetPage"})
|
||||||
|
|
||||||
|
def test_enum_becomes_a_real_enum_class(self):
|
||||||
|
models, enums = self.extractor.extract()
|
||||||
|
# A materialised Enum is what makes every generator resolve it by name
|
||||||
|
# instead of falling through to str.
|
||||||
|
self.assertEqual([e.name for e in enums], ["PetStatus"])
|
||||||
|
status = next(f for f in models[0].fields if f.name == "status")
|
||||||
|
self.assertTrue(isinstance(status.type_hint, type))
|
||||||
|
self.assertTrue(issubclass(status.type_hint, Enum))
|
||||||
|
self.assertEqual([m.value for m in status.type_hint], ["available", "sold"])
|
||||||
|
|
||||||
|
def test_required_and_format_and_relations(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
fields = {f.name: f for f in next(m for m in models if m.name == "Pet").fields}
|
||||||
|
|
||||||
|
self.assertEqual(fields["id"].type_hint, "bigint")
|
||||||
|
self.assertTrue(fields["id"].primary_key)
|
||||||
|
self.assertFalse(fields["name"].optional)
|
||||||
|
self.assertTrue(fields["weight"].optional)
|
||||||
|
self.assertEqual(fields["born_on"].type_hint, "datetime")
|
||||||
|
|
||||||
|
self.assertEqual(fields["category"].foreign_key, "Category")
|
||||||
|
self.assertFalse(fields["category"].many)
|
||||||
|
self.assertEqual(fields["tags"].foreign_key, "Category")
|
||||||
|
self.assertTrue(fields["tags"].many)
|
||||||
|
|
||||||
|
def test_endpoints_carry_operation_shape(self):
|
||||||
|
by_id = {e.operation_id: e for e in self.extractor.endpoints()}
|
||||||
|
self.assertEqual(set(by_id), {"listPets", "createPet", "getPet"})
|
||||||
|
|
||||||
|
listing = by_id["listPets"]
|
||||||
|
self.assertEqual(listing.kind, "collection")
|
||||||
|
self.assertEqual(listing.response_model, "Pet")
|
||||||
|
self.assertTrue(listing.response_is_list)
|
||||||
|
# PetPage wraps the array, so a client expects {"items": [...]}.
|
||||||
|
self.assertEqual(listing.envelope_key, "items")
|
||||||
|
|
||||||
|
created = by_id["createPet"]
|
||||||
|
self.assertEqual(created.status, 201)
|
||||||
|
self.assertEqual(created.request_model, "Pet")
|
||||||
|
|
||||||
|
item = by_id["getPet"]
|
||||||
|
self.assertEqual(item.kind, "item")
|
||||||
|
self.assertEqual(item.path_params, ["petId"])
|
||||||
|
self.assertEqual(item.example, {"id": 7, "name": "Rocinante"})
|
||||||
|
|
||||||
|
def test_relations_reach_the_graphgen_schema(self):
|
||||||
|
out = self.dir / "schema.json"
|
||||||
|
GENERATORS["schema"]().generate(self.extractor.extract(), out)
|
||||||
|
schema = json.loads(out.read_text())["models"]
|
||||||
|
self.assertEqual(schema["Pet"]["fields"]["category"]["type"], "FK:Category")
|
||||||
|
self.assertEqual(schema["Pet"]["fields"]["tags"]["type"], "M2M:Category")
|
||||||
|
|
||||||
|
|
||||||
|
class TabularExtractorTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-tabular-"))
|
||||||
|
self.sheets = self.dir / "sheets"
|
||||||
|
self.sheets.mkdir()
|
||||||
|
(self.sheets / "customers.csv").write_text(CUSTOMERS_CSV)
|
||||||
|
(self.sheets / "orders.csv").write_text(ORDERS_CSV)
|
||||||
|
write_ods(
|
||||||
|
self.sheets / "line_items.ods",
|
||||||
|
"line_items",
|
||||||
|
[
|
||||||
|
["line_id", "order_id", "sku", "qty", "note"],
|
||||||
|
["1", "1001", "A-1", "2", "rush"],
|
||||||
|
["2", "1001", "B-2", "1", ""],
|
||||||
|
["3", "1002", "A-1", "5", ""],
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.extractor = TabularExtractor(self.sheets)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
shutil.rmtree(self.dir, ignore_errors=True)
|
||||||
|
|
||||||
|
def test_detects_a_sheet_directory(self):
|
||||||
|
self.assertTrue(self.extractor.detect())
|
||||||
|
empty = self.dir / "empty"
|
||||||
|
empty.mkdir()
|
||||||
|
self.assertFalse(TabularExtractor(empty).detect())
|
||||||
|
|
||||||
|
def test_one_model_per_file_and_sheet(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
self.assertEqual(
|
||||||
|
{m.name for m in models}, {"Customers", "Orders", "LineItems"}
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_types_are_inferred_per_column(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
fields = {
|
||||||
|
f.name: f for f in next(m for m in models if m.name == "Customers").fields
|
||||||
|
}
|
||||||
|
self.assertEqual(fields["id"].type_hint, int)
|
||||||
|
self.assertEqual(fields["name"].type_hint, str)
|
||||||
|
self.assertEqual(fields["active"].type_hint, bool)
|
||||||
|
self.assertEqual(fields["joined"].type_hint, "datetime")
|
||||||
|
self.assertEqual(fields["balance"].type_hint, float)
|
||||||
|
# One blank cell is what makes the column nullable.
|
||||||
|
self.assertTrue(fields["balance"].optional)
|
||||||
|
self.assertFalse(fields["name"].optional)
|
||||||
|
|
||||||
|
def test_keys_are_confirmed_against_the_data(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
by_name = {m.name: m for m in models}
|
||||||
|
|
||||||
|
customers = {f.name: f for f in by_name["Customers"].fields}
|
||||||
|
self.assertTrue(customers["id"].primary_key)
|
||||||
|
|
||||||
|
orders = {f.name: f for f in by_name["Orders"].fields}
|
||||||
|
self.assertTrue(orders["order_id"].primary_key)
|
||||||
|
# customer_id names another sheet, so it is a relation, not a key.
|
||||||
|
self.assertFalse(orders["customer_id"].primary_key)
|
||||||
|
self.assertEqual(orders["customer_id"].foreign_key, "Customers")
|
||||||
|
|
||||||
|
def test_ods_is_read_and_padding_ignored(self):
|
||||||
|
models, _ = self.extractor.extract()
|
||||||
|
line_items = next(m for m in models if m.name == "LineItems")
|
||||||
|
# Five real columns, not the 1023 the padding claims.
|
||||||
|
self.assertEqual([f.name for f in line_items.fields],
|
||||||
|
["line_id", "order_id", "sku", "qty", "note"])
|
||||||
|
fields = {f.name: f for f in line_items.fields}
|
||||||
|
self.assertEqual(fields["qty"].type_hint, int)
|
||||||
|
self.assertTrue(fields["note"].optional)
|
||||||
|
# line_id names the row rather than the sheet, but leads and holds.
|
||||||
|
self.assertTrue(fields["line_id"].primary_key)
|
||||||
|
self.assertEqual(fields["order_id"].foreign_key, "Orders")
|
||||||
|
|
||||||
|
def test_rows_are_kept_and_coerced(self):
|
||||||
|
self.extractor.extract()
|
||||||
|
datasets = {d.model: d for d in self.extractor.datasets()}
|
||||||
|
self.assertEqual(datasets["Customers"].collection, "customers")
|
||||||
|
|
||||||
|
rows = datasets["Customers"].rows
|
||||||
|
self.assertEqual(len(rows), 3)
|
||||||
|
self.assertEqual(rows[0]["id"], 1)
|
||||||
|
self.assertIs(rows[0]["active"], True)
|
||||||
|
self.assertEqual(rows[0]["balance"], 150.5)
|
||||||
|
# A blank cell is null, not the empty string.
|
||||||
|
self.assertIsNone(rows[2]["balance"])
|
||||||
|
self.assertEqual(datasets["LineItems"].rows[1]["note"], None)
|
||||||
|
|
||||||
|
|
||||||
|
class DatagenTargetTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-datagen-"))
|
||||||
|
self.sheets = self.dir / "sheets"
|
||||||
|
self.sheets.mkdir()
|
||||||
|
(self.sheets / "customers.csv").write_text(CUSTOMERS_CSV)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
shutil.rmtree(self.dir, ignore_errors=True)
|
||||||
|
|
||||||
|
def _generate(self, seeded: bool):
|
||||||
|
extractor = TabularExtractor(self.sheets)
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
datasets = extractor.datasets() if seeded else []
|
||||||
|
|
||||||
|
out = self.dir / "gen" / "datagen_demo.py"
|
||||||
|
DatagenGenerator(class_name="DemoGenerator").generate(
|
||||||
|
(models, enums, datasets), out
|
||||||
|
)
|
||||||
|
if seeded:
|
||||||
|
depot = out.parent / "depot"
|
||||||
|
depot.mkdir(exist_ok=True)
|
||||||
|
(depot / "data.json").write_text(
|
||||||
|
json.dumps({d.model: d.rows for d in datasets})
|
||||||
|
)
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
|
||||||
|
spec = importlib.util.spec_from_file_location(f"demo_{seeded}", out)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module.DemoGenerator()
|
||||||
|
|
||||||
|
def test_registered_as_a_target(self):
|
||||||
|
self.assertIn("datagen", GENERATORS)
|
||||||
|
|
||||||
|
def test_synthesises_when_there_are_no_rows(self):
|
||||||
|
generator = self._generate(seeded=False)
|
||||||
|
self.assertEqual(generator.available_models(), ["customers"])
|
||||||
|
|
||||||
|
record = generator.generate("Customers", 1)[0]
|
||||||
|
self.assertEqual(
|
||||||
|
set(record), {"id", "name", "email", "active", "joined", "balance"}
|
||||||
|
)
|
||||||
|
self.assertIsInstance(record["id"], int)
|
||||||
|
self.assertIsInstance(record["active"], bool)
|
||||||
|
self.assertIn("@", record["email"])
|
||||||
|
|
||||||
|
def test_samples_real_rows_when_seeded(self):
|
||||||
|
generator = self._generate(seeded=True)
|
||||||
|
names = {generator.generate("Customers", 1)[0]["name"] for _ in range(25)}
|
||||||
|
self.assertTrue(names <= {"Ada", "Bruno", "Camila"}, names)
|
||||||
|
|
||||||
|
def test_kwargs_override_the_result(self):
|
||||||
|
generator = self._generate(seeded=True)
|
||||||
|
self.assertEqual(generator.generate("Customers", 1, name="Zed")[0]["name"], "Zed")
|
||||||
|
|
||||||
|
def test_exposes_a_graphgen_schema(self):
|
||||||
|
schema = self._generate(seeded=False).schema()
|
||||||
|
self.assertIn("Customers", schema["models"])
|
||||||
|
self.assertTrue(schema["models"]["Customers"]["fields"]["id"]["pk"])
|
||||||
|
|
||||||
|
def test_unknown_model_is_reported(self):
|
||||||
|
generator = self._generate(seeded=False)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
generator.generate("Nope", 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -61,6 +61,11 @@ PYDANTIC_RESOLVERS: dict[Any, Callable[[Any], str]] = {
|
|||||||
"list": lambda base: f"List[{_get_list_inner(base)}]",
|
"list": lambda base: f"List[{_get_list_inner(base)}]",
|
||||||
"enum": lambda base: base.__name__,
|
"enum": lambda base: base.__name__,
|
||||||
"dataclass": lambda base: base.__name__,
|
"dataclass": lambda base: base.__name__,
|
||||||
|
# DB- and spec-shaped hints. Without these an int64 column resolves through
|
||||||
|
# the "unknown -> str" fallback and a BigIntegerField arrives as a string.
|
||||||
|
"bigint": lambda _: "int",
|
||||||
|
"text": lambda _: "str",
|
||||||
|
"bytes": lambda _: "bytes",
|
||||||
}
|
}
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -95,6 +100,11 @@ TS_RESOLVERS: dict[Any, Callable[[Any], str]] = {
|
|||||||
"list": _resolve_ts_list,
|
"list": _resolve_ts_list,
|
||||||
"enum": lambda base: base.__name__,
|
"enum": lambda base: base.__name__,
|
||||||
"dataclass": lambda base: base.__name__,
|
"dataclass": lambda base: base.__name__,
|
||||||
|
# JS has no 64-bit integer literal type, so int64 is a number like any
|
||||||
|
# other; text is a string; binary arrives base64-encoded over JSON.
|
||||||
|
"bigint": lambda _: "number",
|
||||||
|
"text": lambda _: "string",
|
||||||
|
"bytes": lambda _: "string",
|
||||||
}
|
}
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
127
soleprint/station/tools/shuntgen/README.md
Normal file
127
soleprint/station/tools/shuntgen/README.md
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
# shuntgen
|
||||||
|
|
||||||
|
Generates runnable [shunts](../../../artery/shunts/) from the two things people
|
||||||
|
actually have: a service contract, or a folder of spreadsheets.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# a spec you were handed
|
||||||
|
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
|
||||||
|
|
||||||
|
# sheets a client sent
|
||||||
|
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
|
||||||
|
|
||||||
|
python -m station.tools.shuntgen list
|
||||||
|
```
|
||||||
|
|
||||||
|
Run from `soleprint/`, so `station.tools...` resolves. Also available in the
|
||||||
|
browser at `/station/tools/shuntgen/`.
|
||||||
|
|
||||||
|
## What comes out
|
||||||
|
|
||||||
|
```
|
||||||
|
artery/shunts/<name>/
|
||||||
|
main.py builds the app from the spec
|
||||||
|
run.py uvicorn entry point (PORT, or depot/config.json)
|
||||||
|
shunt_runtime.py vendored copy of runtime.py
|
||||||
|
models.py pydantic, via modelgen
|
||||||
|
datagen_<name>.py BaseDataGenerator subclass, via modelgen
|
||||||
|
depot/spec.json routes, collections and schema
|
||||||
|
depot/responses.json pinned overrides — yours, never overwritten
|
||||||
|
depot/config.json delays, error rate, prefill — yours, never overwritten
|
||||||
|
depot/data.json imported rows
|
||||||
|
templates/index.html config UI
|
||||||
|
README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd artery/shunts/books && python run.py
|
||||||
|
curl localhost:8098/customers
|
||||||
|
```
|
||||||
|
|
||||||
|
## Where a response comes from
|
||||||
|
|
||||||
|
First hit wins:
|
||||||
|
|
||||||
|
1. `depot/responses.json` — a pinned override, keyed `"METHOD /path"`
|
||||||
|
2. the store — rows imported from sheets, plus anything POSTed since
|
||||||
|
3. the spec's `example`, if the source document carried one
|
||||||
|
4. `datagen_<name>.py`, synthesising from the schema
|
||||||
|
5. `{}`
|
||||||
|
|
||||||
|
The store is the part that makes it behave like a service: POST something and
|
||||||
|
GET it back, ask for `/pets/7` and get the pet whose id is 7. Collections with
|
||||||
|
no imported rows are prefilled with generated ones (`prefill` in
|
||||||
|
`depot/config.json`) so the first call answers with something.
|
||||||
|
|
||||||
|
## Two sources, one pipeline
|
||||||
|
|
||||||
|
Both inputs are modelgen extractors, so the same shapes also generate pydantic,
|
||||||
|
TypeScript, prisma and a graphgen schema:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript
|
||||||
|
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t schema,datagen
|
||||||
|
```
|
||||||
|
|
||||||
|
| Source | Becomes | Routes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| OpenAPI 3.x / Swagger 2.0 | one model per schema, enums as real Enums, `$ref` as relations | the operations the document declares |
|
||||||
|
| `.csv` / `.tsv` / `.ods` | one model per file or sheet, types inferred per column | five CRUD routes per table |
|
||||||
|
|
||||||
|
Keys and relations are inferred and then checked against the data: an `id`
|
||||||
|
column that is not unique is not treated as a key, and `customer_id` is only a
|
||||||
|
foreign key if a `customers` sheet came with it.
|
||||||
|
|
||||||
|
ODS is read with `zipfile` + `ElementTree` — no odfpy, no pandas — which is what
|
||||||
|
lets modelgen stay dependency-free and publishable on its own.
|
||||||
|
|
||||||
|
## Control endpoints
|
||||||
|
|
||||||
|
Every generated shunt serves these:
|
||||||
|
|
||||||
|
| Endpoint | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /health` | liveness |
|
||||||
|
| `GET /mock/spec` | the routes it was built from |
|
||||||
|
| `GET /mock/stats` | call counts and row counts |
|
||||||
|
| `POST /mock/reset` | restore imported rows, clear counters |
|
||||||
|
| `GET,POST /mock/config` | delays, error rate, `unknown_id`, page size |
|
||||||
|
| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# make it slow and flaky, the way the real thing is
|
||||||
|
curl -X POST localhost:8098/mock/config \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"enable_random_delays": true, "error_rate": 0.2}'
|
||||||
|
|
||||||
|
# make one call answer something specific
|
||||||
|
curl -X POST localhost:8098/mock/responses \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"GET /customers/1": {"id": 1, "name": "PINNED"}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
`unknown_id` decides what an unknown key does: `generate` (the default) invents
|
||||||
|
a record wearing the id that was asked for, `404` refuses it. Generate by
|
||||||
|
default because a client pointed at a fresh shunt should just work; flip it when
|
||||||
|
the error path is what you are testing.
|
||||||
|
|
||||||
|
## Dependency containers
|
||||||
|
|
||||||
|
`--cabinet postgres,redis` writes a `cabinet.json` declaring what the shunt
|
||||||
|
expects. `python build.py --cfg <room>` composes those services into the room's
|
||||||
|
`docker-compose.yml`; on a cluster they install as rig addons of the same name.
|
||||||
|
See [cabinets](../../cabinets/README.md).
|
||||||
|
|
||||||
|
## Regenerating
|
||||||
|
|
||||||
|
Everything is overwritten except `depot/responses.json` and `depot/config.json`.
|
||||||
|
Fixing a bug in `runtime.py` and regenerating fixes it in every shunt, which is
|
||||||
|
why the routes are built from `spec.json` at startup rather than written out as
|
||||||
|
source.
|
||||||
|
|
||||||
|
## Fixtures
|
||||||
|
|
||||||
|
`fixtures/petstore.yaml` and `fixtures/sheets/` exercise the shapes that are
|
||||||
|
easy to get wrong — an enum, a `$ref`, an array of `$ref`, a wrapped collection,
|
||||||
|
a path parameter, an ODS sheet with padding and a blank column, and a foreign
|
||||||
|
key across two files.
|
||||||
29
soleprint/station/tools/shuntgen/__init__.py
Normal file
29
soleprint/station/tools/shuntgen/__init__.py
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
"""
|
||||||
|
Shuntgen - Generate runnable shunts from a service contract or a pile of sheets.
|
||||||
|
|
||||||
|
A shunt is artery's fake connector: a stand-in service that answers like the
|
||||||
|
real one so tests can run without it. Writing one by hand means copying
|
||||||
|
artery/shunts/example/ and filling in responses.json by hand, which is fine for
|
||||||
|
three endpoints and untenable for eighty.
|
||||||
|
|
||||||
|
This tool takes the two things people actually have —
|
||||||
|
|
||||||
|
an OpenAPI/Swagger document the service already exists somewhere
|
||||||
|
a directory of CSV/ODS sheets the data exists, the service does not
|
||||||
|
|
||||||
|
— and emits a shunt that runs: typed models, a data generator, seeded routes,
|
||||||
|
and a config UI.
|
||||||
|
|
||||||
|
Both inputs reach modelgen's IR through its own extractors, so the shapes also
|
||||||
|
generate pydantic, TypeScript, prisma and a graphgen schema for free. This
|
||||||
|
package only adds the emitter.
|
||||||
|
|
||||||
|
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
|
||||||
|
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .emit import ShuntEmitter
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
|
|
||||||
|
__all__ = ["ShuntEmitter"]
|
||||||
239
soleprint/station/tools/shuntgen/__main__.py
Normal file
239
soleprint/station/tools/shuntgen/__main__.py
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
"""
|
||||||
|
Shuntgen CLI.
|
||||||
|
|
||||||
|
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
|
||||||
|
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
|
||||||
|
python -m station.tools.shuntgen list
|
||||||
|
|
||||||
|
Run from the soleprint/ directory so `station.tools...` resolves.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .emit import SPR_ROOT, ShuntEmitter
|
||||||
|
|
||||||
|
|
||||||
|
def _name_for(args, source: Path) -> str:
|
||||||
|
"""The shunt's name: given, else the output folder, else the source."""
|
||||||
|
if getattr(args, "name", None):
|
||||||
|
return args.name
|
||||||
|
output = Path(args.output)
|
||||||
|
if output.name and output.name not in (".", ".."):
|
||||||
|
return output.name
|
||||||
|
return source.stem
|
||||||
|
|
||||||
|
|
||||||
|
def _refuse_to_clobber(output: Path, force: bool) -> None:
|
||||||
|
"""Regenerating is fine; overwriting something that is not a shunt is not."""
|
||||||
|
if not output.exists() or force:
|
||||||
|
return
|
||||||
|
if not any(output.iterdir()):
|
||||||
|
return
|
||||||
|
if (output / "depot" / "spec.json").exists():
|
||||||
|
return
|
||||||
|
print(
|
||||||
|
f"Error: {output} already exists and was not generated by shuntgen.\n"
|
||||||
|
" Pick another path, or pass --force to write into it anyway.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_from_openapi(args):
|
||||||
|
"""Build a shunt from an OpenAPI 3.x / Swagger 2.0 document."""
|
||||||
|
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
|
||||||
|
|
||||||
|
spec_path = Path(args.spec)
|
||||||
|
if not spec_path.exists():
|
||||||
|
print(f"Error: Spec not found: {spec_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
output = Path(args.output)
|
||||||
|
_refuse_to_clobber(output, args.force)
|
||||||
|
|
||||||
|
extractor = OpenAPIExtractor(spec_path)
|
||||||
|
print(f"Reading spec: {spec_path}")
|
||||||
|
try:
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
endpoints = extractor.endpoints()
|
||||||
|
except (RuntimeError, ValueError) as e:
|
||||||
|
print(f"Error: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not endpoints:
|
||||||
|
print(
|
||||||
|
"Error: the spec declares no operations, so there is nothing to serve.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
name = _name_for(args, spec_path)
|
||||||
|
print(f"Found {len(models)} models, {len(endpoints)} endpoints")
|
||||||
|
|
||||||
|
written = ShuntEmitter(
|
||||||
|
name=name,
|
||||||
|
output=output,
|
||||||
|
models=models,
|
||||||
|
enums=enums,
|
||||||
|
endpoints=endpoints,
|
||||||
|
title=args.title,
|
||||||
|
source=spec_path.name,
|
||||||
|
kind="openapi",
|
||||||
|
port=args.port,
|
||||||
|
cabinets=_cabinets(args),
|
||||||
|
).emit()
|
||||||
|
|
||||||
|
_report(written, name, args.port)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_from_tabular(args):
|
||||||
|
"""Build a shunt from a directory of CSV/TSV/ODS sheets."""
|
||||||
|
from ..modelgen.loader.extract.tabular import TabularExtractor
|
||||||
|
|
||||||
|
source_path = Path(args.source)
|
||||||
|
if not source_path.exists():
|
||||||
|
print(f"Error: Source not found: {source_path}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
output = Path(args.output)
|
||||||
|
_refuse_to_clobber(output, args.force)
|
||||||
|
|
||||||
|
extractor = TabularExtractor(source_path)
|
||||||
|
print(f"Reading sheets: {source_path}")
|
||||||
|
try:
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
except ValueError as e:
|
||||||
|
print(f"Error: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
datasets = extractor.datasets()
|
||||||
|
rows = sum(len(d.rows) for d in datasets)
|
||||||
|
name = _name_for(args, source_path)
|
||||||
|
print(f"Found {len(models)} models, {rows} rows")
|
||||||
|
|
||||||
|
written = ShuntEmitter(
|
||||||
|
name=name,
|
||||||
|
output=output,
|
||||||
|
models=models,
|
||||||
|
enums=enums,
|
||||||
|
datasets=datasets,
|
||||||
|
title=args.title,
|
||||||
|
source=source_path.name,
|
||||||
|
kind="tabular",
|
||||||
|
port=args.port,
|
||||||
|
cabinets=_cabinets(args),
|
||||||
|
).emit()
|
||||||
|
|
||||||
|
_report(written, name, args.port)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_list(args):
|
||||||
|
"""List the shunts that exist under artery/shunts/."""
|
||||||
|
shunts_dir = SPR_ROOT / "artery" / "shunts"
|
||||||
|
if not shunts_dir.exists():
|
||||||
|
print(f"No shunts directory at {shunts_dir}")
|
||||||
|
return
|
||||||
|
|
||||||
|
found = False
|
||||||
|
for path in sorted(shunts_dir.iterdir()):
|
||||||
|
if not path.is_dir() or path.name.startswith(("_", ".")):
|
||||||
|
continue
|
||||||
|
found = True
|
||||||
|
spec = path / "depot" / "spec.json"
|
||||||
|
if spec.exists():
|
||||||
|
import json
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(spec.read_text())
|
||||||
|
print(
|
||||||
|
f" {path.name:<20} generated "
|
||||||
|
f"{len(data.get('routes', []))} routes from {data.get('source', '?')}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
print(f" {path.name:<20} hand-written")
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
print(" (none)")
|
||||||
|
|
||||||
|
|
||||||
|
def _cabinets(args) -> list:
|
||||||
|
if not getattr(args, "cabinet", None):
|
||||||
|
return []
|
||||||
|
return [c.strip() for c in args.cabinet.split(",") if c.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _report(output: Path, name: str, port: int) -> None:
|
||||||
|
print(f"\nWrote {output}")
|
||||||
|
print("\nRun it:")
|
||||||
|
print(f" cd {output} && python run.py")
|
||||||
|
print(f" curl localhost:{port}/health")
|
||||||
|
print(f" open http://localhost:{port}/")
|
||||||
|
|
||||||
|
|
||||||
|
def _add_common(parser, port_default: int) -> None:
|
||||||
|
parser.add_argument(
|
||||||
|
"--output", "-o", type=str, required=True,
|
||||||
|
help="Where to write the shunt (e.g. artery/shunts/petstore)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--name", "-n", type=str, default=None,
|
||||||
|
help="Shunt name (default: the output folder's name)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--title", type=str, default=None,
|
||||||
|
help="Display title for the config UI (default: derived from the name)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--port", "-p", type=int, default=port_default,
|
||||||
|
help=f"Default port for run.py (default: {port_default})",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--cabinet", type=str, default=None,
|
||||||
|
help="Comma-separated dependency containers to declare (e.g. postgres,redis)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--force", action="store_true",
|
||||||
|
help="Write into a non-empty directory that shuntgen did not generate",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Shuntgen - generate runnable shunts",
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
)
|
||||||
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
|
openapi_parser = subparsers.add_parser(
|
||||||
|
"from-openapi", help="Generate a shunt from an OpenAPI / Swagger document"
|
||||||
|
)
|
||||||
|
openapi_parser.add_argument(
|
||||||
|
"--spec", "-s", type=str, required=True,
|
||||||
|
help="Path to the spec (.json, .yaml or .yml)",
|
||||||
|
)
|
||||||
|
_add_common(openapi_parser, 8099)
|
||||||
|
openapi_parser.set_defaults(func=cmd_from_openapi)
|
||||||
|
|
||||||
|
tabular_parser = subparsers.add_parser(
|
||||||
|
"from-tabular", help="Generate a shunt from a directory of CSV/TSV/ODS sheets"
|
||||||
|
)
|
||||||
|
tabular_parser.add_argument(
|
||||||
|
"--source", "-s", type=str, required=True,
|
||||||
|
help="Directory of sheets (or a single .csv/.tsv/.ods file)",
|
||||||
|
)
|
||||||
|
_add_common(tabular_parser, 8098)
|
||||||
|
tabular_parser.set_defaults(func=cmd_from_tabular)
|
||||||
|
|
||||||
|
list_parser = subparsers.add_parser("list", help="List shunts under artery/shunts/")
|
||||||
|
list_parser.set_defaults(func=cmd_list)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
args.func(args)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
354
soleprint/station/tools/shuntgen/api.py
Normal file
354
soleprint/station/tools/shuntgen/api.py
Normal file
@@ -0,0 +1,354 @@
|
|||||||
|
"""FastAPI router for shuntgen — generate shunts from a spec or a sheet folder.
|
||||||
|
|
||||||
|
Mounted by run.py under /station, so the routes below live at
|
||||||
|
/station/tools/shuntgen/... , the same shape as datagen and graphgen.
|
||||||
|
|
||||||
|
Generation writes files, so every path is resolved against the soleprint tree
|
||||||
|
and anything pointing outside it is refused. A tool that turns an uploaded file
|
||||||
|
into a directory of Python is not somewhere to be relaxed about paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, UploadFile
|
||||||
|
from fastapi.responses import HTMLResponse
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from .emit import SPR_ROOT, ShuntEmitter
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
router = APIRouter(prefix="/tools/shuntgen", tags=["shuntgen"])
|
||||||
|
|
||||||
|
SHUNTS_DIR = SPR_ROOT / "artery" / "shunts"
|
||||||
|
UPLOAD_DIR = SPR_ROOT / "station" / "tools" / "shuntgen" / "uploads"
|
||||||
|
|
||||||
|
# Room for a large spec, a hard stop well short of anything that would exhaust
|
||||||
|
# memory while being parsed.
|
||||||
|
MAX_UPLOAD_BYTES = 8 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
# Path safety
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _inside(path: Path, root: Path) -> Path:
|
||||||
|
"""Resolve a path and refuse it if it escapes root."""
|
||||||
|
resolved = (root / path).resolve() if not path.is_absolute() else path.resolve()
|
||||||
|
try:
|
||||||
|
resolved.relative_to(root.resolve())
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Path must stay inside {root.name}/: {path}",
|
||||||
|
)
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_name(name: str) -> str:
|
||||||
|
cleaned = "".join(c for c in name if c.isalnum() or c in "-_").strip("-_")
|
||||||
|
if not cleaned:
|
||||||
|
raise HTTPException(status_code=400, detail="Name must contain letters or digits")
|
||||||
|
return cleaned.lower()
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
# Models
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class GenerateRequest(BaseModel):
|
||||||
|
name: str
|
||||||
|
source: str # spec file or sheet directory, relative to soleprint/
|
||||||
|
kind: str = "auto" # "openapi" | "tabular" | "auto"
|
||||||
|
title: Optional[str] = None
|
||||||
|
port: int = 8099
|
||||||
|
cabinets: list[str] = []
|
||||||
|
force: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
# Routes
|
||||||
|
# ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/", response_class=HTMLResponse)
|
||||||
|
def index():
|
||||||
|
html = Path(__file__).parent / "templates" / "index.html"
|
||||||
|
return HTMLResponse(html.read_text() if html.exists() else "<h1>shuntgen</h1>")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/health")
|
||||||
|
def health():
|
||||||
|
return {"status": "ok", "tool": "shuntgen"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/shunts")
|
||||||
|
def list_shunts():
|
||||||
|
"""Every shunt under artery/shunts/, generated or hand-written."""
|
||||||
|
out = []
|
||||||
|
if SHUNTS_DIR.exists():
|
||||||
|
for path in sorted(SHUNTS_DIR.iterdir()):
|
||||||
|
if not path.is_dir() or path.name.startswith(("_", ".")):
|
||||||
|
continue
|
||||||
|
entry: dict[str, Any] = {"name": path.name, "generated": False}
|
||||||
|
spec_file = path / "depot" / "spec.json"
|
||||||
|
if spec_file.exists():
|
||||||
|
try:
|
||||||
|
spec = json.loads(spec_file.read_text())
|
||||||
|
entry.update(
|
||||||
|
generated=True,
|
||||||
|
title=spec.get("title"),
|
||||||
|
kind=spec.get("kind"),
|
||||||
|
source=spec.get("source"),
|
||||||
|
routes=len(spec.get("routes", [])),
|
||||||
|
models=len(spec.get("models", {})),
|
||||||
|
)
|
||||||
|
except (OSError, ValueError) as e:
|
||||||
|
entry["error"] = str(e)
|
||||||
|
out.append(entry)
|
||||||
|
return {"shunts": out}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/sources")
|
||||||
|
def list_sources():
|
||||||
|
"""Candidate inputs: uploaded specs, and sheet folders under uploads/."""
|
||||||
|
specs, sheets = [], []
|
||||||
|
if UPLOAD_DIR.exists():
|
||||||
|
for path in sorted(UPLOAD_DIR.rglob("*")):
|
||||||
|
rel = str(path.relative_to(SPR_ROOT))
|
||||||
|
if path.is_file() and path.suffix.lower() in (".json", ".yaml", ".yml"):
|
||||||
|
specs.append(rel)
|
||||||
|
elif path.is_dir() and any(
|
||||||
|
child.suffix.lower() in (".csv", ".tsv", ".ods")
|
||||||
|
for child in path.iterdir()
|
||||||
|
if child.is_file()
|
||||||
|
):
|
||||||
|
sheets.append(rel)
|
||||||
|
|
||||||
|
fixtures = Path("station/tools/shuntgen/fixtures")
|
||||||
|
if (SPR_ROOT / fixtures).exists():
|
||||||
|
specs.append(str(fixtures / "petstore.yaml"))
|
||||||
|
sheets.append(str(fixtures / "sheets"))
|
||||||
|
|
||||||
|
return {"specs": specs, "sheets": sheets}
|
||||||
|
|
||||||
|
|
||||||
|
def _multipart_available() -> bool:
|
||||||
|
"""Whether FastAPI can accept file uploads in this environment.
|
||||||
|
|
||||||
|
Declaring an UploadFile parameter without python-multipart raises at import
|
||||||
|
time, not request time. Registering it unconditionally would mean one
|
||||||
|
missing optional dependency stops the whole tool from loading — so the
|
||||||
|
route is registered only when it can work, and a stub explains its absence.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import multipart # noqa: F401
|
||||||
|
|
||||||
|
return True
|
||||||
|
except ImportError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _upload(file: UploadFile, folder: str = ""):
|
||||||
|
"""Accept a spec or sheet, into uploads/[folder]/."""
|
||||||
|
if not file.filename:
|
||||||
|
raise HTTPException(status_code=400, detail="No filename")
|
||||||
|
|
||||||
|
suffix = Path(file.filename).suffix.lower()
|
||||||
|
if suffix not in (".json", ".yaml", ".yml", ".csv", ".tsv", ".ods"):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Unsupported file type '{suffix}'. "
|
||||||
|
"Expected a spec (.json/.yaml/.yml) or a sheet (.csv/.tsv/.ods).",
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = await file.read()
|
||||||
|
if len(payload) > MAX_UPLOAD_BYTES:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=413,
|
||||||
|
detail=f"File is larger than {MAX_UPLOAD_BYTES // (1024 * 1024)}MB",
|
||||||
|
)
|
||||||
|
|
||||||
|
target_dir = UPLOAD_DIR
|
||||||
|
if folder:
|
||||||
|
target_dir = _inside(Path("station/tools/shuntgen/uploads") / _safe_name(folder), SPR_ROOT)
|
||||||
|
target_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
target = target_dir / Path(file.filename).name
|
||||||
|
target.write_bytes(payload)
|
||||||
|
return {"path": str(target.relative_to(SPR_ROOT)), "bytes": len(payload)}
|
||||||
|
|
||||||
|
|
||||||
|
if _multipart_available():
|
||||||
|
router.add_api_route("/api/upload", _upload, methods=["POST"], name="upload")
|
||||||
|
else:
|
||||||
|
|
||||||
|
@router.post("/api/upload")
|
||||||
|
def upload_unavailable():
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=501,
|
||||||
|
detail="Uploads need python-multipart. Install it, or point "
|
||||||
|
"'source' at a path on disk — generation itself works either way.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/generate")
|
||||||
|
def generate(req: GenerateRequest):
|
||||||
|
"""Generate a shunt into artery/shunts/<name>/."""
|
||||||
|
name = _safe_name(req.name)
|
||||||
|
source = _inside(Path(req.source), SPR_ROOT)
|
||||||
|
if not source.exists():
|
||||||
|
raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
|
||||||
|
|
||||||
|
output = _inside(Path("artery/shunts") / name, SPR_ROOT)
|
||||||
|
if (
|
||||||
|
output.exists()
|
||||||
|
and any(output.iterdir())
|
||||||
|
and not (output / "depot" / "spec.json").exists()
|
||||||
|
and not req.force
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail=f"artery/shunts/{name} exists and was not generated by shuntgen. "
|
||||||
|
"Pass force to write into it anyway.",
|
||||||
|
)
|
||||||
|
|
||||||
|
kind = req.kind
|
||||||
|
if kind == "auto":
|
||||||
|
kind = "tabular" if source.is_dir() else "openapi"
|
||||||
|
|
||||||
|
try:
|
||||||
|
if kind == "openapi":
|
||||||
|
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
|
||||||
|
|
||||||
|
extractor = OpenAPIExtractor(source)
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
endpoints = extractor.endpoints()
|
||||||
|
datasets = []
|
||||||
|
if not endpoints:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="The spec declares no operations, so there is nothing to serve.",
|
||||||
|
)
|
||||||
|
elif kind == "tabular":
|
||||||
|
from ..modelgen.loader.extract.tabular import TabularExtractor
|
||||||
|
|
||||||
|
extractor = TabularExtractor(source)
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
datasets = extractor.datasets()
|
||||||
|
endpoints = []
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=400, detail=f"Unknown kind: {req.kind}")
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except (RuntimeError, ValueError) as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
log.exception("shuntgen: extraction failed")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
try:
|
||||||
|
written = ShuntEmitter(
|
||||||
|
name=name,
|
||||||
|
output=output,
|
||||||
|
models=models,
|
||||||
|
enums=enums,
|
||||||
|
datasets=datasets,
|
||||||
|
endpoints=endpoints,
|
||||||
|
title=req.title,
|
||||||
|
source=source.name,
|
||||||
|
kind=kind,
|
||||||
|
port=req.port,
|
||||||
|
cabinets=req.cabinets,
|
||||||
|
).emit()
|
||||||
|
except Exception as e:
|
||||||
|
log.exception("shuntgen: emission failed")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
spec = json.loads((written / "depot" / "spec.json").read_text())
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"path": str(written.relative_to(SPR_ROOT)),
|
||||||
|
"kind": kind,
|
||||||
|
"models": len(models),
|
||||||
|
"routes": len(spec.get("routes", [])),
|
||||||
|
"rows": sum(len(d.rows) for d in datasets),
|
||||||
|
"run": f"cd {written.relative_to(SPR_ROOT)} && python run.py",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/preview")
|
||||||
|
def preview(req: GenerateRequest):
|
||||||
|
"""Extract and report what would be generated, writing nothing permanent."""
|
||||||
|
source = _inside(Path(req.source), SPR_ROOT)
|
||||||
|
if not source.exists():
|
||||||
|
raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
|
||||||
|
|
||||||
|
kind = req.kind
|
||||||
|
if kind == "auto":
|
||||||
|
kind = "tabular" if source.is_dir() else "openapi"
|
||||||
|
|
||||||
|
try:
|
||||||
|
if kind == "openapi":
|
||||||
|
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
|
||||||
|
|
||||||
|
extractor = OpenAPIExtractor(source)
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
endpoints, datasets = extractor.endpoints(), []
|
||||||
|
else:
|
||||||
|
from ..modelgen.loader.extract.tabular import TabularExtractor
|
||||||
|
|
||||||
|
extractor = TabularExtractor(source)
|
||||||
|
models, enums = extractor.extract()
|
||||||
|
datasets, endpoints = extractor.datasets(), []
|
||||||
|
except (RuntimeError, ValueError) as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
|
||||||
|
# Emit into a throwaway directory: the routes are worked out by the emitter,
|
||||||
|
# and duplicating that logic here is how a preview drifts from the thing it
|
||||||
|
# is previewing.
|
||||||
|
temp = Path(tempfile.mkdtemp(prefix="shuntgen-preview-"))
|
||||||
|
try:
|
||||||
|
emitter = ShuntEmitter(
|
||||||
|
name=_safe_name(req.name or "preview"),
|
||||||
|
output=temp,
|
||||||
|
models=models,
|
||||||
|
enums=enums,
|
||||||
|
datasets=datasets,
|
||||||
|
endpoints=endpoints,
|
||||||
|
source=source.name,
|
||||||
|
kind=kind,
|
||||||
|
)
|
||||||
|
collections = emitter._collections()
|
||||||
|
routes = emitter._routes(collections)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(temp, ignore_errors=True)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"kind": kind,
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"name": m.name,
|
||||||
|
"fields": len(m.fields),
|
||||||
|
"doc": (m.docstring or "").strip().splitlines()[0] if m.docstring else None,
|
||||||
|
}
|
||||||
|
for m in models
|
||||||
|
],
|
||||||
|
"collections": collections,
|
||||||
|
"routes": [
|
||||||
|
{
|
||||||
|
"method": r["method"],
|
||||||
|
"path": r["path"],
|
||||||
|
"operation": r["operation"],
|
||||||
|
"model": r["model"],
|
||||||
|
}
|
||||||
|
for r in routes
|
||||||
|
],
|
||||||
|
"rows": sum(len(d.rows) for d in datasets),
|
||||||
|
}
|
||||||
462
soleprint/station/tools/shuntgen/emit.py
Normal file
462
soleprint/station/tools/shuntgen/emit.py
Normal file
@@ -0,0 +1,462 @@
|
|||||||
|
"""
|
||||||
|
Shunt emission — turns extracted models into a runnable shunt directory.
|
||||||
|
|
||||||
|
The layout follows the contract in artery/shunts/__init__.py (main.py, a depot,
|
||||||
|
a README) and adds what a generated shunt needs to answer for itself:
|
||||||
|
|
||||||
|
artery/shunts/<name>/
|
||||||
|
main.py builds the app from the spec
|
||||||
|
run.py uvicorn entry point
|
||||||
|
shunt_runtime.py vendored copy of runtime.py
|
||||||
|
models.py pydantic, via modelgen
|
||||||
|
datagen_<name>.py BaseDataGenerator subclass, via modelgen
|
||||||
|
depot/spec.json normalised routes + collections + schema
|
||||||
|
depot/responses.json pinned overrides, the "METHOD /path" map
|
||||||
|
depot/data.json imported rows
|
||||||
|
depot/config.json latency / error-injection knobs
|
||||||
|
templates/index.html config UI
|
||||||
|
cabinet.json declared dependency containers, if any
|
||||||
|
README.md
|
||||||
|
|
||||||
|
Only spec.json and the depot are worth editing by hand; everything else is
|
||||||
|
regenerated. The vendored runtime is a copy rather than an import because a
|
||||||
|
shunt runs standalone, with no soleprint on its path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from ..modelgen.generator import DatagenGenerator, JsonSchemaGenerator, PydanticGenerator
|
||||||
|
from ..modelgen.loader.schema import (
|
||||||
|
DatasetDefinition,
|
||||||
|
EndpointDefinition,
|
||||||
|
ModelDefinition,
|
||||||
|
)
|
||||||
|
|
||||||
|
HERE = Path(__file__).parent
|
||||||
|
# soleprint/ — four levels up from station/tools/shuntgen/emit.py
|
||||||
|
SPR_ROOT = HERE.parents[2]
|
||||||
|
|
||||||
|
# Types whose keys are integers, so the runtime coerces "/pets/7" to 7 before
|
||||||
|
# comparing it with a stored row.
|
||||||
|
_INT_HINTS = {int, "bigint"}
|
||||||
|
|
||||||
|
|
||||||
|
def _pk_of(model: ModelDefinition) -> tuple[Optional[str], str]:
|
||||||
|
"""Return (primary key field name, "int" | "str") for a model."""
|
||||||
|
for field in model.fields:
|
||||||
|
if getattr(field, "primary_key", False):
|
||||||
|
return field.name, "int" if field.type_hint in _INT_HINTS else "str"
|
||||||
|
for field in model.fields:
|
||||||
|
if field.name == "id":
|
||||||
|
return field.name, "int" if field.type_hint in _INT_HINTS else "str"
|
||||||
|
return None, "str"
|
||||||
|
|
||||||
|
|
||||||
|
def _operation(method: str, kind: str) -> str:
|
||||||
|
"""Name what a route does, from its verb and whether it addresses one row."""
|
||||||
|
if method == "GET":
|
||||||
|
return "list" if kind == "collection" else "retrieve" if kind == "item" else "action"
|
||||||
|
if method == "POST":
|
||||||
|
return "create" if kind == "collection" else "action"
|
||||||
|
if method in ("PUT", "PATCH"):
|
||||||
|
return "update" if kind == "item" else "action"
|
||||||
|
if method == "DELETE":
|
||||||
|
return "delete" if kind == "item" else "action"
|
||||||
|
return "action"
|
||||||
|
|
||||||
|
|
||||||
|
def _class_name(name: str) -> str:
|
||||||
|
parts = [p for p in name.replace("-", "_").split("_") if p]
|
||||||
|
return "".join(p[:1].upper() + p[1:] for p in parts) or "Shunt"
|
||||||
|
|
||||||
|
|
||||||
|
def _theme_css() -> str:
|
||||||
|
"""The theme, inlined.
|
||||||
|
|
||||||
|
A shunt serves its own UI on its own port, so it cannot fetch soleprint's
|
||||||
|
/theme.css. Inlining keeps it standalone and keeps one source of truth —
|
||||||
|
regenerating picks up any change to common/theme.
|
||||||
|
"""
|
||||||
|
theme_dir = SPR_ROOT / "common" / "theme"
|
||||||
|
parts: List[str] = []
|
||||||
|
tokens = theme_dir / "tokens.css"
|
||||||
|
if tokens.exists():
|
||||||
|
parts.append(tokens.read_text())
|
||||||
|
for sheet in sorted((theme_dir / "themes").glob("*.css")):
|
||||||
|
parts.append(sheet.read_text())
|
||||||
|
if parts:
|
||||||
|
return "\n".join(parts)
|
||||||
|
return ":root{--bg:#0a0a0a;--surface:#1a1a1a;--border:#333;--text:#e5e5e5;--muted:#a3a3a3;--accent:#d4a574}"
|
||||||
|
|
||||||
|
|
||||||
|
class ShuntEmitter:
|
||||||
|
"""Writes a complete shunt directory."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
name: str,
|
||||||
|
output: Path,
|
||||||
|
models: List[ModelDefinition],
|
||||||
|
enums: Optional[List[Any]] = None,
|
||||||
|
datasets: Optional[List[DatasetDefinition]] = None,
|
||||||
|
endpoints: Optional[List[EndpointDefinition]] = None,
|
||||||
|
title: Optional[str] = None,
|
||||||
|
source: str = "",
|
||||||
|
kind: str = "openapi",
|
||||||
|
port: int = 8099,
|
||||||
|
cabinets: Optional[List[str]] = None,
|
||||||
|
):
|
||||||
|
self.name = name
|
||||||
|
self.output = Path(output)
|
||||||
|
self.models = models
|
||||||
|
self.enums = enums or []
|
||||||
|
self.datasets = datasets or []
|
||||||
|
self.endpoints = endpoints or []
|
||||||
|
self.title = title or _class_name(name)
|
||||||
|
self.source = source
|
||||||
|
self.kind = kind
|
||||||
|
self.port = port
|
||||||
|
self.cabinets = cabinets or []
|
||||||
|
self.by_name = {m.name: m for m in models}
|
||||||
|
|
||||||
|
# ── entry point ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def emit(self) -> Path:
|
||||||
|
self.output.mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.output / "depot").mkdir(exist_ok=True)
|
||||||
|
(self.output / "templates").mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
collections = self._collections()
|
||||||
|
routes = self._routes(collections)
|
||||||
|
|
||||||
|
self._write_models()
|
||||||
|
self._write_generator()
|
||||||
|
self._write_depot(routes, collections)
|
||||||
|
self._write_runtime()
|
||||||
|
self._write_app()
|
||||||
|
self._write_ui(routes)
|
||||||
|
self._write_cabinets()
|
||||||
|
self._write_readme(routes, collections)
|
||||||
|
return self.output
|
||||||
|
|
||||||
|
# ── spec construction ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _collections(self) -> Dict[str, dict]:
|
||||||
|
"""Models the runtime may keep rows for, with the key to match them on."""
|
||||||
|
out: Dict[str, dict] = {}
|
||||||
|
|
||||||
|
for dataset in self.datasets:
|
||||||
|
model = self.by_name.get(dataset.model)
|
||||||
|
if not model:
|
||||||
|
continue
|
||||||
|
pk, pk_type = _pk_of(model)
|
||||||
|
out[dataset.model] = {
|
||||||
|
"path": f"/{dataset.collection or dataset.model.lower()}",
|
||||||
|
"pk": pk or "id",
|
||||||
|
"pk_type": pk_type,
|
||||||
|
"rows": len(dataset.rows),
|
||||||
|
}
|
||||||
|
|
||||||
|
# A spec's models get a collection too, so POSTing to one and GETting it
|
||||||
|
# back works even though no rows were imported.
|
||||||
|
for endpoint in self.endpoints:
|
||||||
|
model_name = endpoint.model
|
||||||
|
if not model_name or model_name in out:
|
||||||
|
continue
|
||||||
|
model = self.by_name.get(model_name)
|
||||||
|
if not model:
|
||||||
|
continue
|
||||||
|
pk, pk_type = _pk_of(model)
|
||||||
|
if not pk:
|
||||||
|
continue
|
||||||
|
out[model_name] = {
|
||||||
|
"path": self._base_path(endpoint.path),
|
||||||
|
"pk": pk,
|
||||||
|
"pk_type": pk_type,
|
||||||
|
"rows": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _base_path(path: str) -> str:
|
||||||
|
"""Trim a path back to its collection — /pets/{petId} -> /pets."""
|
||||||
|
segments = [s for s in path.split("/") if s and not s.startswith("{")]
|
||||||
|
return "/" + "/".join(segments) if segments else "/"
|
||||||
|
|
||||||
|
def _routes(self, collections: Dict[str, dict]) -> List[dict]:
|
||||||
|
if self.endpoints:
|
||||||
|
return [self._from_endpoint(e) for e in self.endpoints]
|
||||||
|
return self._crud_routes(collections)
|
||||||
|
|
||||||
|
def _from_endpoint(self, endpoint: EndpointDefinition) -> dict:
|
||||||
|
return {
|
||||||
|
"method": endpoint.method,
|
||||||
|
"path": endpoint.path,
|
||||||
|
"operation_id": endpoint.operation_id,
|
||||||
|
"summary": endpoint.summary,
|
||||||
|
"operation": _operation(endpoint.method, endpoint.kind),
|
||||||
|
"model": endpoint.model,
|
||||||
|
"request_model": endpoint.request_model,
|
||||||
|
"response_is_list": endpoint.response_is_list,
|
||||||
|
"envelope_key": endpoint.envelope_key,
|
||||||
|
"status": endpoint.status,
|
||||||
|
"path_params": list(endpoint.path_params),
|
||||||
|
"example": endpoint.example,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _crud_routes(self, collections: Dict[str, dict]) -> List[dict]:
|
||||||
|
"""The five routes a table implies, for sources that describe no calls."""
|
||||||
|
routes: List[dict] = []
|
||||||
|
for model_name, meta in collections.items():
|
||||||
|
base = meta["path"]
|
||||||
|
pk = meta["pk"]
|
||||||
|
item = f"{base}/{{{pk}}}"
|
||||||
|
plural = base.strip("/") or model_name.lower()
|
||||||
|
|
||||||
|
def route(method, path, operation, status, summary, params=()):
|
||||||
|
return {
|
||||||
|
"method": method,
|
||||||
|
"path": path,
|
||||||
|
"operation_id": f"{operation}_{plural}".replace("-", "_"),
|
||||||
|
"summary": summary,
|
||||||
|
"operation": operation,
|
||||||
|
"model": model_name,
|
||||||
|
"request_model": model_name if operation in ("create", "update") else None,
|
||||||
|
"response_is_list": operation == "list",
|
||||||
|
"envelope_key": None,
|
||||||
|
"status": status,
|
||||||
|
"path_params": list(params),
|
||||||
|
"example": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
routes.extend([
|
||||||
|
route("GET", base, "list", 200, f"List {plural}"),
|
||||||
|
route("POST", base, "create", 201, f"Create a {model_name}"),
|
||||||
|
route("GET", item, "retrieve", 200, f"Fetch one {model_name}", (pk,)),
|
||||||
|
route("PUT", item, "update", 200, f"Update a {model_name}", (pk,)),
|
||||||
|
route("DELETE", item, "delete", 204, f"Delete a {model_name}", (pk,)),
|
||||||
|
])
|
||||||
|
return routes
|
||||||
|
|
||||||
|
# ── file writers ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _write_models(self) -> None:
|
||||||
|
PydanticGenerator().generate(
|
||||||
|
(self.models, self.enums), self.output / "models.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _generator_module(self) -> str:
|
||||||
|
return f"datagen_{self.name.replace('-', '_')}"
|
||||||
|
|
||||||
|
def _write_generator(self) -> None:
|
||||||
|
DatagenGenerator(
|
||||||
|
class_name=f"{_class_name(self.name)}Generator",
|
||||||
|
depot="depot/data.json",
|
||||||
|
).generate(
|
||||||
|
(self.models, self.enums, self.datasets),
|
||||||
|
self.output / f"{self._generator_module()}.py",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_depot(self, routes: List[dict], collections: Dict[str, dict]) -> None:
|
||||||
|
depot = self.output / "depot"
|
||||||
|
|
||||||
|
schema_path = depot / "schema.json"
|
||||||
|
JsonSchemaGenerator().generate((self.models, self.enums), schema_path)
|
||||||
|
schema = json.loads(schema_path.read_text())
|
||||||
|
|
||||||
|
spec = {
|
||||||
|
"name": self.name,
|
||||||
|
"title": self.title,
|
||||||
|
"kind": self.kind,
|
||||||
|
"source": self.source,
|
||||||
|
"summary": f"Generated by shuntgen from {self.source or self.kind}.",
|
||||||
|
"generator_module": self._generator_module(),
|
||||||
|
"collections": collections,
|
||||||
|
"routes": routes,
|
||||||
|
"models": schema.get("models", {}),
|
||||||
|
}
|
||||||
|
(depot / "spec.json").write_text(json.dumps(spec, indent=2) + "\n")
|
||||||
|
|
||||||
|
data = {d.model: d.rows for d in self.datasets if d.rows}
|
||||||
|
(depot / "data.json").write_text(json.dumps(data, indent=2) + "\n")
|
||||||
|
|
||||||
|
# Left empty on purpose: an override is a deliberate act, and a file
|
||||||
|
# pre-filled with guesses would quietly shadow the generated responses.
|
||||||
|
responses_path = depot / "responses.json"
|
||||||
|
if not responses_path.exists():
|
||||||
|
responses_path.write_text("{}\n")
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"title": self.title,
|
||||||
|
"port": self.port,
|
||||||
|
"enable_random_delays": False,
|
||||||
|
"min_delay_ms": 200,
|
||||||
|
"max_delay_ms": 800,
|
||||||
|
"error_rate": 0.0,
|
||||||
|
# Imported rows are the real thing; only invent when there are none.
|
||||||
|
"prefill": 0 if any(d.rows for d in self.datasets) else 5,
|
||||||
|
"unknown_id": "generate",
|
||||||
|
"page_size": 50,
|
||||||
|
}
|
||||||
|
config_path = depot / "config.json"
|
||||||
|
if not config_path.exists():
|
||||||
|
config_path.write_text(json.dumps(config, indent=2) + "\n")
|
||||||
|
|
||||||
|
def _write_runtime(self) -> None:
|
||||||
|
shutil.copyfile(HERE / "runtime.py", self.output / "shunt_runtime.py")
|
||||||
|
|
||||||
|
def _write_app(self) -> None:
|
||||||
|
(self.output / "main.py").write_text(
|
||||||
|
f'''"""
|
||||||
|
{self.title} shunt — GENERATED.
|
||||||
|
|
||||||
|
The routes live in depot/spec.json and are built at import time by
|
||||||
|
shunt_runtime. Edit the depot, not this file; regenerate with shuntgen.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from shunt_runtime import build_app
|
||||||
|
|
||||||
|
app = build_app(Path(__file__).parent)
|
||||||
|
'''
|
||||||
|
)
|
||||||
|
|
||||||
|
(self.output / "run.py").write_text(
|
||||||
|
f'''"""Run the {self.title} shunt standalone."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
BASE = Path(__file__).parent
|
||||||
|
|
||||||
|
|
||||||
|
def port() -> int:
|
||||||
|
"""PORT wins, then depot/config.json, then the generated default."""
|
||||||
|
if os.getenv("PORT"):
|
||||||
|
return int(os.environ["PORT"])
|
||||||
|
config = BASE / "depot" / "config.json"
|
||||||
|
if config.exists():
|
||||||
|
try:
|
||||||
|
return int(json.loads(config.read_text()).get("port", {self.port}))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
return {self.port}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
chosen = port()
|
||||||
|
print(f"{self.title} shunt on http://localhost:{{chosen}} (UI at /, spec at /mock/spec)")
|
||||||
|
uvicorn.run("main:app", host="0.0.0.0", port=chosen, reload=False)
|
||||||
|
'''
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_ui(self, routes: List[dict]) -> None:
|
||||||
|
template = (HERE / "templates" / "shunt_ui.html").read_text()
|
||||||
|
page = (
|
||||||
|
template.replace("%%THEME_CSS%%", _theme_css())
|
||||||
|
.replace("%%TITLE%%", self.title)
|
||||||
|
.replace("%%NAME%%", self.name)
|
||||||
|
.replace("%%SOURCE%%", self.source or self.kind)
|
||||||
|
.replace("%%ROUTE_COUNT%%", str(len(routes)))
|
||||||
|
)
|
||||||
|
(self.output / "templates" / "index.html").write_text(page)
|
||||||
|
|
||||||
|
def _write_cabinets(self) -> None:
|
||||||
|
if not self.cabinets:
|
||||||
|
return
|
||||||
|
(self.output / "cabinet.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"requires": self.cabinets,
|
||||||
|
"note": (
|
||||||
|
"Dependency containers this shunt expects. "
|
||||||
|
"`python build.py --cfg <room>` composes them into the "
|
||||||
|
"room's docker-compose.yml; on a cluster they install "
|
||||||
|
"as rig addons of the same name."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_readme(self, routes: List[dict], collections: Dict[str, dict]) -> None:
|
||||||
|
lines = [
|
||||||
|
f"# {self.title} shunt",
|
||||||
|
"",
|
||||||
|
f"Generated by shuntgen from `{self.source or self.kind}`.",
|
||||||
|
"",
|
||||||
|
"## Run",
|
||||||
|
"",
|
||||||
|
"```bash",
|
||||||
|
f"python run.py # http://localhost:{self.port}",
|
||||||
|
f"PORT=9000 python run.py # somewhere else",
|
||||||
|
"```",
|
||||||
|
"",
|
||||||
|
"## Routes",
|
||||||
|
"",
|
||||||
|
"| Method | Path | Does |",
|
||||||
|
"| --- | --- | --- |",
|
||||||
|
]
|
||||||
|
for route in routes:
|
||||||
|
lines.append(
|
||||||
|
f"| {route['method']} | `{route['path']}` | {route['operation']} |"
|
||||||
|
)
|
||||||
|
|
||||||
|
lines += [
|
||||||
|
"",
|
||||||
|
"## Control",
|
||||||
|
"",
|
||||||
|
"| Endpoint | Purpose |",
|
||||||
|
"| --- | --- |",
|
||||||
|
"| `GET /health` | liveness |",
|
||||||
|
"| `GET /mock/spec` | the routes this shunt was built from |",
|
||||||
|
"| `GET /mock/stats` | call counts and row counts |",
|
||||||
|
"| `POST /mock/reset` | restore the imported rows, clear counters |",
|
||||||
|
"| `GET,POST /mock/config` | latency and error-injection knobs |",
|
||||||
|
"| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |",
|
||||||
|
"",
|
||||||
|
"## Depot",
|
||||||
|
"",
|
||||||
|
"| File | Purpose |",
|
||||||
|
"| --- | --- |",
|
||||||
|
"| `spec.json` | routes, collections and schema — the source of truth |",
|
||||||
|
"| `responses.json` | pinned overrides, keyed `\"METHOD /path\"`; these win over everything |",
|
||||||
|
"| `data.json` | seed rows, keyed by model |",
|
||||||
|
"| `config.json` | delays, error rate, prefill, page size |",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
if collections:
|
||||||
|
lines += ["## Collections", "", "| Model | Path | Key | Seed rows |", "| --- | --- | --- | --- |"]
|
||||||
|
for model, meta in collections.items():
|
||||||
|
lines.append(
|
||||||
|
f"| {model} | `{meta['path']}` | `{meta['pk']}` | {meta['rows']} |"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
if self.cabinets:
|
||||||
|
lines += [
|
||||||
|
"## Dependencies",
|
||||||
|
"",
|
||||||
|
f"Declares the cabinets: {', '.join(f'`{c}`' for c in self.cabinets)}. "
|
||||||
|
"See `cabinet.json`.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
lines += [
|
||||||
|
"## Regenerating",
|
||||||
|
"",
|
||||||
|
"Everything here except `depot/responses.json` and `depot/config.json` is",
|
||||||
|
"overwritten on regeneration — those two are yours.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
(self.output / "README.md").write_text("\n".join(lines))
|
||||||
147
soleprint/station/tools/shuntgen/fixtures/petstore.yaml
Normal file
147
soleprint/station/tools/shuntgen/fixtures/petstore.yaml
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
# A small OpenAPI 3 document used to exercise shuntgen and the OpenAPI loader.
|
||||||
|
#
|
||||||
|
# Deliberately covers the shapes that are easy to get wrong: an enum, a $ref to
|
||||||
|
# another schema, an array of $ref, a wrapped collection, a path parameter, a
|
||||||
|
# request body, and a response example.
|
||||||
|
openapi: 3.0.3
|
||||||
|
info:
|
||||||
|
title: Petstore
|
||||||
|
version: 1.0.0
|
||||||
|
|
||||||
|
paths:
|
||||||
|
/pets:
|
||||||
|
get:
|
||||||
|
operationId: listPets
|
||||||
|
summary: List all pets
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: A page of pets
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/PetPage"
|
||||||
|
post:
|
||||||
|
operationId: createPet
|
||||||
|
summary: Create a pet
|
||||||
|
requestBody:
|
||||||
|
required: true
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Pet"
|
||||||
|
responses:
|
||||||
|
"201":
|
||||||
|
description: The created pet
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Pet"
|
||||||
|
|
||||||
|
/pets/{petId}:
|
||||||
|
get:
|
||||||
|
operationId: getPet
|
||||||
|
summary: Fetch one pet
|
||||||
|
parameters:
|
||||||
|
- name: petId
|
||||||
|
in: path
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
type: integer
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: The pet
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Pet"
|
||||||
|
example:
|
||||||
|
id: 7
|
||||||
|
name: "Rocinante"
|
||||||
|
status: "available"
|
||||||
|
"404":
|
||||||
|
description: No such pet
|
||||||
|
delete:
|
||||||
|
operationId: deletePet
|
||||||
|
summary: Remove a pet
|
||||||
|
parameters:
|
||||||
|
- name: petId
|
||||||
|
in: path
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
type: integer
|
||||||
|
responses:
|
||||||
|
"204":
|
||||||
|
description: Deleted
|
||||||
|
|
||||||
|
/categories:
|
||||||
|
get:
|
||||||
|
operationId: listCategories
|
||||||
|
summary: List categories
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: Every category
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
$ref: "#/components/schemas/Category"
|
||||||
|
|
||||||
|
components:
|
||||||
|
schemas:
|
||||||
|
Pet:
|
||||||
|
type: object
|
||||||
|
description: An animal available for adoption.
|
||||||
|
required: [id, name]
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: integer
|
||||||
|
format: int64
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
status:
|
||||||
|
type: string
|
||||||
|
enum: [available, pending, sold]
|
||||||
|
weight_kg:
|
||||||
|
type: number
|
||||||
|
neutered:
|
||||||
|
type: boolean
|
||||||
|
born_on:
|
||||||
|
type: string
|
||||||
|
format: date
|
||||||
|
category:
|
||||||
|
$ref: "#/components/schemas/Category"
|
||||||
|
tags:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
$ref: "#/components/schemas/Tag"
|
||||||
|
|
||||||
|
Category:
|
||||||
|
type: object
|
||||||
|
description: A grouping of pets.
|
||||||
|
required: [id, name]
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: integer
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
|
||||||
|
Tag:
|
||||||
|
type: object
|
||||||
|
required: [id]
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: string
|
||||||
|
format: uuid
|
||||||
|
label:
|
||||||
|
type: string
|
||||||
|
|
||||||
|
PetPage:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
items:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
$ref: "#/components/schemas/Pet"
|
||||||
|
total:
|
||||||
|
type: integer
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
id,name,email,city,active,signed_up_at,credit_limit
|
||||||
|
1,Ada Alvarez,ada@example.com,Buenos Aires,true,2024-03-11,15000.50
|
||||||
|
2,Bruno Bianchi,bruno@example.com,Rosario,true,2024-05-02,8000
|
||||||
|
3,Camila Castro,camila@example.com,Cordoba,false,2023-11-27,
|
||||||
|
4,Diego Duarte,diego@example.com,Montevideo,true,2025-01-19,22500.75
|
||||||
|
5,Elena Esposito,elena@example.com,Santiago,false,2025-06-30,3000
|
||||||
|
@@ -0,0 +1,7 @@
|
|||||||
|
invoice_id,customer_id,issued_on,total,currency,paid
|
||||||
|
1001,1,2025-02-03,4200.00,ARS,true
|
||||||
|
1002,1,2025-03-03,3150.25,ARS,false
|
||||||
|
1003,2,2025-03-14,900.00,USD,true
|
||||||
|
1004,4,2025-04-21,12800.40,ARS,false
|
||||||
|
1005,5,2025-05-05,150.00,USD,true
|
||||||
|
1006,2,2025-06-11,7300.10,ARS,true
|
||||||
|
BIN
soleprint/station/tools/shuntgen/fixtures/sheets/line_items.ods
Normal file
BIN
soleprint/station/tools/shuntgen/fixtures/sheets/line_items.ods
Normal file
Binary file not shown.
489
soleprint/station/tools/shuntgen/runtime.py
Normal file
489
soleprint/station/tools/shuntgen/runtime.py
Normal file
@@ -0,0 +1,489 @@
|
|||||||
|
"""
|
||||||
|
Shunt runtime — copied verbatim into every generated shunt as shunt_runtime.py.
|
||||||
|
|
||||||
|
A generated shunt is a spec plus this file. The routes are built at startup
|
||||||
|
from depot/spec.json rather than written out as source, which is what keeps the
|
||||||
|
generated code short enough to read and the behaviour in one reviewable place:
|
||||||
|
fixing a bug here fixes it in every shunt, and regenerating is a copy.
|
||||||
|
|
||||||
|
It imports nothing from soleprint. A shunt runs as its own process on its own
|
||||||
|
port — often in a test harness that has no soleprint on the path — so the only
|
||||||
|
dependencies are fastapi and the standard library.
|
||||||
|
|
||||||
|
Where a response comes from, first hit wins:
|
||||||
|
|
||||||
|
1. depot/responses.json a pinned override, the documented "METHOD /path" map
|
||||||
|
2. the store rows imported from spreadsheets, plus anything POSTed
|
||||||
|
3. the spec example an `example` carried over from the source document
|
||||||
|
4. the generator datagen_<name>.py, synthesising from the schema
|
||||||
|
5. {} nothing else was available
|
||||||
|
|
||||||
|
The store is what makes it behave like a service rather than a random-value
|
||||||
|
faucet: POST something and GET it back, ask for /pets/7 and get the pet whose
|
||||||
|
id is 7.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
|
from fastapi.responses import HTMLResponse, JSONResponse, Response
|
||||||
|
|
||||||
|
DEFAULT_CONFIG: Dict[str, Any] = {
|
||||||
|
"title": "Shunt",
|
||||||
|
"port": 8099,
|
||||||
|
# Latency and failure injection, the knobs the mercadopago shunt proved out.
|
||||||
|
"enable_random_delays": False,
|
||||||
|
"min_delay_ms": 200,
|
||||||
|
"max_delay_ms": 800,
|
||||||
|
"error_rate": 0.0,
|
||||||
|
# Rows to synthesise per collection that arrived with none, so a GET
|
||||||
|
# answers with something on the first call rather than an empty list.
|
||||||
|
"prefill": 5,
|
||||||
|
# "generate" invents a record for an unknown id; "404" refuses it. Generate
|
||||||
|
# by default: a client pointed at a fresh shunt should just work, and a
|
||||||
|
# test that needs the error path can pin it or flip this.
|
||||||
|
"unknown_id": "generate",
|
||||||
|
"page_size": 50,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Shunt:
|
||||||
|
"""Holds a shunt's spec, config, store and counters."""
|
||||||
|
|
||||||
|
def __init__(self, base: Path):
|
||||||
|
self.base = Path(base)
|
||||||
|
self.depot = self.base / "depot"
|
||||||
|
self.spec: Dict[str, Any] = self._read_json("spec.json", {})
|
||||||
|
self.responses: Dict[str, Any] = self._read_json("responses.json", {})
|
||||||
|
self.config: Dict[str, Any] = {
|
||||||
|
**DEFAULT_CONFIG,
|
||||||
|
**self._read_json("config.json", {}),
|
||||||
|
}
|
||||||
|
self.seed: Dict[str, List[dict]] = self._read_json("data.json", {})
|
||||||
|
self.collections: Dict[str, dict] = self.spec.get("collections", {}) or {}
|
||||||
|
self.stats: Dict[str, int] = {}
|
||||||
|
self.generator = self._load_generator()
|
||||||
|
self.store: Dict[str, List[dict]] = {}
|
||||||
|
self.reset()
|
||||||
|
|
||||||
|
# ── loading ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _read_json(self, name: str, fallback: Any) -> Any:
|
||||||
|
path = self.depot / name
|
||||||
|
if not path.exists():
|
||||||
|
return fallback
|
||||||
|
try:
|
||||||
|
return json.loads(path.read_text())
|
||||||
|
except (OSError, ValueError):
|
||||||
|
# A hand-edited responses.json with a stray comma should degrade to
|
||||||
|
# "no overrides", not stop the shunt from booting.
|
||||||
|
return fallback
|
||||||
|
|
||||||
|
def _load_generator(self):
|
||||||
|
"""Import the sibling datagen module, if one was generated."""
|
||||||
|
module_name = self.spec.get("generator_module")
|
||||||
|
if not module_name:
|
||||||
|
return None
|
||||||
|
path = self.base / f"{module_name}.py"
|
||||||
|
if not path.exists():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||||
|
if not spec or not spec.loader:
|
||||||
|
return None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Same discovery rule as station/tools/datagen/api.py: the class whose
|
||||||
|
# name ends in Generator and is not the base.
|
||||||
|
for name, obj in vars(module).items():
|
||||||
|
if (
|
||||||
|
isinstance(obj, type)
|
||||||
|
and name.endswith("Generator")
|
||||||
|
and name != "BaseDataGenerator"
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
return obj()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
def reset(self) -> None:
|
||||||
|
"""Restore the store to its imported state and clear the counters."""
|
||||||
|
self.store = {model: [dict(r) for r in rows] for model, rows in self.seed.items()}
|
||||||
|
self.stats = {}
|
||||||
|
|
||||||
|
prefill = int(self.config.get("prefill") or 0)
|
||||||
|
if not prefill or not self.generator:
|
||||||
|
return
|
||||||
|
for model in self.collections:
|
||||||
|
if self.store.get(model):
|
||||||
|
continue
|
||||||
|
self.store[model] = self.synthesise(model, prefill, sequential_keys=True)
|
||||||
|
|
||||||
|
# ── generation ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def synthesise(
|
||||||
|
self, model: str, count: int = 1, sequential_keys: bool = False
|
||||||
|
) -> List[dict]:
|
||||||
|
"""Build records for a model, or [] if nothing can."""
|
||||||
|
if not self.generator:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
rows = self.generator.generate(model, count)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if sequential_keys:
|
||||||
|
# Prefilled rows are the ones a caller will look up by id, so their
|
||||||
|
# keys have to be predictable — 1..n, not whatever random produced.
|
||||||
|
pk = (self.collections.get(model) or {}).get("pk")
|
||||||
|
pk_type = (self.collections.get(model) or {}).get("pk_type")
|
||||||
|
if pk and pk_type in ("int", "bigint"):
|
||||||
|
for index, row in enumerate(rows, start=1):
|
||||||
|
row[pk] = index
|
||||||
|
return rows
|
||||||
|
|
||||||
|
# ── store operations ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def rows(self, model: str) -> List[dict]:
|
||||||
|
return self.store.setdefault(model, [])
|
||||||
|
|
||||||
|
def coerce_key(self, model: str, raw: Any) -> Any:
|
||||||
|
"""Match a path segment against the key type the rows actually use."""
|
||||||
|
pk_type = (self.collections.get(model) or {}).get("pk_type")
|
||||||
|
if pk_type in ("int", "bigint"):
|
||||||
|
try:
|
||||||
|
return int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return raw
|
||||||
|
return str(raw)
|
||||||
|
|
||||||
|
def find(self, model: str, key: Any) -> Optional[dict]:
|
||||||
|
pk = (self.collections.get(model) or {}).get("pk")
|
||||||
|
if not pk:
|
||||||
|
return None
|
||||||
|
wanted = self.coerce_key(model, key)
|
||||||
|
for row in self.rows(model):
|
||||||
|
if row.get(pk) == wanted or str(row.get(pk)) == str(wanted):
|
||||||
|
return row
|
||||||
|
return None
|
||||||
|
|
||||||
|
def next_key(self, model: str) -> Any:
|
||||||
|
pk_meta = self.collections.get(model) or {}
|
||||||
|
pk, pk_type = pk_meta.get("pk"), pk_meta.get("pk_type")
|
||||||
|
if not pk:
|
||||||
|
return None
|
||||||
|
if pk_type in ("int", "bigint"):
|
||||||
|
keys = [r.get(pk) for r in self.rows(model) if isinstance(r.get(pk), int)]
|
||||||
|
return (max(keys) + 1) if keys else 1
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
# ── request shaping ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def maybe_fail(self) -> None:
|
||||||
|
rate = float(self.config.get("error_rate") or 0)
|
||||||
|
if rate > 0 and random.random() < rate:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail="injected failure (error_rate is above zero)",
|
||||||
|
)
|
||||||
|
|
||||||
|
def maybe_delay(self) -> None:
|
||||||
|
if not self.config.get("enable_random_delays"):
|
||||||
|
return
|
||||||
|
low = int(self.config.get("min_delay_ms") or 0)
|
||||||
|
high = max(low, int(self.config.get("max_delay_ms") or 0))
|
||||||
|
time.sleep(random.randint(low, high) / 1000.0)
|
||||||
|
|
||||||
|
def pinned(self, method: str, template: str, actual: str) -> Any:
|
||||||
|
"""A configured override, matched on the template or the concrete path."""
|
||||||
|
for key in (f"{method} {template}", f"{method} {actual}"):
|
||||||
|
if key in self.responses:
|
||||||
|
return self.responses[key]
|
||||||
|
return None
|
||||||
|
|
||||||
|
def count(self, key: str) -> None:
|
||||||
|
self.stats[key] = self.stats.get(key, 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
# ── route handlers ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _filtered(rows: List[dict], params) -> List[dict]:
|
||||||
|
"""Narrow a collection by any query parameter naming a field."""
|
||||||
|
reserved = {"limit", "offset", "page", "page_size"}
|
||||||
|
out = rows
|
||||||
|
for name, value in params.multi_items():
|
||||||
|
if name in reserved:
|
||||||
|
continue
|
||||||
|
if not out or name not in out[0]:
|
||||||
|
continue
|
||||||
|
out = [r for r in out if str(r.get(name)) == str(value)]
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _paged(rows: List[dict], params, default_size: int):
|
||||||
|
try:
|
||||||
|
limit = int(params.get("limit", params.get("page_size", default_size)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
limit = default_size
|
||||||
|
try:
|
||||||
|
offset = int(params.get("offset", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
offset = 0
|
||||||
|
limit = max(0, min(limit, 1000))
|
||||||
|
return rows[offset : offset + limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_handler(shunt: Shunt, route: dict):
|
||||||
|
"""Build the endpoint function for one route in the spec."""
|
||||||
|
method = route["method"]
|
||||||
|
template = route["path"]
|
||||||
|
model = route.get("model")
|
||||||
|
operation = route.get("operation", "action")
|
||||||
|
envelope = route.get("envelope_key")
|
||||||
|
status = int(route.get("status") or 200)
|
||||||
|
example = route.get("example")
|
||||||
|
key_param = route["path_params"][-1] if route.get("path_params") else None
|
||||||
|
route_key = f"{method} {template}"
|
||||||
|
|
||||||
|
async def handler(request: Request):
|
||||||
|
shunt.count(route_key)
|
||||||
|
shunt.maybe_fail()
|
||||||
|
shunt.maybe_delay()
|
||||||
|
|
||||||
|
override = shunt.pinned(method, template, request.url.path)
|
||||||
|
if override is not None:
|
||||||
|
return JSONResponse(override, status_code=status)
|
||||||
|
|
||||||
|
body: Any = None
|
||||||
|
if method in ("POST", "PUT", "PATCH"):
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
body = None
|
||||||
|
|
||||||
|
known = model in shunt.collections
|
||||||
|
key = request.path_params.get(key_param) if key_param else None
|
||||||
|
|
||||||
|
if known and operation == "list":
|
||||||
|
rows = _filtered(shunt.rows(model), request.query_params)
|
||||||
|
page = _paged(rows, request.query_params, int(shunt.config["page_size"]))
|
||||||
|
if envelope:
|
||||||
|
return JSONResponse({envelope: page, "total": len(rows)}, status_code=status)
|
||||||
|
return JSONResponse(page, status_code=status)
|
||||||
|
|
||||||
|
if known and operation == "retrieve":
|
||||||
|
found = shunt.find(model, key)
|
||||||
|
if found is not None:
|
||||||
|
return JSONResponse(found, status_code=status)
|
||||||
|
if shunt.config.get("unknown_id") == "404":
|
||||||
|
raise HTTPException(status_code=404, detail=f"No {model} with id {key}")
|
||||||
|
# Nothing stored under that id, so answer with the closest thing to
|
||||||
|
# an authored response: the spec's own example if there is one,
|
||||||
|
# otherwise a synthesised record. Either way it wears the id that
|
||||||
|
# was asked for — a record whose id disagrees with its own URL
|
||||||
|
# breaks clients that re-read what they just fetched.
|
||||||
|
pk = (shunt.collections.get(model) or {}).get("pk")
|
||||||
|
if isinstance(example, dict):
|
||||||
|
stand_in = dict(example)
|
||||||
|
else:
|
||||||
|
made = shunt.synthesise(model, 1)
|
||||||
|
stand_in = made[0] if made else None
|
||||||
|
if stand_in is not None:
|
||||||
|
if pk:
|
||||||
|
stand_in[pk] = shunt.coerce_key(model, key)
|
||||||
|
return JSONResponse(stand_in, status_code=status)
|
||||||
|
|
||||||
|
if known and operation == "create":
|
||||||
|
sent = dict(body) if isinstance(body, dict) else {}
|
||||||
|
pk = (shunt.collections.get(model) or {}).get("pk")
|
||||||
|
# Fill the gaps from the schema so a partial POST still comes back
|
||||||
|
# as a complete record.
|
||||||
|
made = shunt.synthesise(model, 1)
|
||||||
|
record = {**(made[0] if made else {}), **sent}
|
||||||
|
# The gap-filler samples an existing row when there are rows to
|
||||||
|
# sample, so it supplies that row's key too. Only a key the caller
|
||||||
|
# actually sent may survive; anything else gets a fresh one, or a
|
||||||
|
# POST silently overwrites an existing record's identity.
|
||||||
|
if pk and not sent.get(pk):
|
||||||
|
record[pk] = shunt.next_key(model)
|
||||||
|
shunt.rows(model).append(record)
|
||||||
|
return JSONResponse(record, status_code=status if status >= 200 else 201)
|
||||||
|
|
||||||
|
if known and operation == "update":
|
||||||
|
found = shunt.find(model, key)
|
||||||
|
if found is None:
|
||||||
|
if shunt.config.get("unknown_id") == "404":
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404, detail=f"No {model} with id {key}"
|
||||||
|
)
|
||||||
|
found = {}
|
||||||
|
pk = (shunt.collections.get(model) or {}).get("pk")
|
||||||
|
if pk:
|
||||||
|
found[pk] = shunt.coerce_key(model, key)
|
||||||
|
shunt.rows(model).append(found)
|
||||||
|
if isinstance(body, dict):
|
||||||
|
found.update(body)
|
||||||
|
return JSONResponse(found, status_code=status)
|
||||||
|
|
||||||
|
if known and operation == "delete":
|
||||||
|
found = shunt.find(model, key)
|
||||||
|
if found is not None:
|
||||||
|
shunt.rows(model).remove(found)
|
||||||
|
elif shunt.config.get("unknown_id") == "404":
|
||||||
|
raise HTTPException(status_code=404, detail=f"No {model} with id {key}")
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
if example is not None:
|
||||||
|
return JSONResponse(example, status_code=status)
|
||||||
|
|
||||||
|
if model:
|
||||||
|
made = shunt.synthesise(model, 1)
|
||||||
|
if made:
|
||||||
|
payload: Any = made[0]
|
||||||
|
if route.get("response_is_list"):
|
||||||
|
rows = shunt.synthesise(model, 3) or made
|
||||||
|
payload = {envelope: rows, "total": len(rows)} if envelope else rows
|
||||||
|
return JSONResponse(payload, status_code=status)
|
||||||
|
|
||||||
|
if status == 204:
|
||||||
|
return Response(status_code=204)
|
||||||
|
return JSONResponse({}, status_code=status)
|
||||||
|
|
||||||
|
handler.__name__ = route.get("operation_id") or re.sub(
|
||||||
|
r"\W+", "_", f"{method}_{template}"
|
||||||
|
).strip("_")
|
||||||
|
return handler
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitise(path: str) -> str:
|
||||||
|
"""Make spec path parameters legal Starlette converters."""
|
||||||
|
|
||||||
|
def fix(match):
|
||||||
|
name = re.sub(r"\W+", "_", match.group(1)).strip("_") or "param"
|
||||||
|
if name[0].isdigit():
|
||||||
|
name = f"p_{name}"
|
||||||
|
return "{" + name + "}"
|
||||||
|
|
||||||
|
return re.sub(r"\{([^}]*)\}", fix, path)
|
||||||
|
|
||||||
|
|
||||||
|
def build_app(base: Path) -> FastAPI:
|
||||||
|
"""Assemble a shunt's FastAPI app from the spec in its depot."""
|
||||||
|
base = Path(base)
|
||||||
|
shunt = Shunt(base)
|
||||||
|
title = shunt.spec.get("title") or shunt.config.get("title") or "Shunt"
|
||||||
|
|
||||||
|
app = FastAPI(title=title, description=shunt.spec.get("summary", ""))
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health():
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"shunt": shunt.spec.get("name", "shunt"),
|
||||||
|
"routes": len(shunt.spec.get("routes", [])),
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.get("/mock/spec")
|
||||||
|
def mock_spec():
|
||||||
|
return shunt.spec
|
||||||
|
|
||||||
|
@app.get("/mock/stats")
|
||||||
|
def mock_stats():
|
||||||
|
return {
|
||||||
|
"calls": shunt.stats,
|
||||||
|
"total": sum(shunt.stats.values()),
|
||||||
|
"rows": {model: len(rows) for model, rows in shunt.store.items()},
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.post("/mock/reset")
|
||||||
|
@app.get("/mock/reset")
|
||||||
|
def mock_reset():
|
||||||
|
shunt.reset()
|
||||||
|
return {"status": "reset", "rows": {m: len(r) for m, r in shunt.store.items()}}
|
||||||
|
|
||||||
|
@app.get("/mock/config")
|
||||||
|
def mock_config_get():
|
||||||
|
return shunt.config
|
||||||
|
|
||||||
|
@app.post("/mock/config")
|
||||||
|
async def mock_config_set(request: Request):
|
||||||
|
try:
|
||||||
|
incoming = await request.json()
|
||||||
|
except Exception:
|
||||||
|
raise HTTPException(status_code=400, detail="Body must be a JSON object")
|
||||||
|
if not isinstance(incoming, dict):
|
||||||
|
raise HTTPException(status_code=400, detail="Body must be a JSON object")
|
||||||
|
# Only known knobs, so a typo is reported rather than silently stored.
|
||||||
|
unknown = set(incoming) - set(DEFAULT_CONFIG)
|
||||||
|
if unknown:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400, detail=f"Unknown settings: {sorted(unknown)}"
|
||||||
|
)
|
||||||
|
shunt.config.update(incoming)
|
||||||
|
return shunt.config
|
||||||
|
|
||||||
|
@app.get("/mock/responses")
|
||||||
|
def mock_responses():
|
||||||
|
return shunt.responses
|
||||||
|
|
||||||
|
@app.post("/mock/responses")
|
||||||
|
async def mock_responses_set(request: Request):
|
||||||
|
"""Pin or clear an override without restarting — set a key to null to drop it."""
|
||||||
|
try:
|
||||||
|
incoming = await request.json()
|
||||||
|
except Exception:
|
||||||
|
raise HTTPException(status_code=400, detail="Body must be a JSON object")
|
||||||
|
if not isinstance(incoming, dict):
|
||||||
|
raise HTTPException(status_code=400, detail="Body must be a JSON object")
|
||||||
|
for key, value in incoming.items():
|
||||||
|
if value is None:
|
||||||
|
shunt.responses.pop(key, None)
|
||||||
|
else:
|
||||||
|
shunt.responses[key] = value
|
||||||
|
return shunt.responses
|
||||||
|
|
||||||
|
# Static segments before parameterised ones, so /pets/search is not
|
||||||
|
# swallowed by /pets/{petId} when a spec declares both.
|
||||||
|
routes = sorted(
|
||||||
|
shunt.spec.get("routes", []),
|
||||||
|
key=lambda r: (len(r.get("path_params") or []), r.get("path", "")),
|
||||||
|
)
|
||||||
|
|
||||||
|
claimed = set()
|
||||||
|
for route in routes:
|
||||||
|
path = _sanitise(route["path"])
|
||||||
|
key = (route["method"], path)
|
||||||
|
if key in claimed:
|
||||||
|
continue
|
||||||
|
claimed.add(key)
|
||||||
|
app.add_api_route(
|
||||||
|
path,
|
||||||
|
_make_handler(shunt, route),
|
||||||
|
methods=[route["method"]],
|
||||||
|
name=route.get("operation_id") or None,
|
||||||
|
summary=route.get("summary") or None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if ("GET", "/") not in claimed:
|
||||||
|
|
||||||
|
@app.get("/", response_class=HTMLResponse)
|
||||||
|
def config_ui():
|
||||||
|
page = base / "templates" / "index.html"
|
||||||
|
if page.exists():
|
||||||
|
return HTMLResponse(page.read_text())
|
||||||
|
return HTMLResponse(f"<h1>{title}</h1><p>See /mock/spec and /health.</p>")
|
||||||
|
|
||||||
|
app.state.shunt = shunt
|
||||||
|
return app
|
||||||
294
soleprint/station/tools/shuntgen/templates/index.html
Normal file
294
soleprint/station/tools/shuntgen/templates/index.html
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en" data-theme="soleprint">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>shuntgen</title>
|
||||||
|
<link rel="stylesheet" href="/theme.css">
|
||||||
|
<style>
|
||||||
|
* { box-sizing: border-box; }
|
||||||
|
body { margin: 0; padding: var(--space-6); max-width: 1200px; }
|
||||||
|
|
||||||
|
header { display: flex; align-items: baseline; gap: var(--space-3); flex-wrap: wrap;
|
||||||
|
padding-bottom: var(--space-4); border-bottom: var(--hairline) solid var(--border); }
|
||||||
|
h1 { margin: 0; font-size: 20px; color: var(--accent-text); }
|
||||||
|
.sub { color: var(--muted); font-family: var(--font-mono); font-size: 12px; }
|
||||||
|
|
||||||
|
h2 { font-size: 13px; color: var(--muted); margin: var(--space-6) 0 var(--space-3); }
|
||||||
|
|
||||||
|
.cols { display: grid; grid-template-columns: 340px 1fr; gap: var(--space-4);
|
||||||
|
align-items: start; margin-top: var(--space-4); }
|
||||||
|
@media (max-width: 860px) { .cols { grid-template-columns: 1fr; } }
|
||||||
|
|
||||||
|
.panel { background: var(--surface); border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius-lg); padding: var(--space-4); }
|
||||||
|
|
||||||
|
label { display: block; font-size: 11px; color: var(--muted); margin-top: var(--space-3);
|
||||||
|
text-transform: uppercase; letter-spacing: var(--label-spacing); }
|
||||||
|
input, select { width: 100%; padding: 6px 8px; margin-top: 4px;
|
||||||
|
font-family: var(--font-mono); font-size: 12px; }
|
||||||
|
input[type="file"] { padding: 4px; font-family: var(--font-ui); }
|
||||||
|
.row { display: flex; gap: var(--space-2); }
|
||||||
|
.row > * { flex: 1; }
|
||||||
|
.actions { display: flex; gap: var(--space-2); margin-top: var(--space-4); }
|
||||||
|
button { padding: 7px 14px; font-size: 12px; }
|
||||||
|
button.primary { background: var(--accent); color: var(--bg); border-color: var(--accent); }
|
||||||
|
button.primary:hover:not(:disabled) { background: var(--accent-dim); color: var(--bg); }
|
||||||
|
|
||||||
|
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||||
|
th { text-align: left; padding: 6px 8px; color: var(--muted); font-weight: 600;
|
||||||
|
font-size: 10px; text-transform: uppercase; letter-spacing: var(--label-spacing);
|
||||||
|
border-bottom: var(--hairline) solid var(--border); }
|
||||||
|
td { padding: 6px 8px; border-bottom: var(--hairline) solid var(--border);
|
||||||
|
font-family: var(--font-mono); }
|
||||||
|
tr:hover td { background: var(--bg-2); }
|
||||||
|
|
||||||
|
.verb { font-weight: 600; font-size: 10px; padding: 1px 6px; border-radius: var(--radius-sm);
|
||||||
|
border: 1px solid currentColor; }
|
||||||
|
.GET { color: var(--status-info); }
|
||||||
|
.POST { color: var(--status-ok); }
|
||||||
|
.PUT, .PATCH { color: var(--status-warn); }
|
||||||
|
.DELETE { color: var(--status-error); }
|
||||||
|
|
||||||
|
.msg { margin-top: var(--space-3); padding: var(--space-3); border-radius: var(--radius);
|
||||||
|
font-size: 12px; border: var(--hairline) solid var(--border); display: none; }
|
||||||
|
.msg.show { display: block; }
|
||||||
|
.msg.ok { border-color: var(--status-ok); color: var(--status-ok); }
|
||||||
|
.msg.bad { border-color: var(--status-error); color: var(--status-error); }
|
||||||
|
.msg code { color: var(--text); }
|
||||||
|
|
||||||
|
.empty { color: var(--dim); font-size: 12px; padding: var(--space-3); }
|
||||||
|
.note { color: var(--dim); font-size: 11px; margin-top: var(--space-2); }
|
||||||
|
.chip { display: inline-block; font-family: var(--font-mono); font-size: 10px;
|
||||||
|
padding: 1px 6px; border-radius: var(--radius-sm);
|
||||||
|
border: var(--hairline) solid var(--border); color: var(--muted); }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<h1>shuntgen</h1>
|
||||||
|
<span class="sub">a spec or a folder of sheets → a running fake service</span>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div class="cols">
|
||||||
|
<div>
|
||||||
|
<h2>Source</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<label for="source">Spec or sheet folder</label>
|
||||||
|
<select id="source"><option value="">loading…</option></select>
|
||||||
|
|
||||||
|
<label for="upload">…or upload</label>
|
||||||
|
<input type="file" id="upload"
|
||||||
|
accept=".json,.yaml,.yml,.csv,.tsv,.ods">
|
||||||
|
<p class="note">
|
||||||
|
A spec (.json/.yaml) becomes routes. Sheets (.csv/.tsv/.ods) become
|
||||||
|
models with CRUD over their real rows — upload them into a folder
|
||||||
|
to import several at once.
|
||||||
|
</p>
|
||||||
|
<label for="folder">Upload into folder (sheets)</label>
|
||||||
|
<input type="text" id="folder" placeholder="e.g. invoicing">
|
||||||
|
|
||||||
|
<h2>Shunt</h2>
|
||||||
|
<label for="name">Name</label>
|
||||||
|
<input type="text" id="name" placeholder="petstore">
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div>
|
||||||
|
<label for="port">Port</label>
|
||||||
|
<input type="number" id="port" value="8099">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="kind">Kind</label>
|
||||||
|
<select id="kind">
|
||||||
|
<option value="auto">auto</option>
|
||||||
|
<option value="openapi">openapi</option>
|
||||||
|
<option value="tabular">tabular</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<label for="cabinets">Cabinets (comma-separated)</label>
|
||||||
|
<input type="text" id="cabinets" placeholder="postgres,redis">
|
||||||
|
|
||||||
|
<div class="actions">
|
||||||
|
<button type="button" id="preview">Preview</button>
|
||||||
|
<button type="button" id="generate" class="primary">Generate</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="msg" id="msg"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<h2>What it will serve</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<table id="routes">
|
||||||
|
<thead>
|
||||||
|
<tr><th>Method</th><th>Path</th><th>Does</th><th>Model</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody><tr><td colspan="4" class="empty">pick a source and preview</td></tr></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Existing shunts</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<table id="shunts">
|
||||||
|
<thead>
|
||||||
|
<tr><th>Name</th><th>Kind</th><th>Source</th><th>Routes</th><th>Models</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody><tr><td colspan="5" class="empty">loading…</td></tr></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script src="/theme.js" defer></script>
|
||||||
|
<script>
|
||||||
|
const BASE = "/station/tools/shuntgen";
|
||||||
|
const $ = (id) => document.getElementById(id);
|
||||||
|
|
||||||
|
function say(text, ok) {
|
||||||
|
const box = $("msg");
|
||||||
|
box.className = "msg show " + (ok ? "ok" : "bad");
|
||||||
|
box.innerHTML = text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function body() {
|
||||||
|
return {
|
||||||
|
name: $("name").value.trim() || "preview",
|
||||||
|
source: $("source").value,
|
||||||
|
kind: $("kind").value,
|
||||||
|
port: Number($("port").value) || 8099,
|
||||||
|
cabinets: $("cabinets").value.split(",").map(s => s.trim()).filter(Boolean),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function post(path, payload) {
|
||||||
|
const response = await fetch(BASE + path, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
const data = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok) throw new Error(data.detail || response.statusText);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cell(text) {
|
||||||
|
const td = document.createElement("td");
|
||||||
|
td.textContent = text == null ? "" : String(text);
|
||||||
|
return td;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fill(table, rows, columns, emptyText) {
|
||||||
|
const tbody = table.querySelector("tbody");
|
||||||
|
tbody.replaceChildren();
|
||||||
|
if (!rows.length) {
|
||||||
|
const tr = document.createElement("tr");
|
||||||
|
const td = cell(emptyText);
|
||||||
|
td.colSpan = columns;
|
||||||
|
td.className = "empty";
|
||||||
|
tr.appendChild(td);
|
||||||
|
tbody.appendChild(tr);
|
||||||
|
}
|
||||||
|
return tbody;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadSources() {
|
||||||
|
const data = await (await fetch(BASE + "/api/sources")).json();
|
||||||
|
const select = $("source");
|
||||||
|
select.replaceChildren();
|
||||||
|
const all = [...data.specs.map(p => ["spec", p]), ...data.sheets.map(p => ["sheets", p])];
|
||||||
|
if (!all.length) {
|
||||||
|
select.appendChild(new Option("nothing uploaded yet", ""));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const [kind, path] of all) {
|
||||||
|
select.appendChild(new Option(`${kind} · ${path}`, path));
|
||||||
|
}
|
||||||
|
if (!$("name").value) {
|
||||||
|
const guess = all[0][1].split("/").pop().replace(/\.[^.]+$/, "");
|
||||||
|
$("name").placeholder = guess;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadShunts() {
|
||||||
|
const data = await (await fetch(BASE + "/api/shunts")).json();
|
||||||
|
const tbody = fill($("shunts"), data.shunts, 5, "no shunts yet");
|
||||||
|
for (const shunt of data.shunts) {
|
||||||
|
const tr = document.createElement("tr");
|
||||||
|
tr.appendChild(cell(shunt.name));
|
||||||
|
tr.appendChild(cell(shunt.generated ? shunt.kind : "hand-written"));
|
||||||
|
tr.appendChild(cell(shunt.source));
|
||||||
|
tr.appendChild(cell(shunt.routes));
|
||||||
|
tr.appendChild(cell(shunt.models));
|
||||||
|
tbody.appendChild(tr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showRoutes(routes) {
|
||||||
|
const tbody = fill($("routes"), routes, 4, "nothing to serve");
|
||||||
|
for (const route of routes) {
|
||||||
|
const tr = document.createElement("tr");
|
||||||
|
const verb = document.createElement("td");
|
||||||
|
const badge = document.createElement("span");
|
||||||
|
badge.className = "verb " + route.method;
|
||||||
|
badge.textContent = route.method;
|
||||||
|
verb.appendChild(badge);
|
||||||
|
tr.appendChild(verb);
|
||||||
|
tr.appendChild(cell(route.path));
|
||||||
|
tr.appendChild(cell(route.operation));
|
||||||
|
tr.appendChild(cell(route.model));
|
||||||
|
tbody.appendChild(tr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$("upload").addEventListener("change", async (event) => {
|
||||||
|
const file = event.target.files[0];
|
||||||
|
if (!file) return;
|
||||||
|
const form = new FormData();
|
||||||
|
form.append("file", file);
|
||||||
|
const folder = $("folder").value.trim();
|
||||||
|
const url = BASE + "/api/upload" + (folder ? "?folder=" + encodeURIComponent(folder) : "");
|
||||||
|
const response = await fetch(url, { method: "POST", body: form });
|
||||||
|
const data = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok) {
|
||||||
|
say(data.detail || "Upload failed", false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
say("Uploaded <code>" + data.path + "</code>", true);
|
||||||
|
await loadSources();
|
||||||
|
$("source").value = folder
|
||||||
|
? data.path.split("/").slice(0, -1).join("/")
|
||||||
|
: data.path;
|
||||||
|
});
|
||||||
|
|
||||||
|
$("preview").addEventListener("click", async () => {
|
||||||
|
if (!$("source").value) { say("Pick or upload a source first", false); return; }
|
||||||
|
try {
|
||||||
|
const data = await post("/api/preview", body());
|
||||||
|
showRoutes(data.routes);
|
||||||
|
say(
|
||||||
|
`${data.models.length} models, ${data.routes.length} routes` +
|
||||||
|
(data.rows ? `, ${data.rows} rows` : "") + ` (${data.kind})`, true
|
||||||
|
);
|
||||||
|
} catch (error) { say(error.message, false); }
|
||||||
|
});
|
||||||
|
|
||||||
|
$("generate").addEventListener("click", async () => {
|
||||||
|
if (!$("source").value) { say("Pick or upload a source first", false); return; }
|
||||||
|
if (!$("name").value.trim()) { say("Give the shunt a name", false); return; }
|
||||||
|
try {
|
||||||
|
const data = await post("/api/generate", body());
|
||||||
|
say(
|
||||||
|
`Wrote <code>${data.path}</code> — ${data.routes} routes. ` +
|
||||||
|
`Run it with <code>${data.run}</code>`, true
|
||||||
|
);
|
||||||
|
await loadShunts();
|
||||||
|
} catch (error) { say(error.message, false); }
|
||||||
|
});
|
||||||
|
|
||||||
|
Promise.all([loadSources(), loadShunts()]).catch((e) => say(e.message, false));
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
242
soleprint/station/tools/shuntgen/templates/shunt_ui.html
Normal file
242
soleprint/station/tools/shuntgen/templates/shunt_ui.html
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!--
|
||||||
|
Config UI for a generated shunt. Emitted into <shunt>/templates/index.html
|
||||||
|
with the theme inlined, because a shunt serves this on its own port and
|
||||||
|
cannot reach soleprint's /theme.css.
|
||||||
|
|
||||||
|
Placeholders are filled by emit.py: %%THEME_CSS%%, %%TITLE%%, %%NAME%%,
|
||||||
|
%%SOURCE%%, %%ROUTE_COUNT%%.
|
||||||
|
-->
|
||||||
|
<html lang="en" data-theme="soleprint">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>%%TITLE%% shunt</title>
|
||||||
|
<style>
|
||||||
|
%%THEME_CSS%%
|
||||||
|
|
||||||
|
* { box-sizing: border-box; }
|
||||||
|
body { margin: 0; padding: var(--space-6); max-width: 1100px; }
|
||||||
|
|
||||||
|
header { display: flex; align-items: baseline; gap: var(--space-3); flex-wrap: wrap;
|
||||||
|
padding-bottom: var(--space-4); border-bottom: var(--hairline) solid var(--border); }
|
||||||
|
h1 { margin: 0; font-size: 20px; color: var(--accent-text); }
|
||||||
|
.sub { color: var(--muted); font-family: var(--font-mono); font-size: 12px; }
|
||||||
|
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--status-ok);
|
||||||
|
display: inline-block; margin-right: 6px; }
|
||||||
|
|
||||||
|
h2 { font-size: 13px; color: var(--muted); margin: var(--space-6) 0 var(--space-3); }
|
||||||
|
|
||||||
|
.panel { background: var(--surface); border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius-lg); padding: var(--space-4); }
|
||||||
|
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
|
||||||
|
gap: var(--space-3); }
|
||||||
|
|
||||||
|
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||||
|
th { text-align: left; padding: 6px 8px; color: var(--muted); font-weight: 600;
|
||||||
|
font-size: 10px; text-transform: uppercase; letter-spacing: var(--label-spacing);
|
||||||
|
border-bottom: var(--hairline) solid var(--border); }
|
||||||
|
td { padding: 6px 8px; border-bottom: var(--hairline) solid var(--border);
|
||||||
|
font-family: var(--font-mono); }
|
||||||
|
tr:hover td { background: var(--bg-2); }
|
||||||
|
|
||||||
|
.verb { font-weight: 600; font-size: 10px; padding: 1px 6px; border-radius: var(--radius-sm);
|
||||||
|
border: 1px solid currentColor; }
|
||||||
|
.GET { color: var(--status-info); }
|
||||||
|
.POST { color: var(--status-ok); }
|
||||||
|
.PUT, .PATCH { color: var(--status-warn); }
|
||||||
|
.DELETE { color: var(--status-error); }
|
||||||
|
|
||||||
|
a.route { color: var(--text); text-decoration: none; border-bottom: 1px solid transparent; }
|
||||||
|
a.route:hover { color: var(--accent-text); border-bottom-color: var(--accent); }
|
||||||
|
|
||||||
|
label { display: flex; align-items: center; justify-content: space-between;
|
||||||
|
gap: var(--space-2); font-size: 12px; color: var(--muted); padding: 4px 0; }
|
||||||
|
input[type="number"], input[type="text"], select { padding: 4px 8px; width: 130px;
|
||||||
|
font-family: var(--font-mono); font-size: 12px; }
|
||||||
|
input[type="checkbox"] { width: auto; }
|
||||||
|
.actions { display: flex; gap: var(--space-2); margin-top: var(--space-3); }
|
||||||
|
button { padding: 6px 12px; font-size: 12px; }
|
||||||
|
|
||||||
|
pre { background: var(--bg); border: var(--hairline) solid var(--border);
|
||||||
|
border-radius: var(--radius); padding: var(--space-3); overflow-x: auto;
|
||||||
|
font-size: 11px; margin: 0; max-height: 320px; }
|
||||||
|
.note { color: var(--dim); font-size: 11px; margin-top: var(--space-2); }
|
||||||
|
.empty { color: var(--dim); font-size: 12px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<h1>%%TITLE%%</h1>
|
||||||
|
<span class="sub"><span class="dot"></span>shunt · %%NAME%%</span>
|
||||||
|
<span class="sub">%%ROUTE_COUNT%% routes from %%SOURCE%%</span>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<h2>Routes</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<table id="routes">
|
||||||
|
<thead>
|
||||||
|
<tr><th>Method</th><th>Path</th><th>Does</th><th>Model</th><th>Calls</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody><tr><td colspan="5" class="empty">loading…</td></tr></tbody>
|
||||||
|
</table>
|
||||||
|
<p class="note">GET routes are links — they open against this shunt.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid" style="margin-top: var(--space-6)">
|
||||||
|
<div>
|
||||||
|
<h2>Behaviour</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<label>Random delays <input type="checkbox" id="enable_random_delays"></label>
|
||||||
|
<label>Min delay (ms) <input type="number" id="min_delay_ms" min="0"></label>
|
||||||
|
<label>Max delay (ms) <input type="number" id="max_delay_ms" min="0"></label>
|
||||||
|
<label>Error rate (0–1) <input type="number" id="error_rate" min="0" max="1" step="0.05"></label>
|
||||||
|
<label>Unknown id
|
||||||
|
<select id="unknown_id">
|
||||||
|
<option value="generate">generate</option>
|
||||||
|
<option value="404">404</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>Page size <input type="number" id="page_size" min="1"></label>
|
||||||
|
<div class="actions">
|
||||||
|
<button type="button" id="save">Apply</button>
|
||||||
|
<button type="button" id="reset">Reset data</button>
|
||||||
|
</div>
|
||||||
|
<p class="note">Applies immediately, in memory. Edit depot/config.json to persist.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<h2>Stored rows</h2>
|
||||||
|
<div class="panel"><pre id="stats">loading…</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Pinned responses</h2>
|
||||||
|
<div class="panel">
|
||||||
|
<pre id="responses">loading…</pre>
|
||||||
|
<p class="note">
|
||||||
|
Keys are <code>"METHOD /path"</code> and win over everything else.
|
||||||
|
Edit <code>depot/responses.json</code>, or POST to <code>/mock/responses</code>.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const KNOBS = ["enable_random_delays", "min_delay_ms", "max_delay_ms",
|
||||||
|
"error_rate", "unknown_id", "page_size"];
|
||||||
|
|
||||||
|
async function get(path) {
|
||||||
|
const response = await fetch(path);
|
||||||
|
if (!response.ok) throw new Error(path + " -> " + response.status);
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
function cell(text) {
|
||||||
|
const td = document.createElement("td");
|
||||||
|
td.textContent = text == null ? "" : String(text);
|
||||||
|
return td;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadRoutes() {
|
||||||
|
const [spec, stats] = await Promise.all([get("/mock/spec"), get("/mock/stats")]);
|
||||||
|
const body = document.querySelector("#routes tbody");
|
||||||
|
body.replaceChildren();
|
||||||
|
|
||||||
|
if (!spec.routes || !spec.routes.length) {
|
||||||
|
const row = document.createElement("tr");
|
||||||
|
const td = cell("no routes in spec.json");
|
||||||
|
td.colSpan = 5;
|
||||||
|
td.className = "empty";
|
||||||
|
row.appendChild(td);
|
||||||
|
body.appendChild(row);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const route of spec.routes) {
|
||||||
|
const row = document.createElement("tr");
|
||||||
|
|
||||||
|
const verb = document.createElement("td");
|
||||||
|
const badge = document.createElement("span");
|
||||||
|
badge.className = "verb " + route.method;
|
||||||
|
badge.textContent = route.method;
|
||||||
|
verb.appendChild(badge);
|
||||||
|
row.appendChild(verb);
|
||||||
|
|
||||||
|
const path = document.createElement("td");
|
||||||
|
if (route.method === "GET" && !route.path.includes("{")) {
|
||||||
|
const link = document.createElement("a");
|
||||||
|
link.className = "route";
|
||||||
|
link.href = route.path;
|
||||||
|
link.textContent = route.path;
|
||||||
|
path.appendChild(link);
|
||||||
|
} else {
|
||||||
|
path.textContent = route.path;
|
||||||
|
}
|
||||||
|
row.appendChild(path);
|
||||||
|
|
||||||
|
row.appendChild(cell(route.operation));
|
||||||
|
row.appendChild(cell(route.model));
|
||||||
|
row.appendChild(cell(stats.calls[route.method + " " + route.path] || 0));
|
||||||
|
body.appendChild(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadConfig() {
|
||||||
|
const config = await get("/mock/config");
|
||||||
|
for (const knob of KNOBS) {
|
||||||
|
const input = document.getElementById(knob);
|
||||||
|
if (!input) continue;
|
||||||
|
if (input.type === "checkbox") input.checked = Boolean(config[knob]);
|
||||||
|
else input.value = config[knob];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadStats() {
|
||||||
|
const stats = await get("/mock/stats");
|
||||||
|
document.getElementById("stats").textContent = JSON.stringify(stats.rows, null, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadResponses() {
|
||||||
|
const pinned = await get("/mock/responses");
|
||||||
|
const target = document.getElementById("responses");
|
||||||
|
target.textContent = Object.keys(pinned).length
|
||||||
|
? JSON.stringify(pinned, null, 2)
|
||||||
|
: "{} — nothing pinned; responses come from the store, examples, or the generator";
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById("save").addEventListener("click", async () => {
|
||||||
|
const payload = {};
|
||||||
|
for (const knob of KNOBS) {
|
||||||
|
const input = document.getElementById(knob);
|
||||||
|
if (!input) continue;
|
||||||
|
if (input.type === "checkbox") payload[knob] = input.checked;
|
||||||
|
else if (input.type === "number") payload[knob] = Number(input.value);
|
||||||
|
else payload[knob] = input.value;
|
||||||
|
}
|
||||||
|
const response = await fetch("/mock/config", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
alert("Could not apply: " + (await response.text()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await loadConfig();
|
||||||
|
});
|
||||||
|
|
||||||
|
document.getElementById("reset").addEventListener("click", async () => {
|
||||||
|
await fetch("/mock/reset", { method: "POST" });
|
||||||
|
await Promise.all([loadStats(), loadRoutes()]);
|
||||||
|
});
|
||||||
|
|
||||||
|
function refresh() {
|
||||||
|
return Promise.all([loadRoutes(), loadConfig(), loadStats(), loadResponses()])
|
||||||
|
.catch((error) => console.error(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
refresh();
|
||||||
|
setInterval(() => Promise.all([loadRoutes(), loadStats()]).catch(() => {}), 5000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in New Issue
Block a user