mariano/soleprint
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates 33.1 84

Browse Source
This commit is contained in:
buenosairesam
2026-08-10 05:36:32 -03:00
parent 0b04516cbb
commit 9a6337e493
55 changed files with 6387 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -19,6 +19,10 @@ dist/
# Generated runnable instance (entirely gitignored - regenerate with build.py)
gen/
# Specs and sheets uploaded through shuntgen's UI. Inputs someone dropped in a
# browser, not source — and often a client's real data.
soleprint/station/tools/shuntgen/uploads/
# Room configurations (separate repo - contains credentials and room-specific data)
# Keep cfg/standalone/ and cfg/sample/ as templates, ignore actual rooms
cfg/amar/

220
build.py
View File

@@ -317,6 +317,221 @@ def copy_cfg(output_dir: Path, room: str):
copy_path(item, output_dir / item.name)
def load_cabinets(room: str) -> list[dict]:
"""The dependency containers a room asked for, in the order it listed them.
Read from cfg/<room>/data/cabinets.json — the same shape and the same place
as its sibling data/*.json files, so nothing new has to know about it.
Entries are {"name": "postgres"} and may carry an "env" override.
"""
path = SPR_ROOT / "cfg" / room / "data" / "cabinets.json"
if not path.exists():
return []
try:
entries = json.loads(path.read_text())
except ValueError as e:
log.warning(f" cabinets.json is not valid JSON, ignoring: {e}")
return []
out = []
for entry in entries if isinstance(entries, list) else []:
if isinstance(entry, str):
entry = {"name": entry}
if isinstance(entry, dict) and entry.get("name"):
out.append(entry)
return out
def resolve_cabinets(requested: list[dict]) -> list[dict]:
"""Expand each request into its definition, pulling in what it depends on.
Airflow without postgres is a container that exits on boot, so a cabinet's
depends_on is added for you rather than left as something to remember.
"""
cabinets_dir = SPR_ROOT / "soleprint" / "station" / "cabinets"
resolved: dict[str, dict] = {}
def add(name: str, overrides: dict) -> None:
if name in resolved:
# Already pulled in as somebody's dependency. The room asking for it
# by name is the more specific statement, so its env still applies —
# otherwise declaring airflow before postgres would silently drop
# postgres's settings.
if overrides.get("env"):
resolved[name]["env"] = {
**resolved[name].get("env", {}),
**overrides["env"],
}
return
definition_path = cabinets_dir / name / "cabinet.json"
if not definition_path.exists():
available = sorted(
p.name for p in cabinets_dir.iterdir() if p.is_dir()
) if cabinets_dir.exists() else []
log.warning(f" no such cabinet: {name} (available: {', '.join(available) or 'none'})")
return
try:
definition = json.loads(definition_path.read_text())
except ValueError as e:
log.warning(f" cabinet {name} has invalid cabinet.json: {e}")
return
# Mark it claimed before recursing, so a dependency cycle terminates.
resolved[name] = definition
for dependency in definition.get("depends_on", []) or []:
add(dependency, {})
definition["env"] = {**definition.get("env", {}), **overrides.get("env", {})}
for entry in requested:
add(entry["name"], entry)
# Dependencies first, so compose reads in the order things start.
ordered: list[dict] = []
seen: set[str] = set()
def emit(name: str) -> None:
if name in seen or name not in resolved:
return
seen.add(name)
for dependency in resolved[name].get("depends_on", []) or []:
emit(dependency)
ordered.append(resolved[name])
for name in resolved:
emit(name)
return ordered
def compose_cabinets(output_dir: Path, room: str):
"""Merge the room's cabinets into its docker-compose.yml and .env.example.
This is the compile step for dependencies: a room declares postgres, and the
built instance comes out with postgres in its compose file rather than with
instructions for adding it.
"""
requested = load_cabinets(room)
if not requested:
return
try:
import yaml
except ImportError:
log.warning(
" cabinets need PyYAML to merge into docker-compose.yml "
"(pip install pyyaml) — skipping"
)
return
cabinets = resolve_cabinets(requested)
if not cabinets:
return
compose_path = output_dir / "docker-compose.yml"
if not compose_path.exists():
log.warning(
f" no docker-compose.yml in {output_dir.name}, "
f"so there is nothing to merge {len(cabinets)} cabinet(s) into"
)
return
original = compose_path.read_text()
# A YAML round-trip drops every comment, and the room's compose file leads
# with the one that says how to run it. Keep the header block; the rest is
# generated anyway.
header = []
for line in original.splitlines():
if line.startswith("#") or not line.strip():
header.append(line)
else:
break
while header and not header[-1].strip():
header.pop()
compose = yaml.safe_load(original) or {}
services = compose.setdefault("services", {})
volumes = compose.setdefault("volumes", {}) or {}
cabinets_dir = SPR_ROOT / "soleprint" / "station" / "cabinets"
added, skipped = [], []
for cabinet in cabinets:
name = cabinet["name"]
service_name = cabinet.get("service", name)
# The room's own compose file is the authority. A room that already
# declares `db` has arranged it deliberately, and silently replacing it
# would be the worst possible outcome of switching a cabinet on.
if service_name in services:
skipped.append(service_name)
continue
fragment_path = cabinets_dir / name / "service.yml"
if not fragment_path.exists():
log.warning(f" cabinet {name} has no service.yml")
continue
fragment = yaml.safe_load(fragment_path.read_text()) or {}
for key, value in fragment.items():
if key in services:
skipped.append(key)
continue
services[key] = value
added.append(key)
for volume in cabinet.get("volumes", []) or []:
volumes.setdefault(volume, None)
if volumes:
compose["volumes"] = volumes
rendered = yaml.safe_dump(compose, sort_keys=False, default_flow_style=False)
banner = f"# Cabinets merged in by build.py: {', '.join(c['name'] for c in cabinets)}.\n"
preamble = ("\n".join(header) + "\n" + banner + "\n") if header else banner + "\n"
compose_path.write_text(preamble + rendered)
if added:
log.info(f" cabinets: {', '.join(added)}")
if skipped:
log.info(f" cabinets already declared by the room, left alone: {', '.join(skipped)}")
_append_cabinet_env(output_dir, cabinets)
def _append_cabinet_env(output_dir: Path, cabinets: list[dict]):
"""Add each cabinet's settings to .env.example, without touching .env."""
example = output_dir / ".env.example"
existing = example.read_text() if example.exists() else ""
# Match whole settings, not substrings: `POSTGRES_DB=` appears inside
# `MY_POSTGRES_DB=`, and a substring test would decide the setting was
# already there and skip it.
declared = {
line.split("=", 1)[0].strip()
for line in existing.splitlines()
if "=" in line and not line.lstrip().startswith("#")
}
lines = []
for cabinet in cabinets:
env = cabinet.get("env", {})
if not env:
continue
block = [f"\n# ── {cabinet.get('title', cabinet['name'])} (cabinet) ──"]
for note in cabinet.get("notes", []) or []:
block.append(f"# {note}")
wrote = False
for key, value in env.items():
if key in declared:
continue
block.append(f"{key}={value}")
declared.add(key)
wrote = True
if wrote:
lines.extend(block)
if lines:
example.write_text(existing.rstrip("\n") + "\n" + "\n".join(lines) + "\n")
def build_soleprint(output_dir: Path, room: str):
"""Build soleprint folder with core + room config merged."""
soleprint = SPR_ROOT / "soleprint"
@@ -348,6 +563,11 @@ def build_soleprint(output_dir: Path, room: str):
# Room config (includes merging room-specific artery/atlas/station)
copy_cfg(output_dir, room)
# Dependency containers the room asked for, merged into its compose file.
# After copy_cfg, because the compose file being merged into is the room's.
log.info("Composing cabinets...")
compose_cabinets(output_dir, room)
# Generate models
log.info("Generating models...")
if not generate_models(output_dir, room):

124
docs/data/en/export.md Normal file
View File

@@ -0,0 +1,124 @@
# Export / Compile
Soleprint's source tree is not what runs. `build.py` compiles the framework plus
a room's configuration into a self-contained instance under `gen/<room>/`, and
that directory is what a container boots, what `deploy.sh` rsyncs, and what the
cluster manifests point at.
Everything below is a `make` target, and every target is one script in `ctrl/`.
The logic lives in the scripts, never in the Makefile.
```bash
make build # cfg/standalone -> gen/standalone
make build sample # cfg/sample -> gen/sample
make start # run it
```
## The targets
| Command | Runs | Does |
| --- | --- | --- |
| `make build [<room>\|all\|models]` | `ctrl/build.sh` | compile a room into `gen/` |
| `make start [<room>] [-d]` | `ctrl/start.sh` | run a built room's compose stack |
| `make stop [<room>]` | `ctrl/stop.sh` | stop it |
| `make cluster [up\|down\|status]` | `ctrl/cluster.sh` | the shared kind cluster |
| `make component [list\|publish\|sync\|watch\|diff]` | `ctrl/spr.py` | publish a distributable component |
| `make deploy` | `ctrl/deploy.sh` | rsync `gen/standalone` to the server and restart |
Bare words pass straight through, so `make build sample` becomes
`ctrl/build.sh sample`. Anything starting with a dash would be eaten by make
itself, so those go through `ARGS`:
```bash
make deploy ARGS="--build"
make component ARGS="publish soleprint-ui /tmp/out --dist"
```
## What a build does
`python build.py --cfg <room>` runs these in order:
1. **Clean** `gen/<room>/`. A build is not incremental — a stale file left
behind is worse than a slow build.
2. **Copy the framework.** `main.py`, `run.py`, `index.html`, `Dockerfile`,
`requirements.txt`, `dataloader/`, `common/`, and the three systems
(`artery/`, `atlas/`, `station/`).
3. **Merge the room** (`copy_cfg`). `cfg/<room>/config.json` lands in `cfg/`,
`data/*.json` in `data/`, and anything under `cfg/<room>/soleprint/artery|atlas|station/`
is merged *over* the framework copy — which is how a room adds its own vein,
shunt, tool or generator without forking the tree.
4. **Compose cabinets.** The dependency containers the room declared in
`data/cabinets.json` are merged into its `docker-compose.yml`. See
[Cabinets](#station-cabinets).
5. **Generate models.** modelgen reads the room's `config.json` and writes
`models/pydantic/__init__.py`.
6. **Render k8s** (optional). When the room's config enables it,
`soleprint/ctrl/k8s/` writes manifests and lifecycle scripts.
## What comes out
A standalone room is a flat instance:
```
gen/standalone/
run.py main.py Dockerfile docker-compose.yml
artery/ atlas/ station/ common/
cfg/config.json
data/*.json
models/pydantic/
```
A **managed** room — one that wraps an existing application — is three folders
instead, because soleprint sits beside the app rather than containing it:
```
gen/<room>/
<app>/ the application's repos, plus its ctrl scripts
link/ bridge code between the two
soleprint/ the instance, exactly as above
```
`build.py` picks between them on whether the room's `config.json` has a
`managed` block. `gen/` is gitignored in full: it is an artifact, and the way to
change it is to change `cfg/<room>/` and rebuild.
## Distributing components
Rooms are compiled; *components* are published. `registry.json` lists what can
be shipped out of this repo on its own:
```bash
make component # list
make component ARGS="publish soleprint-ui /tmp/out --dist"
make component ARGS="watch soleprint-ui ../unt/ui/framework"
```
`--dist` copies only the built bundle — `dist/**` plus `package.json`,
`README.md` and `LICENSE` — rather than the source. It refuses to publish an
empty `dist/`, because a component whose bundle was never built is the failure
that shows up later as a container that starts and renders nothing:
```
no build at soleprint/common/ui
build it first: cd soleprint/common/ui && pnpm build
```
Each publish leaves a `.spr` stamp in the destination recording name, version,
type, source and mode, so a copy can say where it came from.
## Deploying
```bash
make deploy ARGS="--build" # rebuild, sync, restart
make deploy ARGS="--sync-only" # sync, leave it running
```
`deploy.sh` rsyncs `gen/standalone/` and runs `docker compose up -d --build` on
the far side. `.env` is excluded, so server secrets stay on the server.
## Running without building
`python run.py` from `soleprint/` serves every subsystem on one port (12000 by
default) straight from the source tree. It is for developing the framework
itself; a room's `cfg/config.json` does not exist there, so the landing pages
fall back to their defaults. Rooms use docker.

107
docs/data/en/station-cabinets.md Normal file
View File

@@ -0,0 +1,107 @@
# Cabinets
A cabinet is a **dependency container** a room can switch on: postgres, redis,
airflow. The vocabulary already had the word — `execution.container` in every
room's `config.json` is *"Cabinet — tool container"* — and until now nothing
stood behind it.
The problem it solves: a generated artifact knows what it needs and had no way
to say so. A shunt built from a client's spreadsheets holds its rows in memory
happily, but the moment you want them to survive a restart you need postgres,
and wiring postgres in meant hand-editing a room's `docker-compose.yml` and then
hand-editing the cluster too. A cabinet is that declaration, made once and read
by both paths.
## Switching one on
Add `cfg/<room>/data/cabinets.json`, the same shape as its sibling `data/*.json`
files:
```json
[
{ "name": "postgres" },
{ "name": "redis" },
{ "name": "airflow", "env": { "AIRFLOW_ADMIN_PASSWORD": "change-me" } }
]
```
Then build. The compose merge is a step in [Export / Compile](#export):
```bash
python build.py --cfg sample
cd gen/sample && docker compose up -d
```
`build.py` merges each cabinet's compose fragment into the room's
`docker-compose.yml`, declares its named volumes, and appends its settings to
`.env.example` — never to `.env`.
**A service the room already declares wins.** `cfg/amar/docker-compose.yml`
ships its own `db`; switching the postgres cabinet on will not replace it. The
build says so when it skips one:
```
Composing cabinets...
cabinets: redis, airflow
cabinets already declared by the room, left alone: postgres
```
Dependencies come along automatically. Airflow without a metadata database is a
container that exits on boot, so asking for `airflow` brings `postgres` and
`redis` with it, ordered so compose reads them before the thing that needs them.
## On a cluster
Every cabinet names a `rig_addon`. Where a room runs on kind rather than
compose, the same dependency installs as a rig addon of that name:
```bash
cd rig
PROFILE=data make cluster up
PROFILE=data make addons install
kubectl -n data port-forward svc/postgres 5432:5432
kubectl -n data port-forward svc/airflow 8080:8080
```
The two paths are deliberately separate — compose for a laptop, manifests for a
cluster — and `rig_addon` is the thread between them, so the room declares the
dependency once either way. The addons generate their own passwords on first
install and keep them across re-runs, so re-running never rotates a credential
out from under something already connected.
## What ships
| Cabinet | Image | Notes |
| --- | --- | --- |
| `postgres` | `postgres:16-alpine` | healthcheck wired, so `depends_on: service_healthy` works |
| `redis` | `redis:7-alpine` | cache, and the broker for anything queue-shaped |
| `airflow` | `apache/airflow:2.10.4` | one container on `standalone`; needs postgres and redis |
## Writing one
```
soleprint/station/cabinets/<name>/
cabinet.json what it is, what it needs, what it exports
service.yml the compose service, verbatim
```
`cabinet.json`:
| Key | Purpose |
| --- | --- |
| `name` | must match the directory |
| `title`, `description` | shown on the station index |
| `service` | the key to merge under in `services:` (defaults to `name`) |
| `env` | settings and defaults, written to `.env.example` |
| `volumes` | named volumes to declare at the top level |
| `depends_on` | other cabinets that must come with it |
| `rig_addon` | the matching `rig/ctrl/addons/<name>.sh`, if there is one |
| `notes` | lines written into `.env.example` as comments |
`service.yml` is a plain compose fragment with one top-level key — the service
name. It stays YAML rather than being generated from JSON so it reads like the
file it becomes, and so anything compose supports is available without this tool
learning about it first.
Adding a cabinet is adding a directory. Nothing dispatches on the name.

103
docs/data/en/station-datagen.md
View File

@@ -1,6 +1,8 @@
# Datagen
Test data generator using faker. Produces realistic, domain-specific data for testing and development.
Test data generator. Produces realistic, domain-specific records for testing and
development, from generators a room writes or that
[modelgen](#station-modelgen) writes for it.
**Status:** live
@@ -8,50 +10,95 @@ Test data generator using faker. Produces realistic, domain-specific data for te
## What It Does
Datagen generates fake but realistic data. Names, emails, addresses, transactions -- whatever your domain needs. It uses Python's faker library under the hood.
Datagen hands out instances of a room's models. Core ships the base class, the
discovery, the HTTP API and the browser UI; the generators themselves belong to
a room, because what counts as realistic is a property of the domain.
Core datagen is a placeholder. The real work happens in room-specific generators.
Generation is stdlib `random`, `uuid` and `datetime` — **not** faker, which is
not a dependency of this repo.
## Structure
```
soleprint/station/tools/datagen/ # Core (base classes, placeholder)
cfg/<room>/soleprint/station/tools/datagen/ # Room-specific generators
soleprint/station/tools/datagen/ # base class, api, UI
cfg/<room>/soleprint/station/tools/datagen/ # the room's generators
```
After build, both merge into `gen/<room>/station/tools/datagen/`.
## Pattern
## The contract
Rooms subclass a base generator and provide domain-specific data factories:
A generator subclasses `BaseDataGenerator` and defines **one method per model,
named after it**. There is no registration step: the method name *is* the model
name.
```python
from station.tools.datagen.base import BaseGenerator
from station.tools.datagen.base import BaseDataGenerator
class RoomDataGenerator(BaseGenerator):
def generate_customers(self, count=10):
return [self.fake_customer() for _ in range(count)]
class RoomDataGenerator(BaseDataGenerator):
def customer(self, **kwargs):
return {"id": str(uuid4()), "name": ..., "email": ..., **kwargs}
def fake_customer(self):
return {
"name": self.faker.name(),
"email": self.faker.email(),
"phone": self.faker.phone_number(),
}
def invoice(self, customer_id=None, **kwargs):
return {"id": str(uuid4()), "customer_id": customer_id, **kwargs}
```
The base class provides:
| Method | Does |
| --- | --- |
| `generate(model, count=1, **kwargs)` | calls the matching method `count` times; `kwargs` pass through to every call |
| `available_models()` | the method names, which are the model names |
| `schema()` | override to return a graphgen-compatible schema |
Discovery is by convention too: any `*.py` in the datagen directory whose first
class ends in `Generator` is loaded and keyed by its filename.
## Generating a generator
Writing one by hand is optional. modelgen's `datagen` target emits the whole
class from a schema — and when the schema came from spreadsheets, the generated
class **samples the real rows** rather than inventing values:
```bash
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t datagen
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t datagen
```
Each room defines what data it needs. Core provides the faker instance and base class. Rooms provide the factories.
This is how [shuntgen](#station-shuntgen) fills a generated shunt.
## Room Configuration
## HTTP API
Room generators live in `cfg/<room>/soleprint/station/tools/datagen/`. They are fully self-contained -- they define their own models, factories, and output formats.
Mounted under `/station/tools/datagen/`:
The core module provides:
- Base generator class with faker instance
- CLI entry point
- Output formatting (JSON, CSV)
| Route | Returns |
| --- | --- |
| `GET /` | the browser UI |
| `GET /api/generators` | loaded generator files and their models |
| `GET /api/models?generator=` | model names |
| `POST /api/generate` | `{model, count, generator?, kwargs?}` → the records |
| `GET /api/schema?generator=` | the generator's schema, if it exposes one |
## Feeding graphgen
A generator that overrides `schema()` is surfaced at `/api/schema` in the format
[graphgen](#station-graphgen) reads, so the same definition draws the diagram:
```python
def schema(self):
return {
"models": {
"Invoice": {
"doc": "A billed order.",
"fields": {
"id": {"type": "UUID", "pk": True},
"customer_id": {"type": "FK:Customer"},
"total": {"type": "float"},
},
}
}
}
```
Rooms provide:
- Domain-specific generator subclasses
- Field definitions and relationships
- Volume and distribution configuration
`FK:<Model>` and `M2M:<Model>` are how relations are written. modelgen's
`datagen` target emits this method for you.

117
docs/data/en/station-modelgen.md
View File

@@ -1,54 +1,111 @@
# Modelgen
Generates platform-specific models from JSON Schema. Reads schema once, writes models for multiple targets.
Multi-source, multi-target model generator. Reads a schema from wherever it
already lives, and writes it out for every stack that needs it.
**Status:** dev
**Status:** live
---
## What It Does
Modelgen takes a JSON Schema definition and produces model code for different platforms:
Everything passes through one intermediate representation — `ModelDefinition`,
`FieldDefinition`, `EnumDefinition`. **Loaders** fill it, **generators** emit
from it, and the two sides do not know about each other. Adding an input means
one extractor and every output comes with it; adding an output means one
generator and every input already feeds it.
- **Pydantic** -- Python data validation models
- **Django ORM** -- Django model classes
- **Prisma** -- Prisma schema definitions
```
dataclasses ─┐ ┌─ pydantic
Django │ ├─ django
SQLAlchemy ├──▶ ModelDefinition ──▶├─ sqlmodel
a live DB │ FieldDefinition ├─ typescript
OpenAPI │ EnumDefinition ├─ protobuf
CSV/ODS ─┘ ├─ prisma
├─ strawberry
├─ schema (graphgen)
└─ datagen
```
One schema, multiple outputs.
Core is **pure standard library**. It is published as `soleprint-modelgen` and
installs with no dependencies; live-database extraction is an extra
(`pip install "soleprint-modelgen[db]"`), and YAML specs need PyYAML.
## Extractors
## Sources
Modelgen also works in reverse. Extractors read existing codebases and produce a normalized schema representation:
| Command | Reads |
| --- | --- |
| `from-schema` | Python dataclasses in a `schema/` folder |
| `from-config` | a room's `config.json` |
| `extract` | a Django or SQLAlchemy codebase (`--framework auto` detects) |
| `from-db` | a live database, any SQLAlchemy dialect |
| `from-openapi` | an OpenAPI 3.x / Swagger 2.0 document |
| `from-tabular` | a directory of `.csv` / `.tsv` / `.ods` spreadsheets |
- **Django extractor** -- reads Django model files
- **SQLAlchemy extractor** -- reads SQLAlchemy model files
- **Prisma extractor** -- reads Prisma schema files
```bash
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript,schema
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t pydantic,datagen
python -m station.tools.modelgen extract -s /path/to/django -o out/ -t prisma
python -m station.tools.modelgen from-db -u postgresql://… -o out/ -t typescript
python -m station.tools.modelgen list-formats
```
Extractors feed into graphgen for visualization.
### From a spec
## Output
`components.schemas` (or Swagger's `definitions`) become models. `$ref` chains
and `allOf` are resolved, enums are materialised as real `Enum` classes so every
target names them properly, and a referenced object becomes a relation rather
than a nested type — the same call the database extractor makes, and what keeps
the generated code valid for every target.
Generated models are written to `gen/<room>/models/`.
The parse also yields the *operations*, which is what
[shuntgen](#station-shuntgen) turns into routes.
```
gen/<room>/models/
├── pydantic/
├── django/
└── prisma/
```
### From spreadsheets
## CLI
One model per CSV file, one per sheet in an ODS workbook. Column types are
inferred from the values actually present, and a blank cell makes the column
optional. Keys and relations are inferred by name and then confirmed against the
data: an `id` column that is not unique is not treated as a key, and
`customer_id` is only a foreign key if a `customers` sheet came with it.
```bash
python -m modelgen
```
The rows are kept, not just the shape — which is what lets the `datagen` target
sample real values instead of inventing them.
Reads from `schema.json` (the project source of truth) and writes to the configured output directory.
ODS is read with `zipfile` and `ElementTree`. No odfpy, no pandas: the
dependency-free promise is what makes this package publishable on its own.
## Shared Distribution
## Targets
Modelgen is also distributed as a shared component via `ctrl/spr.py`. This allows other projects to use model generation without running full soleprint.
`pydantic`, `django`, `sqlmodel`, `typescript` (`ts`), `protobuf` (`proto`),
`prisma`, `strawberry`, `schema` (`jsonschema`), `datagen`.
## Schema Source
Two are worth calling out:
- **`schema`** writes the graphgen-compatible `schema.json` — the portable
artifact [graphgen](#station-graphgen) and databrowse read directly.
Relations come out as `FK:<Model>` and `M2M:<Model>`.
- **`datagen`** writes a `BaseDataGenerator` subclass for
[datagen](#station-datagen), including its `schema()` override. Given
spreadsheet rows it samples them; otherwise it synthesises from the types.
Multiple targets in one run get one file each, named `models_<target><ext>`.
## In a build
`build.py` calls modelgen during every room build, writing
`gen/<room>/models/pydantic/__init__.py` from the room's `config.json`. See
[Export / Compile](#export).
## Tests
```bash
cd soleprint/station/tools
python -m unittest modelgen.tests.test_extractors
```
The source of truth is `schema.json` at the project root. All model generation starts from this file. Room-specific schema extensions live in `cfg/<room>/models/`.
stdlib `unittest`, no pytest, and every input is built in a temp directory — the
tests have to pass with nothing installed and nothing else in the tree. Run them
from `station/tools/`, not from inside `modelgen/`: the package ships a
`types.py`, and putting its own directory on `sys.path` shadows the standard
library module of that name.

129
docs/data/en/station-shuntgen.md Normal file
View File

@@ -0,0 +1,129 @@
# Shuntgen
Generates runnable [shunts](#artery-shunts) from the two things people actually
have: a service contract, or a folder of spreadsheets.
Writing a shunt by hand means copying `artery/shunts/example/` and filling in
`responses.json` entry by entry. That is fine for three endpoints and untenable
for eighty — and it is the wrong work anyway, because the endpoints are already
described in the spec somebody handed you.
```bash
# a spec you were handed
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
# sheets a client sent
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
python -m station.tools.shuntgen list
```
Run from `soleprint/`. Also in the browser at `/station/tools/shuntgen/`, where
you can upload a spec, preview the routes it would serve, and generate.
## What comes out
```
artery/shunts/<name>/
main.py builds the app from the spec
run.py uvicorn entry point (PORT, or depot/config.json)
shunt_runtime.py vendored runtime — no soleprint import
models.py pydantic, via modelgen
datagen_<name>.py BaseDataGenerator subclass, via modelgen
depot/spec.json routes, collections and schema
depot/responses.json pinned overrides — yours, never overwritten
depot/config.json delays, error rate, prefill — yours, never overwritten
depot/data.json imported rows
templates/index.html config UI
README.md
```
```bash
cd artery/shunts/books && python run.py
curl localhost:8098/customers
```
The routes are built at startup from `spec.json` rather than written out as
source. That keeps the generated code short enough to read, and puts the
behaviour in one reviewable place: fixing `runtime.py` fixes every shunt, and
regenerating is a copy.
## Where a response comes from
First hit wins:
1. `depot/responses.json` — a pinned override, keyed `"METHOD /path"`
2. the store — rows imported from sheets, plus anything POSTed since
3. the spec's `example`, if the source document carried one
4. `datagen_<name>.py`, synthesising from the schema
5. `{}`
The store is what makes it behave like a service rather than a random-value
faucet: POST something and GET it back, ask for `/pets/7` and get the pet whose
id is 7. Collections that arrived with no rows are prefilled with generated
ones, so the first call answers with something.
## Two sources, one pipeline
Both inputs are [modelgen](#station-modelgen) extractors, so the same shapes
also generate pydantic, TypeScript, prisma and a
[graphgen](#station-graphgen) schema:
```bash
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t schema,datagen
```
| Source | Becomes | Routes |
| --- | --- | --- |
| OpenAPI 3.x / Swagger 2.0 | one model per schema; enums become real Enums, `$ref` becomes a relation | the operations the document declares |
| `.csv` / `.tsv` / `.ods` | one model per file or sheet, types inferred per column | five CRUD routes per table |
Keys and relations are inferred by name and then **checked against the data**:
an `id` column that is not unique is not treated as a key, and `customer_id` is
only a foreign key if a `customers` sheet came with it.
ODS is read with `zipfile` and `ElementTree` — no odfpy, no pandas — which is
what lets modelgen stay dependency-free and publishable on its own.
## Control endpoints
Every generated shunt serves these:
| Endpoint | Purpose |
| --- | --- |
| `GET /health` | liveness |
| `GET /mock/spec` | the routes it was built from |
| `GET /mock/stats` | call counts and row counts |
| `POST /mock/reset` | restore the imported rows, clear counters |
| `GET,POST /mock/config` | delays, error rate, `unknown_id`, page size |
| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |
```bash
# make it slow and flaky, the way the real thing is
curl -X POST localhost:8098/mock/config \
-H 'content-type: application/json' \
-d '{"enable_random_delays": true, "error_rate": 0.2}'
# make one call answer something specific
curl -X POST localhost:8098/mock/responses \
-H 'content-type: application/json' \
-d '{"GET /customers/1": {"id": 1, "name": "PINNED"}}'
```
`unknown_id` decides what an unknown key does: `generate` (the default) invents
a record wearing the id that was asked for; `404` refuses it. Generate by
default, because a client pointed at a fresh shunt should just work — flip it
when the error path is what you are testing.
## Dependency containers
`--cabinet postgres,redis` writes a `cabinet.json` declaring what the shunt
expects. `build.py` composes those services into the room's compose file, and on
a cluster they install as rig addons of the same name. See
[Cabinets](#station-cabinets).
## Regenerating
Everything is overwritten except `depot/responses.json` and `depot/config.json`.
Those two are yours.

3
docs/data/topics.json
View File

@@ -22,8 +22,11 @@
{"id": "station-datagen", "title": {"en": "↳ Datagen"}, "sub": true},
{"id": "station-modelgen", "title": {"en": "↳ Modelgen"}, "sub": true},
{"id": "station-graphgen", "title": {"en": "↳ Graphgen"}, "sub": true},
{"id": "station-shuntgen", "title": {"en": "↳ Shuntgen"}, "sub": true},
{"id": "station-databrowse", "title": {"en": "↳ Databrowse"}, "sub": true},
{"id": "station-cabinets", "title": {"en": "↳ Cabinets"}, "sub": true},
{"id": "components", "title": {"en": "Shared Components"}},
{"id": "export", "title": {"en": "Export / Compile"}},
{"id": "deployment", "title": {"en": "Deployment"}}
]

120
soleprint/artery/index.html
View File

@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
@@ -14,7 +14,7 @@
box-sizing: border-box;
}
html {
background: #0a0a0a;
background: var(--bg);
}
body {
font-family:
@@ -25,8 +25,8 @@
margin: 0 auto;
padding: 2rem 1rem;
line-height: 1.6;
color: #e5e5e5;
background: #b91c1c;
color: var(--text);
background: var(--system-accent);
}
header {
display: flex;
@@ -51,7 +51,7 @@
padding-bottom: 2rem;
}
section {
background: #0a0a0a;
background: var(--bg);
padding: 1.5rem;
margin: 1.5rem 0;
border-radius: 12px;
@@ -59,23 +59,23 @@
section h2 {
margin: 0 0 1rem 0;
font-size: 1.2rem;
color: #fca5a5;
color: var(--system-accent-text);
}
.composition {
background: #1a1a1a;
border: 2px solid #b91c1c;
background: var(--surface);
border: 2px solid var(--system-accent);
padding: 1rem;
border-radius: 12px;
}
.composition h3 {
margin: 0 0 0.75rem 0;
font-size: 1.1rem;
color: #fca5a5;
color: var(--system-accent-text);
}
.composition > p {
margin: 0 0 1rem 0;
font-size: 0.9rem;
color: #a3a3a3;
color: var(--muted);
}
.components {
display: grid;
@@ -83,20 +83,20 @@
gap: 0.75rem;
}
.component {
background: #0a0a0a;
border: 1px solid #3f3f3f;
background: var(--bg);
border: 1px solid var(--border-strong);
padding: 0.75rem;
border-radius: 8px;
}
.component h4 {
margin: 0 0 0.25rem 0;
font-size: 0.95rem;
color: #fca5a5;
color: var(--system-accent-text);
}
.component p {
margin: 0;
font-size: 0.85rem;
color: #a3a3a3;
color: var(--muted);
}
.veins {
display: grid;
@@ -104,8 +104,8 @@
gap: 1rem;
}
.vein {
background: #1a1a1a;
border: 1px solid #3f3f3f;
background: var(--surface);
border: 1px solid var(--border-strong);
padding: 1rem;
border-radius: 8px;
text-align: center;
@@ -113,16 +113,16 @@
transition: all 0.15s;
}
.vein:hover {
background: #2a2a2a;
background: var(--border);
}
.vein.selected {
border-color: #b91c1c;
border-color: var(--system-accent);
border-width: 2px;
background: #1a1a1a;
background: var(--surface);
}
.vein.active {
background: #b91c1c;
border-color: #b91c1c;
background: var(--system-accent);
border-color: var(--system-accent);
}
.vein.active h3 {
color: white;
@@ -139,12 +139,12 @@
cursor: not-allowed;
}
.vein.disabled:hover {
background: #1a1a1a;
background: var(--surface);
}
.vein h3 {
margin: 0;
font-size: 1rem;
color: #e5e5e5;
color: var(--text);
}
.endpoints {
list-style: none;
@@ -153,7 +153,7 @@
}
.endpoints li {
padding: 0.75rem 0;
border-bottom: 1px solid #3f3f3f;
border-bottom: 1px solid var(--border-strong);
display: flex;
justify-content: space-between;
flex-wrap: wrap;
@@ -164,18 +164,18 @@
}
.endpoints code {
font-family: monospace;
background: #1a1a1a;
background: var(--surface);
padding: 0.25rem 0.5rem;
border-radius: 4px;
color: #fca5a5;
color: var(--system-accent-text);
}
.endpoints .desc {
color: #a3a3a3;
color: var(--muted);
font-size: 0.9rem;
}
code {
background: #2a2a2a;
color: #fca5a5;
background: var(--border);
color: var(--system-accent-text);
padding: 0.1rem 0.3rem;
border-radius: 3px;
font-size: 0.85rem;
@@ -210,41 +210,41 @@
display: block;
font-weight: 500;
margin-bottom: 0.5rem;
color: #e5e5e5;
color: var(--text);
}
.api-form input[type="text"] {
width: 100%;
padding: 0.75rem;
border: 1px solid #3f3f3f;
border: 1px solid var(--border-strong);
border-radius: 8px;
font-family: monospace;
font-size: 0.9rem;
background: #1a1a1a;
color: #e5e5e5;
background: var(--surface);
color: var(--text);
}
.api-form input[type="text"]:focus,
.api-form select:focus {
outline: none;
border-color: #b91c1c;
border-color: var(--system-accent);
}
.api-form select {
width: 100%;
padding: 0.75rem;
border: 1px solid #3f3f3f;
border: 1px solid var(--border-strong);
border-radius: 8px;
font-size: 0.9rem;
background: #1a1a1a;
color: #e5e5e5;
background: var(--surface);
color: var(--text);
cursor: pointer;
}
.api-form select:disabled {
background: #0a0a0a;
color: #666;
background: var(--bg);
color: var(--dim);
cursor: not-allowed;
}
.api-form input:disabled {
background: #0a0a0a;
color: #666;
background: var(--bg);
color: var(--dim);
cursor: not-allowed;
}
.api-controls {
@@ -254,7 +254,7 @@
margin-top: 1rem;
}
.api-controls button {
background: #b91c1c;
background: var(--system-accent);
color: white;
border: none;
padding: 0.75rem 1.5rem;
@@ -271,17 +271,17 @@
cursor: not-allowed;
}
.tab-button {
background: #1a1a1a !important;
border: 1px solid #3f3f3f !important;
color: #e5e5e5 !important;
background: var(--surface) !important;
border: 1px solid var(--border-strong) !important;
color: var(--text) !important;
}
.tab-button:hover {
background: #2a2a2a !important;
background: var(--border) !important;
}
.tab-button.active {
border-color: white !important;
border-width: 2px !important;
background: #b91c1c !important;
background: var(--system-accent) !important;
color: white !important;
}
.tab-button.active:hover {
@@ -291,19 +291,19 @@
font-size: 2rem;
font-weight: bold;
padding: 2rem;
color: #e5e5e5;
color: var(--text);
animation: pulse 2s ease-in-out infinite;
}
.epic-status.error {
color: #fca5a5;
color: var(--system-accent-text);
}
@keyframes pulse {
0%,
100% {
color: #0a0a0a;
color: var(--bg);
}
50% {
color: #b91c1c;
color: var(--system-accent);
}
}
.api-controls label {
@@ -312,7 +312,7 @@
gap: 0.5rem;
font-size: 0.9rem;
cursor: pointer;
color: #e5e5e5;
color: var(--text);
}
.output-container {
position: relative;
@@ -324,8 +324,8 @@
}
.output-area {
background: #1a1a1a;
color: #e5e5e5;
background: var(--surface);
color: var(--text);
padding: 1rem;
padding-top: 2.5rem;
border-radius: 8px;
@@ -337,7 +337,7 @@
position: relative;
}
.output-area.error {
color: #fca5a5;
color: var(--system-accent-text);
}
.output-area.scrollable {
max-height: 1000px;
@@ -353,7 +353,7 @@
margin-top: 1.5rem;
}
.attachments-container h3 {
color: #fca5a5;
color: var(--system-accent-text);
margin: 0 0 1rem 0;
font-size: 1rem;
}
@@ -370,10 +370,15 @@
}
.attachment-label {
font-size: 0.85rem;
color: #a3a3a3;
color: var(--muted);
margin-bottom: 0.5rem;
}
</style>
<link rel="stylesheet" href="/theme.css">
<style>
/* Artery keeps its own colour under every theme. */
:root { --system-accent: #b91c1c; --system-accent-text: #fca5a5; }
</style>
</head>
<body>
<header style="position: relative">
@@ -1824,5 +1829,6 @@
}
});
</script>
<script src="/theme.js" defer></script>
</body>
</html>

11
soleprint/artery/shunts/mercadopago/api/routes.py
View File

@@ -7,15 +7,8 @@ from fastapi.responses import JSONResponse
from typing import Optional, List, Dict, Any
from pydantic import BaseModel
# Import datagen from ward/tools
import sys
from pathlib import Path
ward_tools_path = Path(__file__).parent.parent.parent.parent.parent / "ward" / "tools"
sys.path.insert(0, str(ward_tools_path))
from datagen.mercadopago import MercadoPagoDataGenerator
from ..core.config import settings
from core.config import settings
from datagen import MercadoPagoDataGenerator
router = APIRouter()

182
soleprint/artery/shunts/mercadopago/datagen.py Normal file
View File

@@ -0,0 +1,182 @@
"""
MercadoPago response shapes.
This lived under ward/tools/datagen/ before that tree was renamed to station/,
and the module was lost in the move — api/routes.py has been importing a path
that does not exist since. It belongs here rather than back in station/tools:
a shunt runs as its own process, and its payload shapes are part of it.
Amounts are in whole currency units, matching what the real API returns for
ARS. Every method is static; the routes hold the state, this holds the shapes.
"""
import random
import uuid
from datetime import datetime, timedelta, timezone
CURRENCY = "ARS"
SITE = "MLA" # MercadoPago Argentina
PAYMENT_METHODS = ["visa", "master", "amex", "account_money", "rapipago"]
PAYMENT_TYPES = ["credit_card", "debit_card", "account_money", "ticket"]
STATUS_DETAIL = {
"approved": "accredited",
"pending": "pending_contingency",
"in_process": "pending_review_manual",
"rejected": "cc_rejected_insufficient_amount",
"cancelled": "expired",
}
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _later(minutes: int) -> str:
return (datetime.now(timezone.utc) + timedelta(minutes=minutes)).isoformat()
def _numeric_id() -> int:
"""A MercadoPago-sized numeric id — the routes look these up as ints."""
return random.randint(1_000_000_000, 9_999_999_999)
class MercadoPagoDataGenerator:
"""Builds MercadoPago-shaped payloads for the shunt to hand back."""
@staticmethod
def preference(
description: str = "Payment",
total: float = 0.0,
external_reference: str | None = None,
) -> dict:
preference_id = f"{random.randint(100000000, 999999999)}-{uuid.uuid4()}"
return {
"id": preference_id,
"client_id": str(random.randint(1_000_000_000_000_000, 9_999_999_999_999_999)),
"collector_id": _numeric_id(),
"date_created": _now(),
"expires": False,
"external_reference": external_reference or "",
"init_point": f"https://www.mercadopago.com.ar/checkout/v1/redirect?pref_id={preference_id}",
"sandbox_init_point": (
f"https://sandbox.mercadopago.com.ar/checkout/v1/redirect?pref_id={preference_id}"
),
"items": [
{
"id": str(uuid.uuid4()),
"title": description,
"description": description,
"quantity": 1,
"unit_price": total,
"currency_id": CURRENCY,
}
],
"marketplace": "NONE",
"marketplace_fee": 0,
"operation_type": "regular_payment",
"site_id": SITE,
"total_amount": total,
}
@staticmethod
def payment(
transaction_amount: float = 0.0,
description: str = "Payment",
status: str = "approved",
application_fee: float | None = None,
) -> dict:
method = random.choice(PAYMENT_METHODS)
fee = round(transaction_amount * 0.0579, 2) if transaction_amount else 0.0
return {
"id": _numeric_id(),
"date_created": _now(),
"date_approved": _now() if status == "approved" else None,
"date_last_updated": _now(),
"money_release_date": _later(60 * 24 * 14) if status == "approved" else None,
"operation_type": "regular_payment",
"payment_method_id": method,
"payment_type_id": random.choice(PAYMENT_TYPES),
"status": status,
"status_detail": STATUS_DETAIL.get(status, "accredited"),
"currency_id": CURRENCY,
"description": description,
"live_mode": False,
"collector_id": _numeric_id(),
"payer": {
"id": str(_numeric_id()),
"email": f"test_user_{random.randint(1000, 99999)}@testuser.com",
"identification": {"type": "DNI", "number": str(random.randint(10_000_000, 45_000_000))},
"type": "customer",
},
"transaction_amount": transaction_amount,
"transaction_amount_refunded": 0,
"installments": 1,
"transaction_details": {
"net_received_amount": (
round(transaction_amount - fee, 2) if status == "approved" else 0
),
"total_paid_amount": transaction_amount,
"overpaid_amount": 0,
"installment_amount": transaction_amount,
},
"fee_details": (
[{"type": "mercadopago_fee", "amount": fee, "fee_payer": "collector"}]
if status == "approved"
else []
),
"application_fee": application_fee,
"captured": status == "approved",
"external_reference": None,
}
@staticmethod
def merchant_order(
preference_id: str = "",
total: float = 0.0,
paid_amount: float = 0.0,
) -> dict:
return {
"id": _numeric_id(),
"status": "closed" if paid_amount >= total and total else "opened",
"external_reference": "",
"preference_id": preference_id,
"payments": [],
"shipments": [],
"date_created": _now(),
"last_updated": _now(),
"site_id": SITE,
"total_amount": total,
"paid_amount": paid_amount,
"refunded_amount": 0,
"order_status": "paid" if paid_amount >= total and total else "payment_required",
}
@staticmethod
def oauth_token() -> dict:
return {
"access_token": f"APP_USR-{uuid.uuid4().hex}",
"token_type": "Bearer",
"expires_in": 15552000,
"scope": "offline_access read write",
"user_id": _numeric_id(),
"refresh_token": f"TG-{uuid.uuid4().hex}",
"public_key": f"APP_USR-{uuid.uuid4()}",
"live_mode": False,
}
@staticmethod
def webhook_notification(topic: str = "payment", resource_id: str = "") -> dict:
return {
"id": _numeric_id(),
"live_mode": False,
"type": topic,
"date_created": _now(),
"application_id": _numeric_id(),
"user_id": _numeric_id(),
"version": 1,
"api_version": "v1",
"action": f"{topic}.updated",
"data": {"id": str(resource_id)},
}

6
soleprint/artery/shunts/mercadopago/main.py
View File

@@ -5,8 +5,10 @@ from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from fastapi.middleware.cors import CORSMiddleware
from .api.routes import router
from .core.config import settings
# Absolute imports: a shunt is started from its own directory (`python run.py`),
# not imported as a package, so relative imports have no parent to resolve.
from api.routes import router
from core.config import settings
app = FastAPI(
title="MercadoPago (MOCK)",

26
soleprint/atlas/index.html
View File

@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
@@ -14,7 +14,7 @@
box-sizing: border-box;
}
html {
background: #0a0a0a;
background: var(--bg);
}
body {
font-family:
@@ -25,7 +25,7 @@
margin: 0 auto;
padding: 2rem 1rem;
line-height: 1.6;
color: #e5e5e5;
color: var(--text);
background: #163528;
}
header {
@@ -59,7 +59,7 @@
section h2 {
margin: 0 0 1rem 0;
font-size: 1.2rem;
color: #86efac;
color: var(--system-accent-text);
}
.composition {
background: #000000;
@@ -70,12 +70,12 @@
.composition h3 {
margin: 0 0 0.75rem 0;
font-size: 1.1rem;
color: #86efac;
color: var(--system-accent-text);
}
.composition > p {
margin: 0 0 1rem 0;
font-size: 0.9rem;
color: #a3a3a3;
color: var(--muted);
}
.components {
display: grid;
@@ -83,7 +83,7 @@
gap: 0.75rem;
}
.component {
background: #0a0a0a;
background: var(--bg);
border: 1px solid #6b665e;
padding: 0.75rem;
border-radius: 8px;
@@ -91,12 +91,12 @@
.component h4 {
margin: 0 0 0.25rem 0;
font-size: 0.95rem;
color: #86efac;
color: var(--system-accent-text);
}
.component p {
margin: 0;
font-size: 0.85rem;
color: #a3a3a3;
color: var(--muted);
}
.books {
list-style: none;
@@ -114,7 +114,7 @@
border-bottom: none;
}
.books a {
color: #86efac;
color: var(--system-accent-text);
text-decoration: none;
font-weight: 500;
}
@@ -138,6 +138,11 @@
opacity: 0.5;
}
</style>
<link rel="stylesheet" href="/theme.css">
<style>
/* Atlas keeps its own colour under every theme. */
:root { --system-accent: #15803d; --system-accent-text: #86efac; }
</style>
</head>
<body>
<header style="position: relative">
@@ -265,5 +270,6 @@
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
</footer>
<script src="/theme.js" defer></script>
</body>
</html>

118
soleprint/common/theme/theme.js Normal file
View File

@@ -0,0 +1,118 @@
/* Theme selection, and the toggle that drives it.
*
* Served by run.py at /theme.js, next to /theme.css. Include both and a page
* is themed; there is nothing else to wire.
*
* <link rel="stylesheet" href="/theme.css">
* <script src="/theme.js" defer></script>
*
* Resolution order, strongest first:
* 1. ?theme=mcrn — a link that carries its own theme
* 2. localStorage — what this browser last chose
* 3. <html data-theme> — what the page was served with
* 4. the server default — framework.theme in cfg/config.json
*
* The attribute is set before first paint when this script is loaded in the
* head; with `defer` the page renders once in the served theme and then
* switches, which is why run.py stamps data-theme into the served HTML.
*/
(function () {
"use strict";
var THEMES = ["soleprint", "mcrn"];
var KEY = "spr-theme";
var root = document.documentElement;
function fromQuery() {
var match = /[?&]theme=([^&#]+)/.exec(window.location.search);
return match ? decodeURIComponent(match[1]) : null;
}
function stored() {
try {
return window.localStorage.getItem(KEY);
} catch (e) {
// Private mode and file:// origins throw on access rather than
// returning null, and a theme is not worth breaking a page over.
return null;
}
}
function resolve() {
var candidates = [fromQuery(), stored(), root.getAttribute("data-theme")];
for (var i = 0; i < candidates.length; i++) {
if (candidates[i] && THEMES.indexOf(candidates[i]) !== -1) {
return candidates[i];
}
}
return THEMES[0];
}
function apply(theme, persist) {
root.setAttribute("data-theme", theme);
if (persist) {
try {
window.localStorage.setItem(KEY, theme);
} catch (e) {
/* see stored() */
}
}
var buttons = document.querySelectorAll("#spr-theme-toggle button");
for (var i = 0; i < buttons.length; i++) {
buttons[i].setAttribute(
"aria-pressed",
buttons[i].dataset.theme === theme ? "true" : "false"
);
}
window.dispatchEvent(new CustomEvent("spr:theme", { detail: theme }));
}
function buildToggle() {
if (document.getElementById("spr-theme-toggle")) return;
// Opt out with <body data-theme-toggle="off"> — the shunt config UIs
// and any embedded view want the theme without the chrome.
if (document.body.dataset.themeToggle === "off") return;
var box = document.createElement("div");
box.id = "spr-theme-toggle";
box.setAttribute("role", "group");
box.setAttribute("aria-label", "Theme");
THEMES.forEach(function (theme) {
var button = document.createElement("button");
button.type = "button";
button.dataset.theme = theme;
button.textContent = theme === "mcrn" ? "MCRN" : "SPR";
button.title = "Switch to the " + theme + " theme";
button.addEventListener("click", function () {
apply(theme, true);
});
box.appendChild(button);
});
document.body.appendChild(box);
}
// Set the attribute immediately; the toggle needs a body to attach to.
apply(resolve(), false);
if (document.readyState === "loading") {
document.addEventListener("DOMContentLoaded", function () {
buildToggle();
apply(root.getAttribute("data-theme"), false);
});
} else {
buildToggle();
apply(root.getAttribute("data-theme"), false);
}
window.sprTheme = {
get: function () {
return root.getAttribute("data-theme");
},
set: function (theme) {
if (THEMES.indexOf(theme) !== -1) apply(theme, true);
},
themes: THEMES.slice(),
};
})();

110
soleprint/common/theme/themes/mcrn.css Normal file
View File

@@ -0,0 +1,110 @@
/* MCRN — the terminal read of the Expanse aesthetic, as built at mariano.mcrn.ar.
*
* The defining choices, all of them load-bearing:
* - zero corner radius, everywhere, including inputs and buttons
* - one hairline border colour and one accent; nothing in between
* - monospace for everything, not just code
* - headings are uppercase, letterspaced, unbolded, and ruled underneath
* - the only hover is a burnt-orange border and a glow behind it
*
* Values are lifted from ~/wdir/mcrn.ar/css/mcrn.css rather than approximated,
* so the two sites read as one system.
*/
[data-theme="mcrn"] {
--bg: #0a0a0a;
--bg-2: #141414;
--surface: #1a1a1a;
--surface-raised: #202020;
--border: #2a2a2a;
--border-strong: #3d3d3d;
--text: #e0e0e0;
--muted: #888;
--dim: #555;
--accent: #d35400;
--accent-dim: #c0392b;
--accent-text: #d35400;
--glow: rgba(211, 84, 0, 0.3);
--status-ok: #2ecc71;
--status-info: #5dade2;
--status-warn: #f39c12;
--status-error: #e74c3c;
--status-idle: #555;
/* Square. This is the single most recognisable thing about the theme, so
* it applies to the small radii too — a 4px input in a 0px page reads as a
* mistake rather than a detail. */
--radius-sm: 0;
--radius: 0;
--radius-lg: 0;
--radius-xl: 0;
--font-ui: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
--font-mono: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
--font-heading: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
--heading-transform: uppercase;
--heading-spacing: 0.1em;
--heading-weight: 400;
--label-spacing: 0.05em;
--speed-fast: 0.15s;
--speed: 0.2s;
/* Glow instead of lift: this theme never moves anything on hover. */
--hover-shadow: 0 0 20px var(--glow);
--hover-lift: none;
--focus-shadow: 0 0 10px var(--glow);
}
/* Section headings carry a rule, and the rule is part of the type. */
[data-theme="mcrn"] h2 {
color: var(--system-accent-text, var(--accent-text));
padding-bottom: var(--space-2);
border-bottom: var(--hairline) solid var(--border);
}
/* Prompt bullets. Lists in this theme are terminal output, not prose. */
[data-theme="mcrn"] ul:not([class*="reset"]) > li::marker {
content: "";
}
[data-theme="mcrn"] .prompt-list {
list-style: none;
padding-left: 0;
}
[data-theme="mcrn"] .prompt-list > li {
position: relative;
padding-left: 1.25rem;
}
[data-theme="mcrn"] .prompt-list > li::before {
content: ">";
position: absolute;
left: 0;
color: var(--accent);
font-weight: 600;
}
/* The active state is a solid fill with the page colour punched out of it —
* the segmented-control move from the portfolio's language toggle. */
[data-theme="mcrn"] button[aria-pressed="true"],
[data-theme="mcrn"] .active,
[data-theme="mcrn"] .selected {
background: var(--accent);
color: var(--bg);
border-color: var(--accent);
}
[data-theme="mcrn"] .card:hover,
[data-theme="mcrn"] .panel:hover,
[data-theme="mcrn"] .system-card:hover,
[data-theme="mcrn"] .tool-card:hover,
[data-theme="mcrn"] .model-card:hover {
border-color: var(--system-accent, var(--accent));
box-shadow: 0 0 20px var(--glow);
transform: none;
}

113
soleprint/common/theme/themes/soleprint.css Normal file
View File

@@ -0,0 +1,113 @@
/* Soleprint — minimal, rounded, neon.
*
* The surface ramp and text colours come from common/ui/src/tokens.css, which
* is the most considered palette in the repo: near-black with a violet cast
* (#0d0d0f -> #26262f) rather than flat grey. The accent is the amber the brand
* page has always used (#d4a574) but which the token file never carried, so
* this is where the two finally agree.
*
* "Neon" here means light around an edge, not saturated fills: a 1px halo in
* the accent, translucent badge fills at 18/66 alpha, and a small lift on
* hover. Restrained enough to sit under dense tool UIs for an hour.
*/
[data-theme="soleprint"] {
--bg: #0d0d0f;
--bg-2: #16161a;
--surface: #16161a;
--surface-raised: #1e1e24;
--border: #2e2e38;
--border-strong: #3d3d4a;
--text: #e8e8f0;
--muted: #8888a0;
--dim: #555568;
--accent: #d4a574;
--accent-dim: #b8956a;
--accent-text: #e0b98d; /* lifted off the fill colour so small text holds up */
--glow: rgba(212, 165, 116, 0.32);
--status-ok: #3ecf8e;
--status-info: #4f9cf9;
--status-warn: #f5a623;
--status-error: #f06565;
--status-idle: #555568;
/* Rounded but restrained — nothing above 12px, so panels read as soft
* rather than as pills. */
--radius-sm: 4px;
--radius: 6px;
--radius-lg: 8px;
--radius-xl: 12px;
--font-ui: "Inter", system-ui, -apple-system, sans-serif;
--font-mono: "JetBrains Mono", "Fira Code", monospace;
--font-heading: "Inter", system-ui, sans-serif;
--heading-transform: none;
--heading-spacing: 0.02em;
--heading-weight: 600;
--label-spacing: 0.04em;
--speed-fast: 0.12s;
--speed: 0.2s;
--hover-shadow: 0 4px 14px rgba(0, 0, 0, 0.45), 0 0 0 1px var(--accent);
--hover-lift: translateY(-2px);
--focus-shadow: 0 0 0 2px var(--glow);
}
[data-theme="soleprint"] .card:hover,
[data-theme="soleprint"] .panel:hover,
[data-theme="soleprint"] .system-card:hover,
[data-theme="soleprint"] .tool-card:hover,
[data-theme="soleprint"] .model-card:hover {
border-color: var(--system-accent, var(--accent));
box-shadow: 0 4px 14px rgba(0, 0, 0, 0.45),
0 0 0 1px var(--system-accent, var(--accent));
transform: translateY(-2px);
}
[data-theme="soleprint"] button[aria-pressed="true"],
[data-theme="soleprint"] .active,
[data-theme="soleprint"] .selected {
border-color: var(--accent);
color: var(--bg);
background: linear-gradient(135deg, var(--accent), var(--accent-dim));
}
/* Uppercase micro-labels — the recurring motif across the station tools. */
[data-theme="soleprint"] .label,
[data-theme="soleprint"] .panel-title {
font-size: var(--font-size-sm);
font-weight: 600;
text-transform: uppercase;
letter-spacing: var(--label-spacing);
color: var(--muted);
}
/* Translucent badges, the <hex>18 fill / <hex>66 border pattern graphgen
* already uses for pk/fk/m2m chips. */
[data-theme="soleprint"] .badge {
border-radius: var(--radius-sm);
padding: 1px 6px;
font-family: var(--font-mono);
font-size: 10px;
letter-spacing: 0.03em;
background: #d4a57418;
border: 1px solid #d4a57466;
color: var(--accent-text);
}
/* The one animated state, carried over from common/ui's tokens.css. */
@keyframes spr-waiting-glow {
0% { box-shadow: 0 0 3px 1px var(--status-info); }
33% { box-shadow: 0 0 3px 1px var(--status-ok); }
66% { box-shadow: 0 0 3px 1px var(--status-warn); }
100% { box-shadow: 0 0 3px 1px var(--status-info); }
}
[data-theme="soleprint"] .waiting {
animation: spr-waiting-glow 2s linear infinite;
outline: 1px solid transparent;
}

217
soleprint/common/theme/tokens.css Normal file
View File

@@ -0,0 +1,217 @@
/* Soleprint theme contract — the variable names every page may rely on.
*
* This file declares the vocabulary and a neutral default. It sets no colours
* of its own worth looking at: themes/*.css supply those, scoped to
* [data-theme="..."] on <html>, and theme.js decides which one is active.
*
* Two naming families existed before this file and both are still in use, so
* both are answered here rather than renamed across a dozen templates:
*
* --bg / --surface / --border / --text / --muted / --amber
* the station tool templates and the docs site
* --surface-0..3 / --text-primary / --text-secondary / --panel-radius
* common/ui's Vue component library
*
* The theme files set the first family; the aliases at the bottom derive the
* second from it. A page that uses either name gets the same colour, and a
* theme author has one set of values to fill in.
*
* Served by run.py at /theme.css together with the theme files — see the
* handler beside /sidebar.css.
*/
@import url("https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;600&display=swap");
:root {
/* ── surfaces ─────────────────────────────────────────────────────── */
--bg: #0a0a0a;
--bg-2: #141414;
--surface: #1a1a1a;
--surface-raised: #242424;
--border: #333;
--border-strong: #4a4a4a;
/* ── text ─────────────────────────────────────────────────────────── */
--text: #e5e5e5;
--muted: #a3a3a3;
--dim: #666;
/* ── accent ───────────────────────────────────────────────────────── */
--accent: #d4a574;
--accent-dim: #b8956a;
--accent-text: #d4a574; /* accent legible on --bg, where the fill is not */
--glow: rgba(212, 165, 116, 0.3);
/* Each subsystem keeps its own colour under every theme, so a page still
* announces which of artery / atlas / station you are looking at. Pages set
* --system-accent; this is the fallback for those that do not. */
--system-accent: var(--accent);
--system-accent-text: var(--accent-text);
/* ── status ───────────────────────────────────────────────────────── */
--status-ok: #3ecf8e;
--status-info: #4f9cf9;
--status-warn: #f5a623;
--status-error: #f06565;
--status-idle: #555568;
/* ── shape ────────────────────────────────────────────────────────── */
--radius-sm: 4px;
--radius: 6px;
--radius-lg: 8px;
--radius-xl: 12px;
--hairline: 1px;
/* ── type ─────────────────────────────────────────────────────────── */
--font-ui: "Inter", system-ui, -apple-system, sans-serif;
--font-mono: "JetBrains Mono", "Fira Code", "SF Mono", monospace;
--font-heading: var(--font-ui);
--font-size-sm: 11px;
--font-size-base: 13px;
--font-size-lg: 15px;
--heading-transform: none;
--heading-spacing: 0;
--heading-weight: 600;
--label-spacing: 0.04em;
/* ── space ────────────────────────────────────────────────────────── */
--space-1: 4px;
--space-2: 8px;
--space-3: 12px;
--space-4: 16px;
--space-6: 24px;
--space-8: 32px;
/* ── motion ───────────────────────────────────────────────────────── */
--speed-fast: 0.12s;
--speed: 0.2s;
--ease: ease;
/* Themes differ most in how a surface reacts, not in what colour it is.
* Both hover treatments are declared here so a page writes one rule and
* the theme decides whether it lifts, glows, or both. */
--hover-shadow: 0 4px 12px var(--glow);
--hover-lift: translateY(-2px);
--focus-shadow: 0 0 0 1px var(--accent);
/* ── aliases for names already in use ─────────────────────────────── */
/* The station tool templates and the docs site call the accent "amber". */
--amber: var(--accent);
--amber-dim: var(--accent-dim);
/* ── aliases for common/ui's token names ──────────────────────────── */
--surface-0: var(--bg);
--surface-1: var(--surface);
--surface-2: var(--surface-raised);
--surface-3: var(--border);
--text-primary: var(--text);
--text-secondary: var(--muted);
--text-dim: var(--dim);
--panel-radius: var(--radius);
--panel-border: var(--hairline) solid var(--border);
--panel-header-height: 36px;
--status-live: var(--status-ok);
--status-processing: var(--status-info);
--status-escalating: var(--status-warn);
}
/* ── element defaults ─────────────────────────────────────────────────── */
body {
background: var(--bg);
color: var(--text);
font-family: var(--font-ui);
line-height: 1.6;
}
h1, h2, h3, h4, h5, h6 {
font-family: var(--font-heading);
font-weight: var(--heading-weight);
text-transform: var(--heading-transform);
letter-spacing: var(--heading-spacing);
}
code, pre, kbd, samp {
font-family: var(--font-mono);
}
a {
color: var(--accent-text);
}
button {
background: var(--surface-raised);
color: var(--text);
border: var(--hairline) solid var(--border);
border-radius: var(--radius);
font-family: inherit;
cursor: pointer;
transition: border-color var(--speed-fast) var(--ease),
color var(--speed-fast) var(--ease),
background var(--speed-fast) var(--ease);
}
button:hover:not(:disabled) {
border-color: var(--accent);
color: var(--accent-text);
}
button:disabled {
opacity: 0.4;
cursor: default;
}
input, select, textarea {
background: var(--bg);
color: var(--text);
border: var(--hairline) solid var(--border);
border-radius: var(--radius-sm);
font-family: inherit;
}
input:focus, select:focus, textarea:focus {
outline: none;
border-color: var(--accent);
box-shadow: var(--focus-shadow);
}
::-webkit-scrollbar { width: 10px; height: 10px; }
::-webkit-scrollbar-thumb { background: var(--border); border-radius: 5px; }
::-webkit-scrollbar-track { background: transparent; }
/* The theme toggle theme.js injects. Fixed rather than placed, so it needs no
* cooperation from the page it lands on. */
#spr-theme-toggle {
position: fixed;
right: var(--space-3);
bottom: var(--space-3);
z-index: 99998;
display: flex;
gap: 0;
border: var(--hairline) solid var(--border);
border-radius: var(--radius);
overflow: hidden;
background: var(--surface);
font-family: var(--font-mono);
font-size: 10px;
}
#spr-theme-toggle button {
border: none;
border-radius: 0;
background: var(--surface);
color: var(--dim);
padding: var(--space-1) var(--space-2);
text-transform: uppercase;
letter-spacing: var(--label-spacing);
}
#spr-theme-toggle button + button {
border-left: var(--hairline) solid var(--border);
}
#spr-theme-toggle button[aria-pressed="true"] {
background: var(--accent);
color: var(--bg);
}

56
soleprint/index.html
View File

@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
@@ -14,7 +14,7 @@
box-sizing: border-box;
}
html {
background: #0a0a0a;
background: var(--bg);
}
body {
font-family:
@@ -25,8 +25,8 @@
margin: 0 auto;
padding: 2rem 1rem;
line-height: 1.6;
color: #e5e5e5;
background: #0a0a0a;
color: var(--text);
background: var(--bg);
}
/* Sidebar styles */
.sidebar {
@@ -35,8 +35,8 @@
left: 0;
width: 60px;
height: 100vh;
background: #1a1a1a;
border-right: 1px solid #333;
background: var(--surface);
border-right: 1px solid var(--border);
display: flex;
flex-direction: column;
align-items: center;
@@ -52,16 +52,16 @@
border-radius: 8px;
margin-bottom: 0.5rem;
text-decoration: none;
color: #a3a3a3;
color: var(--muted);
transition: all 0.2s;
}
a.sidebar-item:hover {
background: #333;
background: var(--border);
color: white;
}
.sidebar-item.active {
background: #d4a574;
color: #0a0a0a;
background: var(--accent);
color: var(--bg);
}
.sidebar-item svg {
width: 24px;
@@ -70,7 +70,7 @@
.sidebar-divider {
width: 32px;
height: 1px;
background: #333;
background: var(--border);
margin: 0.5rem 0;
}
.sidebar-icon {
@@ -89,7 +89,7 @@
.sidebar-item .tooltip {
position: absolute;
left: 70px;
background: #333;
background: var(--border);
color: white;
padding: 0.5rem 0.75rem;
border-radius: 4px;
@@ -121,18 +121,18 @@
color: white;
}
.tagline {
color: #a3a3a3;
color: var(--muted);
margin-bottom: 2rem;
border-bottom: 1px solid #333;
border-bottom: 1px solid var(--border);
padding-bottom: 2rem;
}
.mission {
background: #1a1a1a;
border-left: 3px solid #d4a574;
background: var(--surface);
border-left: 3px solid var(--accent);
padding: 1rem 1.5rem;
margin: 2rem 0;
border-radius: 0 8px 8px 0;
color: #d4a574;
color: var(--accent);
}
.systems {
display: grid;
@@ -171,11 +171,11 @@
.system-info p {
margin: 0;
font-size: 0.9rem;
color: #a3a3a3;
color: var(--muted);
}
.artery {
background: #1a1a1a;
background: var(--surface);
border: 1px solid #b91c1c;
}
.artery h2 {
@@ -186,7 +186,7 @@
}
.atlas {
background: #1a1a1a;
background: var(--surface);
border: 1px solid #15803d;
}
.atlas h2 {
@@ -197,7 +197,7 @@
}
.station {
background: #1a1a1a;
background: var(--surface);
border: 1px solid #1d4ed8;
}
.station h2 {
@@ -210,9 +210,9 @@
footer {
margin-top: 3rem;
padding-top: 1.5rem;
border-top: 1px solid #333;
border-top: 1px solid var(--border);
font-size: 0.85rem;
color: #666;
color: var(--dim);
}
.showcase-container {
@@ -224,8 +224,8 @@
}
.showcase-link {
display: inline-block;
background: linear-gradient(135deg, #d4a574, #b8956a);
color: #0a0a0a;
background: linear-gradient(135deg, var(--accent), var(--accent-dim));
color: var(--bg);
padding: 0.75rem 1.5rem;
border-radius: 8px;
text-decoration: none;
@@ -237,18 +237,19 @@
box-shadow: 0 4px 12px rgba(212, 165, 116, 0.3);
}
.showcase-hint {
color: #666;
color: var(--dim);
font-size: 0.8rem;
text-decoration: none;
}
.showcase-hint:hover {
color: #d4a574;
color: var(--accent);
}
</style>
{% if managed %}
<link rel="stylesheet" href="/sidebar.css">
<script src="/sidebar.js"></script>
{% endif %}
<link rel="stylesheet" href="/theme.css">
</head>
<body{% if managed %} class="has-sidebar"{% endif %}>
@@ -428,5 +429,6 @@
</div>
<footer>soleprint</footer>
<script src="/theme.js" defer></script>
</body>
</html>

6
soleprint/requirements.txt
View File

@@ -5,6 +5,12 @@ pydantic>=2.5.0
pydantic-settings>=2.0.0
httpx>=0.25.0
jinja2>=3.1.0
# YAML: OpenAPI specs for shuntgen, and compose fragments for cabinets.
# modelgen keeps this optional (it imports it lazily) so the pip package stays
# dependency-free; the server always has it.
pyyaml>=6.0
# Multipart form parsing — file uploads in shuntgen's UI.
python-multipart>=0.0.9
# Database (databrowse)
sqlalchemy>=2.0.0

154
soleprint/run.py
View File

@@ -249,19 +249,41 @@ def load_config() -> dict:
return {}
class SafeDict(dict):
"""A dict whose missing keys read as empty rather than exploding a template.
The landing pages ask for two levels at once — `components.composed.title`.
Jinja tolerates one missing level (it returns Undefined, and `or 'Desk'`
catches it) but not two: attribute access on Undefined raises, and the
whole page 500s.
That is not hypothetical. cfg/config.json is written into a room by
build.py, so it does not exist in the source tree at all, and running
`python run.py` from soleprint/ — the documented way to develop — took
/station/ down every time.
"""
def __getattr__(self, name: str):
# The dict's own contents win; only genuinely absent keys become empty.
if name in self:
value = self[name]
return SafeDict(value) if isinstance(value, dict) else value
return SafeDict()
def get_system_config(system_key: str) -> dict:
"""Get system configuration by key (data_flow, documentation, execution)."""
config = load_config()
for system in config.get("systems", []):
if system.get("key") == system_key:
return system
return {}
return SafeDict(system)
return SafeDict()
def get_components(system_key: str) -> dict:
"""Get component definitions for a system."""
config = load_config()
return config.get("components", {}).get(system_key, {})
return SafeDict(config.get("components", {}).get(system_key, {}))
def load_data(filename: str) -> list[dict]:
@@ -461,6 +483,41 @@ def atlas_route(path: str):
# === Station ===
def load_station_cabinets() -> list[dict]:
"""The dependency containers this room switched on.
Same two-step as artery's shunts: what the room declared, else what is on
disk. The room's list is the honest answer — the catalog holds every
cabinet that could be used, not the ones that were.
"""
declared = load_data("cabinets.json")
catalog = SPR_ROOT / "station" / "cabinets"
if declared:
cabinets = declared
else:
cabinets = [
{"name": path.name}
for path in sorted(catalog.iterdir())
if path.is_dir() and not path.name.startswith(("_", "."))
] if catalog.exists() else []
for cabinet in cabinets:
name = cabinet.get("name", "")
definition_path = catalog / name / "cabinet.json"
if definition_path.exists():
try:
definition = json.loads(definition_path.read_text())
for key in ("title", "description", "image", "rig_addon"):
cabinet.setdefault(key, definition.get(key))
except (OSError, ValueError):
pass
cabinet.setdefault("slug", name)
cabinet.setdefault("title", name.replace("-", " ").title())
cabinet.setdefault("status", "declared" if declared else "available")
return cabinets
@app.get("/station", response_class=HTMLResponse)
@app.get("/station/", response_class=HTMLResponse)
def station_index(request: Request):
@@ -486,6 +543,7 @@ def station_index(request: Request):
d["slug"] = d["name"]
d["title"] = d["name"].replace("-", " ").title()
d["status"] = "ready"
cabinets = load_station_cabinets()
from jinja2 import Template
template = Template(html_path.read_text())
@@ -497,6 +555,7 @@ def station_index(request: Request):
tools=tools,
monitors=monitors,
desks=desks,
cabinets=cabinets,
soleprint_url="/",
)
)
@@ -512,24 +571,18 @@ def station_index(request: Request):
)
# Mount station tool routers
# Mount station tool routers.
#
# Broad except on purpose: a tool that cannot load should cost you that tool,
# not the server. Route registration raises more than ImportError — FastAPI
# turns a missing optional dependency into a RuntimeError at decoration time —
# and catching only ImportError meant one such tool took the whole app down.
for _tool in ("tester", "graphgen", "datagen", "shuntgen"):
try:
from station.tools.tester.api import router as tester_router
app.include_router(tester_router, prefix="/station")
except ImportError as e:
print(f"Warning: Could not load tester router: {e}")
try:
from station.tools.graphgen.api import router as graphgen_router
app.include_router(graphgen_router, prefix="/station")
except ImportError as e:
print(f"Warning: Could not load graphgen router: {e}")
try:
from station.tools.datagen.api import router as datagen_router
app.include_router(datagen_router, prefix="/station")
except ImportError as e:
print(f"Warning: Could not load datagen router: {e}")
_module = importlib.import_module(f"station.tools.{_tool}.api")
app.include_router(_module.router, prefix="/station")
except Exception as e:
print(f"Warning: Could not load {_tool} router: {e}")
@app.get("/station/{path:path}")
@@ -541,6 +594,67 @@ def station_route(path: str):
# === Sidebar Wrapper (served at /spr/* when proxied) ===
# === Theme ===
#
# One stylesheet for every page, served the same way the sidebar is. tokens.css
# declares the variables and both theme files ship in the same response, so a
# page can switch themes without a second request and without FOUC.
def get_default_theme() -> str:
"""The theme a page is served in, before the browser has an opinion."""
framework = load_config().get("framework", {})
theme = framework.get("theme", "soleprint")
return theme if theme in ("soleprint", "mcrn") else "soleprint"
@app.get("/theme.css")
def theme_css():
"""Serve the theme contract plus every theme, concatenated."""
from fastapi.responses import Response
theme_dir = SPR_ROOT / "common" / "theme"
parts = []
tokens = theme_dir / "tokens.css"
if tokens.exists():
parts.append(tokens.read_text())
# Sorted so the response is byte-stable and cacheable; the theme files are
# scoped to [data-theme] selectors, so their order carries no meaning.
for sheet in sorted((theme_dir / "themes").glob("*.css")):
parts.append(f"\n/* ── {sheet.stem} ── */\n")
parts.append(sheet.read_text())
if not parts:
return Response(
content="/* theme not found — is common/theme/ present? */",
media_type="text/css",
)
return Response(content="".join(parts), media_type="text/css")
@app.get("/theme.js")
def theme_js():
"""Serve the theme switcher."""
from fastapi.responses import Response
js_path = SPR_ROOT / "common" / "theme" / "theme.js"
if js_path.exists():
return Response(
content=js_path.read_text(), media_type="application/javascript"
)
return Response(
content="/* theme.js not found */", media_type="application/javascript"
)
@app.get("/api/theme")
def theme_config():
"""The server-side default, for pages that render their own <html> tag."""
return {"theme": get_default_theme(), "themes": ["soleprint", "mcrn"]}
@app.get("/sidebar.css")
def sidebar_css():
"""Serve sidebar CSS for injection."""

91
soleprint/station/cabinets/README.md Normal file
View File

@@ -0,0 +1,91 @@
# Cabinets
A cabinet is a **dependency container** a room can switch on: postgres, redis,
airflow. The vocabulary already had the word — `execution.container` in every
room's `config.json` is *"Cabinet — tool container"* — and until now nothing
stood behind it.
The problem it solves is that a generated artifact knows what it needs and had
no way to say so. A shunt built from a client's spreadsheets can hold its rows
in memory, but the moment you want them to survive a restart you need postgres,
and wiring postgres in meant hand-editing a room's `docker-compose.yml` and then
hand-editing the cluster too. A cabinet is that declaration, made once and read
by both paths.
```
soleprint/station/cabinets/<name>/
cabinet.json what it is, what it needs, what it exports
service.yml the compose service, verbatim
```
## Turning one on
Add `cfg/<room>/data/cabinets.json` — the same shape as its sibling `data/*.json`
files:
```json
[
{ "name": "postgres" },
{ "name": "redis" },
{ "name": "airflow", "env": { "AIRFLOW_ADMIN_PASSWORD": "change-me" } }
]
```
Then build. `python build.py --cfg <room>` merges each cabinet's `service.yml`
into the room's `docker-compose.yml` and appends its settings to `.env.example`:
```bash
python build.py --cfg sample
cd gen/sample && docker compose up -d
```
**A service the room already declares wins.** `cfg/amar/docker-compose.yml`
ships its own `db`; switching on the postgres cabinet will not overwrite it.
Build says so when it skips one.
## On a cluster
`cabinet.json` names a `rig_addon`. Where the room runs on kind rather than
compose, the same dependency installs as a rig addon of that name:
```bash
cd rig
PROFILE=data make cluster up
PROFILE=data make addons install
```
The two paths are deliberately separate — compose for a laptop, helm for a
cluster — and `rig_addon` is the thread between them, so a room declares the
dependency once either way.
## Writing one
`cabinet.json`:
| Key | Purpose |
| --- | --- |
| `name` | must match the directory |
| `title`, `description` | shown on the station index |
| `image` | for the record; `service.yml` is what runs |
| `service` | the key to merge under in `services:` (defaults to `name`) |
| `env` | settings and defaults, written to `.env.example` |
| `volumes` | named volumes to declare at the top level |
| `depends_on` | other cabinets that must come with it |
| `rig_addon` | the matching `rig/ctrl/addons/<name>.sh`, if there is one |
| `ports` | host ports it wants, for the collision note in the docs |
`service.yml` is a plain compose fragment — one top-level key, the service name:
```yaml
postgres:
image: postgres:16-alpine
environment:
POSTGRES_DB: ${POSTGRES_DB:-soleprint}
...
```
Kept as YAML rather than generated from JSON so it reads like the compose file it
becomes, and so anything compose supports is available without this tool
learning about it first.
Adding a cabinet is adding a directory. Nothing dispatches on the name.

22
soleprint/station/cabinets/airflow/cabinet.json Normal file
View File

@@ -0,0 +1,22 @@
{
"name": "airflow",
"title": "Apache Airflow",
"description": "Scheduled pipelines. DAGs live in the room, under dags/.",
"image": "apache/airflow:2.10.4",
"service": "airflow",
"rig_addon": "airflow",
"ports": [8080],
"volumes": ["airflow_logs"],
"depends_on": ["postgres", "redis"],
"env": {
"AIRFLOW_PORT": "8080",
"AIRFLOW_ADMIN_USER": "admin",
"AIRFLOW_ADMIN_PASSWORD": "change-me",
"AIRFLOW_DAGS_DIR": "./dags"
},
"notes": [
"Brings postgres and redis with it — Airflow needs a metadata database and a broker, and will not start without both.",
"LocalExecutor by default: one container, no separate worker. Switch to CeleryExecutor in service.yml when the room outgrows it.",
"8080 collides with almost everything. Set AIRFLOW_PORT in the room's .env."
]
}

37
soleprint/station/cabinets/airflow/service.yml Normal file
View File

@@ -0,0 +1,37 @@
# Airflow — scheduler and webserver in one container, on LocalExecutor.
#
# One container rather than the five the official compose file ships, because a
# room switching this on wants pipelines, not a distributed deployment. The
# metadata database is the postgres cabinet, so the two arrive together; moving
# to CeleryExecutor is changing the executor here and adding a worker service.
airflow:
image: apache/airflow:2.10.4
container_name: ${DEPLOYMENT_NAME:-soleprint}_airflow
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: >-
postgresql+psycopg2://${POSTGRES_USER:-soleprint}:${POSTGRES_PASSWORD:-change-me}@postgres:5432/${POSTGRES_DB:-soleprint}
AIRFLOW__CELERY__BROKER_URL: redis://redis:6379/0
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
# Without a fixed key, every restart invalidates stored connections.
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-}
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "true"
_AIRFLOW_DB_MIGRATE: "true"
_AIRFLOW_WWW_USER_CREATE: "true"
_AIRFLOW_WWW_USER_USERNAME: ${AIRFLOW_ADMIN_USER:-admin}
_AIRFLOW_WWW_USER_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-change-me}
volumes:
- ${AIRFLOW_DAGS_DIR:-./dags}:/opt/airflow/dags
- airflow_logs:/opt/airflow/logs
ports:
- "${AIRFLOW_PORT:-8080}:8080"
# `standalone` runs the migration, creates the admin user, and starts both
# the scheduler and the webserver — the whole first-boot sequence that the
# official compose file spreads across an init container and four services.
command: standalone
restart: unless-stopped

20
soleprint/station/cabinets/postgres/cabinet.json Normal file
View File

@@ -0,0 +1,20 @@
{
"name": "postgres",
"title": "PostgreSQL",
"description": "Relational database. Backs rooms that need their data to outlive a restart.",
"image": "postgres:16-alpine",
"service": "postgres",
"rig_addon": "postgres",
"ports": [5432],
"volumes": ["pgdata"],
"env": {
"POSTGRES_DB": "soleprint",
"POSTGRES_USER": "soleprint",
"POSTGRES_PASSWORD": "change-me",
"POSTGRES_PORT": "5432"
},
"notes": [
"POSTGRES_PASSWORD is a placeholder. Set the real one in the room's .env, which is gitignored.",
"The connection string other services want is postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}"
]
}

24
soleprint/station/cabinets/postgres/service.yml Normal file
View File

@@ -0,0 +1,24 @@
# PostgreSQL — merged into a room's docker-compose.yml when the room asks for it.
#
# The healthcheck is not decoration: anything with `depends_on: condition:
# service_healthy` waits on it, and without one a backend races the database on
# every cold start and fails its first migration.
postgres:
image: postgres:16-alpine
container_name: ${DEPLOYMENT_NAME:-soleprint}_postgres
environment:
POSTGRES_DB: ${POSTGRES_DB:-soleprint}
POSTGRES_USER: ${POSTGRES_USER:-soleprint}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-change-me}
volumes:
- pgdata:/var/lib/postgresql/data
ports:
# Bound on the host so psql and databrowse can reach it from outside the
# compose network. Override POSTGRES_PORT when 5432 is already taken.
- "${POSTGRES_PORT:-5432}:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-soleprint} -d ${POSTGRES_DB:-soleprint}"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped

17
soleprint/station/cabinets/redis/cabinet.json Normal file
View File

@@ -0,0 +1,17 @@
{
"name": "redis",
"title": "Redis",
"description": "In-memory store. Cache, and the broker Celery and Airflow run their queues on.",
"image": "redis:7-alpine",
"service": "redis",
"rig_addon": "redis",
"ports": [6379],
"volumes": ["redisdata"],
"env": {
"REDIS_PORT": "6379"
},
"notes": [
"The URL other services want is redis://redis:6379/0",
"Airflow depends on this one; switching airflow on brings it along."
]
}

14
soleprint/station/cabinets/redis/service.yml Normal file
View File

@@ -0,0 +1,14 @@
# Redis — cache, and the broker for anything queue-shaped in the room.
redis:
image: redis:7-alpine
container_name: ${DEPLOYMENT_NAME:-soleprint}_redis
volumes:
- redisdata:/data
ports:
- "${REDIS_PORT:-6379}:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped

66
soleprint/station/index.html
View File

@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
@@ -14,7 +14,7 @@
box-sizing: border-box;
}
html {
background: #0a0a0a;
background: var(--bg);
}
body {
font-family:
@@ -25,8 +25,8 @@
margin: 0 auto;
padding: 2rem 1rem;
line-height: 1.6;
color: #e5e5e5;
background: #1d4ed8;
color: var(--text);
background: var(--system-accent);
}
header {
display: flex;
@@ -51,7 +51,7 @@
padding-bottom: 2rem;
}
section {
background: #0a0a0a;
background: var(--bg);
padding: 1.5rem;
margin: 1.5rem 0;
border-radius: 12px;
@@ -59,23 +59,23 @@
section h2 {
margin: 0 0 1rem 0;
font-size: 1.2rem;
color: #93c5fd;
color: var(--system-accent-text);
}
.composition {
background: #1a1a1a;
border: 2px solid #1d4ed8;
background: var(--surface);
border: 2px solid var(--system-accent);
padding: 1rem;
border-radius: 12px;
}
.composition h3 {
margin: 0 0 0.75rem 0;
font-size: 1.1rem;
color: #93c5fd;
color: var(--system-accent-text);
}
.composition > p {
margin: 0 0 1rem 0;
font-size: 0.9rem;
color: #a3a3a3;
color: var(--muted);
}
.components {
display: grid;
@@ -83,20 +83,20 @@
gap: 0.75rem;
}
.component {
background: #0a0a0a;
border: 1px solid #3f3f3f;
background: var(--bg);
border: 1px solid var(--border-strong);
padding: 0.75rem;
border-radius: 8px;
}
.component h4 {
margin: 0 0 0.25rem 0;
font-size: 0.95rem;
color: #93c5fd;
color: var(--system-accent-text);
}
.component p {
margin: 0;
font-size: 0.85rem;
color: #a3a3a3;
color: var(--muted);
}
.tables {
list-style: none;
@@ -105,7 +105,7 @@
}
.tables li {
padding: 0.75rem 0;
border-bottom: 1px solid #3f3f3f;
border-bottom: 1px solid var(--border-strong);
display: flex;
justify-content: space-between;
align-items: center;
@@ -116,32 +116,32 @@
.tables .name {
font-weight: 500;
text-decoration: none;
color: #e5e5e5;
color: var(--text);
}
.tables a.name:hover {
color: #93c5fd;
color: var(--system-accent-text);
}
.status {
font-size: 0.75rem;
padding: 0.2rem 0.5rem;
border-radius: 4px;
text-transform: uppercase;
background: #2a2a2a;
color: #a3a3a3;
background: var(--border);
color: var(--muted);
}
.health {
display: inline-block;
margin-top: 1rem;
padding: 0.5rem 1rem;
background: #1a1a1a;
border: 1px solid #3f3f3f;
background: var(--surface);
border: 1px solid var(--border-strong);
border-radius: 4px;
font-family: monospace;
color: #93c5fd;
color: var(--system-accent-text);
text-decoration: none;
}
.health:hover {
background: #2a2a2a;
background: var(--border);
}
footer {
margin-top: 3rem;
@@ -157,6 +157,11 @@
opacity: 0.5;
}
</style>
<link rel="stylesheet" href="/theme.css">
<style>
/* Station keeps its own colour under every theme. */
:root { --system-accent: #1d4ed8; --system-accent-text: #93c5fd; }
</style>
</head>
<body>
<header style="position: relative">
@@ -217,6 +222,20 @@
</ul>
</section>
<section>
<h2>{{ (components.container.plural or 'cabinets')|title }}</h2>
<ul class="tables">
{% for cabinet in cabinets %}
<li>
<span class="name">{{ cabinet.title }}</span
><span class="status">{{ cabinet.status }}</span>
</li>
{% else %}
<li><span class="name">--</span></li>
{% endfor %}
</ul>
</section>
<section>
<h2>{{ (components.watcher.plural or 'monitors')|title }}</h2>
<ul class="tables">
@@ -263,5 +282,6 @@
{% if soleprint_url %}<a href="{{ soleprint_url }}">← Soleprint</a
>{% else %}<span class="disabled">← Soleprint</span>{% endif %}
</footer>
<script src="/theme.js" defer></script>
</body>
</html>

19
soleprint/station/tools/datagen/templates/index.html
View File

@@ -1,27 +1,19 @@
<!DOCTYPE html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>datagen — Test Data Generator</title>
<!-- Palette, fonts and the theme switcher. The :root block that used to sit
here was one of eight copies that had already drifted apart. -->
<link rel="stylesheet" href="/theme.css">
<style>
:root {
--bg: #0a0a0a;
--surface: #1a1a1a;
--border: #333;
--text: #e5e5e5;
--muted: #a3a3a3;
--dim: #666;
--amber: #d4a574;
--amber-dim: #b8956a;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
background: var(--bg);
color: var(--text);
font-family: system-ui, -apple-system, sans-serif;
font-family: var(--font-ui);
font-size: 13px;
height: 100vh;
display: flex;
@@ -493,5 +485,6 @@ document.addEventListener('keydown', e => {
init();
</script>
<script src="/theme.js" defer></script>
</body>
</html>

18
soleprint/station/tools/graphgen/templates/index.html
View File

@@ -1,27 +1,18 @@
<!DOCTYPE html>
<html lang="en">
<html lang="en" data-theme="soleprint">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>graphgen — Schema Explorer</title>
<!-- Palette, fonts and the theme switcher — see common/theme/. -->
<link rel="stylesheet" href="/theme.css">
<style>
:root {
--bg: #0a0a0a;
--surface: #1a1a1a;
--border: #333;
--text: #e5e5e5;
--muted: #a3a3a3;
--dim: #666;
--amber: #d4a574;
--amber-dim: #b8956a;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
background: var(--bg);
color: var(--text);
font-family: system-ui, -apple-system, sans-serif;
font-family: var(--font-ui);
font-size: 13px;
overflow: hidden;
height: 100vh;
@@ -725,5 +716,6 @@ function svgEl(tag) {
applyViewport();
init();
</script>
<script src="/theme.js" defer></script>
</body>
</html>

148
soleprint/station/tools/modelgen/__main__.py
View File

@@ -6,7 +6,10 @@ Generates typed models from various sources to various formats.
Input sources:
- from-config: Configuration files (soleprint config.json style)
- from-schema: Python dataclasses in schema/ folder
- extract: Existing codebases (Django, SQLAlchemy, Prisma)
- extract: Existing codebases (Django, SQLAlchemy)
- from-db: A live database, any SQLAlchemy dialect
- from-openapi: An OpenAPI 3.x / Swagger 2.0 document
- from-tabular: A directory of CSV/TSV/ODS spreadsheets
Output formats:
- pydantic: Pydantic BaseModel classes
@@ -14,12 +17,16 @@ Output formats:
- typescript: TypeScript interfaces
- protobuf: Protocol Buffer definitions
- prisma: Prisma schema
- schema: graphgen-compatible schema.json
- datagen: BaseDataGenerator subclass for station's datagen tool
Usage:
python -m soleprint.station.tools.modelgen --help
python -m soleprint.station.tools.modelgen from-config -c config.json -o models.py
python -m soleprint.station.tools.modelgen from-schema -o models/ --targets pydantic,typescript
python -m soleprint.station.tools.modelgen extract --source /path/to/django --targets pydantic
python -m soleprint.station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,schema
python -m soleprint.station.tools.modelgen from-tabular -s ./sheets -o out/ -t pydantic,datagen
python -m soleprint.station.tools.modelgen generate --config schema/modelgen.json
"""
@@ -225,6 +232,88 @@ def cmd_from_db(args):
print("Done!")
def _emit(payload, targets_arg: str, output: str) -> None:
"""Run one extraction result through every requested target.
The three older commands each carry their own copy of this loop; the two
below share it, because they also have to pass datasets through and a
fourth copy would be a fourth place to keep in step.
"""
targets = [t.strip() for t in targets_arg.split(",") if t.strip()]
output_dir = Path(output)
for target in targets:
if target not in GENERATORS:
print(f"Warning: Unknown target '{target}', skipping", file=sys.stderr)
continue
generator = GENERATORS[target]()
ext = generator.file_extension()
# Determine output filename (use target name to avoid overwrites)
if len(targets) == 1 and output.endswith(ext):
output_file = output_dir
else:
output_file = output_dir / f"models_{target}{ext}"
print(f"Generating {target} to: {output_file}")
generator.generate(payload, output_file)
print("Done!")
def cmd_from_openapi(args):
"""Generate models from an OpenAPI 3.x / Swagger 2.0 document."""
from .loader.extract.openapi import OpenAPIExtractor
spec_path = Path(args.spec)
if not spec_path.exists():
print(f"Error: Spec not found: {spec_path}", file=sys.stderr)
sys.exit(1)
extractor = OpenAPIExtractor(spec_path)
print(f"Reading spec: {spec_path}")
try:
models, enums = extractor.extract()
except (RuntimeError, ValueError) as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
endpoints = extractor.endpoints()
print(
f"Extracted {len(models)} models, {len(enums)} enums, "
f"{len(endpoints)} endpoints"
)
_emit((models, enums), args.targets, args.output)
def cmd_from_tabular(args):
"""Generate models from a directory of CSV/TSV/ODS spreadsheets."""
from .loader.extract.tabular import TabularExtractor
source_path = Path(args.source)
if not source_path.exists():
print(f"Error: Source not found: {source_path}", file=sys.stderr)
sys.exit(1)
extractor = TabularExtractor(source_path)
print(f"Reading sheets: {source_path}")
try:
models, enums = extractor.extract()
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
datasets = extractor.datasets()
rows = sum(len(d.rows) for d in datasets)
print(f"Extracted {len(models)} models, {rows} rows")
# Datasets ride along as a third element: the datagen target seeds from
# them, and every other target ignores the extra slot.
_emit((models, enums, datasets), args.targets, args.output)
def cmd_generate(args):
"""Generate all targets from a JSON config file."""
import json
@@ -430,6 +519,63 @@ def main():
)
db_parser.set_defaults(func=cmd_from_db)
# from-openapi command (service contract -> models)
openapi_parser = subparsers.add_parser(
"from-openapi",
help="Generate models from an OpenAPI 3.x / Swagger 2.0 document",
)
openapi_parser.add_argument(
"--spec",
"-s",
type=str,
required=True,
help="Path to the spec (.json, .yaml or .yml; YAML needs PyYAML)",
)
openapi_parser.add_argument(
"--output",
"-o",
type=str,
required=True,
help="Output path (file or directory)",
)
openapi_parser.add_argument(
"--targets",
"-t",
type=str,
default="pydantic",
help=f"Comma-separated output targets ({formats_str})",
)
openapi_parser.set_defaults(func=cmd_from_openapi)
# from-tabular command (spreadsheets -> models + rows)
tabular_parser = subparsers.add_parser(
"from-tabular",
help="Generate models from a directory of CSV/TSV/ODS spreadsheets",
)
tabular_parser.add_argument(
"--source",
"-s",
type=str,
required=True,
help="Directory of sheets (or a single .csv/.tsv/.ods file)",
)
tabular_parser.add_argument(
"--output",
"-o",
type=str,
required=True,
help="Output path (file or directory)",
)
tabular_parser.add_argument(
"--targets",
"-t",
type=str,
default="pydantic",
help=f"Comma-separated output targets ({formats_str}). "
"The datagen target seeds from the imported rows.",
)
tabular_parser.set_defaults(func=cmd_from_tabular)
# generate command (config-driven multi-target)
gen_parser = subparsers.add_parser(
"generate",

4
soleprint/station/tools/modelgen/generator/__init__.py
View File

@@ -8,11 +8,13 @@ Supported generators:
- ProtobufGenerator: Protocol Buffer definitions
- PrismaGenerator: Prisma schema
- StrawberryGenerator: Strawberry type/input/enum classes
- DatagenGenerator: BaseDataGenerator subclass for station's datagen tool
"""
from typing import Dict, Type
from .base import BaseGenerator
from .datagen import DatagenGenerator
from .django import DjangoGenerator
from .jsonschema import JsonSchemaGenerator
from .prisma import PrismaGenerator
@@ -35,10 +37,12 @@ GENERATORS: Dict[str, Type[BaseGenerator]] = {
"strawberry": StrawberryGenerator,
"schema": JsonSchemaGenerator,
"jsonschema": JsonSchemaGenerator, # Alias
"datagen": DatagenGenerator,
}
__all__ = [
"BaseGenerator",
"DatagenGenerator",
"PydanticGenerator",
"DjangoGenerator",
"StrawberryGenerator",

399
soleprint/station/tools/modelgen/generator/datagen.py Normal file
View File

@@ -0,0 +1,399 @@
"""
Datagen Generator
Emits a BaseDataGenerator subclass — the bridge between modelgen (which knows
the shapes) and datagen (which hands out instances of them).
The contract it targets is datagen/base.py: a method per model, named the way
`generate()` looks it up (`model.lower()`), plus a `schema()` override in the
graphgen-compatible format so the result also renders in graphgen with no extra
work.
Two modes, chosen by what the loader had:
- No rows → every field is synthesised from its type.
- Rows present → the generated class samples depot/data.json, so a shunt
built from a spreadsheet answers with the real values and
only invents where it must.
Synthesis uses random/uuid/datetime from the standard library rather than
faker, which is not a dependency of this repo despite what datagen's README
examples suggest.
"""
import json
from enum import Enum
from pathlib import Path
from pprint import pformat
from typing import Any, Dict, List, Optional
from ..helpers import unwrap_optional
from ..loader.schema import ModelDefinition
from .base import BaseGenerator
# Field-name heuristics. A column called `email` deserves something that looks
# like an email — a mock whose every string is "string_4" is hard to read and
# hard to demo.
_NAME_HINTS = (
("email", '"user{n}@example.com".format(n=random.randint(1, 999))'),
("phone", '"+1-555-{n:04d}".format(n=random.randint(0, 9999))'),
("url", '"https://example.com/{n}".format(n=random.randint(1, 999))'),
("slug", 'random.choice(_WORDS) + "-" + str(random.randint(1, 99))'),
("first_name", "random.choice(_FIRST_NAMES)"),
("last_name", "random.choice(_LAST_NAMES)"),
("name", 'random.choice(_FIRST_NAMES) + " " + random.choice(_LAST_NAMES)'),
("title", 'random.choice(_WORDS).title() + " " + random.choice(_WORDS)'),
("description", '" ".join(random.choices(_WORDS, k=8))'),
("address", '"{n} ".format(n=random.randint(1, 9999)) + random.choice(_WORDS).title() + " St"'),
("city", "random.choice(_CITIES)"),
("country", "random.choice(_COUNTRIES)"),
("currency", 'random.choice(["USD", "EUR", "ARS", "GBP"])'),
("status", 'random.choice(["active", "pending", "closed"])'),
("code", '"".join(random.choices("ABCDEFGHJKLMNPQRSTUVWXYZ0123456789", k=6))'),
("token", "uuid.uuid4().hex"),
("color", '"#{n:06x}".format(n=random.randint(0, 0xFFFFFF))'),
)
_HEADER_HELPERS = '''
_WORDS = [
"alpha", "bravo", "cobalt", "delta", "ember", "falcon", "granite", "harbor",
"indigo", "juniper", "kestrel", "lumen", "meridian", "nimbus", "onyx",
]
_FIRST_NAMES = ["Ada", "Bruno", "Camila", "Diego", "Elena", "Facundo", "Gabriela", "Hugo"]
_LAST_NAMES = ["Alvarez", "Bianchi", "Castro", "Duarte", "Esposito", "Ferrari", "Gomez"]
_CITIES = ["Buenos Aires", "Rosario", "Cordoba", "Montevideo", "Santiago", "Lisbon"]
_COUNTRIES = ["AR", "UY", "CL", "BR", "PT", "ES"]
'''
_FALLBACK_BASE = '''
# Standalone fallback: a generated generator has to keep working inside a
# shunt, which runs as its own process with no soleprint on the path. This
# mirrors station/tools/datagen/base.py.
class BaseDataGenerator: # type: ignore[no-redef]
"""Minimal stand-in for station.tools.datagen.base.BaseDataGenerator."""
_RESERVED = frozenset({"generate", "available_models", "schema"})
def generate(self, model: str, count: int = 1, **kwargs) -> list:
method = getattr(self, model.lower(), None)
if method is None or not callable(method):
raise ValueError(
f"No generator for '{model}'. Available: {self.available_models()}"
)
return [method(**kwargs) for _ in range(count)]
def available_models(self) -> list:
return sorted(
name for name in dir(self)
if not name.startswith("_")
and name not in self._RESERVED
and callable(getattr(self, name))
)
def schema(self):
return None
'''
class DatagenGenerator(BaseGenerator):
"""Generates a BaseDataGenerator subclass from model definitions."""
def __init__(
self,
name_map: Optional[Dict[str, str]] = None,
class_name: Optional[str] = None,
depot: str = "depot/data.json",
):
super().__init__(name_map)
self.class_name = class_name
# Relative to the generated file, so the shunt directory stays movable.
self.depot = depot
def file_extension(self) -> str:
return ".py"
def generate(self, models, output_path: Path) -> None:
model_defs, _enums, datasets = self._unpack(models)
output_path = Path(output_path)
if output_path.suffix != ".py":
output_path = output_path / "generated_datagen.py"
output_path.parent.mkdir(parents=True, exist_ok=True)
class_name = self.class_name or self._derive_class_name(output_path)
seeded = {d.model for d in datasets if d.rows}
model_names = {self.map_name(m.name) for m in model_defs}
lines = self._header(class_name, bool(seeded))
for model_def in model_defs:
lines.extend(self._model_method(model_def, model_names, seeded))
lines.append("")
lines.extend(self._schema_method(model_defs, model_names))
lines.append("")
lines.extend(self._helpers(bool(seeded)))
output_path.write_text("\n".join(lines))
# ── input handling ─────────────────────────────────────────────────────
@staticmethod
def _unpack(models) -> tuple:
"""Accept the shapes the other generators accept, plus datasets."""
if isinstance(models, tuple):
model_defs = list(models[0])
enum_defs = list(models[1]) if len(models) > 1 else []
datasets = list(models[2]) if len(models) > 2 else []
return model_defs, enum_defs, datasets
if hasattr(models, "models"):
# SchemaLoader
model_defs = list(models.models) + list(getattr(models, "api_models", []))
return model_defs, list(getattr(models, "enums", [])), []
if isinstance(models, list):
return list(models), [], []
raise ValueError(f"Unsupported input type: {type(models)}")
@staticmethod
def _derive_class_name(output_path: Path) -> str:
stem = output_path.stem.replace("datagen_", "").replace("_datagen", "")
parts = [p for p in stem.replace("-", "_").split("_") if p]
base = "".join(p[:1].upper() + p[1:] for p in parts) or "Generated"
return f"{base}Generator"
# ── emission ───────────────────────────────────────────────────────────
def _header(self, class_name: str, seeded: bool) -> List[str]:
lines = [
'"""',
"Data generator - GENERATED FILE",
"",
"Do not edit directly. Regenerate using modelgen (target: datagen).",
'"""',
"",
"import json",
"import random",
"import uuid",
"from datetime import datetime, timedelta, timezone",
"from pathlib import Path",
"",
"try:",
" from station.tools.datagen.base import BaseDataGenerator",
"except ImportError: # pragma: no cover - standalone use",
_FALLBACK_BASE.strip("\n"),
"",
_HEADER_HELPERS.strip("\n"),
"",
"",
f"class {class_name}(BaseDataGenerator):",
' """Generated from a modelgen schema."""',
"",
]
if seeded:
lines.extend([
f' _DEPOT = Path(__file__).parent / "{self.depot}"',
"",
" def __init__(self):",
" self._seed = {}",
" if self._DEPOT.exists():",
" try:",
" self._seed = json.loads(self._DEPOT.read_text())",
" except (OSError, ValueError):",
" self._seed = {}",
"",
])
return lines
def _model_method(
self, model_def: ModelDefinition, model_names: set, seeded: set
) -> List[str]:
mapped = self.map_name(model_def.name)
doc = (model_def.docstring or mapped).strip().splitlines()[0]
lines = [
f" def {mapped.lower()}(self, **kwargs) -> dict:",
f' """{doc}"""',
]
if model_def.name in seeded:
# Real rows first; synthesis is the fallback for when the depot is
# missing or a caller asked for more rows than were imported.
lines.append(f' record = self._sample("{model_def.name}")')
lines.append(" if record is None:")
indent = " "
else:
indent = " "
if model_def.fields:
lines.append(f"{indent}record = {{")
for field in model_def.fields:
value = self._value(field, model_names)
lines.append(f'{indent} "{field.name}": {value},')
lines.append(f"{indent}}}")
else:
lines.append(f"{indent}record = {{}}")
lines.append(" record.update(kwargs)")
lines.append(" return record")
return lines
def _value(self, field: Any, model_names: set) -> str:
base, _ = unwrap_optional(field.type_hint)
name = field.name.lower()
if getattr(field, "primary_key", False):
if base in (int, "bigint"):
return "self._next_id()"
return "str(uuid.uuid4())"
fk = getattr(field, "foreign_key", None)
if fk:
target = self.map_name(fk)
# How a relation is carried depends on the shape it was found in.
# A `customer_id` column holds a key; a `category` property in a
# spec holds the object itself, and answering that one with a key
# would be the wrong shape, not merely a dull value.
if base == "dict":
return f'self._nested("{target}")'
if base == "list":
return f'self._nested_list("{target}")'
if base in (int, "bigint"):
# A plausible existing key, not a fresh one: a foreign key that
# never matches anything makes the mock useless for joins.
return "random.randint(1, 100)"
return "str(uuid.uuid4())"
if isinstance(base, type) and issubclass(base, Enum):
values = json.dumps([m.value for m in base])
return f"random.choice({values})"
if base is bool:
return "random.choice([True, False])"
if base in (int, "bigint"):
return "random.randint(1, 10000)"
if base is float:
return "round(random.uniform(1, 10000), 2)"
if base == "UUID":
return "str(uuid.uuid4())"
if base == "datetime":
return "self._recent()"
if base == "dict":
return "{}"
if base == "list":
return "[]"
if base == "bytes":
return '""'
for hint, expression in _NAME_HINTS:
if hint in name:
return expression
return 'random.choice(_WORDS) + "-" + str(random.randint(1, 999))'
def _schema_method(
self, model_defs: List[ModelDefinition], model_names: set
) -> List[str]:
schema: Dict[str, Any] = {"models": {}}
for model_def in model_defs:
mapped = self.map_name(model_def.name)
entry: Dict[str, Any] = {}
if model_def.docstring:
entry["doc"] = model_def.docstring.strip().splitlines()[0]
fields: Dict[str, Any] = {}
for field in model_def.fields:
fields[field.name] = self._schema_field(field, model_names)
entry["fields"] = fields
schema["models"][mapped] = entry
# pformat, not json.dumps: the result is spliced into a source file, so
# it has to be a Python literal. Post-processing JSON with string
# replacement would rewrite "true" inside a docstring too.
body = pformat(schema, indent=4, width=88, sort_dicts=False)
indented = "\n".join(f" {line}" for line in body.splitlines())
return [
" def schema(self) -> dict:",
' """Graphgen-compatible schema — surfaced at /tools/datagen/api/schema."""',
f" return {indented.lstrip()}",
]
def _schema_field(self, field: Any, model_names: set) -> Dict[str, Any]:
base, is_optional = unwrap_optional(field.type_hint)
fk = getattr(field, "foreign_key", None)
if fk:
relation = "M2M" if getattr(field, "many", False) else "FK"
type_value = f"{relation}:{self.map_name(fk)}"
elif isinstance(base, type) and issubclass(base, Enum):
type_value = base.__name__
elif isinstance(base, str):
type_value = base
elif hasattr(base, "__name__"):
type_value = base.__name__
else:
type_value = "Any"
out: Dict[str, Any] = {
"type": type_value,
"nullable": bool(getattr(field, "optional", False) or is_optional),
}
if getattr(field, "primary_key", False):
out["pk"] = True
if getattr(field, "unique", False):
out["unique"] = True
return out
@staticmethod
def _helpers(seeded: bool) -> List[str]:
lines = [
" # ── helpers ────────────────────────────────────────────────",
"",
" _counter = 0",
"",
" def _next_id(self) -> int:",
' """Monotonic ids, so generated relations can be joined."""',
" type(self)._counter += 1",
" return type(self)._counter",
"",
" @staticmethod",
" def _recent() -> str:",
' """An ISO timestamp within the last 90 days."""',
" moment = datetime.now(timezone.utc) - timedelta(",
" days=random.randint(0, 90), seconds=random.randint(0, 86399)",
" )",
" return moment.isoformat()",
"",
" # Nesting is capped rather than followed: schemas refer to each",
" # other in cycles, and an uncapped expansion recurses forever.",
" _depth = 0",
" _MAX_DEPTH = 2",
"",
" def _nested(self, model: str):",
' """One related object, or None once the depth cap is reached."""',
" if type(self)._depth >= self._MAX_DEPTH:",
" return None",
" method = getattr(self, model.lower(), None)",
" if not callable(method):",
" return None",
" type(self)._depth += 1",
" try:",
" return method()",
" finally:",
" type(self)._depth -= 1",
"",
" def _nested_list(self, model: str, count: int = 2) -> list:",
' """A short list of related objects."""',
" items = [self._nested(model) for _ in range(count)]",
" return [item for item in items if item is not None]",
]
if seeded:
lines.extend([
"",
" def _sample(self, model: str):",
' """A copy of a real imported row, or None if there are none."""',
" rows = self._seed.get(model) or []",
" if not rows:",
" return None",
" return dict(random.choice(rows))",
])
lines.append("")
return lines

3
soleprint/station/tools/modelgen/generator/jsonschema.py
View File

@@ -84,7 +84,8 @@ class JsonSchemaGenerator(BaseGenerator):
# Resolve the relationship-aware type string graphgen expects.
if fk_target:
type_value = f"FK:{self.map_name(fk_target)}"
relation = "M2M" if getattr(field, "many", False) else "FK"
type_value = f"{relation}:{self.map_name(fk_target)}"
elif type_str in model_names:
type_value = f"FK:{type_str}"
elif type_str == "M2M":

5
soleprint/station/tools/modelgen/generator/protobuf.py
View File

@@ -27,8 +27,9 @@ class ProtobufGenerator(BaseGenerator):
if hasattr(models, "grpc_messages"):
# SchemaLoader with gRPC definitions
content = self._generate_from_loader(models)
elif isinstance(models, tuple) and len(models) >= 3:
# (messages, service_def) tuple
elif isinstance(models, tuple) and len(models) >= 2:
# (models, enums, ...) tuple — same first two slots as every other
# generator; extra slots (datasets) belong to targets that use them.
content = self._generate_from_definitions(models[0], models[1])
elif isinstance(models, list):
# List of dataclasses (MPR style)

14
soleprint/station/tools/modelgen/loader/__init__.py
View File

@@ -8,8 +8,16 @@ Supported loaders:
"""
from .config import ConfigLoader, load_config
from .extract import EXTRACTORS, BaseExtractor, DjangoExtractor
from .extract import (
EXTRACTORS,
BaseExtractor,
DjangoExtractor,
OpenAPIExtractor,
TabularExtractor,
)
from .schema import (
DatasetDefinition,
EndpointDefinition,
EnumDefinition,
FieldDefinition,
GrpcServiceDefinition,
@@ -30,8 +38,12 @@ __all__ = [
"FieldDefinition",
"EnumDefinition",
"GrpcServiceDefinition",
"EndpointDefinition",
"DatasetDefinition",
# Extractors
"BaseExtractor",
"DjangoExtractor",
"OpenAPIExtractor",
"TabularExtractor",
"EXTRACTORS",
]

20
soleprint/station/tools/modelgen/loader/extract/__init__.py
View File

@@ -1,29 +1,41 @@
"""
Extractors - Extract model definitions from existing codebases.
Supported frameworks:
Supported sources:
- Django: Extract from Django ORM models
- SQLAlchemy: Extract from SQLAlchemy models (planned)
- Prisma: Extract from Prisma schema (planned)
- SQLAlchemy: Extract from SQLAlchemy models
- OpenAPI: Extract from an OpenAPI 3.x / Swagger 2.0 document
- Tabular: Extract from a directory of CSV/TSV/ODS spreadsheets
"""
from typing import Dict, Type
from .base import BaseExtractor
from .django import DjangoExtractor
from .openapi import OpenAPIExtractor
from .sqlalchemy_models import SqlAlchemyExtractor
from .tabular import TabularExtractor
# Registry of code-source extractors (auto-detectable via detect()).
# Registry of source extractors (auto-detectable via detect()).
#
# Ordering matters for `--framework auto`: detection runs in insertion order and
# stops at the first match, so the two that inspect a *file* come after the two
# that inspect a *source tree* and cannot be confused with them.
#
# Note: live-database introspection lives in database.py (DatabaseExtractor),
# invoked explicitly via the `from-db` command since it takes a URL, not a path.
EXTRACTORS: Dict[str, Type[BaseExtractor]] = {
"django": DjangoExtractor,
"sqlalchemy": SqlAlchemyExtractor,
"openapi": OpenAPIExtractor,
"tabular": TabularExtractor,
}
__all__ = [
"BaseExtractor",
"DjangoExtractor",
"SqlAlchemyExtractor",
"OpenAPIExtractor",
"TabularExtractor",
"EXTRACTORS",
]

440
soleprint/station/tools/modelgen/loader/extract/openapi.py Normal file
View File

@@ -0,0 +1,440 @@
"""
OpenAPI Extractor
Reads an OpenAPI 3.x or Swagger 2.0 document and produces modelgen's IR.
One parse yields two products:
extractor = OpenAPIExtractor("api.yaml")
models, enums = extractor.extract() # the shapes (BaseExtractor contract)
endpoints = extractor.endpoints() # the routes (what shuntgen consumes)
Only the first is part of the BaseExtractor contract; a spec is the one input
that describes calls as well as shapes, so endpoints() is an addition rather
than a widening of the ABC.
Referenced schemas become foreign_key metadata rather than nested types, which
is the same call DatabaseExtractor makes (see database.py) and for the same
reason: it keeps every generator emitting valid code, and graphgen still draws
the edge. Enums are materialised as real Enum classes so the existing
PYDANTIC_RESOLVERS/TS_RESOLVERS "enum" branch resolves them by name.
YAML is optional. JSON specs parse with the stdlib alone — which is what keeps
modelgen installable with no dependencies — and a .yaml spec needs PyYAML.
"""
import json
import re
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from ..schema import EndpointDefinition, EnumDefinition, FieldDefinition, ModelDefinition
from .base import BaseExtractor
_YAML_HINT = (
"Reading a YAML spec requires PyYAML. Install it with:\n"
" pip install pyyaml\n"
"(JSON specs need nothing beyond the standard library.)"
)
# OpenAPI `format` is more specific than `type`, so it wins where both are set.
_FORMAT_HINTS: Dict[str, Any] = {
"date-time": "datetime",
"date": "datetime",
"uuid": "UUID",
"binary": "bytes",
"byte": "bytes",
"int64": "bigint",
}
_TYPE_HINTS: Dict[str, Any] = {
"string": str,
"integer": int,
"number": float,
"boolean": bool,
"array": "list",
"object": "dict",
}
# Response codes worth reading a body off, best first. "default" is last
# because a spec that defines it usually means the error case.
_STATUS_PREFERENCE = ("200", "201", "2XX", "202", "203", "204", "default")
def _to_model_name(name: str) -> str:
"""Convert a schema name to PascalCase (pet_category -> PetCategory)."""
parts = [p for p in re.split(r"[^0-9a-zA-Z]+", name) if p]
return "".join(p[:1].upper() + p[1:] for p in parts) or name
def _to_member_name(value: Any) -> str:
"""Convert an enum value to a legal Python identifier."""
name = re.sub(r"[^0-9a-zA-Z]+", "_", str(value)).strip("_").upper()
if not name:
return "EMPTY"
if name[0].isdigit():
return f"V_{name}"
return name
def _load_document(path: Path) -> dict:
"""Parse a spec file. JSON via stdlib; YAML only if PyYAML is installed."""
text = path.read_text()
if path.suffix.lower() in (".yaml", ".yml"):
try:
import yaml
except ImportError as e: # pragma: no cover - exercised only without the extra
raise RuntimeError(_YAML_HINT) from e
return yaml.safe_load(text) or {}
try:
return json.loads(text)
except ValueError:
# A spec is often handed over with no extension or a wrong one, so fall
# back to YAML rather than failing on what is really a naming mistake.
try:
import yaml
except ImportError as e:
raise ValueError(
f"{path} is not valid JSON, and YAML support is unavailable.\n{_YAML_HINT}"
) from e
return yaml.safe_load(text) or {}
class OpenAPIExtractor(BaseExtractor):
"""Extracts modelgen IR from an OpenAPI 3.x / Swagger 2.0 document."""
def __init__(self, source_path):
super().__init__(source_path)
self._doc: Optional[dict] = None
self._enums: Dict[str, EnumDefinition] = {}
self._enum_types: Dict[str, type] = {}
# ── document access ────────────────────────────────────────────────────
@property
def doc(self) -> dict:
if self._doc is None:
self._doc = _load_document(self.source_path)
return self._doc
@property
def is_swagger2(self) -> bool:
return "swagger" in self.doc and "openapi" not in self.doc
def _schemas(self) -> Dict[str, dict]:
"""Named schemas, wherever this spec version keeps them."""
if self.is_swagger2:
return self.doc.get("definitions", {}) or {}
return (self.doc.get("components", {}) or {}).get("schemas", {}) or {}
def detect(self) -> bool:
"""True when the source is a parseable document declaring a spec version."""
if not self.source_path.is_file():
return False
if self.source_path.suffix.lower() not in (".json", ".yaml", ".yml"):
return False
try:
doc = self.doc
except Exception:
return False
return isinstance(doc, dict) and ("openapi" in doc or "swagger" in doc)
# ── $ref plumbing ──────────────────────────────────────────────────────
@staticmethod
def _ref_name(ref: str) -> str:
return _to_model_name(ref.rsplit("/", 1)[-1])
def _deref(self, schema: Any, _seen: Optional[set] = None) -> dict:
"""Follow $ref chains to the schema they land on."""
if not isinstance(schema, dict):
return {}
seen = _seen or set()
ref = schema.get("$ref")
if not ref or ref in seen:
return schema
seen.add(ref)
target = self.doc
for part in ref.lstrip("#/").split("/"):
if not isinstance(target, dict):
return {}
target = target.get(part, {})
return self._deref(target, seen)
def _flatten(self, schema: dict) -> Tuple[Dict[str, dict], List[str]]:
"""Resolve allOf/$ref into one property map plus the required names."""
schema = self._deref(schema)
props: Dict[str, dict] = {}
required: List[str] = []
for sub in schema.get("allOf", []) or []:
sub_props, sub_required = self._flatten(sub)
props.update(sub_props)
required.extend(sub_required)
# anyOf/oneOf describe alternatives, not a single shape. Taking the
# first branch's properties beats emitting an empty model — a shunt
# answering with one valid variant is more useful than one answering {}.
for key in ("oneOf", "anyOf"):
for sub in (schema.get(key) or [])[:1]:
sub_props, sub_required = self._flatten(sub)
props.update(sub_props)
required.extend(sub_required)
props.update(schema.get("properties", {}) or {})
required.extend(schema.get("required", []) or [])
return props, required
# ── type resolution ────────────────────────────────────────────────────
def _register_enum(self, owner: str, field_name: str, values: List[Any]) -> type:
"""Materialise a spec enum as a real Enum class, deduped by value set."""
name = f"{_to_model_name(owner)}{_to_model_name(field_name)}"
pairs = [(_to_member_name(v), str(v)) for v in values]
# Two fields can legitimately declare the same enum name with different
# members; suffix rather than let the second silently win.
existing = self._enums.get(name)
if existing and existing.values != pairs:
suffix = 2
while f"{name}{suffix}" in self._enums and (
self._enums[f"{name}{suffix}"].values != pairs
):
suffix += 1
name = f"{name}{suffix}"
if name not in self._enums:
self._enums[name] = EnumDefinition(name=name, values=pairs)
self._enum_types[name] = Enum(name, pairs)
return self._enum_types[name]
def _resolve(
self, owner: str, field_name: str, prop: dict
) -> Tuple[Any, Optional[str], bool]:
"""Map a property schema to (type_hint, foreign_key, many)."""
ref = prop.get("$ref")
if ref:
target = self._ref_name(ref)
resolved = self._deref(prop)
# A $ref to an enum or a plain scalar is not a relationship — only
# a ref to an object shape is.
if resolved.get("enum"):
return self._register_enum(owner, field_name, resolved["enum"]), None, False
if resolved.get("type") in _TYPE_HINTS and resolved.get("type") != "object":
return self._resolve(owner, field_name, resolved)
return "dict", target, False
if prop.get("enum") and prop.get("type", "string") == "string":
return self._register_enum(owner, field_name, prop["enum"]), None, False
if prop.get("allOf") or prop.get("oneOf") or prop.get("anyOf"):
branch = (
(prop.get("allOf") or prop.get("oneOf") or prop.get("anyOf")) or [{}]
)[0]
if branch.get("$ref"):
return "dict", self._ref_name(branch["$ref"]), False
return self._resolve(owner, field_name, self._deref(branch))
prop_type = prop.get("type")
if prop_type == "array":
items = prop.get("items", {}) or {}
if items.get("$ref"):
inner = self._deref(items)
if inner.get("type", "object") == "object":
return "list", self._ref_name(items["$ref"]), True
return "list", None, False
fmt = prop.get("format")
if fmt in _FORMAT_HINTS:
return _FORMAT_HINTS[fmt], None, False
return _TYPE_HINTS.get(prop_type, str), None, False
# ── extraction ─────────────────────────────────────────────────────────
def extract(self) -> Tuple[List[ModelDefinition], List[EnumDefinition]]:
self._enums = {}
self._enum_types = {}
models: List[ModelDefinition] = []
for raw_name, schema in self._schemas().items():
model_name = _to_model_name(raw_name)
schema = schema or {}
# A top-level enum is a type, not a shape — register it and skip.
if schema.get("enum") and not schema.get("properties"):
self._register_enum(model_name, "", schema["enum"])
continue
props, required = self._flatten(schema)
required_set = set(required)
fields: List[FieldDefinition] = []
for prop_name, prop in props.items():
prop = prop or {}
type_hint, fk, many = self._resolve(model_name, prop_name, prop)
is_pk = prop_name == "id" or prop_name == f"{raw_name.lower()}_id"
fields.append(
FieldDefinition(
name=prop_name,
type_hint=type_hint,
default=prop.get("default"),
optional=prop_name not in required_set and not is_pk,
primary_key=is_pk,
foreign_key=fk,
many=many,
)
)
models.append(
ModelDefinition(
name=model_name,
fields=fields,
docstring=schema.get("description") or schema.get("title"),
)
)
return models, list(self._enums.values())
# ── endpoints ──────────────────────────────────────────────────────────
def endpoints(self) -> List[EndpointDefinition]:
"""Every operation in the document, normalised for routing."""
out: List[EndpointDefinition] = []
base = self.doc.get("basePath", "") if self.is_swagger2 else ""
for path, item in (self.doc.get("paths", {}) or {}).items():
if not isinstance(item, dict):
continue
shared_params = item.get("parameters", []) or []
for method, op in item.items():
if method.lower() not in (
"get", "post", "put", "patch", "delete", "head", "options"
):
continue
if not isinstance(op, dict):
continue
status, response_schema, example = self._response(op)
request_model = self._request_model(op, shared_params)
response_model, is_list, envelope = self._schema_target(response_schema)
full_path = f"{base.rstrip('/')}{path}" if base else path
model = response_model or request_model
if is_list:
kind = "collection"
elif re.search(r"\{[^}]+\}$", full_path):
kind = "item"
elif method.lower() == "post" and model:
kind = "collection"
else:
kind = "action"
out.append(
EndpointDefinition(
method=method.upper(),
path=full_path,
operation_id=op.get("operationId"),
summary=op.get("summary") or op.get("description"),
kind=kind,
model=model,
request_model=request_model,
response_model=response_model,
response_is_list=is_list,
envelope_key=envelope,
status=status,
path_params=re.findall(r"\{([^}]+)\}", full_path),
example=example,
)
)
return out
def _response(self, op: dict) -> Tuple[int, dict, Any]:
"""Pick the success response and return (status, schema, example)."""
responses = op.get("responses", {}) or {}
codes = [str(c) for c in responses]
chosen = next((c for c in _STATUS_PREFERENCE if c in codes), None)
if chosen is None:
chosen = next((c for c in sorted(codes) if c.startswith("2")), None)
if chosen is None:
return 200, {}, None
body = responses.get(chosen) or responses.get(int(chosen), {}) or {}
status = 200 if chosen in ("default", "2XX") else int(chosen)
if self.is_swagger2:
return status, body.get("schema", {}) or {}, body.get("examples")
content = body.get("content", {}) or {}
media = content.get("application/json") or next(
(v for k, v in content.items() if "json" in k), {}
)
example = media.get("example")
if example is None:
examples = media.get("examples") or {}
first = next(iter(examples.values()), None)
if isinstance(first, dict):
example = first.get("value")
return status, media.get("schema", {}) or {}, example
def _request_model(self, op: dict, shared_params: list) -> Optional[str]:
if self.is_swagger2:
params = list(shared_params) + list(op.get("parameters", []) or [])
for param in params:
if isinstance(param, dict) and param.get("in") == "body":
name, _, _ = self._schema_target(param.get("schema", {}) or {})
return name
return None
body = op.get("requestBody") or {}
content = self._deref(body).get("content", {}) or {}
media = content.get("application/json") or next(
(v for k, v in content.items() if "json" in k), {}
)
name, _, _ = self._schema_target(media.get("schema", {}) or {})
return name
def _schema_target(
self, schema: dict
) -> Tuple[Optional[str], bool, Optional[str]]:
"""Return (model name, is_list, envelope key) for a request/response schema."""
if not isinstance(schema, dict) or not schema:
return None, False, None
if schema.get("$ref"):
resolved = self._deref(schema)
# A named wrapper (PetPage) is still a wrapper; look through it.
if resolved.get("type") == "array" or resolved.get("properties"):
inner = self._schema_target(resolved)
if inner[0]:
return inner
return self._ref_name(schema["$ref"]), False, None
if schema.get("type") == "array":
items = schema.get("items", {}) or {}
if items.get("$ref"):
return self._ref_name(items["$ref"]), True, None
return None, True, None
# A wrapped collection — {"items": [...], "total": n} and friends.
for prop_name, prop in (schema.get("properties", {}) or {}).items():
if prop_name in ("items", "results", "data") and (prop or {}).get(
"type"
) == "array":
inner = (prop.get("items") or {}).get("$ref")
if inner:
return self._ref_name(inner), True, prop_name
for key in ("allOf", "oneOf", "anyOf"):
for sub in (schema.get(key) or [])[:1]:
return self._schema_target(sub)
return None, False, None

450
soleprint/station/tools/modelgen/loader/extract/tabular.py Normal file
View File

@@ -0,0 +1,450 @@
"""
Tabular Extractor
Turns a directory of spreadsheets into modelgen's IR — one model per CSV file,
one per sheet inside an ODS workbook — and keeps the rows.
extractor = TabularExtractor("./sheets")
models, enums = extractor.extract() # the shapes (BaseExtractor contract)
datasets = extractor.datasets() # the rows (what shuntgen seeds with)
ODS is read with zipfile + ElementTree rather than odfpy or pandas. An .ods is
a zip with an XML part in it, and reading it directly is what lets modelgen keep
its "no dependencies" promise — the same promise that makes it publishable as a
standalone pip package.
Types are inferred per column from the values actually present, and blanks make
a column optional. Keys are inferred by name and confirmed by the data: an `id`
column that is neither unique nor complete is not treated as a primary key, and
a `customer_id` column is only a foreign key if a matching table exists.
"""
import csv
import re
import zipfile
from datetime import date, datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from uuid import UUID
from xml.etree import ElementTree
from ..schema import (
DatasetDefinition,
EnumDefinition,
FieldDefinition,
ModelDefinition,
)
from .base import BaseExtractor
SUFFIXES = (".csv", ".tsv", ".ods")
_TABLE = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}"
_OFFICE = "{urn:oasis:names:tc:opendocument:xmlns:office:1.0}"
# ODS pads every sheet out to the full grid with repeat counts in the thousands.
# Expanding those verbatim would turn a 5-column sheet into a 1024-column one,
# so repeats are honoured only up to a width a real sheet could plausibly have.
_MAX_COLS = 512
_MAX_ROWS = 100_000
_TRUE = {"true", "yes", "y", "t"}
_FALSE = {"false", "no", "n", "f"}
# Ordered widest-last: the first format that parses every value in the column
# wins, so a stricter pattern must be offered before a looser one.
_DATE_FORMATS = ("%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%m/%d")
_DATETIME_FORMATS = (
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%dT%H:%M",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
)
def _to_model_name(name: str) -> str:
"""Convert a file or sheet name to PascalCase (line_items -> LineItems)."""
parts = [p for p in re.split(r"[^0-9a-zA-Z]+", name) if p]
return "".join(p[:1].upper() + p[1:] for p in parts) or name
def _to_slug(name: str) -> str:
"""Convert a file or sheet name to a url-safe collection name."""
slug = re.sub(r"[^0-9a-zA-Z]+", "-", name).strip("-").lower()
return slug or "items"
def _singular(word: str) -> str:
"""Crude singulariser — enough to match a `customer_id` to a `customers` sheet."""
word = word.lower()
if word.endswith("ies") and len(word) > 3:
return word[:-3] + "y"
for ending in ("ches", "shes", "sses", "xes", "zes"):
if word.endswith(ending):
return word[: -len(ending) + 1]
if word.endswith("s") and not word.endswith("ss"):
return word[:-1]
return word
# ── readers ────────────────────────────────────────────────────────────────
def _read_delimited(path: Path) -> List[List[str]]:
"""Read a CSV/TSV into a grid of raw strings."""
delimiter = "\t" if path.suffix.lower() == ".tsv" else ","
with path.open(newline="", encoding="utf-8-sig") as fh:
return [row for row in csv.reader(fh, delimiter=delimiter)]
def _cell_text(cell: ElementTree.Element) -> str:
"""The displayed text of an ODS cell, typed value preferred over its label."""
value_type = cell.get(f"{_OFFICE}value-type")
for attr in ("value", "date-value", "time-value", "boolean-value"):
raw = cell.get(f"{_OFFICE}{attr}")
if raw is not None:
return raw
if value_type == "string" or value_type is None:
return "".join(cell.itertext()).strip()
return "".join(cell.itertext()).strip()
def _read_ods(path: Path) -> Dict[str, List[List[str]]]:
"""Read an ODS workbook into {sheet name: grid of raw strings}."""
with zipfile.ZipFile(path) as archive:
try:
content = archive.read("content.xml")
except KeyError as e:
raise ValueError(f"{path} is not a readable ODS file (no content.xml)") from e
root = ElementTree.fromstring(content)
sheets: Dict[str, List[List[str]]] = {}
for table in root.iter(f"{_TABLE}table"):
name = table.get(f"{_TABLE}name") or f"sheet{len(sheets) + 1}"
grid: List[List[str]] = []
for row in table.iter(f"{_TABLE}table-row"):
cells: List[str] = []
for cell in row:
if cell.tag not in (
f"{_TABLE}table-cell",
f"{_TABLE}covered-table-cell",
):
continue
repeat = int(cell.get(f"{_TABLE}number-columns-repeated", 1) or 1)
text = "" if cell.tag.endswith("covered-table-cell") else _cell_text(cell)
# Padding is always empty and always repeated; a real repeated
# value is worth expanding, an empty run at the end is not.
if not text and repeat > 1 and len(cells) + repeat > _MAX_COLS:
break
cells.extend([text] * min(repeat, _MAX_COLS - len(cells)))
if len(cells) >= _MAX_COLS:
break
while cells and not cells[-1].strip():
cells.pop()
row_repeat = int(row.get(f"{_TABLE}number-rows-repeated", 1) or 1)
if not cells:
# A repeated blank row is padding; a single one may be a gap.
if row_repeat == 1 and grid:
grid.append([])
continue
for _ in range(min(row_repeat, _MAX_ROWS - len(grid))):
grid.append(list(cells))
if grid:
sheets[name] = grid
return sheets
# ── inference ──────────────────────────────────────────────────────────────
def _parse_bool(value: str) -> Optional[bool]:
lowered = value.strip().lower()
if lowered in _TRUE:
return True
if lowered in _FALSE:
return False
return None
def _parse_int(value: str) -> Optional[int]:
text = value.strip()
# A float-shaped string must not read as an int, or a column of prices
# silently truncates.
if not re.fullmatch(r"[+-]?\d+", text):
return None
try:
return int(text)
except ValueError:
return None
def _parse_float(value: str) -> Optional[float]:
try:
return float(value.strip())
except ValueError:
return None
def _parse_uuid(value: str) -> Optional[str]:
try:
return str(UUID(value.strip()))
except (ValueError, AttributeError):
return None
def _parse_temporal(value: str) -> Optional[str]:
"""Parse a date/datetime into an ISO string, or None."""
text = value.strip()
if not text:
return None
for fmt in _DATETIME_FORMATS:
try:
return datetime.strptime(text, fmt).isoformat()
except ValueError:
continue
for fmt in _DATE_FORMATS:
try:
return datetime.strptime(text, fmt).date().isoformat()
except ValueError:
continue
# ODS writes ISO-8601 with a timezone or fractional seconds; let the
# stdlib parser have the ones the explicit formats miss.
try:
return datetime.fromisoformat(text.replace("Z", "+00:00")).isoformat()
except ValueError:
return None
# Each entry is (type hint, parser). Order is the precedence: the first type
# every value in the column satisfies wins, so narrow types come first.
_CANDIDATES: List[Tuple[Any, Any]] = [
(bool, _parse_bool),
(int, _parse_int),
(float, _parse_float),
("UUID", _parse_uuid),
("datetime", _parse_temporal),
]
def _infer_column(values: List[str]) -> Tuple[Any, Any]:
"""Return (type_hint, coercer) for a column, from its non-blank values."""
present = [v for v in values if v is not None and str(v).strip() != ""]
if not present:
return str, lambda v: v
for type_hint, parser in _CANDIDATES:
if all(parser(v) is not None for v in present):
return type_hint, parser
return str, lambda v: v.strip() if isinstance(v, str) else v
# ── extractor ──────────────────────────────────────────────────────────────
class TabularExtractor(BaseExtractor):
"""Extracts modelgen IR and seed rows from a directory of spreadsheets."""
def __init__(self, source_path):
super().__init__(source_path)
self._datasets: List[DatasetDefinition] = []
def detect(self) -> bool:
"""True when the source is a directory holding at least one sheet."""
if self.source_path.is_file():
return self.source_path.suffix.lower() in SUFFIXES
if not self.source_path.is_dir():
return False
return any(self._sources())
def _sources(self) -> List[Path]:
if self.source_path.is_file():
return [self.source_path]
return sorted(
p
for p in self.source_path.iterdir()
if p.is_file() and p.suffix.lower() in SUFFIXES and not p.name.startswith(".")
)
def datasets(self) -> List[DatasetDefinition]:
"""Seed rows harvested by the last extract() call."""
if not self._datasets:
self.extract()
return self._datasets
def extract(self) -> Tuple[List[ModelDefinition], List[EnumDefinition]]:
grids: List[Tuple[str, str, List[List[str]]]] = [] # (label, source, grid)
for path in self._sources():
if path.suffix.lower() == ".ods":
for sheet_name, grid in _read_ods(path).items():
grids.append((sheet_name, f"{path.name}#{sheet_name}", grid))
else:
grids.append((path.stem, path.name, _read_delimited(path)))
if not grids:
raise ValueError(f"No .csv/.tsv/.ods files found in {self.source_path}")
# Names have to be known before fields, because a foreign key is only a
# foreign key when the table it points at is one of the others.
known: Dict[str, str] = {}
for label, _, _ in grids:
model_name = _to_model_name(label)
known[_singular(label)] = model_name
known[label.lower()] = model_name
models: List[ModelDefinition] = []
self._datasets = []
for label, source, grid in grids:
parsed = self._build(label, source, grid, known)
if parsed:
model, dataset = parsed
models.append(model)
self._datasets.append(dataset)
return models, []
def _build(
self,
label: str,
source: str,
grid: List[List[str]],
known: Dict[str, str],
) -> Optional[Tuple[ModelDefinition, DatasetDefinition]]:
rows = [r for r in grid if any(str(c).strip() for c in r)]
if not rows:
return None
headers = self._headers(rows[0])
if not headers:
return None
body = rows[1:]
columns: Dict[str, List[str]] = {h: [] for h in headers}
for row in body:
for index, header in enumerate(headers):
columns[header].append(row[index] if index < len(row) else "")
model_name = _to_model_name(label)
pk = self._primary_key(label, model_name, headers, columns, len(body), known)
fields: List[FieldDefinition] = []
coercers: Dict[str, Any] = {}
for header in headers:
values = columns[header]
type_hint, coercer = _infer_column(values)
coercers[header] = coercer
is_pk = header == pk
fk = None if is_pk else self._foreign_key(header, model_name, known)
has_blank = any(str(v).strip() == "" for v in values)
fields.append(
FieldDefinition(
name=header,
type_hint=type_hint,
default=None,
optional=has_blank and not is_pk,
primary_key=is_pk,
foreign_key=fk,
unique=is_pk,
)
)
typed_rows: List[Dict[str, Any]] = []
for row in body:
record: Dict[str, Any] = {}
for index, header in enumerate(headers):
raw = row[index] if index < len(row) else ""
if str(raw).strip() == "":
record[header] = None
continue
parsed = coercers[header](raw)
record[header] = raw.strip() if parsed is None else parsed
typed_rows.append(record)
model = ModelDefinition(
name=model_name,
fields=fields,
docstring=f"Imported from {source} ({len(typed_rows)} rows).",
)
dataset = DatasetDefinition(
model=model_name,
rows=typed_rows,
source=source,
collection=_to_slug(label),
)
return model, dataset
@staticmethod
def _headers(row: List[str]) -> List[str]:
"""Normalise the header row, filling blanks and de-duplicating."""
headers: List[str] = []
seen: Dict[str, int] = {}
for index, raw in enumerate(row):
name = re.sub(r"[^0-9a-zA-Z]+", "_", str(raw).strip()).strip("_").lower()
if not name:
name = f"column_{index + 1}"
if name[0].isdigit():
name = f"c_{name}"
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 1
headers.append(name)
return headers
@classmethod
def _primary_key(
cls,
label: str,
model_name: str,
headers: List[str],
columns: Dict[str, List[str]],
row_count: int,
known: Dict[str, str],
) -> Optional[str]:
"""Pick the key column — named like one, and unique and complete in fact."""
def holds(header: str) -> bool:
values = [str(v).strip() for v in columns[header]]
return bool(row_count) and all(values) and len(set(values)) == row_count
stem = _singular(label)
for candidate in ("id", f"{stem}_id", f"{label.lower()}_id"):
if candidate in headers and holds(candidate):
return candidate
# A sheet may name its key after the row rather than the table —
# `line_id` in a `line_items` sheet. Accept the leading column when it
# is shaped like a key, holds like one, and does not point elsewhere.
if headers:
first = headers[0]
if (
(first == "id" or first.endswith("_id"))
and cls._foreign_key(first, model_name, known) is None
and holds(first)
):
return first
return None
@staticmethod
def _foreign_key(
header: str, model_name: str, known: Dict[str, str]
) -> Optional[str]:
"""A `<thing>_id` column pointing at another sheet in the same import."""
if not header.endswith("_id"):
return None
stem = header[:-3]
target = known.get(_singular(stem)) or known.get(stem)
if target and target != model_name:
return target
return None

51
soleprint/station/tools/modelgen/loader/schema.py
View File

@@ -13,7 +13,7 @@ Expects the folder to have an __init__.py that exports:
import dataclasses as dc
import importlib.util
import sys
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, get_type_hints
@@ -32,6 +32,10 @@ class FieldDefinition:
primary_key: bool = False
foreign_key: Optional[str] = None # target model name
unique: bool = False
# True when foreign_key points at many rows rather than one — an array of
# $ref, a m2m table. graphgen renders the two differently, so losing the
# distinction would draw every collection as a single edge.
many: bool = False
@dataclass
@@ -60,6 +64,51 @@ class GrpcServiceDefinition:
methods: List[Dict[str, Any]]
@dataclass
class EndpointDefinition:
"""Represents one HTTP operation on a service.
Models describe the shapes a service passes around; endpoints describe how
it is called. Loaders that read a service contract (OpenAPI) or infer one
(tabular) emit these alongside the models, and shuntgen turns them into
routes. Loaders that only see shapes — dataclasses, a Django app — emit
none, which is why this is not part of the BaseExtractor contract.
"""
method: str # GET, POST, ...
path: str # /pets/{petId}
operation_id: Optional[str] = None
summary: Optional[str] = None
# "collection" returns/accepts many, "item" one, "action" neither.
kind: str = "item"
model: Optional[str] = None # what this operation is about
request_model: Optional[str] = None
response_model: Optional[str] = None
response_is_list: bool = False
# Set when the collection arrives wrapped — {"items": [...], "total": n}
# rather than a bare array. Answering a wrapped endpoint with an array is
# the kind of mismatch a client only discovers at parse time.
envelope_key: Optional[str] = None
status: int = 200
path_params: List[str] = field(default_factory=list)
example: Any = None # response example carried through from the source
@dataclass
class DatasetDefinition:
"""Concrete rows harvested next to a model.
Importing a spreadsheet yields both a shape and the data that shaped it.
Throwing the rows away would mean generating a service that answers with
invented values when the real ones were right there.
"""
model: str
rows: List[Dict[str, Any]] = field(default_factory=list)
source: Optional[str] = None # file the rows came from
collection: Optional[str] = None # url-facing name, e.g. "customers"
class SchemaLoader:
"""Loads model definitions from Python dataclasses in schema/ folder."""

18
soleprint/station/tools/modelgen/tests/__init__.py Normal file
View File

@@ -0,0 +1,18 @@
"""
Tests for modelgen.
Run from station/tools/, not from inside modelgen/ — the package ships a
types.py, and having its own directory on sys.path shadows the standard
library module of that name:
cd soleprint/station/tools
python -m unittest modelgen.tests.test_extractors
stdlib unittest only, and every input is built in a temp directory. modelgen is
published as a standalone pip package, so its tests have to pass with nothing
installed and nothing else in the tree.
These are unit tests of the loaders and generators. They are not contract
tests — those talk HTTP, belong to a room, and are never committed to core
(see station/tools/tester/tests/README.md).
"""

406
soleprint/station/tools/modelgen/tests/test_extractors.py Normal file
View File

@@ -0,0 +1,406 @@
"""
Tests for the OpenAPI and tabular extractors and the datagen target.
stdlib unittest and nothing else, and every input is built in a temp directory
rather than read from a fixture file — modelgen is published as a standalone
package, so its tests have to pass with nothing installed and nothing else in
the tree.
cd soleprint/station/tools && python -m unittest modelgen.tests.test_extractors
Run it from station/tools/, not from inside modelgen/: modelgen ships a
types.py, and putting the package's own directory on sys.path shadows the
standard library module of that name.
"""
import json
import shutil
import sys
import tempfile
import unittest
import zipfile
from enum import Enum
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1].parent))
from modelgen.generator import GENERATORS, DatagenGenerator # noqa: E402
from modelgen.loader.extract.openapi import OpenAPIExtractor # noqa: E402
from modelgen.loader.extract.tabular import TabularExtractor # noqa: E402
SPEC = {
"openapi": "3.0.3",
"info": {"title": "Zoo", "version": "1.0.0"},
"paths": {
"/pets": {
"get": {
"operationId": "listPets",
"responses": {
"200": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/PetPage"}
}
}
}
},
},
"post": {
"operationId": "createPet",
"requestBody": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/Pet"}
}
}
},
"responses": {
"201": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/Pet"}
}
}
}
},
},
},
"/pets/{petId}": {
"get": {
"operationId": "getPet",
"responses": {
"200": {
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/Pet"},
"example": {"id": 7, "name": "Rocinante"},
}
}
}
},
}
},
},
"components": {
"schemas": {
"Pet": {
"type": "object",
"description": "An animal.",
"required": ["id", "name"],
"properties": {
"id": {"type": "integer", "format": "int64"},
"name": {"type": "string"},
"status": {"type": "string", "enum": ["available", "sold"]},
"weight": {"type": "number"},
"born_on": {"type": "string", "format": "date"},
"category": {"$ref": "#/components/schemas/Category"},
"tags": {
"type": "array",
"items": {"$ref": "#/components/schemas/Category"},
},
},
},
"Category": {
"type": "object",
"required": ["id"],
"properties": {
"id": {"type": "integer"},
"name": {"type": "string"},
},
},
"PetPage": {
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {"$ref": "#/components/schemas/Pet"},
},
"total": {"type": "integer"},
},
},
}
},
}
CUSTOMERS_CSV = """id,name,email,active,joined,balance
1,Ada,ada@example.com,true,2024-03-11,150.5
2,Bruno,bruno@example.com,false,2024-05-02,80
3,Camila,camila@example.com,true,2023-11-27,
"""
ORDERS_CSV = """order_id,customer_id,total,placed
1001,1,42.00,2025-02-03
1002,2,15.25,2025-03-14
"""
def write_ods(path: Path, sheet: str, rows: list[list[str]]) -> None:
"""A minimal ODS, padded the way a real one is."""
ns = (
'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" '
'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"'
)
def cell(value: str) -> str:
if value == "":
return "<table:table-cell/>"
return (
'<table:table-cell office:value-type="string">'
f"<text:p>{value}</text:p></table:table-cell>"
)
body = "".join(
"<table:table-row>"
+ "".join(cell(c) for c in row)
+ '<table:table-cell table:number-columns-repeated="1018"/>'
"</table:table-row>"
for row in rows
)
content = (
'<?xml version="1.0" encoding="UTF-8"?>'
f"<office:document-content {ns}>"
"<office:body><office:spreadsheet>"
f'<table:table table:name="{sheet}">{body}'
'<table:table-row table:number-rows-repeated="1048570">'
'<table:table-cell table:number-columns-repeated="1024"/></table:table-row>'
"</table:table></office:spreadsheet></office:body></office:document-content>"
)
with zipfile.ZipFile(path, "w") as archive:
archive.writestr("content.xml", content)
class OpenAPIExtractorTest(unittest.TestCase):
def setUp(self):
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-openapi-"))
self.spec = self.dir / "zoo.json"
self.spec.write_text(json.dumps(SPEC))
self.extractor = OpenAPIExtractor(self.spec)
def tearDown(self):
shutil.rmtree(self.dir, ignore_errors=True)
def test_detects_a_spec(self):
self.assertTrue(self.extractor.detect())
def test_does_not_detect_arbitrary_json(self):
other = self.dir / "not-a-spec.json"
other.write_text('{"hello": "world"}')
self.assertFalse(OpenAPIExtractor(other).detect())
def test_extracts_every_object_schema(self):
models, _ = self.extractor.extract()
self.assertEqual({m.name for m in models}, {"Pet", "Category", "PetPage"})
def test_enum_becomes_a_real_enum_class(self):
models, enums = self.extractor.extract()
# A materialised Enum is what makes every generator resolve it by name
# instead of falling through to str.
self.assertEqual([e.name for e in enums], ["PetStatus"])
status = next(f for f in models[0].fields if f.name == "status")
self.assertTrue(isinstance(status.type_hint, type))
self.assertTrue(issubclass(status.type_hint, Enum))
self.assertEqual([m.value for m in status.type_hint], ["available", "sold"])
def test_required_and_format_and_relations(self):
models, _ = self.extractor.extract()
fields = {f.name: f for f in next(m for m in models if m.name == "Pet").fields}
self.assertEqual(fields["id"].type_hint, "bigint")
self.assertTrue(fields["id"].primary_key)
self.assertFalse(fields["name"].optional)
self.assertTrue(fields["weight"].optional)
self.assertEqual(fields["born_on"].type_hint, "datetime")
self.assertEqual(fields["category"].foreign_key, "Category")
self.assertFalse(fields["category"].many)
self.assertEqual(fields["tags"].foreign_key, "Category")
self.assertTrue(fields["tags"].many)
def test_endpoints_carry_operation_shape(self):
by_id = {e.operation_id: e for e in self.extractor.endpoints()}
self.assertEqual(set(by_id), {"listPets", "createPet", "getPet"})
listing = by_id["listPets"]
self.assertEqual(listing.kind, "collection")
self.assertEqual(listing.response_model, "Pet")
self.assertTrue(listing.response_is_list)
# PetPage wraps the array, so a client expects {"items": [...]}.
self.assertEqual(listing.envelope_key, "items")
created = by_id["createPet"]
self.assertEqual(created.status, 201)
self.assertEqual(created.request_model, "Pet")
item = by_id["getPet"]
self.assertEqual(item.kind, "item")
self.assertEqual(item.path_params, ["petId"])
self.assertEqual(item.example, {"id": 7, "name": "Rocinante"})
def test_relations_reach_the_graphgen_schema(self):
out = self.dir / "schema.json"
GENERATORS["schema"]().generate(self.extractor.extract(), out)
schema = json.loads(out.read_text())["models"]
self.assertEqual(schema["Pet"]["fields"]["category"]["type"], "FK:Category")
self.assertEqual(schema["Pet"]["fields"]["tags"]["type"], "M2M:Category")
class TabularExtractorTest(unittest.TestCase):
def setUp(self):
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-tabular-"))
self.sheets = self.dir / "sheets"
self.sheets.mkdir()
(self.sheets / "customers.csv").write_text(CUSTOMERS_CSV)
(self.sheets / "orders.csv").write_text(ORDERS_CSV)
write_ods(
self.sheets / "line_items.ods",
"line_items",
[
["line_id", "order_id", "sku", "qty", "note"],
["1", "1001", "A-1", "2", "rush"],
["2", "1001", "B-2", "1", ""],
["3", "1002", "A-1", "5", ""],
],
)
self.extractor = TabularExtractor(self.sheets)
def tearDown(self):
shutil.rmtree(self.dir, ignore_errors=True)
def test_detects_a_sheet_directory(self):
self.assertTrue(self.extractor.detect())
empty = self.dir / "empty"
empty.mkdir()
self.assertFalse(TabularExtractor(empty).detect())
def test_one_model_per_file_and_sheet(self):
models, _ = self.extractor.extract()
self.assertEqual(
{m.name for m in models}, {"Customers", "Orders", "LineItems"}
)
def test_types_are_inferred_per_column(self):
models, _ = self.extractor.extract()
fields = {
f.name: f for f in next(m for m in models if m.name == "Customers").fields
}
self.assertEqual(fields["id"].type_hint, int)
self.assertEqual(fields["name"].type_hint, str)
self.assertEqual(fields["active"].type_hint, bool)
self.assertEqual(fields["joined"].type_hint, "datetime")
self.assertEqual(fields["balance"].type_hint, float)
# One blank cell is what makes the column nullable.
self.assertTrue(fields["balance"].optional)
self.assertFalse(fields["name"].optional)
def test_keys_are_confirmed_against_the_data(self):
models, _ = self.extractor.extract()
by_name = {m.name: m for m in models}
customers = {f.name: f for f in by_name["Customers"].fields}
self.assertTrue(customers["id"].primary_key)
orders = {f.name: f for f in by_name["Orders"].fields}
self.assertTrue(orders["order_id"].primary_key)
# customer_id names another sheet, so it is a relation, not a key.
self.assertFalse(orders["customer_id"].primary_key)
self.assertEqual(orders["customer_id"].foreign_key, "Customers")
def test_ods_is_read_and_padding_ignored(self):
models, _ = self.extractor.extract()
line_items = next(m for m in models if m.name == "LineItems")
# Five real columns, not the 1023 the padding claims.
self.assertEqual([f.name for f in line_items.fields],
["line_id", "order_id", "sku", "qty", "note"])
fields = {f.name: f for f in line_items.fields}
self.assertEqual(fields["qty"].type_hint, int)
self.assertTrue(fields["note"].optional)
# line_id names the row rather than the sheet, but leads and holds.
self.assertTrue(fields["line_id"].primary_key)
self.assertEqual(fields["order_id"].foreign_key, "Orders")
def test_rows_are_kept_and_coerced(self):
self.extractor.extract()
datasets = {d.model: d for d in self.extractor.datasets()}
self.assertEqual(datasets["Customers"].collection, "customers")
rows = datasets["Customers"].rows
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0]["id"], 1)
self.assertIs(rows[0]["active"], True)
self.assertEqual(rows[0]["balance"], 150.5)
# A blank cell is null, not the empty string.
self.assertIsNone(rows[2]["balance"])
self.assertEqual(datasets["LineItems"].rows[1]["note"], None)
class DatagenTargetTest(unittest.TestCase):
def setUp(self):
self.dir = Path(tempfile.mkdtemp(prefix="modelgen-datagen-"))
self.sheets = self.dir / "sheets"
self.sheets.mkdir()
(self.sheets / "customers.csv").write_text(CUSTOMERS_CSV)
def tearDown(self):
shutil.rmtree(self.dir, ignore_errors=True)
def _generate(self, seeded: bool):
extractor = TabularExtractor(self.sheets)
models, enums = extractor.extract()
datasets = extractor.datasets() if seeded else []
out = self.dir / "gen" / "datagen_demo.py"
DatagenGenerator(class_name="DemoGenerator").generate(
(models, enums, datasets), out
)
if seeded:
depot = out.parent / "depot"
depot.mkdir(exist_ok=True)
(depot / "data.json").write_text(
json.dumps({d.model: d.rows for d in datasets})
)
import importlib.util
spec = importlib.util.spec_from_file_location(f"demo_{seeded}", out)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.DemoGenerator()
def test_registered_as_a_target(self):
self.assertIn("datagen", GENERATORS)
def test_synthesises_when_there_are_no_rows(self):
generator = self._generate(seeded=False)
self.assertEqual(generator.available_models(), ["customers"])
record = generator.generate("Customers", 1)[0]
self.assertEqual(
set(record), {"id", "name", "email", "active", "joined", "balance"}
)
self.assertIsInstance(record["id"], int)
self.assertIsInstance(record["active"], bool)
self.assertIn("@", record["email"])
def test_samples_real_rows_when_seeded(self):
generator = self._generate(seeded=True)
names = {generator.generate("Customers", 1)[0]["name"] for _ in range(25)}
self.assertTrue(names <= {"Ada", "Bruno", "Camila"}, names)
def test_kwargs_override_the_result(self):
generator = self._generate(seeded=True)
self.assertEqual(generator.generate("Customers", 1, name="Zed")[0]["name"], "Zed")
def test_exposes_a_graphgen_schema(self):
schema = self._generate(seeded=False).schema()
self.assertIn("Customers", schema["models"])
self.assertTrue(schema["models"]["Customers"]["fields"]["id"]["pk"])
def test_unknown_model_is_reported(self):
generator = self._generate(seeded=False)
with self.assertRaises(ValueError):
generator.generate("Nope", 1)
if __name__ == "__main__":
unittest.main()

10
soleprint/station/tools/modelgen/types.py
View File

@@ -61,6 +61,11 @@ PYDANTIC_RESOLVERS: dict[Any, Callable[[Any], str]] = {
"list": lambda base: f"List[{_get_list_inner(base)}]",
"enum": lambda base: base.__name__,
"dataclass": lambda base: base.__name__,
# DB- and spec-shaped hints. Without these an int64 column resolves through
# the "unknown -> str" fallback and a BigIntegerField arrives as a string.
"bigint": lambda _: "int",
"text": lambda _: "str",
"bytes": lambda _: "bytes",
}
# =============================================================================
@@ -95,6 +100,11 @@ TS_RESOLVERS: dict[Any, Callable[[Any], str]] = {
"list": _resolve_ts_list,
"enum": lambda base: base.__name__,
"dataclass": lambda base: base.__name__,
# JS has no 64-bit integer literal type, so int64 is a number like any
# other; text is a string; binary arrives base64-encoded over JSON.
"bigint": lambda _: "number",
"text": lambda _: "string",
"bytes": lambda _: "string",
}
# =============================================================================

127
soleprint/station/tools/shuntgen/README.md Normal file
View File

@@ -0,0 +1,127 @@
# shuntgen
Generates runnable [shunts](../../../artery/shunts/) from the two things people
actually have: a service contract, or a folder of spreadsheets.
```bash
# a spec you were handed
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
# sheets a client sent
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
python -m station.tools.shuntgen list
```
Run from `soleprint/`, so `station.tools...` resolves. Also available in the
browser at `/station/tools/shuntgen/`.
## What comes out
```
artery/shunts/<name>/
main.py builds the app from the spec
run.py uvicorn entry point (PORT, or depot/config.json)
shunt_runtime.py vendored copy of runtime.py
models.py pydantic, via modelgen
datagen_<name>.py BaseDataGenerator subclass, via modelgen
depot/spec.json routes, collections and schema
depot/responses.json pinned overrides — yours, never overwritten
depot/config.json delays, error rate, prefill — yours, never overwritten
depot/data.json imported rows
templates/index.html config UI
README.md
```
```bash
cd artery/shunts/books && python run.py
curl localhost:8098/customers
```
## Where a response comes from
First hit wins:
1. `depot/responses.json` — a pinned override, keyed `"METHOD /path"`
2. the store — rows imported from sheets, plus anything POSTed since
3. the spec's `example`, if the source document carried one
4. `datagen_<name>.py`, synthesising from the schema
5. `{}`
The store is the part that makes it behave like a service: POST something and
GET it back, ask for `/pets/7` and get the pet whose id is 7. Collections with
no imported rows are prefilled with generated ones (`prefill` in
`depot/config.json`) so the first call answers with something.
## Two sources, one pipeline
Both inputs are modelgen extractors, so the same shapes also generate pydantic,
TypeScript, prisma and a graphgen schema:
```bash
python -m station.tools.modelgen from-openapi -s api.yaml -o out/ -t pydantic,typescript
python -m station.tools.modelgen from-tabular -s ./sheets -o out/ -t schema,datagen
```
| Source | Becomes | Routes |
| --- | --- | --- |
| OpenAPI 3.x / Swagger 2.0 | one model per schema, enums as real Enums, `$ref` as relations | the operations the document declares |
| `.csv` / `.tsv` / `.ods` | one model per file or sheet, types inferred per column | five CRUD routes per table |
Keys and relations are inferred and then checked against the data: an `id`
column that is not unique is not treated as a key, and `customer_id` is only a
foreign key if a `customers` sheet came with it.
ODS is read with `zipfile` + `ElementTree` — no odfpy, no pandas — which is what
lets modelgen stay dependency-free and publishable on its own.
## Control endpoints
Every generated shunt serves these:
| Endpoint | Purpose |
| --- | --- |
| `GET /health` | liveness |
| `GET /mock/spec` | the routes it was built from |
| `GET /mock/stats` | call counts and row counts |
| `POST /mock/reset` | restore imported rows, clear counters |
| `GET,POST /mock/config` | delays, error rate, `unknown_id`, page size |
| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |
```bash
# make it slow and flaky, the way the real thing is
curl -X POST localhost:8098/mock/config \
-H 'content-type: application/json' \
-d '{"enable_random_delays": true, "error_rate": 0.2}'
# make one call answer something specific
curl -X POST localhost:8098/mock/responses \
-H 'content-type: application/json' \
-d '{"GET /customers/1": {"id": 1, "name": "PINNED"}}'
```
`unknown_id` decides what an unknown key does: `generate` (the default) invents
a record wearing the id that was asked for, `404` refuses it. Generate by
default because a client pointed at a fresh shunt should just work; flip it when
the error path is what you are testing.
## Dependency containers
`--cabinet postgres,redis` writes a `cabinet.json` declaring what the shunt
expects. `python build.py --cfg <room>` composes those services into the room's
`docker-compose.yml`; on a cluster they install as rig addons of the same name.
See [cabinets](../../cabinets/README.md).
## Regenerating
Everything is overwritten except `depot/responses.json` and `depot/config.json`.
Fixing a bug in `runtime.py` and regenerating fixes it in every shunt, which is
why the routes are built from `spec.json` at startup rather than written out as
source.
## Fixtures
`fixtures/petstore.yaml` and `fixtures/sheets/` exercise the shapes that are
easy to get wrong — an enum, a `$ref`, an array of `$ref`, a wrapped collection,
a path parameter, an ODS sheet with padding and a blank column, and a foreign
key across two files.

29
soleprint/station/tools/shuntgen/__init__.py Normal file
View File

@@ -0,0 +1,29 @@
"""
Shuntgen - Generate runnable shunts from a service contract or a pile of sheets.
A shunt is artery's fake connector: a stand-in service that answers like the
real one so tests can run without it. Writing one by hand means copying
artery/shunts/example/ and filling in responses.json by hand, which is fine for
three endpoints and untenable for eighty.
This tool takes the two things people actually have —
an OpenAPI/Swagger document the service already exists somewhere
a directory of CSV/ODS sheets the data exists, the service does not
— and emits a shunt that runs: typed models, a data generator, seeded routes,
and a config UI.
Both inputs reach modelgen's IR through its own extractors, so the shapes also
generate pydantic, TypeScript, prisma and a graphgen schema for free. This
package only adds the emitter.
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
"""
from .emit import ShuntEmitter
__version__ = "0.1.0"
__all__ = ["ShuntEmitter"]

239
soleprint/station/tools/shuntgen/__main__.py Normal file
View File

@@ -0,0 +1,239 @@
"""
Shuntgen CLI.
python -m station.tools.shuntgen from-openapi -s api.yaml -o artery/shunts/petstore
python -m station.tools.shuntgen from-tabular -s ./sheets -o artery/shunts/books
python -m station.tools.shuntgen list
Run from the soleprint/ directory so `station.tools...` resolves.
"""
import argparse
import sys
from pathlib import Path
from .emit import SPR_ROOT, ShuntEmitter
def _name_for(args, source: Path) -> str:
"""The shunt's name: given, else the output folder, else the source."""
if getattr(args, "name", None):
return args.name
output = Path(args.output)
if output.name and output.name not in (".", ".."):
return output.name
return source.stem
def _refuse_to_clobber(output: Path, force: bool) -> None:
"""Regenerating is fine; overwriting something that is not a shunt is not."""
if not output.exists() or force:
return
if not any(output.iterdir()):
return
if (output / "depot" / "spec.json").exists():
return
print(
f"Error: {output} already exists and was not generated by shuntgen.\n"
" Pick another path, or pass --force to write into it anyway.",
file=sys.stderr,
)
sys.exit(1)
def cmd_from_openapi(args):
"""Build a shunt from an OpenAPI 3.x / Swagger 2.0 document."""
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
spec_path = Path(args.spec)
if not spec_path.exists():
print(f"Error: Spec not found: {spec_path}", file=sys.stderr)
sys.exit(1)
output = Path(args.output)
_refuse_to_clobber(output, args.force)
extractor = OpenAPIExtractor(spec_path)
print(f"Reading spec: {spec_path}")
try:
models, enums = extractor.extract()
endpoints = extractor.endpoints()
except (RuntimeError, ValueError) as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
if not endpoints:
print(
"Error: the spec declares no operations, so there is nothing to serve.",
file=sys.stderr,
)
sys.exit(1)
name = _name_for(args, spec_path)
print(f"Found {len(models)} models, {len(endpoints)} endpoints")
written = ShuntEmitter(
name=name,
output=output,
models=models,
enums=enums,
endpoints=endpoints,
title=args.title,
source=spec_path.name,
kind="openapi",
port=args.port,
cabinets=_cabinets(args),
).emit()
_report(written, name, args.port)
def cmd_from_tabular(args):
"""Build a shunt from a directory of CSV/TSV/ODS sheets."""
from ..modelgen.loader.extract.tabular import TabularExtractor
source_path = Path(args.source)
if not source_path.exists():
print(f"Error: Source not found: {source_path}", file=sys.stderr)
sys.exit(1)
output = Path(args.output)
_refuse_to_clobber(output, args.force)
extractor = TabularExtractor(source_path)
print(f"Reading sheets: {source_path}")
try:
models, enums = extractor.extract()
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
datasets = extractor.datasets()
rows = sum(len(d.rows) for d in datasets)
name = _name_for(args, source_path)
print(f"Found {len(models)} models, {rows} rows")
written = ShuntEmitter(
name=name,
output=output,
models=models,
enums=enums,
datasets=datasets,
title=args.title,
source=source_path.name,
kind="tabular",
port=args.port,
cabinets=_cabinets(args),
).emit()
_report(written, name, args.port)
def cmd_list(args):
"""List the shunts that exist under artery/shunts/."""
shunts_dir = SPR_ROOT / "artery" / "shunts"
if not shunts_dir.exists():
print(f"No shunts directory at {shunts_dir}")
return
found = False
for path in sorted(shunts_dir.iterdir()):
if not path.is_dir() or path.name.startswith(("_", ".")):
continue
found = True
spec = path / "depot" / "spec.json"
if spec.exists():
import json
try:
data = json.loads(spec.read_text())
print(
f" {path.name:<20} generated "
f"{len(data.get('routes', []))} routes from {data.get('source', '?')}"
)
continue
except (OSError, ValueError):
pass
print(f" {path.name:<20} hand-written")
if not found:
print(" (none)")
def _cabinets(args) -> list:
if not getattr(args, "cabinet", None):
return []
return [c.strip() for c in args.cabinet.split(",") if c.strip()]
def _report(output: Path, name: str, port: int) -> None:
print(f"\nWrote {output}")
print("\nRun it:")
print(f" cd {output} && python run.py")
print(f" curl localhost:{port}/health")
print(f" open http://localhost:{port}/")
def _add_common(parser, port_default: int) -> None:
parser.add_argument(
"--output", "-o", type=str, required=True,
help="Where to write the shunt (e.g. artery/shunts/petstore)",
)
parser.add_argument(
"--name", "-n", type=str, default=None,
help="Shunt name (default: the output folder's name)",
)
parser.add_argument(
"--title", type=str, default=None,
help="Display title for the config UI (default: derived from the name)",
)
parser.add_argument(
"--port", "-p", type=int, default=port_default,
help=f"Default port for run.py (default: {port_default})",
)
parser.add_argument(
"--cabinet", type=str, default=None,
help="Comma-separated dependency containers to declare (e.g. postgres,redis)",
)
parser.add_argument(
"--force", action="store_true",
help="Write into a non-empty directory that shuntgen did not generate",
)
def main():
parser = argparse.ArgumentParser(
description="Shuntgen - generate runnable shunts",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
subparsers = parser.add_subparsers(dest="command", required=True)
openapi_parser = subparsers.add_parser(
"from-openapi", help="Generate a shunt from an OpenAPI / Swagger document"
)
openapi_parser.add_argument(
"--spec", "-s", type=str, required=True,
help="Path to the spec (.json, .yaml or .yml)",
)
_add_common(openapi_parser, 8099)
openapi_parser.set_defaults(func=cmd_from_openapi)
tabular_parser = subparsers.add_parser(
"from-tabular", help="Generate a shunt from a directory of CSV/TSV/ODS sheets"
)
tabular_parser.add_argument(
"--source", "-s", type=str, required=True,
help="Directory of sheets (or a single .csv/.tsv/.ods file)",
)
_add_common(tabular_parser, 8098)
tabular_parser.set_defaults(func=cmd_from_tabular)
list_parser = subparsers.add_parser("list", help="List shunts under artery/shunts/")
list_parser.set_defaults(func=cmd_list)
args = parser.parse_args()
args.func(args)
if __name__ == "__main__":
main()

354
soleprint/station/tools/shuntgen/api.py Normal file
View File

@@ -0,0 +1,354 @@
"""FastAPI router for shuntgen — generate shunts from a spec or a sheet folder.
Mounted by run.py under /station, so the routes below live at
/station/tools/shuntgen/... , the same shape as datagen and graphgen.
Generation writes files, so every path is resolved against the soleprint tree
and anything pointing outside it is refused. A tool that turns an uploaded file
into a directory of Python is not somewhere to be relaxed about paths.
"""
import json
import logging
import shutil
import tempfile
from pathlib import Path
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, UploadFile
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from .emit import SPR_ROOT, ShuntEmitter
log = logging.getLogger(__name__)
router = APIRouter(prefix="/tools/shuntgen", tags=["shuntgen"])
SHUNTS_DIR = SPR_ROOT / "artery" / "shunts"
UPLOAD_DIR = SPR_ROOT / "station" / "tools" / "shuntgen" / "uploads"
# Room for a large spec, a hard stop well short of anything that would exhaust
# memory while being parsed.
MAX_UPLOAD_BYTES = 8 * 1024 * 1024
# ─────────────────────────────────────────────────────
# Path safety
# ─────────────────────────────────────────────────────
def _inside(path: Path, root: Path) -> Path:
"""Resolve a path and refuse it if it escapes root."""
resolved = (root / path).resolve() if not path.is_absolute() else path.resolve()
try:
resolved.relative_to(root.resolve())
except ValueError:
raise HTTPException(
status_code=400,
detail=f"Path must stay inside {root.name}/: {path}",
)
return resolved
def _safe_name(name: str) -> str:
cleaned = "".join(c for c in name if c.isalnum() or c in "-_").strip("-_")
if not cleaned:
raise HTTPException(status_code=400, detail="Name must contain letters or digits")
return cleaned.lower()
# ─────────────────────────────────────────────────────
# Models
# ─────────────────────────────────────────────────────
class GenerateRequest(BaseModel):
name: str
source: str # spec file or sheet directory, relative to soleprint/
kind: str = "auto" # "openapi" | "tabular" | "auto"
title: Optional[str] = None
port: int = 8099
cabinets: list[str] = []
force: bool = False
# ─────────────────────────────────────────────────────
# Routes
# ─────────────────────────────────────────────────────
@router.get("/", response_class=HTMLResponse)
def index():
html = Path(__file__).parent / "templates" / "index.html"
return HTMLResponse(html.read_text() if html.exists() else "<h1>shuntgen</h1>")
@router.get("/health")
def health():
return {"status": "ok", "tool": "shuntgen"}
@router.get("/api/shunts")
def list_shunts():
"""Every shunt under artery/shunts/, generated or hand-written."""
out = []
if SHUNTS_DIR.exists():
for path in sorted(SHUNTS_DIR.iterdir()):
if not path.is_dir() or path.name.startswith(("_", ".")):
continue
entry: dict[str, Any] = {"name": path.name, "generated": False}
spec_file = path / "depot" / "spec.json"
if spec_file.exists():
try:
spec = json.loads(spec_file.read_text())
entry.update(
generated=True,
title=spec.get("title"),
kind=spec.get("kind"),
source=spec.get("source"),
routes=len(spec.get("routes", [])),
models=len(spec.get("models", {})),
)
except (OSError, ValueError) as e:
entry["error"] = str(e)
out.append(entry)
return {"shunts": out}
@router.get("/api/sources")
def list_sources():
"""Candidate inputs: uploaded specs, and sheet folders under uploads/."""
specs, sheets = [], []
if UPLOAD_DIR.exists():
for path in sorted(UPLOAD_DIR.rglob("*")):
rel = str(path.relative_to(SPR_ROOT))
if path.is_file() and path.suffix.lower() in (".json", ".yaml", ".yml"):
specs.append(rel)
elif path.is_dir() and any(
child.suffix.lower() in (".csv", ".tsv", ".ods")
for child in path.iterdir()
if child.is_file()
):
sheets.append(rel)
fixtures = Path("station/tools/shuntgen/fixtures")
if (SPR_ROOT / fixtures).exists():
specs.append(str(fixtures / "petstore.yaml"))
sheets.append(str(fixtures / "sheets"))
return {"specs": specs, "sheets": sheets}
def _multipart_available() -> bool:
"""Whether FastAPI can accept file uploads in this environment.
Declaring an UploadFile parameter without python-multipart raises at import
time, not request time. Registering it unconditionally would mean one
missing optional dependency stops the whole tool from loading — so the
route is registered only when it can work, and a stub explains its absence.
"""
try:
import multipart # noqa: F401
return True
except ImportError:
return False
async def _upload(file: UploadFile, folder: str = ""):
"""Accept a spec or sheet, into uploads/[folder]/."""
if not file.filename:
raise HTTPException(status_code=400, detail="No filename")
suffix = Path(file.filename).suffix.lower()
if suffix not in (".json", ".yaml", ".yml", ".csv", ".tsv", ".ods"):
raise HTTPException(
status_code=400,
detail=f"Unsupported file type '{suffix}'. "
"Expected a spec (.json/.yaml/.yml) or a sheet (.csv/.tsv/.ods).",
)
payload = await file.read()
if len(payload) > MAX_UPLOAD_BYTES:
raise HTTPException(
status_code=413,
detail=f"File is larger than {MAX_UPLOAD_BYTES // (1024 * 1024)}MB",
)
target_dir = UPLOAD_DIR
if folder:
target_dir = _inside(Path("station/tools/shuntgen/uploads") / _safe_name(folder), SPR_ROOT)
target_dir.mkdir(parents=True, exist_ok=True)
target = target_dir / Path(file.filename).name
target.write_bytes(payload)
return {"path": str(target.relative_to(SPR_ROOT)), "bytes": len(payload)}
if _multipart_available():
router.add_api_route("/api/upload", _upload, methods=["POST"], name="upload")
else:
@router.post("/api/upload")
def upload_unavailable():
raise HTTPException(
status_code=501,
detail="Uploads need python-multipart. Install it, or point "
"'source' at a path on disk — generation itself works either way.",
)
@router.post("/api/generate")
def generate(req: GenerateRequest):
"""Generate a shunt into artery/shunts/<name>/."""
name = _safe_name(req.name)
source = _inside(Path(req.source), SPR_ROOT)
if not source.exists():
raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
output = _inside(Path("artery/shunts") / name, SPR_ROOT)
if (
output.exists()
and any(output.iterdir())
and not (output / "depot" / "spec.json").exists()
and not req.force
):
raise HTTPException(
status_code=409,
detail=f"artery/shunts/{name} exists and was not generated by shuntgen. "
"Pass force to write into it anyway.",
)
kind = req.kind
if kind == "auto":
kind = "tabular" if source.is_dir() else "openapi"
try:
if kind == "openapi":
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
extractor = OpenAPIExtractor(source)
models, enums = extractor.extract()
endpoints = extractor.endpoints()
datasets = []
if not endpoints:
raise HTTPException(
status_code=400,
detail="The spec declares no operations, so there is nothing to serve.",
)
elif kind == "tabular":
from ..modelgen.loader.extract.tabular import TabularExtractor
extractor = TabularExtractor(source)
models, enums = extractor.extract()
datasets = extractor.datasets()
endpoints = []
else:
raise HTTPException(status_code=400, detail=f"Unknown kind: {req.kind}")
except HTTPException:
raise
except (RuntimeError, ValueError) as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
log.exception("shuntgen: extraction failed")
raise HTTPException(status_code=500, detail=str(e))
try:
written = ShuntEmitter(
name=name,
output=output,
models=models,
enums=enums,
datasets=datasets,
endpoints=endpoints,
title=req.title,
source=source.name,
kind=kind,
port=req.port,
cabinets=req.cabinets,
).emit()
except Exception as e:
log.exception("shuntgen: emission failed")
raise HTTPException(status_code=500, detail=str(e))
spec = json.loads((written / "depot" / "spec.json").read_text())
return {
"name": name,
"path": str(written.relative_to(SPR_ROOT)),
"kind": kind,
"models": len(models),
"routes": len(spec.get("routes", [])),
"rows": sum(len(d.rows) for d in datasets),
"run": f"cd {written.relative_to(SPR_ROOT)} && python run.py",
}
@router.post("/api/preview")
def preview(req: GenerateRequest):
"""Extract and report what would be generated, writing nothing permanent."""
source = _inside(Path(req.source), SPR_ROOT)
if not source.exists():
raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
kind = req.kind
if kind == "auto":
kind = "tabular" if source.is_dir() else "openapi"
try:
if kind == "openapi":
from ..modelgen.loader.extract.openapi import OpenAPIExtractor
extractor = OpenAPIExtractor(source)
models, enums = extractor.extract()
endpoints, datasets = extractor.endpoints(), []
else:
from ..modelgen.loader.extract.tabular import TabularExtractor
extractor = TabularExtractor(source)
models, enums = extractor.extract()
datasets, endpoints = extractor.datasets(), []
except (RuntimeError, ValueError) as e:
raise HTTPException(status_code=400, detail=str(e))
# Emit into a throwaway directory: the routes are worked out by the emitter,
# and duplicating that logic here is how a preview drifts from the thing it
# is previewing.
temp = Path(tempfile.mkdtemp(prefix="shuntgen-preview-"))
try:
emitter = ShuntEmitter(
name=_safe_name(req.name or "preview"),
output=temp,
models=models,
enums=enums,
datasets=datasets,
endpoints=endpoints,
source=source.name,
kind=kind,
)
collections = emitter._collections()
routes = emitter._routes(collections)
finally:
shutil.rmtree(temp, ignore_errors=True)
return {
"kind": kind,
"models": [
{
"name": m.name,
"fields": len(m.fields),
"doc": (m.docstring or "").strip().splitlines()[0] if m.docstring else None,
}
for m in models
],
"collections": collections,
"routes": [
{
"method": r["method"],
"path": r["path"],
"operation": r["operation"],
"model": r["model"],
}
for r in routes
],
"rows": sum(len(d.rows) for d in datasets),
}

462
soleprint/station/tools/shuntgen/emit.py Normal file
View File

@@ -0,0 +1,462 @@
"""
Shunt emission — turns extracted models into a runnable shunt directory.
The layout follows the contract in artery/shunts/__init__.py (main.py, a depot,
a README) and adds what a generated shunt needs to answer for itself:
artery/shunts/<name>/
main.py builds the app from the spec
run.py uvicorn entry point
shunt_runtime.py vendored copy of runtime.py
models.py pydantic, via modelgen
datagen_<name>.py BaseDataGenerator subclass, via modelgen
depot/spec.json normalised routes + collections + schema
depot/responses.json pinned overrides, the "METHOD /path" map
depot/data.json imported rows
depot/config.json latency / error-injection knobs
templates/index.html config UI
cabinet.json declared dependency containers, if any
README.md
Only spec.json and the depot are worth editing by hand; everything else is
regenerated. The vendored runtime is a copy rather than an import because a
shunt runs standalone, with no soleprint on its path.
"""
import json
import shutil
from pathlib import Path
from typing import Any, Dict, List, Optional
from ..modelgen.generator import DatagenGenerator, JsonSchemaGenerator, PydanticGenerator
from ..modelgen.loader.schema import (
DatasetDefinition,
EndpointDefinition,
ModelDefinition,
)
HERE = Path(__file__).parent
# soleprint/ — four levels up from station/tools/shuntgen/emit.py
SPR_ROOT = HERE.parents[2]
# Types whose keys are integers, so the runtime coerces "/pets/7" to 7 before
# comparing it with a stored row.
_INT_HINTS = {int, "bigint"}
def _pk_of(model: ModelDefinition) -> tuple[Optional[str], str]:
"""Return (primary key field name, "int" | "str") for a model."""
for field in model.fields:
if getattr(field, "primary_key", False):
return field.name, "int" if field.type_hint in _INT_HINTS else "str"
for field in model.fields:
if field.name == "id":
return field.name, "int" if field.type_hint in _INT_HINTS else "str"
return None, "str"
def _operation(method: str, kind: str) -> str:
"""Name what a route does, from its verb and whether it addresses one row."""
if method == "GET":
return "list" if kind == "collection" else "retrieve" if kind == "item" else "action"
if method == "POST":
return "create" if kind == "collection" else "action"
if method in ("PUT", "PATCH"):
return "update" if kind == "item" else "action"
if method == "DELETE":
return "delete" if kind == "item" else "action"
return "action"
def _class_name(name: str) -> str:
parts = [p for p in name.replace("-", "_").split("_") if p]
return "".join(p[:1].upper() + p[1:] for p in parts) or "Shunt"
def _theme_css() -> str:
"""The theme, inlined.
A shunt serves its own UI on its own port, so it cannot fetch soleprint's
/theme.css. Inlining keeps it standalone and keeps one source of truth —
regenerating picks up any change to common/theme.
"""
theme_dir = SPR_ROOT / "common" / "theme"
parts: List[str] = []
tokens = theme_dir / "tokens.css"
if tokens.exists():
parts.append(tokens.read_text())
for sheet in sorted((theme_dir / "themes").glob("*.css")):
parts.append(sheet.read_text())
if parts:
return "\n".join(parts)
return ":root{--bg:#0a0a0a;--surface:#1a1a1a;--border:#333;--text:#e5e5e5;--muted:#a3a3a3;--accent:#d4a574}"
class ShuntEmitter:
"""Writes a complete shunt directory."""
def __init__(
self,
name: str,
output: Path,
models: List[ModelDefinition],
enums: Optional[List[Any]] = None,
datasets: Optional[List[DatasetDefinition]] = None,
endpoints: Optional[List[EndpointDefinition]] = None,
title: Optional[str] = None,
source: str = "",
kind: str = "openapi",
port: int = 8099,
cabinets: Optional[List[str]] = None,
):
self.name = name
self.output = Path(output)
self.models = models
self.enums = enums or []
self.datasets = datasets or []
self.endpoints = endpoints or []
self.title = title or _class_name(name)
self.source = source
self.kind = kind
self.port = port
self.cabinets = cabinets or []
self.by_name = {m.name: m for m in models}
# ── entry point ────────────────────────────────────────────────────────
def emit(self) -> Path:
self.output.mkdir(parents=True, exist_ok=True)
(self.output / "depot").mkdir(exist_ok=True)
(self.output / "templates").mkdir(exist_ok=True)
collections = self._collections()
routes = self._routes(collections)
self._write_models()
self._write_generator()
self._write_depot(routes, collections)
self._write_runtime()
self._write_app()
self._write_ui(routes)
self._write_cabinets()
self._write_readme(routes, collections)
return self.output
# ── spec construction ──────────────────────────────────────────────────
def _collections(self) -> Dict[str, dict]:
"""Models the runtime may keep rows for, with the key to match them on."""
out: Dict[str, dict] = {}
for dataset in self.datasets:
model = self.by_name.get(dataset.model)
if not model:
continue
pk, pk_type = _pk_of(model)
out[dataset.model] = {
"path": f"/{dataset.collection or dataset.model.lower()}",
"pk": pk or "id",
"pk_type": pk_type,
"rows": len(dataset.rows),
}
# A spec's models get a collection too, so POSTing to one and GETting it
# back works even though no rows were imported.
for endpoint in self.endpoints:
model_name = endpoint.model
if not model_name or model_name in out:
continue
model = self.by_name.get(model_name)
if not model:
continue
pk, pk_type = _pk_of(model)
if not pk:
continue
out[model_name] = {
"path": self._base_path(endpoint.path),
"pk": pk,
"pk_type": pk_type,
"rows": 0,
}
return out
@staticmethod
def _base_path(path: str) -> str:
"""Trim a path back to its collection — /pets/{petId} -> /pets."""
segments = [s for s in path.split("/") if s and not s.startswith("{")]
return "/" + "/".join(segments) if segments else "/"
def _routes(self, collections: Dict[str, dict]) -> List[dict]:
if self.endpoints:
return [self._from_endpoint(e) for e in self.endpoints]
return self._crud_routes(collections)
def _from_endpoint(self, endpoint: EndpointDefinition) -> dict:
return {
"method": endpoint.method,
"path": endpoint.path,
"operation_id": endpoint.operation_id,
"summary": endpoint.summary,
"operation": _operation(endpoint.method, endpoint.kind),
"model": endpoint.model,
"request_model": endpoint.request_model,
"response_is_list": endpoint.response_is_list,
"envelope_key": endpoint.envelope_key,
"status": endpoint.status,
"path_params": list(endpoint.path_params),
"example": endpoint.example,
}
def _crud_routes(self, collections: Dict[str, dict]) -> List[dict]:
"""The five routes a table implies, for sources that describe no calls."""
routes: List[dict] = []
for model_name, meta in collections.items():
base = meta["path"]
pk = meta["pk"]
item = f"{base}/{{{pk}}}"
plural = base.strip("/") or model_name.lower()
def route(method, path, operation, status, summary, params=()):
return {
"method": method,
"path": path,
"operation_id": f"{operation}_{plural}".replace("-", "_"),
"summary": summary,
"operation": operation,
"model": model_name,
"request_model": model_name if operation in ("create", "update") else None,
"response_is_list": operation == "list",
"envelope_key": None,
"status": status,
"path_params": list(params),
"example": None,
}
routes.extend([
route("GET", base, "list", 200, f"List {plural}"),
route("POST", base, "create", 201, f"Create a {model_name}"),
route("GET", item, "retrieve", 200, f"Fetch one {model_name}", (pk,)),
route("PUT", item, "update", 200, f"Update a {model_name}", (pk,)),
route("DELETE", item, "delete", 204, f"Delete a {model_name}", (pk,)),
])
return routes
# ── file writers ───────────────────────────────────────────────────────
def _write_models(self) -> None:
PydanticGenerator().generate(
(self.models, self.enums), self.output / "models.py"
)
def _generator_module(self) -> str:
return f"datagen_{self.name.replace('-', '_')}"
def _write_generator(self) -> None:
DatagenGenerator(
class_name=f"{_class_name(self.name)}Generator",
depot="depot/data.json",
).generate(
(self.models, self.enums, self.datasets),
self.output / f"{self._generator_module()}.py",
)
def _write_depot(self, routes: List[dict], collections: Dict[str, dict]) -> None:
depot = self.output / "depot"
schema_path = depot / "schema.json"
JsonSchemaGenerator().generate((self.models, self.enums), schema_path)
schema = json.loads(schema_path.read_text())
spec = {
"name": self.name,
"title": self.title,
"kind": self.kind,
"source": self.source,
"summary": f"Generated by shuntgen from {self.source or self.kind}.",
"generator_module": self._generator_module(),
"collections": collections,
"routes": routes,
"models": schema.get("models", {}),
}
(depot / "spec.json").write_text(json.dumps(spec, indent=2) + "\n")
data = {d.model: d.rows for d in self.datasets if d.rows}
(depot / "data.json").write_text(json.dumps(data, indent=2) + "\n")
# Left empty on purpose: an override is a deliberate act, and a file
# pre-filled with guesses would quietly shadow the generated responses.
responses_path = depot / "responses.json"
if not responses_path.exists():
responses_path.write_text("{}\n")
config = {
"title": self.title,
"port": self.port,
"enable_random_delays": False,
"min_delay_ms": 200,
"max_delay_ms": 800,
"error_rate": 0.0,
# Imported rows are the real thing; only invent when there are none.
"prefill": 0 if any(d.rows for d in self.datasets) else 5,
"unknown_id": "generate",
"page_size": 50,
}
config_path = depot / "config.json"
if not config_path.exists():
config_path.write_text(json.dumps(config, indent=2) + "\n")
def _write_runtime(self) -> None:
shutil.copyfile(HERE / "runtime.py", self.output / "shunt_runtime.py")
def _write_app(self) -> None:
(self.output / "main.py").write_text(
f'''"""
{self.title} shunt — GENERATED.
The routes live in depot/spec.json and are built at import time by
shunt_runtime. Edit the depot, not this file; regenerate with shuntgen.
"""
from pathlib import Path
from shunt_runtime import build_app
app = build_app(Path(__file__).parent)
'''
)
(self.output / "run.py").write_text(
f'''"""Run the {self.title} shunt standalone."""
import json
import os
from pathlib import Path
import uvicorn
BASE = Path(__file__).parent
def port() -> int:
"""PORT wins, then depot/config.json, then the generated default."""
if os.getenv("PORT"):
return int(os.environ["PORT"])
config = BASE / "depot" / "config.json"
if config.exists():
try:
return int(json.loads(config.read_text()).get("port", {self.port}))
except (OSError, ValueError):
pass
return {self.port}
if __name__ == "__main__":
chosen = port()
print(f"{self.title} shunt on http://localhost:{{chosen}} (UI at /, spec at /mock/spec)")
uvicorn.run("main:app", host="0.0.0.0", port=chosen, reload=False)
'''
)
def _write_ui(self, routes: List[dict]) -> None:
template = (HERE / "templates" / "shunt_ui.html").read_text()
page = (
template.replace("%%THEME_CSS%%", _theme_css())
.replace("%%TITLE%%", self.title)
.replace("%%NAME%%", self.name)
.replace("%%SOURCE%%", self.source or self.kind)
.replace("%%ROUTE_COUNT%%", str(len(routes)))
)
(self.output / "templates" / "index.html").write_text(page)
def _write_cabinets(self) -> None:
if not self.cabinets:
return
(self.output / "cabinet.json").write_text(
json.dumps(
{
"requires": self.cabinets,
"note": (
"Dependency containers this shunt expects. "
"`python build.py --cfg <room>` composes them into the "
"room's docker-compose.yml; on a cluster they install "
"as rig addons of the same name."
),
},
indent=2,
)
+ "\n"
)
def _write_readme(self, routes: List[dict], collections: Dict[str, dict]) -> None:
lines = [
f"# {self.title} shunt",
"",
f"Generated by shuntgen from `{self.source or self.kind}`.",
"",
"## Run",
"",
"```bash",
f"python run.py # http://localhost:{self.port}",
f"PORT=9000 python run.py # somewhere else",
"```",
"",
"## Routes",
"",
"| Method | Path | Does |",
"| --- | --- | --- |",
]
for route in routes:
lines.append(
f"| {route['method']} | `{route['path']}` | {route['operation']} |"
)
lines += [
"",
"## Control",
"",
"| Endpoint | Purpose |",
"| --- | --- |",
"| `GET /health` | liveness |",
"| `GET /mock/spec` | the routes this shunt was built from |",
"| `GET /mock/stats` | call counts and row counts |",
"| `POST /mock/reset` | restore the imported rows, clear counters |",
"| `GET,POST /mock/config` | latency and error-injection knobs |",
"| `GET,POST /mock/responses` | pin an override; set a key to `null` to drop it |",
"",
"## Depot",
"",
"| File | Purpose |",
"| --- | --- |",
"| `spec.json` | routes, collections and schema — the source of truth |",
"| `responses.json` | pinned overrides, keyed `\"METHOD /path\"`; these win over everything |",
"| `data.json` | seed rows, keyed by model |",
"| `config.json` | delays, error rate, prefill, page size |",
"",
]
if collections:
lines += ["## Collections", "", "| Model | Path | Key | Seed rows |", "| --- | --- | --- | --- |"]
for model, meta in collections.items():
lines.append(
f"| {model} | `{meta['path']}` | `{meta['pk']}` | {meta['rows']} |"
)
lines.append("")
if self.cabinets:
lines += [
"## Dependencies",
"",
f"Declares the cabinets: {', '.join(f'`{c}`' for c in self.cabinets)}. "
"See `cabinet.json`.",
"",
]
lines += [
"## Regenerating",
"",
"Everything here except `depot/responses.json` and `depot/config.json` is",
"overwritten on regeneration — those two are yours.",
"",
]
(self.output / "README.md").write_text("\n".join(lines))

147
soleprint/station/tools/shuntgen/fixtures/petstore.yaml Normal file
View File

@@ -0,0 +1,147 @@
# A small OpenAPI 3 document used to exercise shuntgen and the OpenAPI loader.
#
# Deliberately covers the shapes that are easy to get wrong: an enum, a $ref to
# another schema, an array of $ref, a wrapped collection, a path parameter, a
# request body, and a response example.
openapi: 3.0.3
info:
title: Petstore
version: 1.0.0
paths:
/pets:
get:
operationId: listPets
summary: List all pets
responses:
"200":
description: A page of pets
content:
application/json:
schema:
$ref: "#/components/schemas/PetPage"
post:
operationId: createPet
summary: Create a pet
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/Pet"
responses:
"201":
description: The created pet
content:
application/json:
schema:
$ref: "#/components/schemas/Pet"
/pets/{petId}:
get:
operationId: getPet
summary: Fetch one pet
parameters:
- name: petId
in: path
required: true
schema:
type: integer
responses:
"200":
description: The pet
content:
application/json:
schema:
$ref: "#/components/schemas/Pet"
example:
id: 7
name: "Rocinante"
status: "available"
"404":
description: No such pet
delete:
operationId: deletePet
summary: Remove a pet
parameters:
- name: petId
in: path
required: true
schema:
type: integer
responses:
"204":
description: Deleted
/categories:
get:
operationId: listCategories
summary: List categories
responses:
"200":
description: Every category
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Category"
components:
schemas:
Pet:
type: object
description: An animal available for adoption.
required: [id, name]
properties:
id:
type: integer
format: int64
name:
type: string
status:
type: string
enum: [available, pending, sold]
weight_kg:
type: number
neutered:
type: boolean
born_on:
type: string
format: date
category:
$ref: "#/components/schemas/Category"
tags:
type: array
items:
$ref: "#/components/schemas/Tag"
Category:
type: object
description: A grouping of pets.
required: [id, name]
properties:
id:
type: integer
name:
type: string
Tag:
type: object
required: [id]
properties:
id:
type: string
format: uuid
label:
type: string
PetPage:
type: object
properties:
items:
type: array
items:
$ref: "#/components/schemas/Pet"
total:
type: integer

6
soleprint/station/tools/shuntgen/fixtures/sheets/customers.csv Normal file
View File

@@ -0,0 +1,6 @@
id,name,email,city,active,signed_up_at,credit_limit
1,Ada Alvarez,ada@example.com,Buenos Aires,true,2024-03-11,15000.50
2,Bruno Bianchi,bruno@example.com,Rosario,true,2024-05-02,8000
3,Camila Castro,camila@example.com,Cordoba,false,2023-11-27,
4,Diego Duarte,diego@example.com,Montevideo,true,2025-01-19,22500.75
5,Elena Esposito,elena@example.com,Santiago,false,2025-06-30,3000
1 id name email city active signed_up_at credit_limit
2 1 Ada Alvarez ada@example.com Buenos Aires true 2024-03-11 15000.50
3 2 Bruno Bianchi bruno@example.com Rosario true 2024-05-02 8000
4 3 Camila Castro camila@example.com Cordoba false 2023-11-27
5 4 Diego Duarte diego@example.com Montevideo true 2025-01-19 22500.75
6 5 Elena Esposito elena@example.com Santiago false 2025-06-30 3000

7
soleprint/station/tools/shuntgen/fixtures/sheets/invoices.csv Normal file
View File

@@ -0,0 +1,7 @@
invoice_id,customer_id,issued_on,total,currency,paid
1001,1,2025-02-03,4200.00,ARS,true
1002,1,2025-03-03,3150.25,ARS,false
1003,2,2025-03-14,900.00,USD,true
1004,4,2025-04-21,12800.40,ARS,false
1005,5,2025-05-05,150.00,USD,true
1006,2,2025-06-11,7300.10,ARS,true
1 invoice_id customer_id issued_on total currency paid
2 1001 1 2025-02-03 4200.00 ARS true
3 1002 1 2025-03-03 3150.25 ARS false
4 1003 2 2025-03-14 900.00 USD true
5 1004 4 2025-04-21 12800.40 ARS false
6 1005 5 2025-05-05 150.00 USD true
7 1006 2 2025-06-11 7300.10 ARS true

BIN
soleprint/station/tools/shuntgen/fixtures/sheets/line_items.ods Normal file
View File

Binary file not shown.

489
soleprint/station/tools/shuntgen/runtime.py Normal file
View File

@@ -0,0 +1,489 @@
"""
Shunt runtime — copied verbatim into every generated shunt as shunt_runtime.py.
A generated shunt is a spec plus this file. The routes are built at startup
from depot/spec.json rather than written out as source, which is what keeps the
generated code short enough to read and the behaviour in one reviewable place:
fixing a bug here fixes it in every shunt, and regenerating is a copy.
It imports nothing from soleprint. A shunt runs as its own process on its own
port — often in a test harness that has no soleprint on the path — so the only
dependencies are fastapi and the standard library.
Where a response comes from, first hit wins:
1. depot/responses.json a pinned override, the documented "METHOD /path" map
2. the store rows imported from spreadsheets, plus anything POSTed
3. the spec example an `example` carried over from the source document
4. the generator datagen_<name>.py, synthesising from the schema
5. {} nothing else was available
The store is what makes it behave like a service rather than a random-value
faucet: POST something and GET it back, ask for /pets/7 and get the pet whose
id is 7.
"""
import importlib.util
import json
import random
import re
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, Response
DEFAULT_CONFIG: Dict[str, Any] = {
"title": "Shunt",
"port": 8099,
# Latency and failure injection, the knobs the mercadopago shunt proved out.
"enable_random_delays": False,
"min_delay_ms": 200,
"max_delay_ms": 800,
"error_rate": 0.0,
# Rows to synthesise per collection that arrived with none, so a GET
# answers with something on the first call rather than an empty list.
"prefill": 5,
# "generate" invents a record for an unknown id; "404" refuses it. Generate
# by default: a client pointed at a fresh shunt should just work, and a
# test that needs the error path can pin it or flip this.
"unknown_id": "generate",
"page_size": 50,
}
class Shunt:
"""Holds a shunt's spec, config, store and counters."""
def __init__(self, base: Path):
self.base = Path(base)
self.depot = self.base / "depot"
self.spec: Dict[str, Any] = self._read_json("spec.json", {})
self.responses: Dict[str, Any] = self._read_json("responses.json", {})
self.config: Dict[str, Any] = {
**DEFAULT_CONFIG,
**self._read_json("config.json", {}),
}
self.seed: Dict[str, List[dict]] = self._read_json("data.json", {})
self.collections: Dict[str, dict] = self.spec.get("collections", {}) or {}
self.stats: Dict[str, int] = {}
self.generator = self._load_generator()
self.store: Dict[str, List[dict]] = {}
self.reset()
# ── loading ────────────────────────────────────────────────────────────
def _read_json(self, name: str, fallback: Any) -> Any:
path = self.depot / name
if not path.exists():
return fallback
try:
return json.loads(path.read_text())
except (OSError, ValueError):
# A hand-edited responses.json with a stray comma should degrade to
# "no overrides", not stop the shunt from booting.
return fallback
def _load_generator(self):
"""Import the sibling datagen module, if one was generated."""
module_name = self.spec.get("generator_module")
if not module_name:
return None
path = self.base / f"{module_name}.py"
if not path.exists():
return None
try:
spec = importlib.util.spec_from_file_location(module_name, path)
if not spec or not spec.loader:
return None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
except Exception:
return None
# Same discovery rule as station/tools/datagen/api.py: the class whose
# name ends in Generator and is not the base.
for name, obj in vars(module).items():
if (
isinstance(obj, type)
and name.endswith("Generator")
and name != "BaseDataGenerator"
):
try:
return obj()
except Exception:
return None
return None
def reset(self) -> None:
"""Restore the store to its imported state and clear the counters."""
self.store = {model: [dict(r) for r in rows] for model, rows in self.seed.items()}
self.stats = {}
prefill = int(self.config.get("prefill") or 0)
if not prefill or not self.generator:
return
for model in self.collections:
if self.store.get(model):
continue
self.store[model] = self.synthesise(model, prefill, sequential_keys=True)
# ── generation ─────────────────────────────────────────────────────────
def synthesise(
self, model: str, count: int = 1, sequential_keys: bool = False
) -> List[dict]:
"""Build records for a model, or [] if nothing can."""
if not self.generator:
return []
try:
rows = self.generator.generate(model, count)
except Exception:
return []
if sequential_keys:
# Prefilled rows are the ones a caller will look up by id, so their
# keys have to be predictable — 1..n, not whatever random produced.
pk = (self.collections.get(model) or {}).get("pk")
pk_type = (self.collections.get(model) or {}).get("pk_type")
if pk and pk_type in ("int", "bigint"):
for index, row in enumerate(rows, start=1):
row[pk] = index
return rows
# ── store operations ───────────────────────────────────────────────────
def rows(self, model: str) -> List[dict]:
return self.store.setdefault(model, [])
def coerce_key(self, model: str, raw: Any) -> Any:
"""Match a path segment against the key type the rows actually use."""
pk_type = (self.collections.get(model) or {}).get("pk_type")
if pk_type in ("int", "bigint"):
try:
return int(raw)
except (TypeError, ValueError):
return raw
return str(raw)
def find(self, model: str, key: Any) -> Optional[dict]:
pk = (self.collections.get(model) or {}).get("pk")
if not pk:
return None
wanted = self.coerce_key(model, key)
for row in self.rows(model):
if row.get(pk) == wanted or str(row.get(pk)) == str(wanted):
return row
return None
def next_key(self, model: str) -> Any:
pk_meta = self.collections.get(model) or {}
pk, pk_type = pk_meta.get("pk"), pk_meta.get("pk_type")
if not pk:
return None
if pk_type in ("int", "bigint"):
keys = [r.get(pk) for r in self.rows(model) if isinstance(r.get(pk), int)]
return (max(keys) + 1) if keys else 1
return str(uuid.uuid4())
# ── request shaping ────────────────────────────────────────────────────
def maybe_fail(self) -> None:
rate = float(self.config.get("error_rate") or 0)
if rate > 0 and random.random() < rate:
raise HTTPException(
status_code=500,
detail="injected failure (error_rate is above zero)",
)
def maybe_delay(self) -> None:
if not self.config.get("enable_random_delays"):
return
low = int(self.config.get("min_delay_ms") or 0)
high = max(low, int(self.config.get("max_delay_ms") or 0))
time.sleep(random.randint(low, high) / 1000.0)
def pinned(self, method: str, template: str, actual: str) -> Any:
"""A configured override, matched on the template or the concrete path."""
for key in (f"{method} {template}", f"{method} {actual}"):
if key in self.responses:
return self.responses[key]
return None
def count(self, key: str) -> None:
self.stats[key] = self.stats.get(key, 0) + 1
# ── route handlers ─────────────────────────────────────────────────────────
def _filtered(rows: List[dict], params) -> List[dict]:
"""Narrow a collection by any query parameter naming a field."""
reserved = {"limit", "offset", "page", "page_size"}
out = rows
for name, value in params.multi_items():
if name in reserved:
continue
if not out or name not in out[0]:
continue
out = [r for r in out if str(r.get(name)) == str(value)]
return out
def _paged(rows: List[dict], params, default_size: int):
try:
limit = int(params.get("limit", params.get("page_size", default_size)))
except (TypeError, ValueError):
limit = default_size
try:
offset = int(params.get("offset", 0))
except (TypeError, ValueError):
offset = 0
limit = max(0, min(limit, 1000))
return rows[offset : offset + limit]
def _make_handler(shunt: Shunt, route: dict):
"""Build the endpoint function for one route in the spec."""
method = route["method"]
template = route["path"]
model = route.get("model")
operation = route.get("operation", "action")
envelope = route.get("envelope_key")
status = int(route.get("status") or 200)
example = route.get("example")
key_param = route["path_params"][-1] if route.get("path_params") else None
route_key = f"{method} {template}"
async def handler(request: Request):
shunt.count(route_key)
shunt.maybe_fail()
shunt.maybe_delay()
override = shunt.pinned(method, template, request.url.path)
if override is not None:
return JSONResponse(override, status_code=status)
body: Any = None
if method in ("POST", "PUT", "PATCH"):
try:
body = await request.json()
except Exception:
body = None
known = model in shunt.collections
key = request.path_params.get(key_param) if key_param else None
if known and operation == "list":
rows = _filtered(shunt.rows(model), request.query_params)
page = _paged(rows, request.query_params, int(shunt.config["page_size"]))
if envelope:
return JSONResponse({envelope: page, "total": len(rows)}, status_code=status)
return JSONResponse(page, status_code=status)
if known and operation == "retrieve":
found = shunt.find(model, key)
if found is not None:
return JSONResponse(found, status_code=status)
if shunt.config.get("unknown_id") == "404":
raise HTTPException(status_code=404, detail=f"No {model} with id {key}")
# Nothing stored under that id, so answer with the closest thing to
# an authored response: the spec's own example if there is one,
# otherwise a synthesised record. Either way it wears the id that
# was asked for — a record whose id disagrees with its own URL
# breaks clients that re-read what they just fetched.
pk = (shunt.collections.get(model) or {}).get("pk")
if isinstance(example, dict):
stand_in = dict(example)
else:
made = shunt.synthesise(model, 1)
stand_in = made[0] if made else None
if stand_in is not None:
if pk:
stand_in[pk] = shunt.coerce_key(model, key)
return JSONResponse(stand_in, status_code=status)
if known and operation == "create":
sent = dict(body) if isinstance(body, dict) else {}
pk = (shunt.collections.get(model) or {}).get("pk")
# Fill the gaps from the schema so a partial POST still comes back
# as a complete record.
made = shunt.synthesise(model, 1)
record = {**(made[0] if made else {}), **sent}
# The gap-filler samples an existing row when there are rows to
# sample, so it supplies that row's key too. Only a key the caller
# actually sent may survive; anything else gets a fresh one, or a
# POST silently overwrites an existing record's identity.
if pk and not sent.get(pk):
record[pk] = shunt.next_key(model)
shunt.rows(model).append(record)
return JSONResponse(record, status_code=status if status >= 200 else 201)
if known and operation == "update":
found = shunt.find(model, key)
if found is None:
if shunt.config.get("unknown_id") == "404":
raise HTTPException(
status_code=404, detail=f"No {model} with id {key}"
)
found = {}
pk = (shunt.collections.get(model) or {}).get("pk")
if pk:
found[pk] = shunt.coerce_key(model, key)
shunt.rows(model).append(found)
if isinstance(body, dict):
found.update(body)
return JSONResponse(found, status_code=status)
if known and operation == "delete":
found = shunt.find(model, key)
if found is not None:
shunt.rows(model).remove(found)
elif shunt.config.get("unknown_id") == "404":
raise HTTPException(status_code=404, detail=f"No {model} with id {key}")
return Response(status_code=204)
if example is not None:
return JSONResponse(example, status_code=status)
if model:
made = shunt.synthesise(model, 1)
if made:
payload: Any = made[0]
if route.get("response_is_list"):
rows = shunt.synthesise(model, 3) or made
payload = {envelope: rows, "total": len(rows)} if envelope else rows
return JSONResponse(payload, status_code=status)
if status == 204:
return Response(status_code=204)
return JSONResponse({}, status_code=status)
handler.__name__ = route.get("operation_id") or re.sub(
r"\W+", "_", f"{method}_{template}"
).strip("_")
return handler
def _sanitise(path: str) -> str:
"""Make spec path parameters legal Starlette converters."""
def fix(match):
name = re.sub(r"\W+", "_", match.group(1)).strip("_") or "param"
if name[0].isdigit():
name = f"p_{name}"
return "{" + name + "}"
return re.sub(r"\{([^}]*)\}", fix, path)
def build_app(base: Path) -> FastAPI:
"""Assemble a shunt's FastAPI app from the spec in its depot."""
base = Path(base)
shunt = Shunt(base)
title = shunt.spec.get("title") or shunt.config.get("title") or "Shunt"
app = FastAPI(title=title, description=shunt.spec.get("summary", ""))
@app.get("/health")
def health():
return {
"status": "ok",
"shunt": shunt.spec.get("name", "shunt"),
"routes": len(shunt.spec.get("routes", [])),
}
@app.get("/mock/spec")
def mock_spec():
return shunt.spec
@app.get("/mock/stats")
def mock_stats():
return {
"calls": shunt.stats,
"total": sum(shunt.stats.values()),
"rows": {model: len(rows) for model, rows in shunt.store.items()},
}
@app.post("/mock/reset")
@app.get("/mock/reset")
def mock_reset():
shunt.reset()
return {"status": "reset", "rows": {m: len(r) for m, r in shunt.store.items()}}
@app.get("/mock/config")
def mock_config_get():
return shunt.config
@app.post("/mock/config")
async def mock_config_set(request: Request):
try:
incoming = await request.json()
except Exception:
raise HTTPException(status_code=400, detail="Body must be a JSON object")
if not isinstance(incoming, dict):
raise HTTPException(status_code=400, detail="Body must be a JSON object")
# Only known knobs, so a typo is reported rather than silently stored.
unknown = set(incoming) - set(DEFAULT_CONFIG)
if unknown:
raise HTTPException(
status_code=400, detail=f"Unknown settings: {sorted(unknown)}"
)
shunt.config.update(incoming)
return shunt.config
@app.get("/mock/responses")
def mock_responses():
return shunt.responses
@app.post("/mock/responses")
async def mock_responses_set(request: Request):
"""Pin or clear an override without restarting — set a key to null to drop it."""
try:
incoming = await request.json()
except Exception:
raise HTTPException(status_code=400, detail="Body must be a JSON object")
if not isinstance(incoming, dict):
raise HTTPException(status_code=400, detail="Body must be a JSON object")
for key, value in incoming.items():
if value is None:
shunt.responses.pop(key, None)
else:
shunt.responses[key] = value
return shunt.responses
# Static segments before parameterised ones, so /pets/search is not
# swallowed by /pets/{petId} when a spec declares both.
routes = sorted(
shunt.spec.get("routes", []),
key=lambda r: (len(r.get("path_params") or []), r.get("path", "")),
)
claimed = set()
for route in routes:
path = _sanitise(route["path"])
key = (route["method"], path)
if key in claimed:
continue
claimed.add(key)
app.add_api_route(
path,
_make_handler(shunt, route),
methods=[route["method"]],
name=route.get("operation_id") or None,
summary=route.get("summary") or None,
)
if ("GET", "/") not in claimed:
@app.get("/", response_class=HTMLResponse)
def config_ui():
page = base / "templates" / "index.html"
if page.exists():
return HTMLResponse(page.read_text())
return HTMLResponse(f"<h1>{title}</h1><p>See /mock/spec and /health.</p>")
app.state.shunt = shunt
return app

294
soleprint/station/tools/shuntgen/templates/index.html Normal file
View File

@@ -0,0 +1,294 @@
<!DOCTYPE html>
<html lang="en" data-theme="soleprint">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>shuntgen</title>
<link rel="stylesheet" href="/theme.css">
<style>
* { box-sizing: border-box; }
body { margin: 0; padding: var(--space-6); max-width: 1200px; }
header { display: flex; align-items: baseline; gap: var(--space-3); flex-wrap: wrap;
padding-bottom: var(--space-4); border-bottom: var(--hairline) solid var(--border); }
h1 { margin: 0; font-size: 20px; color: var(--accent-text); }
.sub { color: var(--muted); font-family: var(--font-mono); font-size: 12px; }
h2 { font-size: 13px; color: var(--muted); margin: var(--space-6) 0 var(--space-3); }
.cols { display: grid; grid-template-columns: 340px 1fr; gap: var(--space-4);
align-items: start; margin-top: var(--space-4); }
@media (max-width: 860px) { .cols { grid-template-columns: 1fr; } }
.panel { background: var(--surface); border: var(--hairline) solid var(--border);
border-radius: var(--radius-lg); padding: var(--space-4); }
label { display: block; font-size: 11px; color: var(--muted); margin-top: var(--space-3);
text-transform: uppercase; letter-spacing: var(--label-spacing); }
input, select { width: 100%; padding: 6px 8px; margin-top: 4px;
font-family: var(--font-mono); font-size: 12px; }
input[type="file"] { padding: 4px; font-family: var(--font-ui); }
.row { display: flex; gap: var(--space-2); }
.row > * { flex: 1; }
.actions { display: flex; gap: var(--space-2); margin-top: var(--space-4); }
button { padding: 7px 14px; font-size: 12px; }
button.primary { background: var(--accent); color: var(--bg); border-color: var(--accent); }
button.primary:hover:not(:disabled) { background: var(--accent-dim); color: var(--bg); }
table { width: 100%; border-collapse: collapse; font-size: 12px; }
th { text-align: left; padding: 6px 8px; color: var(--muted); font-weight: 600;
font-size: 10px; text-transform: uppercase; letter-spacing: var(--label-spacing);
border-bottom: var(--hairline) solid var(--border); }
td { padding: 6px 8px; border-bottom: var(--hairline) solid var(--border);
font-family: var(--font-mono); }
tr:hover td { background: var(--bg-2); }
.verb { font-weight: 600; font-size: 10px; padding: 1px 6px; border-radius: var(--radius-sm);
border: 1px solid currentColor; }
.GET { color: var(--status-info); }
.POST { color: var(--status-ok); }
.PUT, .PATCH { color: var(--status-warn); }
.DELETE { color: var(--status-error); }
.msg { margin-top: var(--space-3); padding: var(--space-3); border-radius: var(--radius);
font-size: 12px; border: var(--hairline) solid var(--border); display: none; }
.msg.show { display: block; }
.msg.ok { border-color: var(--status-ok); color: var(--status-ok); }
.msg.bad { border-color: var(--status-error); color: var(--status-error); }
.msg code { color: var(--text); }
.empty { color: var(--dim); font-size: 12px; padding: var(--space-3); }
.note { color: var(--dim); font-size: 11px; margin-top: var(--space-2); }
.chip { display: inline-block; font-family: var(--font-mono); font-size: 10px;
padding: 1px 6px; border-radius: var(--radius-sm);
border: var(--hairline) solid var(--border); color: var(--muted); }
</style>
</head>
<body>
<header>
<h1>shuntgen</h1>
<span class="sub">a spec or a folder of sheets &rarr; a running fake service</span>
</header>
<div class="cols">
<div>
<h2>Source</h2>
<div class="panel">
<label for="source">Spec or sheet folder</label>
<select id="source"><option value="">loading…</option></select>
<label for="upload">…or upload</label>
<input type="file" id="upload"
accept=".json,.yaml,.yml,.csv,.tsv,.ods">
<p class="note">
A spec (.json/.yaml) becomes routes. Sheets (.csv/.tsv/.ods) become
models with CRUD over their real rows — upload them into a folder
to import several at once.
</p>
<label for="folder">Upload into folder (sheets)</label>
<input type="text" id="folder" placeholder="e.g. invoicing">
<h2>Shunt</h2>
<label for="name">Name</label>
<input type="text" id="name" placeholder="petstore">
<div class="row">
<div>
<label for="port">Port</label>
<input type="number" id="port" value="8099">
</div>
<div>
<label for="kind">Kind</label>
<select id="kind">
<option value="auto">auto</option>
<option value="openapi">openapi</option>
<option value="tabular">tabular</option>
</select>
</div>
</div>
<label for="cabinets">Cabinets (comma-separated)</label>
<input type="text" id="cabinets" placeholder="postgres,redis">
<div class="actions">
<button type="button" id="preview">Preview</button>
<button type="button" id="generate" class="primary">Generate</button>
</div>
<div class="msg" id="msg"></div>
</div>
</div>
<div>
<h2>What it will serve</h2>
<div class="panel">
<table id="routes">
<thead>
<tr><th>Method</th><th>Path</th><th>Does</th><th>Model</th></tr>
</thead>
<tbody><tr><td colspan="4" class="empty">pick a source and preview</td></tr></tbody>
</table>
</div>
<h2>Existing shunts</h2>
<div class="panel">
<table id="shunts">
<thead>
<tr><th>Name</th><th>Kind</th><th>Source</th><th>Routes</th><th>Models</th></tr>
</thead>
<tbody><tr><td colspan="5" class="empty">loading…</td></tr></tbody>
</table>
</div>
</div>
</div>
<script src="/theme.js" defer></script>
<script>
const BASE = "/station/tools/shuntgen";
const $ = (id) => document.getElementById(id);
function say(text, ok) {
const box = $("msg");
box.className = "msg show " + (ok ? "ok" : "bad");
box.innerHTML = text;
}
function body() {
return {
name: $("name").value.trim() || "preview",
source: $("source").value,
kind: $("kind").value,
port: Number($("port").value) || 8099,
cabinets: $("cabinets").value.split(",").map(s => s.trim()).filter(Boolean),
};
}
async function post(path, payload) {
const response = await fetch(BASE + path, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
});
const data = await response.json().catch(() => ({}));
if (!response.ok) throw new Error(data.detail || response.statusText);
return data;
}
function cell(text) {
const td = document.createElement("td");
td.textContent = text == null ? "" : String(text);
return td;
}
function fill(table, rows, columns, emptyText) {
const tbody = table.querySelector("tbody");
tbody.replaceChildren();
if (!rows.length) {
const tr = document.createElement("tr");
const td = cell(emptyText);
td.colSpan = columns;
td.className = "empty";
tr.appendChild(td);
tbody.appendChild(tr);
}
return tbody;
}
async function loadSources() {
const data = await (await fetch(BASE + "/api/sources")).json();
const select = $("source");
select.replaceChildren();
const all = [...data.specs.map(p => ["spec", p]), ...data.sheets.map(p => ["sheets", p])];
if (!all.length) {
select.appendChild(new Option("nothing uploaded yet", ""));
return;
}
for (const [kind, path] of all) {
select.appendChild(new Option(`${kind} · ${path}`, path));
}
if (!$("name").value) {
const guess = all[0][1].split("/").pop().replace(/\.[^.]+$/, "");
$("name").placeholder = guess;
}
}
async function loadShunts() {
const data = await (await fetch(BASE + "/api/shunts")).json();
const tbody = fill($("shunts"), data.shunts, 5, "no shunts yet");
for (const shunt of data.shunts) {
const tr = document.createElement("tr");
tr.appendChild(cell(shunt.name));
tr.appendChild(cell(shunt.generated ? shunt.kind : "hand-written"));
tr.appendChild(cell(shunt.source));
tr.appendChild(cell(shunt.routes));
tr.appendChild(cell(shunt.models));
tbody.appendChild(tr);
}
}
function showRoutes(routes) {
const tbody = fill($("routes"), routes, 4, "nothing to serve");
for (const route of routes) {
const tr = document.createElement("tr");
const verb = document.createElement("td");
const badge = document.createElement("span");
badge.className = "verb " + route.method;
badge.textContent = route.method;
verb.appendChild(badge);
tr.appendChild(verb);
tr.appendChild(cell(route.path));
tr.appendChild(cell(route.operation));
tr.appendChild(cell(route.model));
tbody.appendChild(tr);
}
}
$("upload").addEventListener("change", async (event) => {
const file = event.target.files[0];
if (!file) return;
const form = new FormData();
form.append("file", file);
const folder = $("folder").value.trim();
const url = BASE + "/api/upload" + (folder ? "?folder=" + encodeURIComponent(folder) : "");
const response = await fetch(url, { method: "POST", body: form });
const data = await response.json().catch(() => ({}));
if (!response.ok) {
say(data.detail || "Upload failed", false);
return;
}
say("Uploaded <code>" + data.path + "</code>", true);
await loadSources();
$("source").value = folder
? data.path.split("/").slice(0, -1).join("/")
: data.path;
});
$("preview").addEventListener("click", async () => {
if (!$("source").value) { say("Pick or upload a source first", false); return; }
try {
const data = await post("/api/preview", body());
showRoutes(data.routes);
say(
`${data.models.length} models, ${data.routes.length} routes` +
(data.rows ? `, ${data.rows} rows` : "") + ` (${data.kind})`, true
);
} catch (error) { say(error.message, false); }
});
$("generate").addEventListener("click", async () => {
if (!$("source").value) { say("Pick or upload a source first", false); return; }
if (!$("name").value.trim()) { say("Give the shunt a name", false); return; }
try {
const data = await post("/api/generate", body());
say(
`Wrote <code>${data.path}</code> — ${data.routes} routes. ` +
`Run it with <code>${data.run}</code>`, true
);
await loadShunts();
} catch (error) { say(error.message, false); }
});
Promise.all([loadSources(), loadShunts()]).catch((e) => say(e.message, false));
</script>
</body>
</html>

242
soleprint/station/tools/shuntgen/templates/shunt_ui.html Normal file
View File

@@ -0,0 +1,242 @@
<!DOCTYPE html>
<!--
Config UI for a generated shunt. Emitted into <shunt>/templates/index.html
with the theme inlined, because a shunt serves this on its own port and
cannot reach soleprint's /theme.css.
Placeholders are filled by emit.py: %%THEME_CSS%%, %%TITLE%%, %%NAME%%,
%%SOURCE%%, %%ROUTE_COUNT%%.
-->
<html lang="en" data-theme="soleprint">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>%%TITLE%% shunt</title>
<style>
%%THEME_CSS%%
* { box-sizing: border-box; }
body { margin: 0; padding: var(--space-6); max-width: 1100px; }
header { display: flex; align-items: baseline; gap: var(--space-3); flex-wrap: wrap;
padding-bottom: var(--space-4); border-bottom: var(--hairline) solid var(--border); }
h1 { margin: 0; font-size: 20px; color: var(--accent-text); }
.sub { color: var(--muted); font-family: var(--font-mono); font-size: 12px; }
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--status-ok);
display: inline-block; margin-right: 6px; }
h2 { font-size: 13px; color: var(--muted); margin: var(--space-6) 0 var(--space-3); }
.panel { background: var(--surface); border: var(--hairline) solid var(--border);
border-radius: var(--radius-lg); padding: var(--space-4); }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
gap: var(--space-3); }
table { width: 100%; border-collapse: collapse; font-size: 12px; }
th { text-align: left; padding: 6px 8px; color: var(--muted); font-weight: 600;
font-size: 10px; text-transform: uppercase; letter-spacing: var(--label-spacing);
border-bottom: var(--hairline) solid var(--border); }
td { padding: 6px 8px; border-bottom: var(--hairline) solid var(--border);
font-family: var(--font-mono); }
tr:hover td { background: var(--bg-2); }
.verb { font-weight: 600; font-size: 10px; padding: 1px 6px; border-radius: var(--radius-sm);
border: 1px solid currentColor; }
.GET { color: var(--status-info); }
.POST { color: var(--status-ok); }
.PUT, .PATCH { color: var(--status-warn); }
.DELETE { color: var(--status-error); }
a.route { color: var(--text); text-decoration: none; border-bottom: 1px solid transparent; }
a.route:hover { color: var(--accent-text); border-bottom-color: var(--accent); }
label { display: flex; align-items: center; justify-content: space-between;
gap: var(--space-2); font-size: 12px; color: var(--muted); padding: 4px 0; }
input[type="number"], input[type="text"], select { padding: 4px 8px; width: 130px;
font-family: var(--font-mono); font-size: 12px; }
input[type="checkbox"] { width: auto; }
.actions { display: flex; gap: var(--space-2); margin-top: var(--space-3); }
button { padding: 6px 12px; font-size: 12px; }
pre { background: var(--bg); border: var(--hairline) solid var(--border);
border-radius: var(--radius); padding: var(--space-3); overflow-x: auto;
font-size: 11px; margin: 0; max-height: 320px; }
.note { color: var(--dim); font-size: 11px; margin-top: var(--space-2); }
.empty { color: var(--dim); font-size: 12px; }
</style>
</head>
<body>
<header>
<h1>%%TITLE%%</h1>
<span class="sub"><span class="dot"></span>shunt &middot; %%NAME%%</span>
<span class="sub">%%ROUTE_COUNT%% routes from %%SOURCE%%</span>
</header>
<h2>Routes</h2>
<div class="panel">
<table id="routes">
<thead>
<tr><th>Method</th><th>Path</th><th>Does</th><th>Model</th><th>Calls</th></tr>
</thead>
<tbody><tr><td colspan="5" class="empty">loading…</td></tr></tbody>
</table>
<p class="note">GET routes are links — they open against this shunt.</p>
</div>
<div class="grid" style="margin-top: var(--space-6)">
<div>
<h2>Behaviour</h2>
<div class="panel">
<label>Random delays <input type="checkbox" id="enable_random_delays"></label>
<label>Min delay (ms) <input type="number" id="min_delay_ms" min="0"></label>
<label>Max delay (ms) <input type="number" id="max_delay_ms" min="0"></label>
<label>Error rate (0–1) <input type="number" id="error_rate" min="0" max="1" step="0.05"></label>
<label>Unknown id
<select id="unknown_id">
<option value="generate">generate</option>
<option value="404">404</option>
</select>
</label>
<label>Page size <input type="number" id="page_size" min="1"></label>
<div class="actions">
<button type="button" id="save">Apply</button>
<button type="button" id="reset">Reset data</button>
</div>
<p class="note">Applies immediately, in memory. Edit depot/config.json to persist.</p>
</div>
</div>
<div>
<h2>Stored rows</h2>
<div class="panel"><pre id="stats">loading…</pre></div>
</div>
</div>
<h2>Pinned responses</h2>
<div class="panel">
<pre id="responses">loading…</pre>
<p class="note">
Keys are <code>"METHOD /path"</code> and win over everything else.
Edit <code>depot/responses.json</code>, or POST to <code>/mock/responses</code>.
</p>
</div>
<script>
const KNOBS = ["enable_random_delays", "min_delay_ms", "max_delay_ms",
"error_rate", "unknown_id", "page_size"];
async function get(path) {
const response = await fetch(path);
if (!response.ok) throw new Error(path + " -> " + response.status);
return response.json();
}
function cell(text) {
const td = document.createElement("td");
td.textContent = text == null ? "" : String(text);
return td;
}
async function loadRoutes() {
const [spec, stats] = await Promise.all([get("/mock/spec"), get("/mock/stats")]);
const body = document.querySelector("#routes tbody");
body.replaceChildren();
if (!spec.routes || !spec.routes.length) {
const row = document.createElement("tr");
const td = cell("no routes in spec.json");
td.colSpan = 5;
td.className = "empty";
row.appendChild(td);
body.appendChild(row);
return;
}
for (const route of spec.routes) {
const row = document.createElement("tr");
const verb = document.createElement("td");
const badge = document.createElement("span");
badge.className = "verb " + route.method;
badge.textContent = route.method;
verb.appendChild(badge);
row.appendChild(verb);
const path = document.createElement("td");
if (route.method === "GET" && !route.path.includes("{")) {
const link = document.createElement("a");
link.className = "route";
link.href = route.path;
link.textContent = route.path;
path.appendChild(link);
} else {
path.textContent = route.path;
}
row.appendChild(path);
row.appendChild(cell(route.operation));
row.appendChild(cell(route.model));
row.appendChild(cell(stats.calls[route.method + " " + route.path] || 0));
body.appendChild(row);
}
}
async function loadConfig() {
const config = await get("/mock/config");
for (const knob of KNOBS) {
const input = document.getElementById(knob);
if (!input) continue;
if (input.type === "checkbox") input.checked = Boolean(config[knob]);
else input.value = config[knob];
}
}
async function loadStats() {
const stats = await get("/mock/stats");
document.getElementById("stats").textContent = JSON.stringify(stats.rows, null, 2);
}
async function loadResponses() {
const pinned = await get("/mock/responses");
const target = document.getElementById("responses");
target.textContent = Object.keys(pinned).length
? JSON.stringify(pinned, null, 2)
: "{} — nothing pinned; responses come from the store, examples, or the generator";
}
document.getElementById("save").addEventListener("click", async () => {
const payload = {};
for (const knob of KNOBS) {
const input = document.getElementById(knob);
if (!input) continue;
if (input.type === "checkbox") payload[knob] = input.checked;
else if (input.type === "number") payload[knob] = Number(input.value);
else payload[knob] = input.value;
}
const response = await fetch("/mock/config", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
});
if (!response.ok) {
alert("Could not apply: " + (await response.text()));
return;
}
await loadConfig();
});
document.getElementById("reset").addEventListener("click", async () => {
await fetch("/mock/reset", { method: "POST" });
await Promise.all([loadStats(), loadRoutes()]);
});
function refresh() {
return Promise.all([loadRoutes(), loadConfig(), loadStats(), loadResponses()])
.catch((error) => console.error(error));
}
refresh();
setInterval(() => Promise.all([loadRoutes(), loadStats()]).catch(() => {}), 5000);
</script>
</body>
</html>