verbose live UI + tool-level SSE events + Groq default + regression tests

api/datasets/README.md

@@ -0,0 +1,23 @@
+# Datasets
+
+One subdirectory per dataset. The folder name is the dataset's identity —
+used as:
+
+- the `NVI_DATASET` env var value that selects it,
+- the Postgres schema where its tables live,
+- the SQLite filename in `ctrl/seed/data/<name>.sqlite` that the loader
+  reads from.
+
+## Adding a new dataset
+
+1. Place the source SQLite at `ctrl/seed/data/<name>.sqlite` (or adapt
+   `ctrl/seed/download.sh` to fetch it).
+2. Create `api/datasets/<name>/`:
+   - `schema_docs.yaml` — table and column descriptions.
+   - `metrics.yaml` — canonical SQL for business terms.
+3. Set `NVI_DATASET=<name>` and restart the api.
+4. `make seed` will load `<name>.sqlite` into Postgres schema `<name>`.
+
+The Plan/Analyses/Tools/Runtime code references no dataset name; only the
+loader (`api.tools.schema.load_schema`) and the engine (`api.tools.db`)
+consult the active-dataset setting.

api/datasets/__init__.py

@@ -0,0 +1,40 @@
+"""Dataset registry — the only dataset-specific surface in api/.
+
+Each subdirectory is one dataset. The folder name doubles as the Postgres
+schema name, the SQLite filename under `ctrl/seed/data/<name>.sqlite`, and
+the value of the `NVI_DATASET` env var that selects it.
+
+Inside each dataset folder:
+  schema_docs.yaml   — human descriptions for tables/columns
+  metrics.yaml       — canonical SQL fragments for business terms
+
+Both files are consumed by `api.tools.schema.load_schema()`. The Plan,
+Analyses, Tools, and Runtime code reference no dataset name — they read
+the active dataset through `settings.dataset`.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from api.config import get_settings
+
+DATASETS_DIR = Path(__file__).resolve().parent
+
+
+def active_dir() -> Path:
+    return DATASETS_DIR / get_settings().dataset
+
+
+def available() -> list[str]:
+    return sorted(p.name for p in DATASETS_DIR.iterdir() if p.is_dir() and not p.name.startswith("_"))
+
+
+@lru_cache(maxsize=None)
+def read_yaml(dataset: str, filename: str) -> dict[str, Any]:
+    """Load one YAML file from a dataset folder. Cached per (dataset, file)."""
+    return yaml.safe_load((DATASETS_DIR / dataset / filename).read_text()) or {}

api/datasets/financial/metrics.yaml

@@ -0,0 +1,50 @@
+# Domain metrics for the financial schema. The planner reads this so it can
+# resolve a business term ("default rate", "loan volume") to a canonical SQL
+# expression instead of guessing.
+
+schema: financial
+
+metrics:
+  loan_default_rate:
+    description: Fraction of finished loans that ended in default (status 'B'). Excludes still-running loans.
+    sql: "AVG(CASE WHEN status = 'B' THEN 1.0 WHEN status = 'A' THEN 0.0 END)"
+    from_table: loan
+    filter: "status IN ('A','B')"
+    unit: ratio
+
+  at_risk_loan_share:
+    description: Of currently running loans, the share that are in debt (status 'D').
+    sql: "AVG(CASE WHEN status = 'D' THEN 1.0 WHEN status = 'C' THEN 0.0 END)"
+    from_table: loan
+    filter: "status IN ('C','D')"
+    unit: ratio
+
+  loan_volume:
+    description: Total CZK lent.
+    sql: "SUM(amount)"
+    from_table: loan
+    unit: CZK
+
+  average_loan_amount:
+    description: Mean loan size in CZK.
+    sql: "AVG(amount)"
+    from_table: loan
+    unit: CZK
+
+  transaction_volume:
+    description: Total CZK moved through transactions (credits + debits, absolute).
+    sql: "SUM(amount)"
+    from_table: trans
+    unit: CZK
+
+  active_accounts:
+    description: Distinct accounts with at least one transaction in the period.
+    sql: "COUNT(DISTINCT account_id)"
+    from_table: trans
+    unit: count
+
+  unemployment_rate_96:
+    description: District unemployment rate in 1996 (already a percentage).
+    sql: "A13"
+    from_table: district
+    unit: percent

api/datasets/financial/schema_docs.yaml

@@ -0,0 +1,93 @@
+# Per-table / per-column human descriptions for the BIRD `financial` schema
+# (PKDD'99 Czech bank). The text-to-SQL tool prepends the relevant subset to
+# the LLM prompt so it can disambiguate cryptic column names like A2, A3, etc.
+
+schema: financial
+
+tables:
+  account:
+    description: One row per bank account. The pivot table — most facts hang off account_id.
+    columns:
+      account_id: Surrogate primary key for the account.
+      district_id: Geographic district the account belongs to (joins to district).
+      frequency: Statement issuance frequency. "POPLATEK MESICNE" = monthly, "POPLATEK TYDNE" = weekly, "POPLATEK PO OBRATU" = on transaction.
+      date: Date the account was opened (YYMMDD as integer).
+
+  client:
+    description: One row per bank customer.
+    columns:
+      client_id: Surrogate primary key for the client.
+      gender: '"M" or "F". Derived from birth_number.'
+      birth_date: Date of birth, normalised. Original birth_number encoded gender by adding 50 to the month field for women.
+      district_id: District the client lives in (joins to district).
+
+  disp:
+    description: Disposition — many-to-many link between client and account. Type tells you whether the client is the OWNER or just a USER of the account.
+    columns:
+      disp_id: Surrogate key.
+      client_id: Joins to client.
+      account_id: Joins to account.
+      type: '"OWNER" or "DISPONENT" (USER). Only owners can request loans / cards.'
+
+  district:
+    description: Demographic and economic data for each Czech district. The A-columns are infamous — most analysis questions referencing demographics use these.
+    columns:
+      district_id: Surrogate key.
+      A2: District name.
+      A3: Region (a grouping of districts).
+      A4: Number of inhabitants.
+      A5: Number of municipalities with population < 499.
+      A6: Number of municipalities with population 500-1999.
+      A7: Number of municipalities with population 2000-9999.
+      A8: Number of municipalities with population > 10000.
+      A9: Number of cities.
+      A10: Ratio of urban inhabitants (percent).
+      A11: Average salary (in CZK).
+      A12: Unemployment rate in 1995 (percent). NULL means missing data.
+      A13: Unemployment rate in 1996 (percent).
+      A14: Number of entrepreneurs per 1000 inhabitants.
+      A15: Number of crimes committed in 1995. NULL means missing data.
+      A16: Number of crimes committed in 1996.
+
+  loan:
+    description: One row per loan granted to an account.
+    columns:
+      loan_id: Surrogate key.
+      account_id: The account the loan was granted to.
+      date: Loan grant date (YYMMDD as integer).
+      amount: Total amount of the loan (CZK).
+      duration: Loan duration in months.
+      payments: Monthly payment (CZK).
+      status: Loan status code. "A" = contract finished, no problems. "B" = contract finished, loan not paid (DEFAULT). "C" = running contract, payments on time. "D" = running contract, client in debt (AT RISK). Defaults are A,B finished or B,D paid-flag attached.
+
+  card:
+    description: One row per credit/debit card issued to a disposition.
+    columns:
+      card_id: Surrogate key.
+      disp_id: The disposition (and via it, the account) the card belongs to.
+      type: '"junior", "classic", or "gold".'
+      issued: Date the card was issued.
+
+  trans:
+    description: Transaction history. Largest table by far (~1M rows). Every credit, debit, transfer.
+    columns:
+      trans_id: Surrogate key.
+      account_id: The account the transaction belongs to.
+      date: Transaction date (YYMMDD as integer).
+      type: '"PRIJEM" (credit) or "VYDAJ" (debit).'
+      operation: Mode of operation, e.g. "VKLAD" (cash credit), "VYBER" (cash withdrawal), "PREVOD Z UCTU" (transfer from another bank), etc.
+      amount: Amount (CZK).
+      balance: Account balance after the transaction.
+      k_symbol: Characterisation of the payment. "POJISTNE" = insurance, "SLUZBY" = household, "UROK" = interest credited, "SANKC. UROK" = penalty interest, "SIPO" = household payment, "DUCHOD" = old-age pension, "UVER" = loan payment.
+      bank: Counterparty bank code (only set for inter-bank transfers).
+      account: Counterparty account number (only set for inter-bank transfers).
+
+  "order":
+    description: Standing payment orders (recurring debits). Note the table name collides with a SQL keyword and must be quoted.
+    columns:
+      order_id: Surrogate key.
+      account_id: The originating account.
+      bank_to: Recipient bank.
+      account_to: Recipient account.
+      amount: Order amount (CZK).
+      k_symbol: Purpose, same vocabulary as trans.k_symbol.