recon-driven sql composer; pick → compose → execute; llm out of structural sql

api/recon/build.py

@@ -80,16 +80,25 @@ def _read_yaml(path: Path) -> dict[str, Any]:
     return yaml.safe_load(path.read_text()) or {}
 
 
-def _parse_column_descs(raw: dict[str, Any]) -> dict[str, str]:
-    """Accept either the flat form `{table.col: desc}` or the nested form
-    `{table: {col: desc}}`. Returns flat form."""
-    out: dict[str, str] = {}
-    for k, v in raw.items():
+def _parse_column_specs(raw: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """Accept either the flat form `{table.col: <spec>}` or the nested form
+    `{table: {col: <spec>}}`. Each `<spec>` is either a bare description
+    string OR a dict `{desc?: str, type?: str}` where `type` is a domain-
+    level semantic type (e.g. "date_yymmdd"). Returns flat form: each value
+    is normalised to `{desc, type}`."""
+    def norm(v: Any) -> dict[str, Any]:
         if isinstance(v, dict):
-            for col, desc in v.items():
-                out[f"{k}.{col}"] = desc
+            return {"desc": v.get("desc") or v.get("description"), "type": v.get("type")}
+        return {"desc": v, "type": None}
+
+    out: dict[str, dict[str, Any]] = {}
+    for k, v in raw.items():
+        if isinstance(v, dict) and not ("desc" in v or "description" in v or "type" in v):
+            # Nested: {table: {col: spec}}.
+            for col, sub in v.items():
+                out[f"{k}.{col}"] = norm(sub)
         else:
-            out[k] = v
+            out[k] = norm(v)
     return out
 
 
@@ -101,19 +110,20 @@ def merge_into_recon(dataset: str, extracted: dict[str, Any]) -> Recon:
     metrics_yaml = _read_yaml(DATASETS_DIR / dataset / "metrics.yaml")
 
     table_descs: dict[str, str] = aug.get("tables", {}) or {}
-    col_descs = _parse_column_descs(aug.get("columns", {}) or {})
+    col_specs = _parse_column_specs(aug.get("columns", {}) or {})
 
     tables: dict[str, Table] = {}
     for tname, t_data in extracted["tables"].items():
-        cols = [
-            Column(
+        cols = []
+        for c in t_data["columns"]:
+            spec = col_specs.get(f"{tname}.{c['name']}", {})
+            cols.append(Column(
                 name=c["name"],
                 sql_type=c["sql_type"],
                 nullable=c["nullable"],
-                description=col_descs.get(f"{tname}.{c['name']}"),
-            )
-            for c in t_data["columns"]
-        ]
+                description=spec.get("desc"),
+                semantic_type=spec.get("type"),
+            ))
         tables[tname] = Table(
             name=tname,
             description=table_descs.get(tname),

api/recon/types.py

@@ -23,6 +23,10 @@ class Column:
     sql_type: str
     nullable: bool
     description: str | None = None
+    # Domain-level type that the composer dispatches on for filter rendering.
+    # Examples: "date_yymmdd" (BIRD's int-encoded YYMMDD dates), "date" (real
+    # postgres date). None = treat literally with the column's sql_type.
+    semantic_type: str | None = None
 
     @classmethod
     def from_inspector(cls, info: dict[str, Any], description: str | None = None) -> "Column":
@@ -40,12 +44,16 @@ class Column:
             "sql_type": self.sql_type,
             "nullable": self.nullable,
             "description": self.description,
+            "semantic_type": self.semantic_type,
         }
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "Column":
-        return cls(name=d["name"], sql_type=d["sql_type"],
-                   nullable=d["nullable"], description=d.get("description"))
+        return cls(
+            name=d["name"], sql_type=d["sql_type"],
+            nullable=d["nullable"], description=d.get("description"),
+            semantic_type=d.get("semantic_type"),
+        )
 
 
 @dataclass
@@ -144,7 +152,7 @@ class Recon:
     column_to_tables: dict[str, list[str]] = field(default_factory=dict)
     relationships: list[Relationship] = field(default_factory=list)
 
-    # ── Read-side helpers used by Analyses + text_to_sql ──
+    # ── Read-side helpers used by Analyses + the composer ──
 
     def table_names(self) -> list[str]:
         return sorted(self.tables)
@@ -156,6 +164,37 @@ class Recon:
         """Tables that have a column with this name (case-sensitive)."""
         return self.column_to_tables.get(column, [])
 
+    def resolve_column(self, ref: str) -> tuple[str, "Column"]:
+        """Resolve a `column` or `table.column` reference to its (table_name,
+        Column) pair. Raises ValueError if the column doesn't exist or is
+        ambiguous (lives on multiple tables) without an explicit `table.` prefix.
+
+        Used by the composer to bind every Pick.group_by entry to a single
+        owning table before SQL composition. Disambiguation is on the caller:
+        if you mean `account.district_id`, say so."""
+        if "." in ref:
+            table, col = ref.split(".", 1)
+            if table not in self.tables:
+                raise ValueError(f"unknown table {table!r} in column ref {ref!r}")
+            for c in self.tables[table].columns:
+                if c.name == col:
+                    return table, c
+            raise ValueError(f"column {col!r} not found on table {table!r}")
+        owners = self.owning_tables(ref)
+        if not owners:
+            raise ValueError(f"no table has a column named {ref!r}")
+        if len(owners) > 1:
+            raise ValueError(
+                f"column {ref!r} is ambiguous (lives in: {', '.join(owners)}); "
+                f"qualify it as 'table.{ref}'"
+            )
+        table = owners[0]
+        for c in self.tables[table].columns:
+            if c.name == ref:
+                return table, c
+        # Index says it's there but the table doesn't — shouldn't happen.
+        raise ValueError(f"column {ref!r} indexed under {table!r} but not found")
+
     def join_path(self, src: str, dst: str) -> list[str] | None:
         """Shortest sequence of tables from src to dst via declared relationships.
         Returns None if no path exists. The path includes both endpoints."""
@@ -220,6 +259,29 @@ class Recon:
             )
         return "\n".join(lines)
 
+    def render_metrics_brief(self) -> str:
+        """One line per metric — name, unit, description. No SQL, no table.
+
+        For the planner: it picks WHICH analyses to run; it doesn't need
+        implementation detail. Saves prompt tokens and cuts the noise the
+        model has to filter through."""
+        if not self.metrics:
+            return "(no metrics defined)"
+        return "\n".join(
+            f"- {m.name} [{m.unit or 'unitless'}]: {m.description}"
+            for m in self.metrics.values()
+        )
+
+    def render_tables_brief(self) -> str:
+        """One line per table — name + description. No columns, no types."""
+        if not self.tables:
+            return "(no tables)"
+        lines: list[str] = []
+        for t in self.tables.values():
+            desc = t.description or "(no description)"
+            lines.append(f"- {t.name}: {desc}")
+        return "\n".join(lines)
+
     def render_relationships(self) -> str:
         if not self.relationships:
             return "(no relationships declared)"