recon-driven sql composer; pick → compose → execute; llm out of structural sql

tests/test_composer_compose.py

@@ -0,0 +1,229 @@
+"""Composer SQL emission — golden-string tests against a representative recon.
+
+The fixture mirrors enough of the financial dataset to exercise: metric
+with default filter, group_by with single hop and multi-hop joins, typed
+date_range filter on a real DATE column, and ORDER BY by both metric and
+dimension. Adjust goldens cautiously — if a string changes, confirm the
+new SQL still semantically matches before updating the test.
+"""
+
+from api.composer import compose
+from api.composer.types import Filter, OrderBy, Pick, PickValidationError
+from api.recon.types import Column, Metric, Recon, Relationship, Table
+
+import pytest
+
+
+def _fixture_recon() -> Recon:
+    cols_account = [
+        Column("account_id", "BIGINT", False),
+        Column("district_id", "BIGINT", True),
+    ]
+    cols_district = [
+        Column("district_id", "BIGINT", False),
+        Column("A2", "TEXT", True),
+        Column("A3", "TEXT", True),
+    ]
+    cols_loan = [
+        Column("loan_id", "BIGINT", False),
+        Column("account_id", "BIGINT", True),
+        Column("amount", "BIGINT", True),
+        Column("status", "TEXT", True),
+        Column("date", "DATE", True),
+    ]
+    rels = [
+        Relationship("account", "district_id", "district", "district_id"),
+        Relationship("loan", "account_id", "account", "account_id"),
+    ]
+    c2t: dict[str, list[str]] = {}
+    for t in [Table("account", "accounts", cols_account),
+              Table("district", "districts", cols_district),
+              Table("loan", "loans", cols_loan)]:
+        for c in t.columns:
+            c2t.setdefault(c.name, []).append(t.name)
+    for k in c2t:
+        c2t[k] = sorted(c2t[k])
+
+    metrics = {
+        "loan_default_rate": Metric(
+            name="loan_default_rate",
+            description="Default rate over finished loans.",
+            sql="AVG(CASE WHEN status = 'B' THEN 1.0 WHEN status = 'A' THEN 0.0 END)",
+            from_table="loan",
+            filter="status IN ('A','B')",
+            unit="ratio",
+        ),
+        "loan_volume": Metric(
+            name="loan_volume",
+            description="Total CZK lent.",
+            sql="SUM(amount)",
+            from_table="loan",
+            filter=None,
+            unit="CZK",
+        ),
+    }
+    return Recon(
+        schema="financial",
+        tables={
+            "account": Table("account", "accounts", cols_account),
+            "district": Table("district", "districts", cols_district),
+            "loan": Table("loan", "loans", cols_loan),
+        },
+        metrics=metrics,
+        column_to_tables=c2t,
+        relationships=rels,
+    )
+
+
+# ── Aggregate, no group_by ──────────────────────────────────
+
+def test_metric_only_no_group_by():
+    r = _fixture_recon()
+    pick = Pick(kind="aggregate", metric="loan_volume")
+    out = compose(pick, r)
+    assert out.sql == 'SELECT SUM("l"."amount") AS "loan_volume" FROM "loan" AS "l"'
+    assert out.used_tables == ["loan"]
+
+
+def test_metric_with_default_filter_no_group_by():
+    r = _fixture_recon()
+    pick = Pick(kind="aggregate", metric="loan_default_rate")
+    out = compose(pick, r)
+    expected = (
+        "SELECT AVG(CASE WHEN \"l\".\"status\" = 'B' THEN 1.0 "
+        "WHEN \"l\".\"status\" = 'A' THEN 0.0 END) AS \"loan_default_rate\" "
+        "FROM \"loan\" AS \"l\" "
+        "WHERE \"l\".\"status\" IN ('A', 'B')"
+    )
+    assert out.sql == expected
+
+
+# ── Group by — single hop ───────────────────────────────────
+
+def test_group_by_multi_hop_join():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_default_rate",
+        group_by=["A2"],
+        limit=10,
+    )
+    out = compose(pick, r)
+    # loan → account → district, A2 owned by district.
+    assert 'FROM "loan" AS "l"' in out.sql
+    assert 'JOIN "account" AS "a" ON "l"."account_id" = "a"."account_id"' in out.sql
+    assert (
+        'JOIN "district" AS "d" ON "a"."district_id" = "d"."district_id"' in out.sql
+    )
+    assert 'GROUP BY "A2"' in out.sql
+    # default ORDER BY metric DESC
+    assert 'ORDER BY "loan_default_rate" DESC' in out.sql
+    assert "LIMIT 10" in out.sql
+    assert set(out.used_tables) == {"loan", "account", "district"}
+
+
+# ── Filters ─────────────────────────────────────────────────
+
+def test_where_equals_filter():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_volume",
+        where=[Filter(column="status", equals="D")],
+    )
+    out = compose(pick, r)
+    assert "WHERE \"l\".\"status\" = 'D'" in out.sql
+
+
+def test_where_in_values_filter():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_volume",
+        where=[Filter(column="status", in_values=["C", "D"])],
+    )
+    out = compose(pick, r)
+    assert "\"l\".\"status\" IN ('C', 'D')" in out.sql
+
+
+def test_where_date_range_on_real_date_column():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_default_rate",
+        where=[Filter(column="date", date_range="1996")],
+    )
+    out = compose(pick, r)
+    assert "\"l\".\"date\" BETWEEN '1996-01-01' AND '1996-12-31'" in out.sql
+    # metric's default filter still ANDed in
+    assert "\"l\".\"status\" IN ('A', 'B')" in out.sql
+
+
+def test_metric_filter_ands_with_pick_filter():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_default_rate",
+        where=[Filter(column="date", date_range="1996")],
+    )
+    out = compose(pick, r)
+    # Both filters in the same WHERE clause, joined by AND.
+    assert " AND " in out.sql
+
+
+# ── ORDER BY by dimension ───────────────────────────────────
+
+def test_order_by_dimension():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_volume",
+        group_by=["A3"],
+        order_by=OrderBy(by="dimension", direction="asc", dimension="A3"),
+    )
+    out = compose(pick, r)
+    assert 'ORDER BY "A3" ASC' in out.sql
+
+
+def test_order_by_dimension_not_in_group_by_raises():
+    r = _fixture_recon()
+    pick = Pick(
+        kind="aggregate",
+        metric="loan_volume",
+        group_by=["A2"],
+        order_by=OrderBy(by="dimension", direction="asc", dimension="A3"),
+    )
+    with pytest.raises(PickValidationError, match="not in group_by"):
+        compose(pick, r)
+
+
+# ── Validation rejections ───────────────────────────────────
+
+def test_unknown_metric_raises():
+    r = _fixture_recon()
+    pick = Pick(kind="aggregate", metric="moon_phase")
+    with pytest.raises(PickValidationError, match="not in recon"):
+        compose(pick, r)
+
+
+def test_unknown_group_by_column_raises():
+    r = _fixture_recon()
+    pick = Pick(kind="aggregate", metric="loan_volume", group_by=["nonsense"])
+    with pytest.raises(ValueError, match="no table has a column named 'nonsense'"):
+        compose(pick, r)
+
+
+def test_ambiguous_group_by_column_raises():
+    r = _fixture_recon()
+    # account_id lives on both account and loan; must be qualified.
+    pick = Pick(kind="aggregate", metric="loan_volume", group_by=["account_id"])
+    with pytest.raises(ValueError, match="ambiguous"):
+        compose(pick, r)
+
+
+def test_qualified_disambiguates():
+    r = _fixture_recon()
+    pick = Pick(kind="aggregate", metric="loan_volume", group_by=["loan.account_id"])
+    out = compose(pick, r)
+    # Output alias becomes "loan_account_id" since the ref was qualified.
+    assert '"loan_account_id"' in out.sql

tests/test_composer_dates.py

@@ -0,0 +1,50 @@
+"""Date-range resolver tests — bounds parsing + dispatch on column type."""
+
+import pytest
+
+from api.composer.dates import _bounds, resolve_date_range
+from api.composer.types import PickValidationError
+from api.recon.types import Column
+
+
+def test_bounds_year():
+    assert _bounds("1996") == ("1996-01-01", "1996-12-31")
+
+
+def test_bounds_quarter_q1():
+    assert _bounds("1996-Q1") == ("1996-01-01", "1996-03-31")
+
+
+def test_bounds_quarter_q3():
+    assert _bounds("1996-Q3") == ("1996-07-01", "1996-09-30")
+
+
+def test_bounds_unsupported_raises():
+    with pytest.raises(PickValidationError, match="unsupported"):
+        _bounds("Q3-1996")
+
+
+def test_resolve_real_date_column():
+    col = Column(name="date", sql_type="DATE", nullable=False)
+    out = resolve_date_range("1996", col, '"l"."date"')
+    assert out == "\"l\".\"date\" BETWEEN '1996-01-01' AND '1996-12-31'"
+
+
+def test_resolve_timestamp_column():
+    col = Column(name="created_at", sql_type="TIMESTAMP WITHOUT TIME ZONE", nullable=True)
+    out = resolve_date_range("1996-Q2", col, '"x"."created_at"')
+    assert out == "\"x\".\"created_at\" BETWEEN '1996-04-01' AND '1996-06-30'"
+
+
+def test_resolve_yymmdd_semantic_type():
+    col = Column(name="date", sql_type="INTEGER", nullable=False,
+                 semantic_type="date_yymmdd")
+    out = resolve_date_range("1996", col, '"l"."date"')
+    assert "TO_DATE(LPAD(" in out
+    assert "BETWEEN '1996-01-01' AND '1996-12-31'" in out
+
+
+def test_resolve_unsupported_column_type_raises():
+    col = Column(name="x", sql_type="TEXT", nullable=True)
+    with pytest.raises(PickValidationError, match="not supported"):
+        resolve_date_range("1996", col, '"x"."x"')

tests/test_events.py

@@ -62,13 +62,13 @@ def test_tool_call_shapes_carry_analysis_field():
     """Tool/LLM events must include the active Analysis name from the
     contextvar — UI groups sub-events by it. None when outside an Analysis."""
     # Outside any Analysis: analysis is None.
-    s = events.tool_call_start("text_to_sql", input={"q": "?"})
+    s = events.tool_call_start("compose_sql", input={"q": "?"})
     assert s["type"] == "tool_call_start"
-    assert s["tool"] == "text_to_sql"
+    assert s["tool"] == "compose_sql"
     assert s["analysis"] is None
     assert s["input"] == {"q": "?"}
 
-    e = events.tool_call_end("text_to_sql", output={"sql": "..."})
+    e = events.tool_call_end("compose_sql", output={"sql": "..."})
     assert e["analysis"] is None
     assert e["output"] == {"sql": "..."}
     assert e["error"] is None

tests/test_imports.py

@@ -21,13 +21,19 @@ MODULES = [
     "api.recon.types",
     "api.recon.build",
     "api.tools.db",
-    "api.tools.text_to_sql",
     "api.tools.execute_sql",
     "api.tools.types",
+    "api.composer",
+    "api.composer.types",
+    "api.composer.compose",
+    "api.composer.dates",
     "api.analyses.base",
     "api.analyses.types",
+    "api.analyses._pick",
+    "api.analyses._narrow",
     "api.analyses.direct_answer",
     "api.analyses.compare_periods",
+    "api.analyses.drill_down",
     "api.analyses.registry",
     "api.plan.planner",
     "api.plan.types",

tests/test_pick_shape.py

@@ -0,0 +1,122 @@
+"""Pick / Filter / OrderBy validation — round-trips and rejection cases."""
+
+import pytest
+
+from api.composer.types import Filter, OrderBy, Pick, PickValidationError
+
+
+# ── Filter ──────────────────────────────────────────────────
+
+def test_filter_equals_roundtrip():
+    f = Filter.from_dict({"column": "status", "equals": "B"})
+    assert f.kind() == "equals"
+    assert f.to_dict() == {"column": "status", "equals": "B"}
+
+
+def test_filter_in_values_roundtrip():
+    f = Filter.from_dict({"column": "status", "in_values": ["A", "B"]})
+    assert f.kind() == "in_values"
+    assert f.to_dict() == {"column": "status", "in_values": ["A", "B"]}
+
+
+def test_filter_between_normalised_to_tuple():
+    f = Filter.from_dict({"column": "amount", "between": [1000, 5000]})
+    assert f.kind() == "between"
+    assert f.between == (1000, 5000)
+    # round-trip emits a list, not a tuple
+    assert f.to_dict() == {"column": "amount", "between": [1000, 5000]}
+
+
+def test_filter_date_range_roundtrip():
+    f = Filter.from_dict({"column": "date", "date_range": "1996"})
+    assert f.kind() == "date_range"
+    assert f.to_dict() == {"column": "date", "date_range": "1996"}
+
+
+def test_filter_missing_value_raises():
+    with pytest.raises(PickValidationError, match="no value field"):
+        Filter(column="x").kind()
+
+
+def test_filter_multiple_value_fields_raises():
+    f = Filter(column="x", equals="A", in_values=["A", "B"])
+    with pytest.raises(PickValidationError, match="multiple value fields"):
+        f.kind()
+
+
+def test_filter_missing_column_raises():
+    with pytest.raises(PickValidationError, match="missing 'column'"):
+        Filter.from_dict({"equals": "B"})
+
+
+def test_filter_between_wrong_arity_raises():
+    with pytest.raises(PickValidationError, match="2-element"):
+        Filter.from_dict({"column": "x", "between": [1, 2, 3]})
+
+
+# ── OrderBy ─────────────────────────────────────────────────
+
+def test_order_by_metric():
+    ob = OrderBy.from_dict({"by": "metric", "direction": "desc"})
+    assert ob.by == "metric"
+    assert ob.direction == "desc"
+    assert ob.to_dict() == {"by": "metric", "direction": "desc"}
+
+
+def test_order_by_dimension_requires_dimension():
+    with pytest.raises(PickValidationError, match="requires order_by.dimension"):
+        OrderBy.from_dict({"by": "dimension"})
+
+
+def test_order_by_invalid_direction_raises():
+    with pytest.raises(PickValidationError, match="direction"):
+        OrderBy.from_dict({"by": "metric", "direction": "sideways"})
+
+
+# ── Pick ────────────────────────────────────────────────────
+
+def test_pick_minimal():
+    p = Pick.from_dict({"kind": "aggregate", "metric": "loan_volume"})
+    assert p.metric == "loan_volume"
+    assert p.group_by == []
+    assert p.where == []
+    assert p.order_by is None
+    assert p.limit is None
+
+
+def test_pick_full_roundtrip():
+    raw = {
+        "kind": "aggregate",
+        "metric": "loan_default_rate",
+        "group_by": ["A2"],
+        "where": [{"column": "date", "date_range": "1996"}],
+        "order_by": {"by": "metric", "direction": "desc"},
+        "limit": 10,
+    }
+    p = Pick.from_dict(raw)
+    assert p.to_dict() == raw
+
+
+def test_pick_rejects_non_aggregate_kind():
+    with pytest.raises(PickValidationError, match="aggregate"):
+        Pick.from_dict({"kind": "ratio", "metric": "x"})
+
+
+def test_pick_missing_metric_raises():
+    with pytest.raises(PickValidationError, match="metric"):
+        Pick.from_dict({"kind": "aggregate"})
+
+
+def test_pick_group_by_must_be_list_of_strings():
+    with pytest.raises(PickValidationError, match="group_by"):
+        Pick.from_dict({"kind": "aggregate", "metric": "x", "group_by": [1, 2]})
+
+
+def test_pick_where_must_be_list():
+    with pytest.raises(PickValidationError, match="where must be a list"):
+        Pick.from_dict({"kind": "aggregate", "metric": "x", "where": {"foo": "bar"}})
+
+
+def test_pick_limit_must_be_int():
+    with pytest.raises(PickValidationError, match="limit"):
+        Pick.from_dict({"kind": "aggregate", "metric": "x", "limit": "ten"})