verbose live UI + tool-level SSE events + Groq default + regression tests

api/analyses/base.py

@@ -0,0 +1,27 @@
+"""Analysis base class.
+
+An Analysis is a self-contained mini-agent. It owns its reasoning pattern
+(CoT, ReAct, plan-and-execute) and exposes a uniform external contract:
+`run(args, question) -> Finding`. The runtime composes Analyses; it doesn't
+know how each one thinks internally.
+
+Public surface is framework-free — no langgraph types leak out.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from api.analyses.types import Finding
+
+
+class Analysis:
+    """Subclass and set `name`, `description`, `args_schema`; then implement run()."""
+
+    name: ClassVar[str] = ""
+    description: ClassVar[str] = ""
+    # JSON-schema-ish description of the args dict — the planner reads this.
+    args_schema: ClassVar[dict[str, Any]] = {}
+
+    async def run(self, args: dict[str, Any], question: str) -> Finding:
+        raise NotImplementedError

api/analyses/compare_periods.py

@@ -0,0 +1,141 @@
+"""compare_periods Analysis — CoT with two queries.
+
+Both SQL queries are generated in one LLM call (paired prompt), then both
+executed, then a single interpretation pass diffs them. Single round-trip
+per LLM step → cheap and bounded latency.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+from api import langfuse_client as lf
+from api.analyses.base import Analysis
+from api.analyses.types import Finding
+from api.llm import chat
+from api.prompts import load, render
+from api.runtime import events
+from api.tools.execute_sql import execute_sql
+from api.tools.schema import load_schema
+
+logger = logging.getLogger("nvi.analyses.compare_periods")
+
+
+class ComparePeriods(Analysis):
+    name = "compare_periods"
+    description = (
+        "Compare a metric across two time windows (e.g. Q2 vs Q3, 1995 vs 1996). "
+        "Generates two parallel SQL queries and diffs the results."
+    )
+    args_schema = {
+        "question": {"type": "string", "description": "The metric / scope to compare."},
+        "period_a": {"type": "string", "description": "First period label, e.g. '1995' or 'Q2 1996'."},
+        "period_b": {"type": "string", "description": "Second period label, e.g. '1996' or 'Q3 1996'."},
+    }
+
+    async def run(self, args: dict[str, Any], question: str) -> Finding:
+        sub_q = args.get("question") or question
+        period_a = args.get("period_a", "")
+        period_b = args.get("period_b", "")
+
+        with lf.span("analysis.compare_periods", input=args) as span:
+            try:
+                await events.publish_current(events.tool_call_start(
+                    "generate_pair", input={"question": sub_q, "period_a": period_a, "period_b": period_b},
+                ))
+                pair = _generate_pair(sub_q, period_a, period_b)
+                await events.publish_current(events.tool_call_end(
+                    "generate_pair",
+                    output={"a": {"label": pair["a"]["label"], "sql": pair["a"]["sql"]},
+                            "b": {"label": pair["b"]["label"], "sql": pair["b"]["sql"]}},
+                ))
+
+                await events.publish_current(events.tool_call_start("execute_sql", input={"label": pair["a"]["label"], "sql": pair["a"]["sql"]}))
+                res_a = execute_sql(pair["a"]["sql"])
+                await events.publish_current(events.tool_call_end(
+                    "execute_sql",
+                    output={"label": pair["a"]["label"], "row_count": res_a.row_count,
+                            "preview": res_a.as_dicts()[:5]},
+                ))
+
+                await events.publish_current(events.tool_call_start("execute_sql", input={"label": pair["b"]["label"], "sql": pair["b"]["sql"]}))
+                res_b = execute_sql(pair["b"]["sql"])
+                await events.publish_current(events.tool_call_end(
+                    "execute_sql",
+                    output={"label": pair["b"]["label"], "row_count": res_b.row_count,
+                            "preview": res_b.as_dicts()[:5]},
+                ))
+
+                interpret_system = load("compare_periods.interpret")
+                interpret_user = render(
+                    "compare_periods.interpret.user",
+                    question=sub_q,
+                    label_a=pair["a"]["label"],
+                    label_b=pair["b"]["label"],
+                    rows_a=repr(res_a.as_dicts()[:20]),
+                    rows_b=repr(res_b.as_dicts()[:20]),
+                )
+                await events.publish_current(events.llm_call(
+                    "compare_periods.interpret",
+                    system_len=len(interpret_system),
+                    user_len=len(interpret_user),
+                ))
+                summary = chat(system=interpret_system, user=interpret_user, max_tokens=512)
+            except Exception as e:
+                logger.exception("compare_periods failed")
+                await events.publish_current(events.tool_call_end("compare_periods", error=str(e)))
+                span.update(output={"error": str(e)})
+                return Finding(analysis=self.name, summary="", error=str(e))
+
+            span.update(output={"summary": summary})
+            return Finding(
+                analysis=self.name,
+                summary=summary,
+                rows=[
+                    {"period": pair["a"]["label"], **r} for r in res_a.as_dicts()[:20]
+                ] + [
+                    {"period": pair["b"]["label"], **r} for r in res_b.as_dicts()[:20]
+                ],
+                sql=[pair["a"]["sql"], pair["b"]["sql"]],
+                metadata={
+                    "label_a": pair["a"]["label"],
+                    "label_b": pair["b"]["label"],
+                    "row_count_a": res_a.row_count,
+                    "row_count_b": res_b.row_count,
+                },
+            )
+
+
+def _generate_pair(question: str, period_a: str, period_b: str) -> dict[str, dict[str, str]]:
+    schema = load_schema()
+    text = chat(
+        system=load("compare_periods.pair"),
+        user=render(
+            "compare_periods.pair.user",
+            question=question,
+            period_a=period_a or "(infer)",
+            period_b=period_b or "(infer)",
+            schema_block=schema.render_tables(),
+            metrics_block=schema.render_metrics(),
+        ),
+        max_tokens=1024,
+    )
+    obj = json.loads(_extract_json(text))
+    for k in ("a", "b"):
+        if k not in obj or "sql" not in obj[k] or "label" not in obj[k]:
+            raise ValueError(f"pair generator returned malformed payload: missing {k}")
+        obj[k]["sql"] = obj[k]["sql"].strip().rstrip(";").strip()
+    return obj
+
+
+def _extract_json(text: str) -> str:
+    m = re.search(r"```(?:json)?\s*(\{.*\})\s*```", text, re.DOTALL)
+    if m:
+        return m.group(1)
+    start, end = text.find("{"), text.rfind("}")
+    if start >= 0 and end > start:
+        return text[start:end + 1]
+    return text

api/analyses/direct_answer.py

@@ -0,0 +1,81 @@
+"""direct_answer Analysis — CoT, one-shot.
+
+Pattern: text-to-SQL → execute → ask the LLM to interpret the rows. No loop.
+For questions that resolve to a single SQL query and a short interpretation.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from api import langfuse_client as lf
+from api.analyses.base import Analysis
+from api.analyses.types import Finding
+from api.llm import chat
+from api.prompts import load, render
+from api.runtime import events
+from api.tools.execute_sql import execute_sql
+from api.tools.text_to_sql import text_to_sql
+
+logger = logging.getLogger("nvi.analyses.direct_answer")
+
+
+class DirectAnswer(Analysis):
+    name = "direct_answer"
+    description = (
+        "Answer a question that resolves to a single SQL query. Best for "
+        "lookup / aggregation questions with a single, well-defined metric."
+    )
+    args_schema = {
+        "question": {"type": "string", "description": "Refined question this Analysis should answer."},
+        "hint_tables": {"type": "array", "items": "string", "description": "Tables the planner thinks are relevant (optional)."},
+    }
+
+    async def run(self, args: dict[str, Any], question: str) -> Finding:
+        sub_q = args.get("question") or question
+        hint_tables = args.get("hint_tables")
+
+        with lf.span("analysis.direct_answer", input={"question": sub_q}) as span:
+            try:
+                await events.publish_current(events.tool_call_start("text_to_sql", input={"question": sub_q}))
+                t2s = text_to_sql(sub_q, hint_tables=hint_tables)
+                await events.publish_current(events.tool_call_end(
+                    "text_to_sql", output={"sql": t2s.sql, "tables": t2s.used_tables},
+                ))
+
+                await events.publish_current(events.tool_call_start("execute_sql", input={"sql": t2s.sql}))
+                result = execute_sql(t2s.sql)
+                await events.publish_current(events.tool_call_end(
+                    "execute_sql",
+                    output={"row_count": result.row_count, "truncated": result.truncated,
+                            "preview": result.as_dicts()[:5]},
+                ))
+
+                interpret_system = load("direct_answer.interpret")
+                interpret_user = render(
+                    "direct_answer.interpret.user",
+                    question=sub_q,
+                    sql=t2s.sql,
+                    rows=repr(result.as_dicts()[:20]),
+                )
+                await events.publish_current(events.llm_call(
+                    "direct_answer.interpret",
+                    system_len=len(interpret_system),
+                    user_len=len(interpret_user),
+                ))
+                summary = chat(system=interpret_system, user=interpret_user, max_tokens=512)
+            except Exception as e:
+                logger.exception("direct_answer failed")
+                await events.publish_current(events.tool_call_end("direct_answer", error=str(e)))
+                span.update(output={"error": str(e)})
+                return Finding(analysis=self.name, summary="", error=str(e))
+
+            span.update(output={"summary": summary, "rows": result.row_count})
+            return Finding(
+                analysis=self.name,
+                summary=summary,
+                rows=result.as_dicts()[:20],
+                sql=[t2s.sql],
+                metadata={"row_count": result.row_count, "truncated": result.truncated},
+            )

api/analyses/registry.py

@@ -0,0 +1,30 @@
+"""Analysis registry — the planner reads this to know what's available."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from api.analyses.base import Analysis
+from api.analyses.compare_periods import ComparePeriods
+from api.analyses.direct_answer import DirectAnswer
+
+REGISTRY: dict[str, Analysis] = {
+    DirectAnswer.name: DirectAnswer(),
+    ComparePeriods.name: ComparePeriods(),
+}
+
+
+def get(name: str) -> Analysis | None:
+    return REGISTRY.get(name)
+
+
+def catalog() -> list[dict[str, Any]]:
+    """Plain-dict catalog for prompt injection into the planner."""
+    return [
+        {
+            "name": a.name,
+            "description": a.description,
+            "args_schema": a.args_schema,
+        }
+        for a in REGISTRY.values()
+    ]

api/analyses/types.py

@@ -0,0 +1,17 @@
+"""Public data shapes for the Analyses layer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Finding:
+    """Structured output every Analysis returns."""
+    analysis: str
+    summary: str
+    rows: list[dict[str, Any]] = field(default_factory=list)
+    sql: list[str] = field(default_factory=list)
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)