diff --git a/ctrl/Tiltfile b/ctrl/Tiltfile index a708843..1d864db 100644 --- a/ctrl/Tiltfile +++ b/ctrl/Tiltfile @@ -63,21 +63,22 @@ k8s_resource('postgres') k8s_resource('api', resource_deps=['postgres']) k8s_resource('ui', resource_deps=['api']) k8s_resource('docs') +# In-cluster Caddy reverse proxy: owns the single NodePort (30060) and routes by +# Host header to ui / api / docs. The host's dnsmasq + system Caddy forward +# *.local.ar to this NodePort — gateway is this cluster's ingress, not the +# multi-cluster host routing. +# NOTE: editing k8s/base/Caddyfile won't roll the pod on its own +# (disableNameSuffixHash keeps the configMap name stable). Apply Caddyfile edits +# with: kubectl --context kind-nvi -n nvi rollout restart deployment/gateway k8s_resource('gateway', resource_deps=['ui', 'api', 'docs']) -# Hot-reload gateway Caddy on Caddyfile edit. configMapGenerator uses -# disableNameSuffixHash so the Deployment template doesn't change → kustomize -# won't roll the pod on its own. This local_resource closes the loop. -local_resource( - 'gateway-reload', - cmd='kubectl --context kind-nvi -n nvi rollout restart deployment/gateway', - deps=['k8s/base/Caddyfile'], - resource_deps=['gateway'], - auto_init=False, -) - -# Group infra objects +# Group infra objects (namespace, configmaps, the .env-backed secret). k8s_resource( - objects=['nvi:namespace', 'nvi-config:configmap', 'gateway-config:configmap'], + objects=[ + 'nvi:namespace', + 'nvi-config:configmap', + 'gateway-config:configmap', + 'nvi-secrets:secret', + ], new_name='infra', ) diff --git a/ctrl/k8s/base/kustomization.yaml b/ctrl/k8s/base/kustomization.yaml index cd992c8..d03652e 100644 --- a/ctrl/k8s/base/kustomization.yaml +++ b/ctrl/k8s/base/kustomization.yaml @@ -13,8 +13,9 @@ resources: - gateway.yaml # Hash suffix disabled so the name stays static — lets Tilt group it under the -# 'infra' resource. Pod doesn't auto-restart on Caddyfile edit; the Tiltfile's -# 'gateway-reload' local_resource closes that loop. +# 'infra' resource. Trade-off: editing the Caddyfile won't roll the gateway pod +# automatically; apply it with +# kubectl --context kind-nvi -n nvi rollout restart deployment/gateway configMapGenerator: - name: gateway-config files: diff --git a/docs/graphs/architecture.dot b/docs/graphs/architecture.dot index f2c3e44..3d77933 100644 --- a/docs/graphs/architecture.dot +++ b/docs/graphs/architecture.dot @@ -21,7 +21,7 @@ digraph nvi { color="#30363d"; fontcolor="#58a6ff"; fontsize=11; - plan [label="planner\n(1 LLM call)"]; + plan [label="planner\n(1 LLM call → ordered Analyses)"]; } subgraph cluster_l2 { @@ -29,37 +29,53 @@ digraph nvi { color="#30363d"; fontcolor="#58a6ff"; fontsize=11; - compare [label="compare_periods\n(CoT)"]; - drill [label="drill_down\n(ReAct)"]; - outl [label="find_outliers\n(CoT)"]; - corr [label="correlate\n(plan-and-execute)"]; + direct [label="direct_answer\n(CoT, one shot)"]; + compare [label="compare_periods\n(CoT, 2 periods)"]; + drill [label="drill_down\n(ReAct loop)"]; + planned [label="find_outliers · correlate\n(planned)", + style="rounded,filled,dashed", fontcolor="#6e7681"]; } subgraph cluster_l1 { - label="L1 — Tools (deterministic)"; + label="L1 — Atomic actions (deterministic, no LLM)"; color="#30363d"; fontcolor="#58a6ff"; fontsize=11; - t2s [label="text_to_sql"]; - exec [label="execute_sql"]; - schema [label="retrieve_schema"]; - py [label="python_sandbox"]; + compose [label="compose(pick, recon)\n→ SQL"]; + exec [label="execute_sql\n→ rows"]; + } + + subgraph cluster_l0 { + label="L0 — Recon + dataset setup (foundation)"; + color="#30363d"; + fontcolor="#58a6ff"; + fontsize=11; + recon [label="recon\nknowledge graph"]; + setup [label="schema_docs.yaml · metrics.yaml\nencodings · DDL extract", fontcolor="#8b949e"]; } warehouse [label="Postgres\nBIRD financial", fillcolor="#388bfd33"]; langfuse [label="Langfuse (lng, WG)", fillcolor="#bf4b8a33", fontcolor="#ffffff"]; synth [label="synthesize\nfindings → answer"]; + // General → specific flow. user -> plan; - plan -> compare; plan -> drill; plan -> outl; plan -> corr; - compare -> t2s; drill -> t2s; outl -> t2s; corr -> t2s; - drill -> schema [style=dashed]; - corr -> py [style=dashed]; - t2s -> exec; + plan -> direct; plan -> compare; plan -> drill; + direct -> compose [label="typed Pick"]; + compare -> compose; + drill -> compose [label="typed Pick"]; + drill -> drill [label="iterate on results", color="#bf4b8a", fontcolor="#bf4b8a"]; + compose -> exec; exec -> warehouse; - compare -> synth; drill -> synth; outl -> synth; corr -> synth; + direct -> synth; compare -> synth; drill -> synth; + // L0 is consulted by every layer above (not a one-way cascade). + setup -> recon [label="make recon"]; + recon -> plan [style=dashed, label="brief names", constraint=false]; + recon -> compose [style=dashed, label="columns · joins · metrics", constraint=false]; + + // Observability. plan -> langfuse [style=dotted, label="spans"]; - compare -> langfuse [style=dotted]; + compose -> langfuse [style=dotted]; exec -> langfuse [style=dotted]; } diff --git a/docs/graphs/architecture.svg b/docs/graphs/architecture.svg index 1b8f49f..5689174 100644 --- a/docs/graphs/architecture.svg +++ b/docs/graphs/architecture.svg @@ -4,231 +4,240 @@ - - + + nvi - + cluster_l3 - -L3 — Plan + +L3 — Plan cluster_l2 - -L2 — Analyses (mini-agents) + +L2 — Analyses (mini-agents) cluster_l1 - -L1 — Tools (deterministic) + +L1 — Atomic actions (deterministic, no LLM) + + +cluster_l0 + +L0 — Recon + dataset setup (foundation) user - -user question + +user question plan - -planner -(1 LLM call) + +planner +(1 LLM call → ordered Analyses) user->plan - - + + + + + +direct + +direct_answer +(CoT, one shot) + + + +plan->direct + + - + compare - -compare_periods -(CoT) + +compare_periods +(CoT, 2 periods) - + plan->compare - - + + - + drill - -drill_down -(ReAct) + +drill_down +(ReAct loop) - -plan->drill - - - - - -outl - -find_outliers -(CoT) - - -plan->outl - - - - - -corr - -correlate -(plan-and-execute) - - - -plan->corr - - +plan->drill + + langfuse - -Langfuse (lng, WG) + +Langfuse (lng, WG) - + plan->langfuse - - -spans + + +spans - + -t2s - -text_to_sql +compose + +compose(pick, recon) +→ SQL - - -compare->t2s - - - - - -compare->langfuse - - + + +direct->compose + + +typed Pick synth - -synthesize -findings → answer + +synthesize +findings → answer + + + +direct->synth + + + + + +compare->compose + + - + compare->synth - - + + - + + +drill->drill + + +iterate on results + + -drill->t2s - - - - - -schema - -retrieve_schema - - - -drill->schema - - +drill->compose + + +typed Pick - + drill->synth - - + + - - -outl->t2s - - - - - -outl->synth - - - - - -corr->t2s - - - - - -py - -python_sandbox - - - -corr->py - - - - - -corr->synth - - + + +planned + +find_outliers · correlate +(planned) exec - -execute_sql + +execute_sql +→ rows - - -t2s->exec - - + + +compose->exec + + + + + +compose->langfuse + + warehouse - -Postgres -BIRD financial + +Postgres +BIRD financial - + exec->warehouse - - + + - + exec->langfuse - - + + + + + +recon + +recon +knowledge graph + + + +recon->plan + + +brief names + + + +recon->compose + + +columns · joins · metrics + + + +setup + +schema_docs.yaml · metrics.yaml +encodings · DDL extract + + + +setup->recon + + +make recon diff --git a/docs/index.html b/docs/index.html index b8847a7..5e1aebf 100644 --- a/docs/index.html +++ b/docs/index.html @@ -256,7 +256,8 @@ Architecture Plan Analyses - Tools + Atomic actions + Recon Runtime Run it Decisions @@ -278,27 +279,53 @@

The shape mirrors the JD's three building blocks - (text-to-SQL, reasoning agents, planners) as three distinct - layers rather than one omnibus prompt. That gives every part a name - and a tight contract, and lets each Analysis pick the simplest - reasoning pattern (CoT, ReAct, plan-and-execute) for what it does - instead of forcing one over everything. + (text-to-SQL, reasoning agents, planners) as distinct layers + rather than one omnibus prompt — Plan (L3), Analyses (L2), + and deterministic atomic actions (L1) — resting on a recon + foundation (L0): the dataset's typed knowledge graph. That gives every + part a name and a tight contract, and lets each Analysis pick the + simplest reasoning pattern (CoT, ReAct, plan-and-execute) for what it + does instead of forcing one over everything. +

+

+ The load-bearing decision: the LLM never authors SQL. It picks a + typed shape from finite, recon-derived sets; the composer walks recon and + emits the query deterministically. Column–table binding, joins, and + quoting are graph traversals, not model guesses — so a question the system + can't express fails cleanly instead of producing plausible-but-wrong SQL.

Architecture

-

Three layers — Plan composes Analyses; Analyses are mini-agents; Tools are deterministic.

+

Four layers — Plan composes Analyses; Analyses are mini-agents; atomic actions are deterministic; recon is the foundation they all consult.

Plan (L3)
-
One LLM call that turns a question into an ordered set of Analyses with intended interpretation. Not agentic by itself — produces a structured artifact the runtime executes.
+
One LLM call that turns a question into an ordered set of Analysis invocations — each with args, an intended interpretation, and an optional fallback. It sees only brief table/metric names and the Analysis catalog, never raw DDL. Stateless: it emits a plan and never observes execution.
Analyses (L2)
-
Each Analysis is a self-contained mini-agent. It picks the simplest reasoning pattern it needs — CoT for one-shot interpretive moves, ReAct for ones that iterate over Tool results, plan-and-execute for compound ones. The Analysis owns its loop.
-
Tools (L1)
-
Deterministic primitives — text_to_sql, execute_sql, retrieve_schema, retrieve_metric_definition, python_sandbox. No LLM reasoning inside. Called from inside Analyses.
+
Self-contained mini-agents. Each picks the simplest reasoning pattern it needs — CoT for a one-shot move, ReAct for ones that iterate over results, plan-and-execute for compound ones — and owns its loop behind a uniform contract, run(args, question) → Finding. The LLM work lives here: choosing a typed Pick and interpreting rows, never writing SQL.
+
Atomic actions (L1)
+
Deterministic, no LLM. compose(pick, recon) → SQL authors the query by walking recon's join graph; execute_sql runs it read-only with a row cap and timeout. Invalid SQL isn't expressible — the composer binds columns and joins from recon, not from model output.
+
Recon (L0)
+
The dataset's typed knowledge graph — tables, columns, metrics, relationships, storage encodings — merged from sparse human YAML and extracted DDL. The foundation every layer above consults. (Sometimes called L4: the setup that exists before any run, not a step inside one.)
+
+

How the layers interconnect

+

+ The layers read general → specific (Plan → Analyses → atomic + actions), but execution is a graph, not a one-way cascade. Later steps + depend on earlier intermediate results: drill_down + loops, choosing its next dimension from the previous slice's rows; a + step's fallback Analysis fires based on whether the primary's + Finding came back empty or errored. And L0 recon is consulted at every + level — the planner reads its brief names, the Pick + validates against it, the composer walks it. So the layers are + interconnected through both the data they share (recon) and the results + they pass forward. +

+
@@ -306,13 +333,23 @@

L3 — one LLM call that decides which Analyses to run.

- The planner gets the user question, the metric catalog, a schema - overview, and the catalog of Analyses. It returns a structured plan: - an ordered list of Analysis invocations with arguments and an intended - interpretation. It does not call tools or query data itself — it's a - composition step. + The planner (api/plan/planner.py) gets the user question + plus three context blocks rendered from recon: brief table names, brief + metric names, and the catalog of Analyses with their args_schema. + Deliberately no DDL — it decides which Analyses to run, not how to + query. One LLM call returns a structured Plan. +

+

Each step in the plan carries:

+ +

+ The planner never calls a tool or touches data — it's a pure composition + step. The runtime (below) walks the steps in order.

-

To be expanded as the planner is built out.

@@ -321,32 +358,86 @@

L2 — each Analysis is its own mini-agent.

- Analyses are the agent layer. Each one wraps a small piece of analytical - intent (compare two periods, drill down by dimension, - find outliers) and decides how to satisfy it — sometimes one - LLM call is enough, sometimes a ReAct loop over Tool results is the - right fit. The Analysis base class only enforces a uniform external - contract (run(state) → Finding); the internal pattern is - a per-Analysis decision. + Analyses are the agent layer. Each wraps a piece of analytical intent and + decides how to satisfy it — sometimes one LLM call is enough, sometimes a + ReAct loop over query results is the right fit. The base class + (api/analyses/base.py) enforces only a uniform external + contract, run(args, question) → Finding; the internal + pattern is a per-Analysis decision and no framework types leak out.

-

Per-Analysis pattern table coming as each Analysis is implemented.

+

+ Within an Analysis, a free-form question becomes a typed Pick + via one LLM call (pick_for_question) over recon-narrowed + candidate sets — that is the only place the model makes a structural + choice, and it chooses from finite known sets, not free text. The + Pick then goes to the deterministic composer (L1). +

+

Implemented Analyses

+
+
direct_answer
+
CoT, one shot. pick → compose → execute → interpret. For single-query lookups and aggregations.
+
compare_periods
+
CoT over two queries. One Pick, cloned with two period filters → compose ×2 → execute ×2 → interpret the pair.
+
drill_down
+
ReAct loop. decide_next (LLM) picks a dimension from the candidate list → build Pick → compose → execute → loop on the slice's results. Zero LLM in the slice path itself — only the dimension choice is a model decision, and it's guarded against the candidate list.
+
find_outliers · correlate
+
Planned. Additive — new Analyses register without touching the runtime or the layers below.
+
-

Tools layer

-

L1 — deterministic primitives.

+

Atomic actions

+

L1 — deterministic, no LLM inside. The composer is the SQL author.

- Tools have no LLM reasoning inside them. text_to_sql is - the one exception that uses an LLM, but tightly: schema-RAG context, - sqlglot validation, retry on parse/runtime errors. execute_sql - runs against Postgres read-only with a row cap and timeout. - retrieve_schema and retrieve_metric_definition - power the semantic layer. python_sandbox handles stats - work that's awkward in SQL. + There is no LLM in this layer — and notably no text_to_sql + tool. An earlier design had the model author SQL with schema-RAG context, + sqlglot validation, and retries; it was removed once the composer landed + (see Decisions). The two atomic actions are: +

+
+
compose
+
compose(pick, recon) → SQL (api/composer/). Builds one sqlglot expression tree from a typed Pick by walking recon: resolve the metric's table, bind each group-by/filter column to its owning table, compute join paths, render filters and dates, emit quoted SQL. Split by concern — aliases, joins, filters, metrics, dates, order, encodings. If a Pick can't be expressed against recon it raises PickValidationError; it never fabricates SQL.
+
execute_sql
+
execute_sql(sql) → rows (api/tools/execute_sql.py). Runs read-only against Postgres (SELECT/WITH only) with a row cap and a statement timeout, returning columns + rows + truncation metadata.
+
+

+ Because column–table binding and joins come from recon rather than model + output, the composer cannot emit SQL that references a column that + doesn't exist. The validator that used to catch the model's mistakes after + the fact is demoted to a paranoid self-check on the composer. +

+
+
+ +
+

Recon

+

L0 — the dataset's typed knowledge graph, and the only dataset-specific surface.

+
+

+ Recon (api/recon/) is the foundation every other layer + consults. It's built in two stages and cached as recon.json: +

+ +

+ The merged Recon exposes the queries downstream code needs + without going back to the model: resolve_column (bare or + table.col → owning table), join_path (shortest + table sequence between two tables), the metric catalog, and storage + encodings (e.g. how a YYMMDD-int column coerces to a date). + Rebuild with make recon after changing the warehouse or the + YAML. +

+

+ Everything dataset-specific lives here. The Plan, Analyses, and atomic + actions reference no dataset name — point them at a different + api/datasets/<name>/ and the same machinery answers + questions over a new warehouse.

-

Per-Tool documentation coming as each Tool is implemented.

@@ -368,10 +459,26 @@ the nodes are, and that's where the interesting reading is.

- Concretely: api/runtime/ contains the langgraph builder - and event tap. api/plan/, api/analyses/, and - api/tools/ contain the domain code — and those files - don't expose langgraph types in their public interfaces. + Concretely the graph is small and linear: +

+ 
START → plan → execute → synthesize → END
+

+ plan runs the planner; execute walks the plan + steps, dispatching each to its Analysis (and to a fallback Analysis when + the primary returns empty or errors); synthesize folds the + Findings into one answer. The whole run streams SSE events + (run_start, plan_ready, + analysis_start/end, tool_call_start/end, + analysis_fallback) that drive the live trace UI, and every + LLM call opens a named Langfuse generation span. +

+

+ api/runtime/runner.py is the one place langgraph appears — + StateGraph is imported there and nowhere else. + api/plan/, api/analyses/, + api/composer/, and api/tools/ contain the + domain code, and none of those files expose langgraph types in their + public interfaces.

@@ -398,7 +505,8 @@ open http://nvi.local.ar

Architecture decisions