chunker ui redo

core/api/chunker_sse.py

@@ -1,8 +1,10 @@
 """
 SSE endpoint for chunker pipeline events.
 
-Bridges gRPC StreamProgress to browser-native EventSource.
-GET /api/chunker/stream/{job_id} → text/event-stream
+Uses Redis as the event bus between Celery workers and the SSE stream.
+Celery worker pushes events via core.events, SSE endpoint polls them.
+
+GET /chunker/stream/{job_id} → text/event-stream
 """
 
 import asyncio
@@ -14,46 +16,39 @@ from typing import AsyncGenerator
 from fastapi import APIRouter
 from starlette.responses import StreamingResponse
 
+from core.events import poll_events
+
 logger = logging.getLogger(__name__)
 
-router = APIRouter(prefix="/api/chunker", tags=["chunker"])
+router = APIRouter(prefix="/chunker", tags=["chunker"])
 
 
 async def _event_generator(job_id: str) -> AsyncGenerator[str, None]:
     """
-    Generate SSE events by polling gRPC job state.
-
-    Yields server-sent events in the format:
-        event: <event_type>
-        data: <json_payload>
+    Generate SSE events by polling Redis for chunk job events.
     """
-    from core.rpc.server import _active_jobs
-
-    last_state = None
+    cursor = 0
     timeout = time.monotonic() + 600  # 10 min max
 
     while time.monotonic() < timeout:
-        job_state = _active_jobs.get(job_id)
+        events, cursor = poll_events(job_id, cursor)
 
-        if job_state is None:
-            # Job not found yet — may not have started
+        if not events:
             yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n"
-            await asyncio.sleep(0.5)
+            await asyncio.sleep(0.1)
             continue
 
-        # Only send if state changed
-        if job_state != last_state:
-            last_state = dict(job_state)
-            event_type = job_state.get("status", "update")
+        for data in events:
+            event_type = data.pop("event", "update")
+            payload = {**data, "job_id": job_id}
 
-            yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n"
+            yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n"
 
-            # End stream when job is terminal
-            if event_type in ("completed", "failed", "cancelled"):
+            if event_type in ("pipeline_complete", "pipeline_error", "cancelled"):
                 yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n"
-                break
+                return
 
-        await asyncio.sleep(0.2)
+        await asyncio.sleep(0.05)
 
     yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n"