mariano/mediaproc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 12

Browse Source
This commit is contained in:
buenosairesam
2026-03-26 06:10:19 -03:00
parent 731964ca10
commit e27cb5bcc3
41 changed files with 2079 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
admin/mpr/media_assets/models.py
View File

@@ -28,6 +28,25 @@ class ChunkJobStatus(models.TextChoices):
FAILED = "failed", "Failed"
CANCELLED = "cancelled", "Cancelled"
class DetectJobStatus(models.TextChoices):
PENDING = "pending", "Pending"
RUNNING = "running", "Running"
PAUSED = "paused", "Paused"
COMPLETED = "completed", "Completed"
FAILED = "failed", "Failed"
CANCELLED = "cancelled", "Cancelled"
class RunType(models.TextChoices):
INITIAL = "initial", "Initial"
REPLAY = "replay", "Replay"
RETRY = "retry", "Retry"
class BrandSource(models.TextChoices):
OCR = "ocr", "Ocr"
VLM = "local_vlm", "Vlm"
CLOUD = "cloud_llm", "Cloud"
MANUAL = "manual", "Manual"
class MediaAsset(models.Model):
"""A video/audio file registered in the system."""
@@ -148,3 +167,104 @@ class ChunkJob(models.Model):
def __str__(self):
return str(self.id)
class DetectJob(models.Model):
"""A detection pipeline job."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
source_asset_id = models.UUIDField()
video_path = models.CharField(max_length=1000)
profile_name = models.CharField(max_length=255)
parent_job_id = models.UUIDField(null=True, blank=True)
run_type = models.CharField(max_length=20, choices=RunType.choices, default=RunType.INITIAL)
replay_from_stage = models.CharField(max_length=255, null=True, blank=True)
config_overrides = models.JSONField(default=dict, blank=True)
status = models.CharField(max_length=20, choices=DetectJobStatus.choices, default=DetectJobStatus.PENDING)
current_stage = models.CharField(max_length=255, null=True, blank=True)
progress = models.FloatField(default=0.0)
error_message = models.TextField(blank=True, default='')
total_detections = models.IntegerField(default=0)
brands_found = models.IntegerField(default=0)
cloud_llm_calls = models.IntegerField(default=0)
estimated_cost_usd = models.FloatField(default=0.0)
celery_task_id = models.CharField(max_length=255, null=True, blank=True)
priority = models.IntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
class Meta:
ordering = ["-created_at"]
def __str__(self):
return str(self.id)
class StageCheckpoint(models.Model):
"""A checkpoint saved after a pipeline stage completes."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
job_id = models.UUIDField()
stage = models.CharField(max_length=255)
stage_index = models.IntegerField()
frames_prefix = models.CharField(max_length=255)
frames_manifest = models.JSONField(default=dict, blank=True)
frames_meta = models.JSONField(default=list, blank=True)
filtered_frame_sequences = models.JSONField(default=list, blank=True)
boxes_by_frame = models.JSONField(default=dict, blank=True)
text_candidates = models.JSONField(default=list, blank=True)
unresolved_candidates = models.JSONField(default=list, blank=True)
detections = models.JSONField(default=list, blank=True)
stats = models.JSONField(default=dict, blank=True)
config_snapshot = models.JSONField(default=dict, blank=True)
config_overrides = models.JSONField(default=dict, blank=True)
video_path = models.CharField(max_length=1000)
profile_name = models.CharField(max_length=255)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ["-created_at"]
def __str__(self):
return str(self.id)
class KnownBrand(models.Model):
"""A brand discovered or registered in the system."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
canonical_name = models.CharField(max_length=255)
aliases = models.JSONField(default=list, blank=True)
first_source = models.CharField(max_length=20, choices=BrandSource.choices, default=BrandSource.OCR)
total_occurrences = models.IntegerField(default=0)
confirmed = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["-created_at"]
def __str__(self):
return str(self.id)
class SourceBrandSighting(models.Model):
"""A brand seen in a specific source (video/asset)."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
source_asset_id = models.UUIDField()
brand_id = models.UUIDField()
brand_name = models.CharField(max_length=255)
first_seen_timestamp = models.FloatField(default=0.0)
last_seen_timestamp = models.FloatField(default=0.0)
occurrences = models.IntegerField(default=0)
detection_source = models.CharField(max_length=20, choices=BrandSource.choices, default=BrandSource.OCR)
avg_confidence = models.FloatField(default=0.0)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ["-created_at"]
def __str__(self):
return str(self.id)

88
core/api/detect_config.py Normal file
View File

@@ -0,0 +1,88 @@
"""
Runtime config endpoint for the detection pipeline.
GET /detect/config — read current config
PUT /detect/config — update config (takes effect on next run)
GET /detect/config/stages — list stage palette with config fields
"""
from __future__ import annotations
import logging
from fastapi import APIRouter
from pydantic import BaseModel
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/detect", tags=["detect"])
# In-memory config — persists until server restart.
# Phase 12+ moves this to DB.
_runtime_config: dict = {}
class ConfigUpdate(BaseModel):
detection: dict | None = None
ocr: dict | None = None
resolver: dict | None = None
escalation: dict | None = None
preprocessing: dict | None = None
class StageConfigInfo(BaseModel):
name: str
label: str
description: str
category: str
config_fields: list[dict]
reads: list[str]
writes: list[str]
@router.get("/config")
def read_config():
return _runtime_config
@router.put("/config")
def write_config(update: ConfigUpdate):
changes = update.model_dump(exclude_none=True)
for section, values in changes.items():
if section not in _runtime_config:
_runtime_config[section] = {}
_runtime_config[section].update(values)
logger.info("Config updated: %s", list(changes.keys()))
return _runtime_config
@router.get("/config/stages", response_model=list[StageConfigInfo])
def list_stage_configs():
"""Return the stage palette with config field metadata for the editor."""
from detect.stages import list_stages
result = []
for stage in list_stages():
info = StageConfigInfo(
name=stage.name,
label=stage.label,
description=stage.description,
category=stage.category,
config_fields=[
{
"name": f.name,
"type": f.type,
"default": f.default,
"description": f.description,
"min": f.min,
"max": f.max,
"options": f.options,
}
for f in stage.config_fields
],
reads=stage.io.reads,
writes=stage.io.writes,
)
result.append(info)
return result

4
core/api/main.py
View File

@@ -26,6 +26,7 @@ from strawberry.fastapi import GraphQLRouter
from core.api.chunker_sse import router as chunker_router
from core.api.detect_sse import router as detect_router
from core.api.detect_replay import router as detect_replay_router
from core.api.detect_config import router as detect_config_router
from core.api.graphql import schema as graphql_schema
CALLBACK_API_KEY = os.environ.get("CALLBACK_API_KEY", "")
@@ -60,6 +61,9 @@ app.include_router(detect_router)
# Detection replay/retry
app.include_router(detect_replay_router)
# Detection config
app.include_router(detect_config_router)
@app.get("/health")
def health():

5
core/schema/modelgen.json
View File

@@ -30,6 +30,11 @@
"target": "typescript",
"output": "ui/detection-app/src/types/sse-contract.ts",
"include": ["detect_views"]
},
{
"target": "typescript",
"output": "ui/detection-app/src/types/store-state.ts",
"include": ["ui_state_views"]
}
]
}

1
core/schema/models/__init__.py
View File

@@ -33,6 +33,7 @@ from .detect_jobs import (
from .media import AssetStatus, MediaAsset
from .presets import BUILTIN_PRESETS, TranscodePreset
from .detect import DETECT_VIEWS # noqa: F401 — discovered by modelgen generic loader
from .ui_state import UI_STATE_VIEWS # noqa: F401 — UI store state types
from .views import ChunkEvent, ChunkOutputFile, PipelineStats, WorkerEvent
# Core domain models - generates Django, Pydantic, TypeScript

139
core/schema/models/ui_state.py Normal file
View File

@@ -0,0 +1,139 @@
"""
UI application state models.
Source of truth for all frontend store state shapes.
Generates TypeScript types via modelgen.
The store implementation (Pinia, etc.) is just the reactive container.
"""
from dataclasses import dataclass, field
from typing import List, Optional
# ---------------------------------------------------------------------------
# Pipeline store
# ---------------------------------------------------------------------------
@dataclass
class NodeState:
"""A pipeline node's current status."""
id: str
status: str = "pending" # pending | running | done | error
has_checkpoint: bool = False
has_region_editor: bool = False # stage works with visual regions
has_config_editor: bool = True # all stages have config
@dataclass
class PipelineState:
"""Full pipeline run state."""
job_id: str = ""
status: str = "idle" # idle | running | paused | completed | error
layout_mode: str = "normal" # normal | bbox_editor | stage_editor
editor_stage: Optional[str] = None # which stage's editor is open
nodes: List[NodeState] = field(default_factory=list)
current_stage: Optional[str] = None
run_id: Optional[str] = None
parent_job_id: Optional[str] = None
run_type: str = "initial" # initial | replay | retry
error: Optional[str] = None
# ---------------------------------------------------------------------------
# Config store
# ---------------------------------------------------------------------------
@dataclass
class DetectionConfigOverrides:
"""Tunable detection stage config."""
model_name: Optional[str] = None
confidence_threshold: Optional[float] = None
target_classes: Optional[List[str]] = None
@dataclass
class OCRConfigOverrides:
"""Tunable OCR stage config."""
languages: Optional[List[str]] = None
min_confidence: Optional[float] = None
@dataclass
class ResolverConfigOverrides:
"""Tunable brand resolver config."""
fuzzy_threshold: Optional[int] = None
@dataclass
class EscalationConfigOverrides:
"""Tunable escalation config."""
vlm_min_confidence: Optional[float] = None
cloud_min_confidence: Optional[float] = None
cloud_provider: Optional[str] = None
@dataclass
class PreprocessingConfigOverrides:
"""Tunable preprocessing config."""
binarize: Optional[bool] = None
deskew: Optional[bool] = None
contrast: Optional[bool] = None
@dataclass
class ConfigOverrides:
"""Aggregated config overrides from all panels."""
detection: Optional[DetectionConfigOverrides] = None
ocr: Optional[OCRConfigOverrides] = None
resolver: Optional[ResolverConfigOverrides] = None
escalation: Optional[EscalationConfigOverrides] = None
preprocessing: Optional[PreprocessingConfigOverrides] = None
@dataclass
class ConfigState:
"""Config store state."""
current: ConfigOverrides = field(default_factory=ConfigOverrides)
pending: ConfigOverrides = field(default_factory=ConfigOverrides)
dirty: bool = False
# ---------------------------------------------------------------------------
# Selection store
# ---------------------------------------------------------------------------
@dataclass
class BboxRegion:
"""A user-drawn bounding box region."""
x: int
y: int
w: int
h: int
@dataclass
class SelectionState:
"""Cross-panel selection state."""
selected_frame: Optional[int] = None
selected_brand: Optional[str] = None
hovered_timestamp: Optional[float] = None
bbox_region: Optional[BboxRegion] = None
# ---------------------------------------------------------------------------
# Export for modelgen
# ---------------------------------------------------------------------------
UI_STATE_VIEWS = [
NodeState,
PipelineState,
DetectionConfigOverrides,
OCRConfigOverrides,
ResolverConfigOverrides,
EscalationConfigOverrides,
PreprocessingConfigOverrides,
ConfigOverrides,
ConfigState,
BboxRegion,
SelectionState,
]

33
detect/graph.py
View File

@@ -18,6 +18,7 @@ from detect.state import DetectState
from detect.stages.frame_extractor import extract_frames
from detect.stages.scene_filter import scene_filter
from detect.stages.yolo_detector import detect_objects
from detect.stages.preprocess import preprocess_regions
from detect.stages.ocr_stage import run_ocr
from detect.stages.brand_resolver import resolve_brands
from detect.stages.vlm_local import escalate_vlm
@@ -31,6 +32,7 @@ NODES = [
"extract_frames",
"filter_scenes",
"detect_objects",
"preprocess",
"run_ocr",
"match_brands",
"escalate_vlm",
@@ -137,6 +139,36 @@ def node_detect_objects(state: DetectState) -> dict:
return {"boxes_by_frame": all_boxes, "stats": stats}
def node_preprocess(state: DetectState) -> dict:
_emit_transition(state, "preprocess", "running")
with trace_node(state, "preprocess") as span:
profile = _get_profile(state)
frames = state.get("filtered_frames", [])
boxes = state.get("boxes_by_frame", {})
job_id = state.get("job_id")
# Get preprocessing config from profile overrides or defaults
overrides = state.get("config_overrides", {})
prep_config = overrides.get("preprocessing", {})
do_contrast = prep_config.get("contrast", True)
do_deskew = prep_config.get("deskew", False)
do_binarize = prep_config.get("binarize", False)
preprocessed = preprocess_regions(
frames, boxes,
do_contrast=do_contrast,
do_deskew=do_deskew,
do_binarize=do_binarize,
inference_url=INFERENCE_URL,
job_id=job_id,
)
span.set_output({"regions_preprocessed": len(preprocessed)})
_emit_transition(state, "preprocess", "done")
return {"preprocessed_crops": preprocessed}
def node_run_ocr(state: DetectState) -> dict:
_emit_transition(state, "run_ocr", "running")
@@ -304,6 +336,7 @@ NODE_FUNCTIONS = [
("extract_frames", node_extract_frames),
("filter_scenes", node_filter_scenes),
("detect_objects", node_detect_objects),
("preprocess", node_preprocess),
("run_ocr", node_run_ocr),
("match_brands", node_match_brands),
("escalate_vlm", node_escalate_vlm),

37
detect/sse_contract.py
View File

@@ -101,3 +101,40 @@ class JobComplete(BaseModel):
"""Final report when pipeline finishes. SSE event: job_complete"""
job_id: str
report: Optional[DetectionReportSummary] = None
class RunContext(BaseModel):
"""Run context injected into all SSE events for grouping."""
run_id: str
parent_job_id: str
run_type: str = "initial"
class CheckpointInfo(BaseModel):
"""Available checkpoint for a stage."""
stage: str
class ReplayRequest(BaseModel):
"""Request to replay pipeline from a specific stage."""
job_id: str
start_stage: str
config_overrides: Optional[Dict[str, Any]] = None
class ReplayResponse(BaseModel):
"""Result of a replay invocation."""
status: str
job_id: str
start_stage: str
detections: int = 0
brands_found: int = 0
class RetryRequest(BaseModel):
"""Request to queue async retry with different config."""
job_id: str
config_overrides: Optional[Dict[str, Any]] = None
start_stage: str = "escalate_vlm"
schedule_seconds: Optional[float] = None
class RetryResponse(BaseModel):
"""Result of queueing a retry task."""
status: str
task_id: str
job_id: str

128
detect/stages/preprocess.py Normal file
View File

@@ -0,0 +1,128 @@
"""
Stage 3.5 — Preprocessing
Runs between YOLO detection and OCR. Applies configurable image
preprocessing to each detected region crop: contrast enhancement,
deskewing, binarization.
Operates on the crops derived from boxes_by_frame, produces
preprocessed_crops keyed by (frame_sequence, box_index).
"""
from __future__ import annotations
import logging
import numpy as np
from detect import emit
from detect.models import BoundingBox, Frame
logger = logging.getLogger(__name__)
def _crop_region(frame: Frame, box: BoundingBox) -> np.ndarray:
h, w = frame.image.shape[:2]
x1 = max(0, box.x)
y1 = max(0, box.y)
x2 = min(w, box.x + box.w)
y2 = min(h, box.y + box.h)
return frame.image[y1:y2, x1:x2]
def preprocess_regions(
frames: list[Frame],
boxes_by_frame: dict[int, list[BoundingBox]],
do_contrast: bool = True,
do_deskew: bool = False,
do_binarize: bool = False,
inference_url: str | None = None,
job_id: str | None = None,
) -> dict[str, np.ndarray]:
"""
Preprocess cropped regions from YOLO detections.
Returns dict keyed by "{frame_seq}_{box_idx}" → preprocessed crop.
These are passed to the OCR stage instead of raw crops.
"""
total_regions = sum(len(boxes) for boxes in boxes_by_frame.values())
any_active = do_contrast or do_deskew or do_binarize
if not any_active:
emit.log(job_id, "Preprocess", "INFO",
f"Preprocessing disabled, passing {total_regions} regions through")
return {}
mode = "remote" if inference_url else "local"
emit.log(job_id, "Preprocess", "INFO",
f"Preprocessing {total_regions} regions (mode={mode}, "
f"contrast={do_contrast}, deskew={do_deskew}, binarize={do_binarize})")
frame_map = {f.sequence: f for f in frames}
preprocessed: dict[str, np.ndarray] = {}
processed_count = 0
for seq, boxes in boxes_by_frame.items():
frame = frame_map.get(seq)
if not frame:
continue
for idx, box in enumerate(boxes):
crop = _crop_region(frame, box)
if crop.size == 0:
continue
key = f"{seq}_{idx}"
if inference_url:
result = _preprocess_remote(crop, inference_url,
do_contrast, do_deskew, do_binarize)
else:
result = _preprocess_local(crop, do_contrast, do_deskew, do_binarize)
preprocessed[key] = result
processed_count += 1
emit.log(job_id, "Preprocess", "INFO",
f"Preprocessed {processed_count} regions")
return preprocessed
def _preprocess_remote(crop: np.ndarray, inference_url: str,
do_contrast: bool, do_deskew: bool, do_binarize: bool) -> np.ndarray:
"""Call GPU server /preprocess endpoint."""
import base64
import io
import requests
from PIL import Image
img = Image.fromarray(crop)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85)
image_b64 = base64.b64encode(buf.getvalue()).decode()
resp = requests.post(
f"{inference_url.rstrip('/')}/preprocess",
json={
"image": image_b64,
"contrast": do_contrast,
"deskew": do_deskew,
"binarize": do_binarize,
},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
result_bytes = base64.b64decode(data["image"])
result_img = Image.open(io.BytesIO(result_bytes)).convert("RGB")
return np.array(result_img)
def _preprocess_local(crop: np.ndarray,
do_contrast: bool, do_deskew: bool, do_binarize: bool) -> np.ndarray:
"""Run preprocessing in-process (requires opencv-python-headless)."""
from gpu.models.preprocess import preprocess
return preprocess(crop, do_binarize=do_binarize, do_deskew=do_deskew, do_contrast=do_contrast)

32
detect/stages/registry/preprocessing.py
View File

@@ -1,4 +1,4 @@
"""Registration for preprocessing stages: frame extraction, scene filter."""
"""Registration for preprocessing stages: frame extraction, scene filter, image preprocessing."""
from detect.stages.base import StageDefinition, StageIO, StageConfigField, register_stage
from ._serializers import serialize_frames, deserialize_frames
@@ -25,6 +25,17 @@ def _deser_filter(data: dict, job_id: str) -> dict:
return {"_filtered_sequences": data["filtered_frame_sequences"]}
def _ser_preprocess(state: dict, job_id: str) -> dict:
# Preprocessed crops are numpy arrays — regenerable from frames + boxes + config
crops = state.get("preprocessed_crops", {})
return {"crop_keys": list(crops.keys()), "count": len(crops)}
def _deser_preprocess(data: dict, job_id: str) -> dict:
# Crops are regenerable — no need to restore from checkpoint
return {"preprocessed_crops": {}}
def register():
extract = StageDefinition(
name="extract_frames",
@@ -55,3 +66,22 @@ def register():
deserialize_fn=_deser_filter,
)
register_stage(scene_filter)
preprocess = StageDefinition(
name="preprocess",
label="Preprocess",
description="Image preprocessing on detected regions before OCR",
category="preprocessing",
io=StageIO(
reads=["filtered_frames", "boxes_by_frame"],
writes=["preprocessed_crops"],
),
config_fields=[
StageConfigField("contrast", "bool", True, "CLAHE contrast enhancement"),
StageConfigField("deskew", "bool", False, "Correct slight rotation"),
StageConfigField("binarize", "bool", False, "Otsu binarization"),
],
serialize_fn=_ser_preprocess,
deserialize_fn=_deser_preprocess,
)
register_stage(preprocess)

1
detect/state.py
View File

@@ -23,6 +23,7 @@ class DetectState(TypedDict, total=False):
frames: list[Frame]
filtered_frames: list[Frame]
boxes_by_frame: dict[int, list[BoundingBox]]
preprocessed_crops: dict # "{frame_seq}_{box_idx}" → np.ndarray
text_candidates: list[TextCandidate]
unresolved_candidates: list[TextCandidate]
detections: list[BrandDetection]

117
gpu/models/preprocess.py Normal file
View File

@@ -0,0 +1,117 @@
"""
Image preprocessing pipeline for crops before OCR.
Each step is independently toggleable via config.
Operates on numpy arrays (BGR or RGB), returns processed array.
"""
from __future__ import annotations
import logging
import numpy as np
logger = logging.getLogger(__name__)
def binarize(image: np.ndarray, threshold: int = 128) -> np.ndarray:
"""Convert to grayscale and apply Otsu binarization."""
import cv2
if len(image.shape) == 3:
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
else:
gray = image
_, binary = cv2.threshold(gray, threshold, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
# Convert back to 3-channel for downstream compatibility
result = cv2.cvtColor(binary, cv2.COLOR_GRAY2RGB)
return result
def deskew(image: np.ndarray) -> np.ndarray:
"""Correct slight rotation using minimum area rectangle."""
import cv2
if len(image.shape) == 3:
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
else:
gray = image
coords = np.column_stack(np.where(gray < 128))
if len(coords) < 10:
return image
rect = cv2.minAreaRect(coords)
angle = rect[-1]
# Normalize angle
if angle < -45:
angle = -(90 + angle)
else:
angle = -angle
if abs(angle) < 0.5:
return image
h, w = image.shape[:2]
center = (w // 2, h // 2)
rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
result = cv2.warpAffine(
image, rotation_matrix, (w, h),
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REPLICATE,
)
return result
def enhance_contrast(image: np.ndarray) -> np.ndarray:
"""Apply CLAHE (adaptive histogram equalization) for contrast normalization."""
import cv2
if len(image.shape) == 3:
lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)
l_channel = lab[:, :, 0]
else:
l_channel = image
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
enhanced = clahe.apply(l_channel)
if len(image.shape) == 3:
lab[:, :, 0] = enhanced
result = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB)
else:
result = enhanced
return result
def preprocess(
image: np.ndarray,
do_binarize: bool = False,
do_deskew: bool = False,
do_contrast: bool = True,
) -> np.ndarray:
"""
Run the preprocessing pipeline on a crop image.
Each step is independently toggleable. Order: contrast → deskew → binarize.
Contrast first (works best on color), binarize last (destroys color info).
"""
result = image
if do_contrast:
result = enhance_contrast(result)
logger.debug("Preprocessing: contrast enhanced")
if do_deskew:
result = deskew(result)
logger.debug("Preprocessing: deskewed")
if do_binarize:
result = binarize(result)
logger.debug("Preprocessing: binarized")
return result

3
gpu/requirements.txt
View File

@@ -25,3 +25,6 @@ paddleocr>=3.0.0
# (all_tied_weights_keys API change). Also needs accelerate for device_map.
transformers>=4.40.0,<5
accelerate>=0.27.0
# Preprocessing (phase 12)
opencv-python-headless>=4.8.0

39
gpu/server.py
View File

@@ -73,6 +73,17 @@ class OCRResponse(BaseModel):
results: list[OCRTextResult]
class PreprocessRequest(BaseModel):
image: str
binarize: bool = False
deskew: bool = False
contrast: bool = True
class PreprocessResponse(BaseModel):
image: str # base64 JPEG of processed image
class VLMRequest(BaseModel):
image: str
prompt: str
@@ -183,6 +194,34 @@ def ocr(req: OCRRequest):
return OCRResponse(results=[OCRTextResult(**r) for r in results])
@app.post("/preprocess", response_model=PreprocessResponse)
def preprocess_image(req: PreprocessRequest):
try:
image = _decode_image(req.image)
except Exception as e:
raise HTTPException(status_code=400, detail=f"Bad image: {e}")
try:
from models.preprocess import preprocess
processed = preprocess(
image,
do_binarize=req.binarize,
do_deskew=req.deskew,
do_contrast=req.contrast,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Preprocessing failed: {e}")
from PIL import Image as PILImage
import io
img = PILImage.fromarray(processed)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=90)
result_b64 = base64.b64encode(buf.getvalue()).decode()
return PreprocessResponse(image=result_b64)
@app.post("/vlm", response_model=VLMResponse)
def vlm(req: VLMRequest):
try:

2
tests/detect/manual/push_pipeline.py
View File

@@ -35,7 +35,7 @@ def push(r, key, event):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="pipeline-test")
parser.add_argument("--job", default=f"pipeline-{int(__import__('time').time()) % 100000}")
parser.add_argument("--port", type=int, default=6382)
parser.add_argument("--delay", type=float, default=0.5)
args = parser.parse_args()

2
tests/detect/manual/run_extract_filter.py
View File

@@ -15,7 +15,7 @@ import sys
# Parse args early so we can set REDIS_URL before imports
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="extract-filter-test")
parser.add_argument("--job", default=f"extract-{int(__import__('time').time()) % 100000}")
parser.add_argument("--port", type=int, default=6382)
args = parser.parse_args()

3
tests/detect/manual/run_graph.py
View File

@@ -13,8 +13,9 @@ import logging
import os
import sys
import time as _time
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="graph-test")
parser.add_argument("--job", default=f"graph-{int(_time.time()) % 100000}")
parser.add_argument("--port", type=int, default=6382)
args = parser.parse_args()

2
tests/detect/manual/test_brand_table_e2e.py
View File

@@ -55,7 +55,7 @@ def push(r, key, event):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="brand-table-test")
parser.add_argument("--job", default=f"brand-{int(__import__('time').time()) % 100000}")
parser.add_argument("--port", type=int, default=6382)
parser.add_argument("--delay", type=float, default=0.6)
args = parser.parse_args()

44
tests/detect/manual/test_escalation_e2e.py
View File

@@ -23,8 +23,8 @@ import redis
logging.basicConfig(level=logging.INFO, format="%(levelname)-7s %(name)s — %(message)s")
logger = logging.getLogger(__name__)
NODES = ["extract_frames", "filter_scenes", "detect_objects", "run_ocr",
"match_brands", "escalate_vlm", "escalate_cloud", "compile_report"]
NODES = ["extract_frames", "filter_scenes", "detect_objects", "preprocess",
"run_ocr", "match_brands", "escalate_vlm", "escalate_cloud", "compile_report"]
def ts():
@@ -70,12 +70,22 @@ def push_stats(r, key, **fields):
push(r, key, base)
_bbox_idx = 0
def push_detection(r, key, brand, conf, source, timestamp, frame_ref, delay):
global _bbox_idx
# Spread fake bboxes across the frame so they don't overlap
col = _bbox_idx % 4
row = _bbox_idx // 4
bbox = {"x": 50 + col * 200, "y": 50 + row * 120, "w": 160, "h": 80}
_bbox_idx += 1
push(r, key, {
"event": "detection",
"brand": brand, "confidence": conf, "source": source,
"timestamp": timestamp, "duration": 0.5,
"content_type": "soccer_broadcast", "frame_ref": frame_ref,
"bbox": bbox,
})
logger.info(" [%s] %s %.2f t=%.1fs", source, brand, conf, timestamp)
time.sleep(delay * 0.3)
@@ -83,7 +93,9 @@ def push_detection(r, key, brand, conf, source, timestamp, frame_ref, delay):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="escalation-test")
import time as _time
default_job = f"escalation-{int(_time.time()) % 100000}"
parser.add_argument("--job", default=default_job)
parser.add_argument("--port", type=int, default=6382)
parser.add_argument("--delay", type=float, default=0.5)
args = parser.parse_args()
@@ -121,6 +133,32 @@ def main():
push(r, key, {"event": "log", "level": "INFO", "stage": "YOLODetector",
"msg": "Running yolov8n on 52 frames"})
time.sleep(delay)
# Push a sample frame with YOLO boxes
import base64, io
from PIL import Image as PILImage, ImageDraw
frame_img = PILImage.new("RGB", (960, 540), "#1a1a2e")
draw = ImageDraw.Draw(frame_img)
draw.rectangle([40, 440, 900, 520], outline="#444", width=2)
draw.text((100, 460), "SPONSOR BOARD AREA", fill="#666")
draw.rectangle([350, 150, 610, 380], outline="#333", width=1)
draw.text((400, 200), "PLAYER", fill="#555")
buf = io.BytesIO()
frame_img.save(buf, "JPEG")
frame_b64 = base64.b64encode(buf.getvalue()).decode()
yolo_boxes = [
{"x": 40, "y": 440, "w": 860, "h": 80, "confidence": 0.92,
"label": "ad_board", "stage": "detect_objects", "source": "yolo"},
{"x": 350, "y": 150, "w": 260, "h": 230, "confidence": 0.87,
"label": "person", "stage": "detect_objects", "source": "yolo"},
{"x": 700, "y": 30, "w": 200, "h": 60, "confidence": 0.78,
"label": "scoreboard", "stage": "detect_objects", "source": "yolo"},
]
push(r, key, {"event": "frame_update", "frame_ref": 25, "timestamp": 12.5,
"jpeg_b64": frame_b64, "boxes": yolo_boxes})
time.sleep(delay)
push_stats(r, key, frames_extracted=180, frames_after_scene_filter=52,
regions_detected=41, processing_time_seconds=14.2)
push_graph(r, key, "detect_objects", "done", delay)

2
tests/detect/manual/test_timeline_cost.py
View File

@@ -85,7 +85,7 @@ def push_stats(r, key, **overrides):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--job", default="timeline-cost-test")
parser.add_argument("--job", default=f"timeline-{int(__import__('time').time()) % 100000}")
parser.add_argument("--port", type=int, default=6382)
parser.add_argument("--delay", type=float, default=0.4)
args = parser.parse_args()

44
tests/detect/test_config_endpoint.py Normal file
View File

@@ -0,0 +1,44 @@
"""Tests for the config endpoint and stage palette."""
from detect.stages import list_stages, get_palette
def test_stage_palette_has_config_fields():
"""Every stage with config fields should be servable by the endpoint."""
stages = list_stages()
stages_with_config = [s for s in stages if s.config_fields]
assert len(stages_with_config) > 0
for stage in stages_with_config:
for field in stage.config_fields:
assert field.name
assert field.type
assert field.default is not None or field.type == "bool"
def test_palette_categories():
palette = get_palette()
expected_categories = {"preprocessing", "detection", "resolution", "escalation", "output"}
actual_categories = set(palette.keys())
assert actual_categories == expected_categories
def test_stage_config_serializable():
"""Config fields should be JSON-serializable for the API response."""
import json
stages = list_stages()
for stage in stages:
data = {
"name": stage.name,
"label": stage.label,
"config_fields": [
{"name": f.name, "type": f.type, "default": f.default}
for f in stage.config_fields
],
}
json_str = json.dumps(data)
assert len(json_str) > 0

84
tests/detect/test_preprocess.py Normal file
View File

@@ -0,0 +1,84 @@
"""Tests for OpenCV preprocessing — runs without GPU."""
import numpy as np
import pytest
try:
import cv2
HAS_CV2 = True
except ImportError:
HAS_CV2 = False
requires_cv2 = pytest.mark.skipif(not HAS_CV2, reason="opencv-python-headless not installed")
# Add gpu/ to path so imports resolve (gpu modules use relative imports)
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "gpu"))
def _make_image(w: int = 200, h: int = 60) -> np.ndarray:
"""White image with black text-like region."""
img = np.ones((h, w, 3), dtype=np.uint8) * 255
img[15:45, 20:180] = 30 # dark band simulating text
return img
@requires_cv2
def test_binarize():
from gpu.models.preprocess import binarize
img = _make_image()
result = binarize(img)
assert result.shape == img.shape
assert result.dtype == np.uint8
# Should be mostly black and white (no grays)
unique_values = np.unique(result)
assert len(unique_values) <= 3 # 0, 255, maybe one more from anti-aliasing
@requires_cv2
def test_enhance_contrast():
from gpu.models.preprocess import enhance_contrast
img = _make_image()
result = enhance_contrast(img)
assert result.shape == img.shape
assert result.dtype == np.uint8
@requires_cv2
def test_deskew_no_rotation():
from gpu.models.preprocess import deskew
img = _make_image()
result = deskew(img)
assert result.shape == img.shape
# Straight image should be unchanged (angle < 0.5 deg)
assert np.allclose(result, img, atol=5)
@requires_cv2
def test_preprocess_pipeline():
from gpu.models.preprocess import preprocess
img = _make_image()
result = preprocess(img, do_binarize=False, do_deskew=False, do_contrast=True)
assert result.shape == img.shape
result = preprocess(img, do_binarize=True, do_deskew=True, do_contrast=True)
assert result.shape[:2] == img.shape[:2] # h, w same; channels may differ then get converted back
@requires_cv2
def test_preprocess_all_disabled():
from gpu.models.preprocess import preprocess
img = _make_image()
result = preprocess(img, do_binarize=False, do_deskew=False, do_contrast=False)
assert np.array_equal(result, img)

4
tests/detect/test_stage_registry.py
View File

@@ -4,8 +4,8 @@ from detect.stages import list_stages, get_stage, get_palette
EXPECTED_STAGES = [
"extract_frames", "filter_scenes", "detect_objects", "run_ocr",
"match_brands", "escalate_vlm", "escalate_cloud", "compile_report",
"extract_frames", "filter_scenes", "detect_objects", "preprocess",
"run_ocr", "match_brands", "escalate_vlm", "escalate_cloud", "compile_report",
]

72
ui/common/types/generated.ts
View File

@@ -7,6 +7,9 @@
export type AssetStatus = "pending" | "ready" | "error";
export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled";
export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled";
export type DetectJobStatus = "pending" | "running" | "paused" | "completed" | "failed" | "cancelled";
export type RunType = "initial" | "replay" | "retry";
export type BrandSource = "ocr" | "local_vlm" | "cloud_llm" | "manual";
export interface MediaAsset {
id: string;
@@ -97,6 +100,75 @@ export interface ChunkJob {
completed_at: string | null;
}
export interface DetectJob {
id: string;
source_asset_id: string;
video_path: string;
profile_name: string;
parent_job_id: string | null;
run_type: RunType;
replay_from_stage: string | null;
config_overrides: Record<string, unknown>;
status: DetectJobStatus;
current_stage: string | null;
progress: number;
error_message: string | null;
total_detections: number;
brands_found: number;
cloud_llm_calls: number;
estimated_cost_usd: number;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface StageCheckpoint {
id: string;
job_id: string;
stage: string;
stage_index: number;
frames_prefix: string;
frames_manifest: Record<string, unknown>;
frames_meta: string[];
filtered_frame_sequences: number[];
boxes_by_frame: Record<string, unknown>;
text_candidates: string[];
unresolved_candidates: string[];
detections: string[];
stats: Record<string, unknown>;
config_snapshot: Record<string, unknown>;
config_overrides: Record<string, unknown>;
video_path: string;
profile_name: string;
created_at: string | null;
}
export interface KnownBrand {
id: string;
canonical_name: string;
aliases: string[];
first_source: BrandSource;
total_occurrences: number;
confirmed: boolean;
created_at: string | null;
updated_at: string | null;
}
export interface SourceBrandSighting {
id: string;
source_asset_id: string;
brand_id: string;
brand_name: string;
first_seen_timestamp: number;
last_seen_timestamp: number;
occurrences: number;
detection_source: BrandSource;
avg_confidence: number;
created_at: string | null;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;

117
ui/detection-app/src/App.vue
View File

@@ -10,6 +10,9 @@ import BrandTablePanel from './panels/BrandTablePanel.vue'
import TimelinePanel from './panels/TimelinePanel.vue'
import CostStatsPanel from './panels/CostStatsPanel.vue'
import type { StatsUpdate, RunContext } from './types/sse-contract'
import { usePipelineStore } from './stores/pipeline'
const pipeline = usePipelineStore()
const jobId = ref(new URLSearchParams(window.location.search).get('job') || 'test-job')
const stats = ref<StatsUpdate | null>(null)
@@ -89,16 +92,17 @@ source.connect()
</div>
<ResizeHandle direction="horizontal" @resize="onPipelineResize" />
<!-- Right area: interactive panels -->
<!-- Right area: mode-dependent content -->
<div class="content-col">
<!-- Row 1: Frame viewer + Funnel -->
<!-- === NORMAL MODE === -->
<template v-if="pipeline.layoutMode === 'normal'">
<div class="viewer-row" :style="{ height: viewerHeight + 'px' }">
<FramePanel :source="source" :status="status" />
<FunnelPanel :source="source" :status="status" />
</div>
<ResizeHandle direction="vertical" @resize="onViewerResize" />
<!-- Row 2: Detections + Stats side by side -->
<div class="detections-stats-row">
<div class="detections-col" :style="{ flex: detectionsFlex }">
<Panel title="Detections" :status="status">
@@ -133,12 +137,54 @@ source.connect()
<CostStatsPanel :source="source" :status="status" />
</div>
</div>
</template>
<!-- === BBOX EDITOR MODE === -->
<template v-else-if="pipeline.layoutMode === 'bbox_editor'">
<Panel :title="`Region Editor — ${pipeline.editorStage?.replace(/_/g, ' ')}`" :status="status">
<div class="editor-placeholder">
<div class="editor-frame">
<FramePanel :source="source" :status="status" />
</div>
<div class="editor-tools">
<p>Stage: <strong>{{ pipeline.editorStage }}</strong></p>
<p>Draw polygons to define regions</p>
<button class="editor-close" @click="pipeline.closeEditor()">✕ Close</button>
</div>
</div>
</Panel>
</template>
<!-- === STAGE EDITOR MODE === -->
<template v-else-if="pipeline.layoutMode === 'stage_editor'">
<Panel :title="`Stage Config — ${pipeline.editorStage?.replace(/_/g, ' ')}`" :status="status">
<div class="editor-placeholder">
<div class="editor-config">
<p>Stage: <strong>{{ pipeline.editorStage }}</strong></p>
<p>Config fields will be auto-generated from stage registry</p>
<button class="editor-close" @click="pipeline.closeEditor()">✕ Close</button>
</div>
</div>
</Panel>
</template>
</div>
</div>
<!-- Bottom: Log (full width) -->
<!-- Bottom bar: Log or Blob viewer depending on mode -->
<div class="log-row">
<template v-if="pipeline.layoutMode === 'bbox_editor'">
<Panel :title="`Blobs — ${pipeline.editorStage?.replace(/_/g, ' ')}`" :status="status">
<div class="blob-viewer">
<div class="blob-placeholder">
Blob viewer: crops, preprocessed images, OCR results for {{ pipeline.editorStage }}
</div>
</div>
</Panel>
</template>
<template v-else>
<LogPanel :source="source" :status="status" />
</template>
</div>
</div>
</template>
@@ -290,4 +336,67 @@ header h1 { font-size: var(--font-size-lg); font-weight: 600; }
}
.empty { color: var(--text-dim); padding: var(--space-6); text-align: center; }
/* Editor placeholders */
.editor-placeholder {
display: flex;
height: 100%;
gap: var(--space-2);
}
.editor-frame {
flex: 1;
min-height: 0;
}
.editor-tools {
width: 200px;
flex-shrink: 0;
padding: var(--space-3);
background: var(--surface-2);
border-radius: var(--panel-radius);
display: flex;
flex-direction: column;
gap: var(--space-2);
font-size: var(--font-size-sm);
color: var(--text-secondary);
}
.editor-config {
padding: var(--space-4);
font-size: var(--font-size-sm);
color: var(--text-secondary);
display: flex;
flex-direction: column;
gap: var(--space-2);
}
.editor-close {
background: var(--surface-3);
border: 1px solid var(--surface-3);
border-radius: 4px;
padding: var(--space-2) var(--space-3);
color: var(--text-secondary);
font-family: var(--font-mono);
font-size: var(--font-size-sm);
cursor: pointer;
margin-top: auto;
}
.editor-close:hover {
background: var(--status-error);
color: #000;
}
.blob-viewer {
height: 100%;
overflow-x: auto;
}
.blob-placeholder {
padding: var(--space-4);
color: var(--text-dim);
text-align: center;
font-size: var(--font-size-sm);
}
</style>

5
ui/detection-app/src/main.ts
View File

@@ -1,4 +1,7 @@
import { createApp } from 'vue'
import { createPinia } from 'pinia'
import App from './App.vue'
createApp(App).mount('#app')
const app = createApp(App)
app.use(createPinia())
app.mount('#app')

241
ui/detection-app/src/panels/FramePanel.vue
View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { ref } from 'vue'
import { ref, computed } from 'vue'
import { Panel } from 'mpr-ui-framework'
import FrameRenderer from 'mpr-ui-framework/src/renderers/FrameRenderer.vue'
import type { FrameBBox } from 'mpr-ui-framework/src/renderers/FrameRenderer.vue'
@@ -11,8 +11,23 @@ const props = defineProps<{
}>()
const imageSrc = ref('')
const boxes = ref<FrameBBox[]>([])
// Per-stage box accumulation
const stageBoxes = ref<Record<string, FrameBBox[]>>({})
const stageStatus = ref<Record<string, string>>({})
// Toggles — multiple can be active at once, all start ON
const activeToggles = ref<Set<string>>(new Set())
const STAGE_TABS = [
{ key: 'detect_objects', label: 'YOLO', color: '#f5a623' },
{ key: 'preprocess', label: 'Prep', color: '#e0e0e0' },
{ key: 'run_ocr', label: 'OCR', color: '#ff8c42' },
{ key: 'match_brands', label: 'Brands', color: '#3ecf8e' },
{ key: 'escalate_vlm', label: 'VLM', color: '#4f9cf9' },
{ key: 'escalate_cloud', label: 'Cloud', color: '#a78bfa' },
]
// Frame updates — store image, replace (not accumulate) boxes per frame
props.source.on<{
frame_ref: number
timestamp: number
@@ -20,12 +35,230 @@ props.source.on<{
boxes: FrameBBox[]
}>('frame_update', (e) => {
imageSrc.value = e.jpeg_b64
boxes.value = e.boxes
// Group incoming boxes by stage, replace previous for that stage
const incoming: Record<string, FrameBBox[]> = {}
for (const box of e.boxes) {
const stage = box.stage || 'detect_objects'
if (!incoming[stage]) incoming[stage] = []
incoming[stage].push(box)
}
for (const [stage, boxes] of Object.entries(incoming)) {
stageBoxes.value[stage] = boxes
ensureToggleOn(stage)
}
})
// Track stage status from graph updates
props.source.on<{ nodes: { id: string; status: string }[] }>('graph_update', (e) => {
for (const node of e.nodes) {
stageStatus.value[node.id] = node.status
}
})
// Detection events also produce boxes
props.source.on<{
brand: string
confidence: number
source: string
timestamp: number
frame_ref: number | null
bbox?: { x: number; y: number; w: number; h: number } | null
}>('detection', (e) => {
if (!e.bbox) return
const stage = sourceToStage(e.source)
const box: FrameBBox = {
x: e.bbox.x,
y: e.bbox.y,
w: e.bbox.w,
h: e.bbox.h,
confidence: e.confidence,
label: e.brand,
resolved_brand: e.brand,
source: e.source,
stage: stage,
}
if (!stageBoxes.value[stage]) {
stageBoxes.value[stage] = []
}
stageBoxes.value[stage].push(box)
ensureToggleOn(stage)
})
function toggleStage(key: string) {
if (activeToggles.value.has(key)) {
activeToggles.value.delete(key)
} else {
activeToggles.value.add(key)
}
// Force reactivity
activeToggles.value = new Set(activeToggles.value)
}
function ensureToggleOn(stage: string) {
if (!activeToggles.value.has(stage)) {
activeToggles.value.add(stage)
activeToggles.value = new Set(activeToggles.value)
}
}
function sourceToStage(source: string): string {
const map: Record<string, string> = {
ocr: 'match_brands',
local_vlm: 'escalate_vlm',
cloud_llm: 'escalate_cloud',
}
return map[source] || 'match_brands'
}
// Filtered boxes — show all toggled-on stages
const visibleBoxes = computed<FrameBBox[]>(() => {
const result: FrameBBox[] = []
for (const [stage, boxes] of Object.entries(stageBoxes.value)) {
if (activeToggles.value.has(stage)) {
result.push(...boxes)
}
}
return result
})
// Which toggles are visible (stage exists in pipeline)
const visibleTabs = computed(() => {
return STAGE_TABS.filter((tab) => {
const status = stageStatus.value[tab.key]
return status !== undefined
})
})
// Whether a toggle has data (boxes available)
function hasData(key: string): boolean {
return (stageBoxes.value[key]?.length || 0) > 0
}
</script>
<template>
<Panel title="Frame Viewer" :status="status">
<FrameRenderer :image-src="imageSrc" :boxes="boxes" />
<div class="frame-panel">
<div class="stage-toggles" v-if="visibleTabs.length > 0">
<button
v-for="tab in visibleTabs"
:key="tab.key"
:class="['stage-toggle', {
active: activeToggles.has(tab.key),
running: stageStatus[tab.key] === 'running',
done: stageStatus[tab.key] === 'done',
disabled: !hasData(tab.key),
}]"
:style="{ '--toggle-color': tab.color }"
:disabled="!hasData(tab.key)"
@click="toggleStage(tab.key)"
>
<span class="toggle-dot" />
{{ tab.label }}
<span class="toggle-count" v-if="stageBoxes[tab.key]?.length">
{{ stageBoxes[tab.key].length }}
</span>
</button>
</div>
<div class="frame-content">
<FrameRenderer :image-src="imageSrc" :boxes="visibleBoxes" />
</div>
</div>
</Panel>
</template>
<style scoped>
.frame-panel {
display: flex;
flex-direction: column;
height: 100%;
}
.stage-toggles {
display: flex;
gap: 2px;
padding: 4px;
flex-shrink: 0;
overflow-x: auto;
background: var(--surface-2);
border-bottom: var(--panel-border);
}
.stage-toggle {
display: flex;
align-items: center;
gap: 4px;
padding: 3px 8px;
border: 1px solid transparent;
border-radius: 3px;
background: transparent;
color: var(--text-dim);
font-family: var(--font-mono);
font-size: 10px;
cursor: pointer;
white-space: nowrap;
transition: all 0.15s;
}
.stage-toggle:hover {
color: var(--text-secondary);
background: var(--surface-3);
}
.stage-toggle.active {
color: var(--text-primary);
border-color: var(--toggle-color, var(--text-dim));
background: var(--surface-3);
}
.stage-toggle:not(.active) {
opacity: 0.5;
}
.stage-toggle:not(.active) .toggle-dot {
background: var(--text-dim);
}
.stage-toggle.disabled {
opacity: 0.3;
cursor: not-allowed;
}
.stage-toggle.disabled:hover {
background: transparent;
color: var(--text-dim);
}
.stage-toggle.running .toggle-dot {
animation: pulse 1s infinite;
}
.toggle-dot {
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--toggle-color, var(--text-dim));
}
.toggle-count {
background: var(--surface-1);
padding: 0 4px;
border-radius: 8px;
font-size: 9px;
}
.frame-content {
flex: 1;
min-height: 0;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
</style>

21
ui/detection-app/src/panels/PipelineGraphPanel.vue
View File

@@ -4,10 +4,11 @@ import { Panel } from 'mpr-ui-framework'
import GraphRenderer from 'mpr-ui-framework/src/renderers/GraphRenderer.vue'
import type { GraphNode } from 'mpr-ui-framework/src/renderers/GraphRenderer.vue'
import type { DataSource } from 'mpr-ui-framework'
import { usePipelineStore } from '../stores/pipeline'
const PIPELINE_NODES = [
'extract_frames', 'filter_scenes', 'detect_objects', 'run_ocr',
'match_brands', 'escalate_vlm', 'escalate_cloud', 'compile_report',
'extract_frames', 'filter_scenes', 'detect_objects', 'preprocess',
'run_ocr', 'match_brands', 'escalate_vlm', 'escalate_cloud', 'compile_report',
]
const props = defineProps<{
@@ -15,6 +16,8 @@ const props = defineProps<{
status?: 'idle' | 'live' | 'processing' | 'error'
}>()
const pipeline = usePipelineStore()
const nodes = ref<GraphNode[]>(
PIPELINE_NODES.map((id) => ({ id, status: 'pending' }))
)
@@ -22,10 +25,22 @@ const nodes = ref<GraphNode[]>(
props.source.on<{ nodes: GraphNode[] }>('graph_update', (e) => {
nodes.value = e.nodes
})
function onOpenRegionEditor(stage: string) {
pipeline.openBBoxEditor(stage)
}
function onOpenStageEditor(stage: string) {
pipeline.openStageEditor(stage)
}
</script>
<template>
<Panel title="Pipeline" :status="status">
<GraphRenderer :nodes="nodes" />
<GraphRenderer
:nodes="nodes"
@open-region-editor="onOpenRegionEditor"
@open-stage-editor="onOpenStageEditor"
/>
</Panel>
</template>

47
ui/detection-app/src/stores/config.ts Normal file
View File

@@ -0,0 +1,47 @@
/**
* Config store — aggregated config from all panels.
*
* Panels write their own config slice (ocr, detection, etc.).
* Pipeline panel reads the full config and triggers replay.
* State shape defined in types/store-state.ts.
*/
import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import type { ConfigState, ConfigOverrides } from '../types/store-state'
export const useConfigStore = defineStore('config', () => {
const current = ref<ConfigOverrides>({})
const pending = ref<ConfigOverrides>({})
const dirty = computed(() => JSON.stringify(pending.value) !== JSON.stringify(current.value))
function updatePending(section: keyof ConfigOverrides, values: Record<string, unknown>) {
pending.value = {
...pending.value,
[section]: { ...(pending.value[section] as Record<string, unknown> || {}), ...values },
}
}
function apply() {
current.value = JSON.parse(JSON.stringify(pending.value))
}
function revert() {
pending.value = JSON.parse(JSON.stringify(current.value))
}
function loadFromServer(config: ConfigOverrides) {
current.value = config
pending.value = JSON.parse(JSON.stringify(config))
}
function getOverrides(): ConfigOverrides {
return JSON.parse(JSON.stringify(pending.value))
}
return {
current, pending, dirty,
updatePending, apply, revert, loadFromServer, getOverrides,
}
})

40
ui/detection-app/src/stores/data.ts Normal file
View File

@@ -0,0 +1,40 @@
/**
* Data store — latest SSE data, replaces inline refs in App.vue.
*
* The SSE DataSource writes here. Panels read from here.
* State shape defined in types/store-state.ts.
*/
import { defineStore } from 'pinia'
import { ref } from 'vue'
import type { DataState } from '../types/store-state'
import type { StatsUpdate, Detection } from '../types/sse-contract'
export const useDataStore = defineStore('data', () => {
const stats = ref<StatsUpdate | null>(null)
const detections = ref<Detection[]>([])
const connectionStatus = ref<'idle' | 'connecting' | 'live' | 'error'>('idle')
function updateStats(s: StatsUpdate) {
stats.value = s
}
function addDetection(d: Detection) {
detections.value.push(d)
}
function setConnectionStatus(s: 'idle' | 'connecting' | 'live' | 'error') {
connectionStatus.value = s
}
function reset() {
stats.value = null
detections.value = []
connectionStatus.value = 'idle'
}
return {
stats, detections, connectionStatus,
updateStats, addDetection, setConnectionStatus, reset,
}
})

13
ui/detection-app/src/stores/index.ts Normal file
View File

@@ -0,0 +1,13 @@
/**
* Store index — re-exports all stores.
*
* State shapes are in types/store-state.ts (the contract).
* These files are the Pinia bindings (the implementation).
* Swap Pinia for anything else by replacing these files,
* keeping the same function signatures.
*/
export { usePipelineStore } from './pipeline'
export { useConfigStore } from './config'
export { useSelectionStore } from './selection'
export { useDataStore } from './data'

96
ui/detection-app/src/stores/pipeline.ts Normal file
View File

@@ -0,0 +1,96 @@
/**
* Pipeline store — run state, transport controls, checkpoint status.
*
* State shape defined in types/store-state.ts.
* This file is just the Pinia binding.
*/
import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import type { NodeState } from '../types/store-state'
import type { CheckpointInfo } from '../types/sse-contract'
export const usePipelineStore = defineStore('pipeline', () => {
const jobId = ref('')
const status = ref<string>('idle')
const nodes = ref<NodeState[]>([])
const currentStage = ref<string | null>(null)
const runId = ref<string | null>(null)
const parentJobId = ref<string | null>(null)
const runType = ref<string>('initial')
const checkpoints = ref<CheckpointInfo[]>([])
const error = ref<string | null>(null)
// Layout mode
const layoutMode = ref<string>('normal') // normal | bbox_editor | stage_editor
const editorStage = ref<string | null>(null) // which stage's editor is open
const isRunning = computed(() => status.value === 'running')
const isPaused = computed(() => status.value === 'paused')
const canReplay = computed(() => checkpoints.value.length > 0)
const isEditing = computed(() => layoutMode.value !== 'normal')
function setJob(id: string) {
jobId.value = id
}
function setStatus(s: string) {
status.value = s
}
function updateNodes(nodeList: NodeState[]) {
nodes.value = nodeList
const running = nodeList.find((n) => n.status === 'running')
currentStage.value = running?.id ?? null
}
function setRunContext(rid: string, parentId: string, rtype: string) {
runId.value = rid
parentJobId.value = parentId
runType.value = rtype
}
function setCheckpoints(list: CheckpointInfo[]) {
checkpoints.value = list
}
function setError(msg: string | null) {
error.value = msg
if (msg) status.value = 'error'
}
function openBBoxEditor(stage: string) {
layoutMode.value = 'bbox_editor'
editorStage.value = stage
}
function openStageEditor(stage: string) {
layoutMode.value = 'stage_editor'
editorStage.value = stage
}
function closeEditor() {
layoutMode.value = 'normal'
editorStage.value = null
}
function reset() {
status.value = 'idle'
layoutMode.value = 'normal'
editorStage.value = null
nodes.value = []
currentStage.value = null
runId.value = null
parentJobId.value = null
runType.value = 'initial'
error.value = null
}
return {
jobId, status, nodes, currentStage, runId, parentJobId, runType,
checkpoints, error, layoutMode, editorStage,
isRunning, isPaused, canReplay, isEditing,
setJob, setStatus, updateNodes, setRunContext, setCheckpoints, setError,
openBBoxEditor, openStageEditor, closeEditor, reset,
}
})

59
ui/detection-app/src/stores/selection.ts Normal file
View File

@@ -0,0 +1,59 @@
/**
* Selection store — cross-panel selection state.
*
* When you click a detection in the table, the frame viewer highlights it.
* When you hover on the timeline, the crosshair syncs across charts.
* When you draw a bbox, it feeds into the config store.
*
* State shape defined in types/store-state.ts.
*/
import { defineStore } from 'pinia'
import { ref } from 'vue'
import type { SelectionState } from '../types/store-state'
import type { Detection } from '../types/sse-contract'
export const useSelectionStore = defineStore('selection', () => {
const selectedFrame = ref<number | null>(null)
const selectedDetection = ref<Detection | null>(null)
const selectedBrand = ref<string | null>(null)
const hoveredTimestamp = ref<number | null>(null)
const bboxRegion = ref<{ x: number; y: number; w: number; h: number } | null>(null)
function selectFrame(seq: number | null) {
selectedFrame.value = seq
}
function selectDetection(det: Detection | null) {
selectedDetection.value = det
if (det) {
selectedBrand.value = det.brand
selectedFrame.value = det.frame_ref
}
}
function selectBrand(brand: string | null) {
selectedBrand.value = brand
}
function hoverTimestamp(ts: number | null) {
hoveredTimestamp.value = ts
}
function setBbox(region: { x: number; y: number; w: number; h: number } | null) {
bboxRegion.value = region
}
function clearAll() {
selectedFrame.value = null
selectedDetection.value = null
selectedBrand.value = null
hoveredTimestamp.value = null
bboxRegion.value = null
}
return {
selectedFrame, selectedDetection, selectedBrand, hoveredTimestamp, bboxRegion,
selectFrame, selectDetection, selectBrand, hoverTimestamp, setBbox, clearAll,
}
})

14
ui/detection-app/src/types/sse-contract.ts
View File

@@ -96,16 +96,12 @@ export interface JobComplete {
report: DetectionReportSummary | null;
}
// --- Run context (injected into all SSE events) ---
export interface RunContext {
run_id: string;
parent_job_id: string;
run_type: 'initial' | 'replay' | 'retry';
run_type: string;
}
// --- Checkpoint API types ---
export interface CheckpointInfo {
stage: string;
}
@@ -113,7 +109,7 @@ export interface CheckpointInfo {
export interface ReplayRequest {
job_id: string;
start_stage: string;
config_overrides?: Record<string, unknown>;
config_overrides: Record<string, unknown> | null;
}
export interface ReplayResponse {
@@ -126,9 +122,9 @@ export interface ReplayResponse {
export interface RetryRequest {
job_id: string;
config_overrides?: Record<string, unknown>;
start_stage?: string;
schedule_seconds?: number;
config_overrides: Record<string, unknown> | null;
start_stage: string;
schedule_seconds: number | null;
}
export interface RetryResponse {

82
ui/detection-app/src/types/store-state.ts Normal file
View File

@@ -0,0 +1,82 @@
/**
* TypeScript Types - GENERATED FILE
*
* Do not edit directly. Regenerate using modelgen.
*/
export interface NodeState {
id: string;
status: string;
has_checkpoint: boolean;
has_region_editor: boolean;
has_config_editor: boolean;
}
export interface PipelineState {
job_id: string;
status: string;
layout_mode: string;
editor_stage: string | null;
nodes: NodeState[];
current_stage: string | null;
run_id: string | null;
parent_job_id: string | null;
run_type: string;
error: string | null;
}
export interface DetectionConfigOverrides {
model_name: string | null;
confidence_threshold: number | null;
target_classes: string[] | null;
}
export interface OCRConfigOverrides {
languages: string[] | null;
min_confidence: number | null;
}
export interface ResolverConfigOverrides {
fuzzy_threshold: number | null;
}
export interface EscalationConfigOverrides {
vlm_min_confidence: number | null;
cloud_min_confidence: number | null;
cloud_provider: string | null;
}
export interface PreprocessingConfigOverrides {
binarize: boolean | null;
deskew: boolean | null;
contrast: boolean | null;
}
export interface ConfigOverrides {
detection: DetectionConfigOverrides | null;
ocr: OCRConfigOverrides | null;
resolver: ResolverConfigOverrides | null;
escalation: EscalationConfigOverrides | null;
preprocessing: PreprocessingConfigOverrides | null;
}
export interface ConfigState {
current: ConfigOverrides;
pending: ConfigOverrides;
dirty: boolean;
}
export interface BboxRegion {
x: number;
y: number;
w: number;
h: number;
}
export interface SelectionState {
selected_frame: number | null;
selected_brand: string | null;
hovered_timestamp: number | null;
bbox_region: BboxRegion | null;
}

7
ui/framework/src/index.ts
View File

@@ -15,3 +15,10 @@ export { default as TimeSeriesRenderer } from './renderers/TimeSeriesRenderer.vu
export { default as GraphRenderer } from './renderers/GraphRenderer.vue'
export { default as FrameRenderer } from './renderers/FrameRenderer.vue'
export { default as TableRenderer } from './renderers/TableRenderer.vue'
// Interaction plugins
export type { InteractionPlugin, PluginContext } from './plugins/InteractionPlugin'
export { BBoxDrawPlugin } from './plugins/BBoxDrawPlugin'
export type { BBoxResult, BBoxCallback } from './plugins/BBoxDrawPlugin'
export { CrosshairPlugin } from './plugins/CrosshairPlugin'
export type { CrosshairCallback } from './plugins/CrosshairPlugin'

88
ui/framework/src/plugins/BBoxDrawPlugin.ts Normal file
View File

@@ -0,0 +1,88 @@
/**
* BBoxDrawPlugin — draw bounding boxes on the frame viewer.
*
* User drags on the canvas to draw a rectangle.
* On pointer up, emits the bbox coordinates via the callback.
* The frame viewer panel feeds this into the selection store.
*/
import type { InteractionPlugin, PluginContext } from './InteractionPlugin'
export interface BBoxResult {
x: number
y: number
w: number
h: number
}
export type BBoxCallback = (bbox: BBoxResult) => void
export class BBoxDrawPlugin implements InteractionPlugin {
name = 'bbox-draw'
private ctx: CanvasRenderingContext2D | null = null
private drawing = false
private startX = 0
private startY = 0
private currentBox: BBoxResult | null = null
private callback: BBoxCallback
constructor(callback: BBoxCallback) {
this.callback = callback
}
onMount(context: PluginContext): void {
this.ctx = context.ctx
}
onUnmount(): void {
this.ctx = null
this.drawing = false
this.currentBox = null
}
onPointerDown(e: PointerEvent): void {
this.drawing = true
this.startX = e.offsetX
this.startY = e.offsetY
this.currentBox = null
}
onPointerMove(e: PointerEvent): void {
if (!this.drawing) return
const x = Math.min(this.startX, e.offsetX)
const y = Math.min(this.startY, e.offsetY)
const w = Math.abs(e.offsetX - this.startX)
const h = Math.abs(e.offsetY - this.startY)
this.currentBox = { x, y, w, h }
}
onPointerUp(_e: PointerEvent): void {
if (!this.drawing) return
this.drawing = false
if (this.currentBox && this.currentBox.w > 5 && this.currentBox.h > 5) {
this.callback(this.currentBox)
}
this.currentBox = null
}
render(ctx: CanvasRenderingContext2D): void {
if (!this.currentBox) return
const box = this.currentBox
ctx.strokeStyle = '#4f9cf9'
ctx.lineWidth = 2
ctx.setLineDash([6, 3])
ctx.strokeRect(box.x, box.y, box.w, box.h)
ctx.setLineDash([])
// Semi-transparent fill
ctx.fillStyle = 'rgba(79, 156, 249, 0.1)'
ctx.fillRect(box.x, box.y, box.w, box.h)
}
}

60
ui/framework/src/plugins/CrosshairPlugin.ts Normal file
View File

@@ -0,0 +1,60 @@
/**
* CrosshairPlugin — synchronized vertical crosshair across time-series panels.
*
* When the user hovers on any panel with this plugin, the crosshair
* position (as a timestamp) is written to the selection store.
* All panels with this plugin render a vertical line at that timestamp.
*/
import type { InteractionPlugin, PluginContext } from './InteractionPlugin'
export type CrosshairCallback = (timestamp: number | null) => void
export class CrosshairPlugin implements InteractionPlugin {
name = 'crosshair'
private width = 0
private callback: CrosshairCallback
/** Current crosshair X position (pixels), set externally from store */
public crosshairX: number | null = null
constructor(callback: CrosshairCallback) {
this.callback = callback
}
onMount(context: PluginContext): void {
this.width = context.width
}
onUnmount(): void {
this.crosshairX = null
}
onPointerMove(e: PointerEvent): void {
// Convert pixel X to normalized position (0-1)
const normalized = e.offsetX / this.width
this.callback(normalized)
}
onPointerDown(_e: PointerEvent): void {
// no-op for crosshair
}
onPointerUp(_e: PointerEvent): void {
this.callback(null)
}
render(ctx: CanvasRenderingContext2D): void {
if (this.crosshairX === null) return
ctx.strokeStyle = '#a78bfa'
ctx.lineWidth = 1
ctx.setLineDash([4, 4])
ctx.beginPath()
ctx.moveTo(this.crosshairX, 0)
ctx.lineTo(this.crosshairX, ctx.canvas.height)
ctx.stroke()
ctx.setLineDash([])
}
}

36
ui/framework/src/plugins/InteractionPlugin.ts Normal file
View File

@@ -0,0 +1,36 @@
/**
* Interaction plugin interface.
*
* Plugins attach to a Panel's overlay canvas. They receive pointer events
* and emit typed results via the callback. The panel handles rendering
* the overlay and routing events to the active plugin.
*/
export interface PluginContext {
/** Canvas element for drawing overlays */
canvas: HTMLCanvasElement
/** 2D rendering context */
ctx: CanvasRenderingContext2D
/** Canvas dimensions (may differ from display size) */
width: number
height: number
}
export interface InteractionPlugin {
/** Unique plugin name */
name: string
/** Called when the plugin is mounted on a panel */
onMount(context: PluginContext): void
/** Called when the plugin is unmounted */
onUnmount(): void
/** Pointer event handlers (optional) */
onPointerDown?(e: PointerEvent): void
onPointerMove?(e: PointerEvent): void
onPointerUp?(e: PointerEvent): void
/** Called each animation frame to render the overlay */
render(ctx: CanvasRenderingContext2D): void
}

42
ui/framework/src/renderers/FrameRenderer.vue
View File

@@ -8,6 +8,10 @@ export interface FrameBBox {
h: number
confidence: number
label: string
resolved_brand?: string | null
source?: string | null
stage?: string | null
ocr_text?: string | null
}
const props = defineProps<{
@@ -46,27 +50,37 @@ function draw() {
const bw = box.w * scale
const bh = box.h * scale
// Box outline
ctx.strokeStyle = confidenceColor(box.confidence)
const color = sourceColor(box)
const resolved = box.resolved_brand || box.ocr_text
// Box outline only — no labels, no percentages
ctx.strokeStyle = color
ctx.lineWidth = 2
if (!resolved) {
ctx.setLineDash([4, 3])
}
ctx.strokeRect(bx, by, bw, bh)
// Label background
const label = `${box.label} ${(box.confidence * 100).toFixed(0)}%`
ctx.font = '11px var(--font-mono)'
const metrics = ctx.measureText(label)
const labelH = 16
ctx.fillStyle = confidenceColor(box.confidence)
ctx.fillRect(bx, by - labelH, metrics.width + 8, labelH)
// Label text
ctx.fillStyle = '#000'
ctx.fillText(label, bx + 4, by - 4)
ctx.setLineDash([])
}
}
img.src = `data:image/jpeg;base64,${props.imageSrc}`
}
const SOURCE_COLORS: Record<string, string> = {
yolo: '#f5a623', // yellow — raw detection
ocr: '#ff8c42', // orange — text extracted
ocr_matched: '#3ecf8e', // green — brand resolved
local_vlm: '#4f9cf9', // blue — VLM resolved
cloud_llm: '#a78bfa', // purple — cloud resolved
unresolved: '#e05252', // red — nothing matched
}
function sourceColor(box: FrameBBox): string {
if (box.resolved_brand) return SOURCE_COLORS.ocr_matched
if (box.source && box.source in SOURCE_COLORS) return SOURCE_COLORS[box.source]
return confidenceColor(box.confidence)
}
function confidenceColor(conf: number): string {
if (conf >= 0.7) return 'var(--conf-high)'
if (conf >= 0.4) return 'var(--conf-mid)'

124
ui/framework/src/renderers/GraphRenderer.vue
View File

@@ -11,8 +11,19 @@ export interface GraphNode {
const props = defineProps<{
nodes: GraphNode[]
/** Stages that have a region editor (bbox/polygon) */
regionStages?: string[]
}>()
const emit = defineEmits<{
'open-region-editor': [stage: string]
'open-stage-editor': [stage: string]
}>()
const regionStageSet = computed(() => new Set(props.regionStages ?? [
'detect_objects', 'run_ocr', 'match_brands', 'escalate_vlm', 'escalate_cloud',
]))
const statusColors: Record<string, string> = {
pending: 'var(--status-idle)',
running: 'var(--status-processing)',
@@ -23,17 +34,15 @@ const statusColors: Record<string, string> = {
const flowNodes = computed(() =>
props.nodes.map((n, i) => ({
id: n.id,
type: 'stage',
position: { x: 20, y: i * 80 },
data: {
label: n.id.replace(/_/g, ' '),
position: { x: 20, y: i * 70 },
style: {
background: statusColors[n.status] ?? statusColors.pending,
color: n.status === 'pending' ? '#ccc' : '#000',
border: 'none',
borderRadius: 'var(--panel-radius)',
fontFamily: 'var(--font-mono)',
fontSize: 'var(--font-size-sm)',
fontWeight: '600',
padding: '8px 16px',
status: n.status,
color: statusColors[n.status] ?? statusColors.pending,
textColor: n.status === 'pending' ? '#888' : '#000',
hasRegionEditor: regionStageSet.value.has(n.id),
isRunning: n.status === 'running',
},
}))
)
@@ -63,7 +72,38 @@ const flowEdges = computed(() => {
:nodes-connectable="false"
:zoom-on-scroll="false"
:pan-on-scroll="false"
/>
>
<template #node-stage="{ data, id }">
<div
class="stage-node"
:class="{ running: data.isRunning }"
:style="{ background: data.color, color: data.textColor }"
>
<span class="stage-label">{{ data.label }}</span>
<span class="stage-actions">
<button
v-if="data.hasRegionEditor"
class="stage-btn region-btn"
title="Region editor"
@click.stop="emit('open-region-editor', id)"
>
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" stroke="currentColor" stroke-width="1.5">
<circle cx="5" cy="5" r="3.5"/><line x1="7.5" y1="7.5" x2="11" y2="11"/>
</svg>
</button>
<button
class="stage-btn config-btn"
title="Stage config"
@click.stop="emit('open-stage-editor', id)"
>
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" stroke="currentColor" stroke-width="1.5">
<circle cx="6" cy="6" r="2"/><path d="M6 1v2M6 9v2M1 6h2M9 6h2M2.5 2.5l1.4 1.4M8.1 8.1l1.4 1.4M2.5 9.5l1.4-1.4M8.1 3.9l1.4-1.4"/>
</svg>
</button>
</span>
</div>
</template>
</VueFlow>
</div>
</template>
@@ -77,4 +117,66 @@ const flowEdges = computed(() => {
.graph-renderer :deep(.vue-flow__background) {
background: transparent;
}
/* Hide default node styling — we use custom template */
.graph-renderer :deep(.vue-flow__node-stage) {
padding: 0;
border: none;
background: transparent;
border-radius: 0;
}
.stage-node {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 10px;
border-radius: var(--panel-radius);
font-family: var(--font-mono);
font-size: var(--font-size-sm);
font-weight: 600;
min-width: 180px;
}
.stage-node.running {
animation: node-pulse 1.5s infinite;
}
.stage-label {
flex: 1;
}
.stage-actions {
display: flex;
gap: 2px;
opacity: 0;
transition: opacity 0.15s;
}
.stage-node:hover .stage-actions {
opacity: 1;
}
.stage-btn {
background: rgba(0, 0, 0, 0.15);
border: none;
border-radius: 3px;
width: 20px;
height: 20px;
font-size: 11px;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
color: inherit;
}
.stage-btn:hover {
background: rgba(0, 0, 0, 0.3);
}
@keyframes node-pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.7; }
}
</style>