From a3b51c458d2751be141b6f04b27d2b618a56fa04 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Fri, 27 Mar 2026 07:20:27 -0300 Subject: [PATCH] timeline --- core/schema/models/__init__.py | 2 - core/schema/models/stages.py | 5 + core/schema/serializers/__init__.py | 5 +- detect/stages/edge_detector.py | 1 + ui/detection-app/src/App.vue | 140 ++++++++-- .../src/components/FrameStrip.vue | 263 ++++++++++++++++++ .../src/components/StageConfigSliders.vue | 95 ++++--- ui/framework/src/cv/index.ts | 22 ++ ui/framework/src/cv/tracks.ts | 181 ++++++++++++ ui/framework/src/renderers/FrameRenderer.vue | 6 +- 10 files changed, 653 insertions(+), 67 deletions(-) create mode 100644 ui/detection-app/src/components/FrameStrip.vue create mode 100644 ui/framework/src/cv/tracks.ts diff --git a/core/schema/models/__init__.py b/core/schema/models/__init__.py index 9a076fb..f085456 100644 --- a/core/schema/models/__init__.py +++ b/core/schema/models/__init__.py @@ -81,8 +81,6 @@ __all__ = [ "Job", "Timeline", "Checkpoint", - "KnownBrand", - "SourceBrandSighting", # Enums "AssetStatus", "JobStatus", diff --git a/core/schema/models/stages.py b/core/schema/models/stages.py index d4bb7d9..d7ba1d4 100644 --- a/core/schema/models/stages.py +++ b/core/schema/models/stages.py @@ -48,6 +48,11 @@ class StageDefinition: io: StageIO = field(default_factory=StageIO) config_fields: List[StageConfigField] = field(default_factory=list) + # The box label this stage produces that should be time-tracked in the editor. + # Set to the label string (e.g. "edge_region") for stages that have a + # meaningful temporal element. None means no motion tracker overlay. + tracks_element: Optional[str] = None + # Legacy fields — used by old registry pattern during migration. # New stages use Stage subclass instead. fn: Any = None diff --git a/core/schema/serializers/__init__.py b/core/schema/serializers/__init__.py index 623b0e8..8866e42 100644 --- a/core/schema/serializers/__init__.py +++ b/core/schema/serializers/__init__.py @@ -1,9 +1,8 @@ """ Model serializers — one module per model group, mirroring core/schema/models/. - models/detect_pipeline.py → serializers/detect_pipeline.py - models/detect_jobs.py → serializers/detect_jobs.py - models/detect.py → serializers/detect.py (SSE events) + models/pipeline.py → serializers/pipeline.py + models/detect.py → serializers/detect.py (SSE events) Common utilities in _common.py. """ diff --git a/detect/stages/edge_detector.py b/detect/stages/edge_detector.py index 063bb1c..85a2e0e 100644 --- a/detect/stages/edge_detector.py +++ b/detect/stages/edge_detector.py @@ -51,6 +51,7 @@ class EdgeDetectionStage(Stage): StageConfigField("edge_pair_max_distance", "int", 200, "Max distance between line pair (px)", min=10, max=500), StageConfigField("edge_pair_min_distance", "int", 15, "Min distance between line pair (px)", min=5, max=200), ], + tracks_element="edge_region", ) def run(self, frames: list[Frame], config: dict) -> dict[int, list[BoundingBox]]: diff --git a/ui/detection-app/src/App.vue b/ui/detection-app/src/App.vue index 90c6d60..b928f85 100644 --- a/ui/detection-app/src/App.vue +++ b/ui/detection-app/src/App.vue @@ -1,5 +1,5 @@ + + + + diff --git a/ui/detection-app/src/components/StageConfigSliders.vue b/ui/detection-app/src/components/StageConfigSliders.vue index e0d440a..69e388f 100644 --- a/ui/detection-app/src/components/StageConfigSliders.vue +++ b/ui/detection-app/src/components/StageConfigSliders.vue @@ -27,12 +27,20 @@ const props = defineProps<{ frameImage?: string | null /** Currently displayed frame sequence number */ frameRef?: number | null + /** All checkpoint frames — when provided, Apply runs on the selection range */ + frames?: Array<{ seq: number; jpeg_b64: string }> + /** Index into frames[] for selection start (default 0) */ + selectionStart?: number + /** Index into frames[] for selection end (default frames.length - 1) */ + selectionEnd?: number }>() const emit = defineEmits<{ 'replay-result': [result: { regions_by_frame: Record debug: Record + frameWidth?: number + frameHeight?: number }] }>() @@ -45,6 +53,7 @@ const debugEnabled = ref(true) const autoApply = ref(true) // auto-run on slider change (fast CV); uncheck for heavy stages const execMode = ref<'local' | 'server'>('local') const execTimeMs = ref(null) +const processingIndex = ref(null) // current frame index during multi-frame run // Config field defaults for detect_edges (used when API is unavailable) const EDGE_DEFAULTS: ConfigField[] = [ @@ -89,6 +98,11 @@ watch(() => props.frameImage, (newVal, oldVal) => { } }) +// Auto-run when selection range changes (strip handle drag) +watch([() => props.selectionStart, () => props.selectionEnd], () => { + if (autoApply.value) onSliderChange() +}) + const numericFields = computed(() => fields.value.filter(f => f.type === 'int' || f.type === 'float')) const boolFields = computed(() => fields.value.filter(f => f.type === 'bool')) @@ -108,7 +122,8 @@ function onSliderChange() { } async function applyDetection() { - if (!props.frameImage) { + const hasFrames = props.frames && props.frames.length > 0 + if (!props.frameImage && !hasFrames) { error.value = 'No frame available' return } @@ -130,14 +145,10 @@ async function applyDetection() { } } -/** Browser-side CV — no network, instant */ +/** Browser-side CV — runs on current frame or selection range */ async function runLocal() { const t0 = performance.now() - // Decode base64 JPEG → ImageData - const imageData = await b64ToImageData(props.frameImage!) - - // Build params from slider values const params: Partial = { cannyLow: values.value['edge_canny_low'] as number, cannyHigh: values.value['edge_canny_high'] as number, @@ -148,38 +159,49 @@ async function runLocal() { pairMinDistance: values.value['edge_pair_min_distance'] as number, } - const frameKey = String(props.frameRef ?? 0) + // Determine which frames to process + const targetFrames = props.frames && props.frames.length > 0 + ? props.frames.slice(props.selectionStart ?? 0, (props.selectionEnd ?? props.frames.length - 1) + 1) + : [{ seq: props.frameRef ?? 0, jpeg_b64: props.frameImage! }] - if (debugEnabled.value) { - const result = await runEdgeDetectionDebug(imageData, params) - execTimeMs.value = Math.round(performance.now() - t0) - regionCount.value = result.regions.length + const regions_by_frame: Record = {} + const debug: Record = {} + let totalRegions = 0 + let frameWidth = 0 + let frameHeight = 0 - // Convert ImageData overlays to base64 for FrameRenderer - const edgeB64 = await imageDataToB64(result.edgeImageData) - const linesB64 = await imageDataToB64(result.linesImageData) + for (let i = 0; i < targetFrames.length; i++) { + processingIndex.value = i + const frame = targetFrames[i] + const imageData = await b64ToImageData(frame.jpeg_b64) + frameWidth = imageData.width + frameHeight = imageData.height + const frameKey = String(frame.seq) - emit('replay-result', { - regions_by_frame: { [frameKey]: result.regions }, - debug: { - [frameKey]: { - edge_overlay_b64: edgeB64, - lines_overlay_b64: linesB64, - horizontal_count: result.horizontalCount, - pair_count: result.pairCount, - }, - }, - }) - } else { - const result = await runEdgeDetection(imageData, params) - execTimeMs.value = Math.round(performance.now() - t0) - regionCount.value = result.regions.length - - emit('replay-result', { - regions_by_frame: { [frameKey]: result.regions }, - debug: {}, - }) + if (debugEnabled.value) { + const result = await runEdgeDetectionDebug(imageData, params) + totalRegions += result.regions.length + const edgeB64 = await imageDataToB64(result.edgeImageData) + const linesB64 = await imageDataToB64(result.linesImageData) + regions_by_frame[frameKey] = result.regions + debug[frameKey] = { + edge_overlay_b64: edgeB64, + lines_overlay_b64: linesB64, + horizontal_count: result.horizontalCount, + pair_count: result.pairCount, + } + } else { + const result = await runEdgeDetection(imageData, params) + totalRegions += result.regions.length + regions_by_frame[frameKey] = result.regions + } } + + processingIndex.value = null + execTimeMs.value = Math.round(performance.now() - t0) + regionCount.value = totalRegions + + emit('replay-result', { regions_by_frame, debug, frameWidth, frameHeight }) } /** Server-side CV — calls GPU box via proxy */ @@ -297,7 +319,10 @@ async function runServer() { Auto - + + {{ processingIndex + 1 }}/{{ frames.length }} + + {{ regionCount }} regions diff --git a/ui/framework/src/cv/index.ts b/ui/framework/src/cv/index.ts index 522b3b1..7106990 100644 --- a/ui/framework/src/cv/index.ts +++ b/ui/framework/src/cv/index.ts @@ -16,6 +16,8 @@ import { detectEdges, detectEdgesDebug, type EdgeRegion, type EdgeDetectionParam export type { EdgeRegion, EdgeDetectionParams } from './edges' export type { EdgeDetectionResult, EdgeDetectionDebugResult } from './edges' +export { matchTracks, renderTracksToImageData } from './tracks' +export type { Track, TrackPoint } from './tracks' /** Run edge detection. Returns bounding boxes. */ export async function runEdgeDetection( @@ -59,6 +61,26 @@ export function b64ToImageData(b64: string): Promise { }) } +/** + * Encode ImageData to base64 PNG string (preserves transparency). + * + * Used for overlays that need a transparent background (e.g. motion tracks). + * Pair with srcFormat: 'png' on the FrameOverlay. + */ +export async function imageDataToPngB64(imageData: ImageData): Promise { + const canvas = new OffscreenCanvas(imageData.width, imageData.height) + const ctx = canvas.getContext('2d')! + ctx.putImageData(imageData, 0, 0) + const blob = await canvas.convertToBlob({ type: 'image/png' }) + const buffer = await blob.arrayBuffer() + const bytes = new Uint8Array(buffer) + let binary = '' + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]) + } + return btoa(binary) +} + /** * Encode ImageData to base64 JPEG string. * diff --git a/ui/framework/src/cv/tracks.ts b/ui/framework/src/cv/tracks.ts new file mode 100644 index 0000000..048d447 --- /dev/null +++ b/ui/framework/src/cv/tracks.ts @@ -0,0 +1,181 @@ +/** + * Motion track matching and rendering. + * + * Matches bounding boxes across frames by IoU (Intersection over Union). + * Uses Hungarian-style optimal assignment so every track gets its best match. + * Renders VFX-style tracks: × at start/end, dotted trace in between. + * Returns ImageData (RGBA, transparent background) for use as a FrameOverlay. + */ + +export type TrackPoint = { seq: number; cx: number; cy: number; w: number; h: number } +export type Track = { id: number; points: TrackPoint[] } + +type Box = { x: number; y: number; w: number; h: number } + +function iou(a: Box, b: Box): number { + const x1 = Math.max(a.x, b.x) + const y1 = Math.max(a.y, b.y) + const x2 = Math.min(a.x + a.w, b.x + b.w) + const y2 = Math.min(a.y + a.h, b.y + b.h) + const inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1) + if (inter === 0) return 0 + const union = a.w * a.h + b.w * b.h - inter + return union > 0 ? inter / union : 0 +} + +/** + * Match bounding boxes across frames by IoU. + * + * For each pair of consecutive frames, computes an IoU cost matrix and + * assigns tracks to boxes using greedy-best-first on IoU score (descending). + * minIoU: minimum overlap to consider a match (default 0.15). + */ +export function matchTracks( + regionsByFrame: Record, + minIoU = 0.15, +): Track[] { + const seqs = Object.keys(regionsByFrame).map(Number).sort((a, b) => a - b) + if (seqs.length === 0) return [] + + let nextId = 0 + const active: Track[] = [] + const finished: Track[] = [] + + for (const seq of seqs) { + const boxes = regionsByFrame[seq] + + if (active.length === 0) { + for (const b of boxes) { + active.push({ + id: nextId++, + points: [{ seq, cx: b.x + b.w / 2, cy: b.y + b.h / 2, w: b.w, h: b.h }], + }) + } + continue + } + + // Build cost matrix: IoU between each active track's last box and each new box + const costs: Array<{ trackIdx: number; boxIdx: number; score: number }> = [] + for (let t = 0; t < active.length; t++) { + const last = active[t].points[active[t].points.length - 1] + const trackBox: Box = { x: last.cx - last.w / 2, y: last.cy - last.h / 2, w: last.w, h: last.h } + for (let b = 0; b < boxes.length; b++) { + const score = iou(trackBox, boxes[b]) + if (score >= minIoU) { + costs.push({ trackIdx: t, boxIdx: b, score }) + } + } + } + + // Greedy-best-first assignment (sorted by descending IoU) + costs.sort((a, b) => b.score - a.score) + const matchedTracks = new Set() + const matchedBoxes = new Set() + const next: Track[] = [] + + for (const { trackIdx, boxIdx } of costs) { + if (matchedTracks.has(trackIdx) || matchedBoxes.has(boxIdx)) continue + matchedTracks.add(trackIdx) + matchedBoxes.add(boxIdx) + const b = boxes[boxIdx] + active[trackIdx].points.push({ + seq, cx: b.x + b.w / 2, cy: b.y + b.h / 2, w: b.w, h: b.h, + }) + next.push(active[trackIdx]) + } + + // Unmatched tracks → finished + for (let t = 0; t < active.length; t++) { + if (!matchedTracks.has(t)) finished.push(active[t]) + } + + // Unmatched boxes → new tracks + for (let b = 0; b < boxes.length; b++) { + if (!matchedBoxes.has(b)) { + const box = boxes[b] + next.push({ + id: nextId++, + points: [{ seq, cx: box.x + box.w / 2, cy: box.y + box.h / 2, w: box.w, h: box.h }], + }) + } + } + + active.length = 0 + active.push(...next) + } + + return [...finished, ...active] +} + +/** + * Render tracks to ImageData (RGBA, transparent background). + * + * Only draws tracks that span ≥2 frames (single-frame detections are skipped). + * × at start, × at end, dotted trace in between. + * Filled dot at currentSeq. + */ +export function renderTracksToImageData( + tracks: Track[], + width: number, + height: number, + currentSeq: number, + color = '#00e5ff', +): ImageData { + const canvas = new OffscreenCanvas(width, height) + const ctx = canvas.getContext('2d')! + + ctx.clearRect(0, 0, width, height) + ctx.strokeStyle = color + ctx.fillStyle = color + + for (const track of tracks) { + const pts = track.points + // Skip single-frame detections — no temporal signal + if (pts.length < 2) continue + + // Dotted trace connecting all centers + ctx.globalAlpha = 0.75 + ctx.lineWidth = 1.5 + ctx.setLineDash([4, 4]) + ctx.beginPath() + ctx.moveTo(pts[0].cx, pts[0].cy) + for (let i = 1; i < pts.length; i++) { + ctx.lineTo(pts[i].cx, pts[i].cy) + } + ctx.stroke() + ctx.setLineDash([]) + + // × at first point + ctx.globalAlpha = 0.9 + ctx.lineWidth = 1.5 + drawX(ctx, pts[0].cx, pts[0].cy, 7) + + // × at last point + drawX(ctx, pts[pts.length - 1].cx, pts[pts.length - 1].cy, 7) + + // Filled dot at current frame + const curr = pts.find(p => p.seq === currentSeq) + if (curr) { + ctx.globalAlpha = 1 + ctx.beginPath() + ctx.arc(curr.cx, curr.cy, 4, 0, Math.PI * 2) + ctx.fill() + } + } + + return ctx.getImageData(0, 0, width, height) +} + +function drawX( + ctx: OffscreenCanvasRenderingContext2D, + cx: number, + cy: number, + size: number, +): void { + ctx.beginPath() + ctx.moveTo(cx - size, cy - size) + ctx.lineTo(cx + size, cy + size) + ctx.moveTo(cx + size, cy - size) + ctx.lineTo(cx - size, cy + size) + ctx.stroke() +} diff --git a/ui/framework/src/renderers/FrameRenderer.vue b/ui/framework/src/renderers/FrameRenderer.vue index d211360..50118c0 100644 --- a/ui/framework/src/renderers/FrameRenderer.vue +++ b/ui/framework/src/renderers/FrameRenderer.vue @@ -15,12 +15,14 @@ export interface FrameBBox { } export interface FrameOverlay { - /** Base64 JPEG image (same dimensions as main image) */ + /** Base64 encoded image (same dimensions as main image) */ src: string label: string visible: boolean /** Opacity 0-1, default 0.5 */ opacity?: number + /** Image format — 'jpeg' (default) or 'png' (supports transparency) */ + srcFormat?: 'jpeg' | 'png' } const props = defineProps<{ @@ -97,7 +99,7 @@ function drawOverlays(ctx: CanvasRenderingContext2D, dx: number, dy: number, dw: // Load async, redraw when ready const overlay = new window.Image() overlay.onload = () => draw() - overlay.src = `data:image/jpeg;base64,${layer.src}` + overlay.src = `data:image/${layer.srcFormat ?? 'jpeg'};base64,${layer.src}` overlayCache.set(layer.src, overlay) } }