mariano/mediaproc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 4

Browse Source
This commit is contained in:
buenosairesam
2026-03-27 22:57:45 -03:00
parent 94c7b21ae5
commit 1c6af767eb
4 changed files with 360 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
ui/detection-app/src/App.vue
View File

@@ -1,8 +1,6 @@
<script setup lang="ts">
import { ref, computed } from 'vue'
import { SSEDataSource, Panel, ResizeHandle } from 'mpr-ui-framework'
import type { FrameOverlay, FrameBBox } from 'mpr-ui-framework'
import { matchTracks, renderTracksToImageData, imageDataToPngB64 } from '@/cv'
import { ref } from 'vue'
import { Panel, ResizeHandle } from 'mpr-ui-framework'
import 'mpr-ui-framework/src/tokens.css'
import LogPanel from './panels/LogPanel.vue'
import FunnelPanel from './panels/FunnelPanel.vue'
@@ -14,46 +12,46 @@ import CostStatsPanel from './panels/CostStatsPanel.vue'
import SourceSelector from './panels/SourceSelector.vue'
import StageConfigSliders from './components/StageConfigSliders.vue'
import FrameStrip from './components/FrameStrip.vue'
import type { StatsUpdate, RunContext } from './types/sse-contract'
import { usePipelineStore } from './stores/pipeline'
import { useSSEConnection } from './composables/useSSEConnection'
import { useCheckpointLoader } from './composables/useCheckpointLoader'
import { useEditorState } from './composables/useEditorState'
const pipeline = usePipelineStore()
const jobParam = new URLSearchParams(window.location.search).get('job')
const jobId = ref(jobParam || '')
const stats = ref<StatsUpdate | null>(null)
const runContext = ref<RunContext | null>(null)
const status = ref<'idle' | 'live' | 'processing' | 'error'>('idle')
const logPanel = ref<{ clear: () => void } | null>(null)
const sseConnected = ref(false)
// No job selected and no hash route → open source selector
if (!jobParam && !window.location.hash.replace(/^#\/?/, '')) {
pipeline.openSourceSelector()
// SSE connection + pipeline status
const {
jobId, stats, runContext, status, sseConnected, source,
stopPipeline, onJobStarted: sseJobStarted,
} = useSSEConnection()
// Checkpoint frames + navigation
const {
currentFrameImage, currentFrameRef,
checkpointFrames, checkpointFrameIndex,
stripSelStart, stripSelEnd, stripSelEndOverride,
setCheckpointFrame: cpSetFrame,
} = useCheckpointLoader(jobId, source)
// Editor overlays + CV result accumulation
const {
editorOverlays, editorBoxes,
updateDisplayForFrame, onReplayResult,
} = useEditorState(currentFrameRef)
// Wire checkpoint frame change to editor display update
function setCheckpointFrame(index: number) {
cpSetFrame(index)
const frame = checkpointFrames.value[index]
if (frame) updateDisplayForFrame(frame.seq)
}
const source = new SSEDataSource({
id: 'detect-stream',
url: jobId.value ? `/api/detect/stream/${jobId.value}` : '',
eventTypes: ['graph_update', 'stats_update', 'frame_update', 'detection', 'log', 'job_complete', 'waiting'],
})
source.on<StatsUpdate>('stats_update', (e) => {
stats.value = e
if (!runContext.value && e.run_id) {
runContext.value = {
run_id: e.run_id!,
parent_job_id: e.parent_job_id!,
run_type: e.run_type ?? 'initial',
// Wire job start to clear log panel
function onJobStarted(newJobId: string) {
logPanel.value?.clear()
sseJobStarted(newJobId)
}
}
})
source.on<{ report?: { status?: string, error?: string } }>('job_complete', (e) => {
if (e.report?.status === 'failed') {
status.value = 'error'
}
})
// Resizable splits
const pipelineWidth = ref(320)
@@ -61,6 +59,7 @@ const detectionsFlex = ref(3)
const viewerHeight = ref(240)
const timelineFlex = ref(1)
const tableFlex = ref(1)
const slidersWidth = ref(210)
function onPipelineResize(delta: number) {
pipelineWidth.value = Math.max(200, Math.min(500, pipelineWidth.value + delta))
@@ -80,247 +79,9 @@ function onTimelineResize(delta: number) {
tableFlex.value = Math.max(0.3, Math.min(3, tableFlex.value - shift))
}
// Editor sliders sidebar width — drag right = shrink sliders (grow frame)
const slidersWidth = ref(210)
function onSlidersResize(delta: number) {
slidersWidth.value = Math.max(210, Math.min(350, slidersWidth.value - delta))
}
// Editor bottom height (overlays bar)
const editorBottomHeight = ref(50)
function onEditorBottomResize(delta: number) {
editorBottomHeight.value = Math.max(36, Math.min(120, editorBottomHeight.value - delta))
}
const statusMap: Record<string, 'idle' | 'live' | 'processing' | 'error'> = {
idle: 'idle',
connecting: 'processing',
live: 'live',
error: 'error',
}
const checkStatus = () => {
if (sseConnected.value) {
status.value = statusMap[source.status.value] ?? 'idle'
}
}
setInterval(checkStatus, 500)
// Only connect SSE for live pipeline runs (no hash route = dashboard mode)
// Scenario URLs use hash routing and load from checkpoint instead
const isScenarioMode = pipeline.isEditing || pipeline.layoutMode !== 'normal'
if (jobId.value && !isScenarioMode) {
source.connect()
sseConnected.value = true
}
async function stopPipeline() {
if (!jobId.value) return
try {
await fetch(`/api/detect/stop/${jobId.value}`, { method: 'POST' })
} catch { /* ignore — UI will see the cancel event via SSE */ }
}
// Current frame image (base64) — tracked for the editor's direct GPU calls
const currentFrameImage = ref<string | null>(null)
const currentFrameRef = ref<number | null>(null)
// All checkpoint frames (for scenario mode — scrubbing)
const checkpointFrames = ref<{ seq: number; timestamp: number; jpeg_b64: string }[]>([])
const checkpointFrameIndex = ref(0)
const checkpointStage = ref<string | null>(null) // which stage the checkpoint is at
// Frame strip selection (indices into checkpointFrames)
const stripSelStart = ref(0)
const stripSelEnd = computed(() =>
stripSelEndOverride.value ?? Math.max(0, checkpointFrames.value.length - 1),
)
const stripSelEndOverride = ref<number | null>(null)
// Per-frame CV results — accumulated across all processed frames
const allFrameRegions = ref<Record<number, Array<{ x: number; y: number; w: number; h: number; confidence: number; label: string }>>>({})
const allFrameDebug = ref<Record<number, { edge_overlay_b64: string; lines_overlay_b64: string }>>({})
const frameDimensions = ref<{ w: number; h: number } | null>(null)
source.on<{ frame_ref: number; jpeg_b64: string }>('frame_update', (e) => {
currentFrameImage.value = e.jpeg_b64
currentFrameRef.value = e.frame_ref
})
// Load checkpoint data when in scenario mode
async function loadCheckpoint(job: string, stage: string) {
try {
const resp = await fetch(`/api/detect/checkpoints/${job}/${stage}`)
if (!resp.ok) return
const data = await resp.json()
checkpointFrames.value = data.frames ?? []
checkpointStage.value = stage
// Show first frame
if (checkpointFrames.value.length > 0) {
checkpointFrameIndex.value = 0
const first = checkpointFrames.value[0]
currentFrameImage.value = first.jpeg_b64
currentFrameRef.value = first.seq
}
// Reset accumulated CV results and strip selection for this checkpoint
allFrameRegions.value = {}
allFrameDebug.value = {}
frameDimensions.value = null
stripSelStart.value = 0
stripSelEndOverride.value = null
status.value = 'idle'
} catch (e) {
console.error('Failed to load checkpoint:', e)
}
}
function setCheckpointFrame(index: number) {
if (index < 0 || index >= checkpointFrames.value.length) return
checkpointFrameIndex.value = index
const frame = checkpointFrames.value[index]
currentFrameImage.value = frame.jpeg_b64
currentFrameRef.value = frame.seq
updateDisplayForFrame(frame.seq)
}
function updateDisplayForFrame(seq: number) {
// Update boxes for this frame from accumulated results
const regions = allFrameRegions.value[seq] ?? []
editorBoxes.value = regions.map(r => ({
x: r.x, y: r.y, w: r.w, h: r.h,
confidence: r.confidence,
label: r.label ?? 'edge_region',
stage: 'detect_edges',
}))
// Update Canny/Hough overlays for this frame (preserving visibility/opacity)
const debug = allFrameDebug.value[seq]
if (debug) {
const overlays: FrameOverlay[] = []
if (debug.edge_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Canny edges')
overlays.push({ src: debug.edge_overlay_b64, label: 'Canny edges', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
}
if (debug.lines_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Hough lines')
overlays.push({ src: debug.lines_overlay_b64, label: 'Hough lines', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
}
// Re-append track overlay if it exists
const trackOverlay = editorOverlays.value.find(o => o.label === 'Motion tracks')
if (trackOverlay) overlays.push(trackOverlay)
editorOverlays.value = overlays
}
// Re-render track overlay with updated currentSeq
if (Object.keys(allFrameRegions.value).length >= 2 && frameDimensions.value) {
updateTrackOverlay(seq)
}
}
async function updateTrackOverlay(currentSeq: number) {
const dims = frameDimensions.value
if (!dims || Object.keys(allFrameRegions.value).length < 2) return
const tracks = matchTracks(allFrameRegions.value)
const imageData = renderTracksToImageData(tracks, dims.w, dims.h, currentSeq)
const b64 = await imageDataToPngB64(imageData)
const existing = editorOverlays.value.find(o => o.label === 'Motion tracks')
const trackOverlay: FrameOverlay = {
src: b64,
label: 'Motion tracks',
visible: existing?.visible ?? true,
opacity: existing?.opacity ?? 0.9,
srcFormat: 'png',
}
editorOverlays.value = [
...editorOverlays.value.filter(o => o.label !== 'Motion tracks'),
trackOverlay,
]
}
// Load checkpoint when in editor mode with a job (scenario URL)
// Uses watch to handle both initial load and navigation
import { watch as vueWatch } from 'vue'
vueWatch(
() => [pipeline.layoutMode, pipeline.editorStage, jobId.value] as const,
([mode, stage, job]) => {
if (mode === 'bbox_editor' && stage && job) {
const stageMap: Record<string, string> = {
detect_edges: 'filter_scenes',
detect_contours: 'detect_edges',
detect_color: 'detect_contours',
merge_regions: 'detect_color',
}
const cpStage = stageMap[stage] ?? 'filter_scenes'
loadCheckpoint(job, cpStage)
}
},
{ immediate: true },
)
// Debug overlays from replay-stage results
const editorOverlays = ref<FrameOverlay[]>([])
// Boxes from edge detection (local or server)
const editorBoxes = ref<FrameBBox[]>([])
type RegionBox = { x: number; y: number; w: number; h: number; confidence: number; label: string }
function onReplayResult(result: {
regions_by_frame?: Record<string, RegionBox[]>
debug?: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
frameWidth?: number
frameHeight?: number
}) {
// Store frame dimensions for track overlay rendering
if (result.frameWidth && result.frameHeight) {
frameDimensions.value = { w: result.frameWidth, h: result.frameHeight }
}
// Merge incoming per-frame regions into accumulated store
if (result.regions_by_frame) {
for (const [seqStr, regions] of Object.entries(result.regions_by_frame)) {
allFrameRegions.value[Number(seqStr)] = regions
}
}
// Merge incoming per-frame debug overlays into accumulated store
if (result.debug) {
for (const [seqStr, dbg] of Object.entries(result.debug)) {
allFrameDebug.value[Number(seqStr)] = {
edge_overlay_b64: dbg.edge_overlay_b64,
lines_overlay_b64: dbg.lines_overlay_b64,
}
}
}
// Update display for the currently shown frame
const currentSeq = currentFrameRef.value ?? 0
updateDisplayForFrame(currentSeq)
}
function onJobStarted(newJobId: string) {
jobId.value = newJobId
// Reset UI state
stats.value = null
runContext.value = null
status.value = 'processing'
logPanel.value?.clear()
pipeline.reset()
pipeline.setStatus('running')
// Update URL without reload
const url = new URL(window.location.href)
url.searchParams.set('job', newJobId)
window.history.pushState({}, '', url.toString())
// Connect SSE to new job
source.disconnect()
source.setUrl(`/api/detect/stream/${newJobId}`)
source.connect()
sseConnected.value = true
}
</script>
<template>

98
ui/detection-app/src/composables/useCheckpointLoader.ts Normal file
View File

@@ -0,0 +1,98 @@
import { ref, computed, watch } from 'vue'
import type { Ref } from 'vue'
import type { DataSource } from 'mpr-ui-framework'
import { usePipelineStore } from '../stores/pipeline'
interface CheckpointFrame {
seq: number
timestamp: number
jpeg_b64: string
}
export function useCheckpointLoader(
jobId: Ref<string>,
source: DataSource,
) {
const pipeline = usePipelineStore()
const currentFrameImage = ref<string | null>(null)
const currentFrameRef = ref<number | null>(null)
const checkpointFrames = ref<CheckpointFrame[]>([])
const checkpointFrameIndex = ref(0)
const checkpointStage = ref<string | null>(null)
const stripSelStart = ref(0)
const stripSelEndOverride = ref<number | null>(null)
const stripSelEnd = computed(() =>
stripSelEndOverride.value ?? Math.max(0, checkpointFrames.value.length - 1),
)
// Track current frame from SSE
source.on<{ frame_ref: number; jpeg_b64: string }>('frame_update', (e) => {
currentFrameImage.value = e.jpeg_b64
currentFrameRef.value = e.frame_ref
})
async function loadCheckpoint(job: string, stage: string) {
try {
const resp = await fetch(`/api/detect/checkpoints/${job}/${stage}`)
if (!resp.ok) return
const data = await resp.json()
checkpointFrames.value = data.frames ?? []
checkpointStage.value = stage
if (checkpointFrames.value.length > 0) {
checkpointFrameIndex.value = 0
const first = checkpointFrames.value[0]
currentFrameImage.value = first.jpeg_b64
currentFrameRef.value = first.seq
}
stripSelStart.value = 0
stripSelEndOverride.value = null
} catch (e) {
console.error('Failed to load checkpoint:', e)
}
}
function setCheckpointFrame(index: number) {
if (index < 0 || index >= checkpointFrames.value.length) return
checkpointFrameIndex.value = index
const frame = checkpointFrames.value[index]
currentFrameImage.value = frame.jpeg_b64
currentFrameRef.value = frame.seq
}
// Auto-load checkpoint when entering editor mode
watch(
() => [pipeline.layoutMode, pipeline.editorStage, jobId.value] as const,
([mode, stage, job]) => {
if (mode === 'bbox_editor' && stage && job) {
const stageMap: Record<string, string> = {
detect_edges: 'filter_scenes',
detect_contours: 'detect_edges',
detect_color: 'detect_contours',
merge_regions: 'detect_color',
}
const cpStage = stageMap[stage] ?? 'filter_scenes'
loadCheckpoint(job, cpStage)
}
},
{ immediate: true },
)
return {
currentFrameImage,
currentFrameRef,
checkpointFrames,
checkpointFrameIndex,
checkpointStage,
stripSelStart,
stripSelEnd,
stripSelEndOverride,
loadCheckpoint,
setCheckpointFrame,
}
}

120
ui/detection-app/src/composables/useEditorState.ts Normal file
View File

@@ -0,0 +1,120 @@
import { ref } from 'vue'
import type { Ref } from 'vue'
import type { FrameOverlay, FrameBBox } from 'mpr-ui-framework'
import { matchTracks, renderTracksToImageData, imageDataToPngB64 } from '@/cv'
export type RegionBox = {
x: number
y: number
w: number
h: number
confidence: number
label: string
}
export function useEditorState(currentFrameRef: Ref<number | null>) {
const editorOverlays = ref<FrameOverlay[]>([])
const editorBoxes = ref<FrameBBox[]>([])
const allFrameRegions = ref<Record<number, RegionBox[]>>({})
const allFrameDebug = ref<Record<number, { edge_overlay_b64: string; lines_overlay_b64: string }>>({})
const frameDimensions = ref<{ w: number; h: number } | null>(null)
function updateDisplayForFrame(seq: number) {
const regions = allFrameRegions.value[seq] ?? []
editorBoxes.value = regions.map(r => ({
x: r.x, y: r.y, w: r.w, h: r.h,
confidence: r.confidence,
label: r.label ?? 'edge_region',
stage: 'detect_edges',
}))
const debug = allFrameDebug.value[seq]
if (debug) {
const overlays: FrameOverlay[] = []
if (debug.edge_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Canny edges')
overlays.push({ src: debug.edge_overlay_b64, label: 'Canny edges', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
}
if (debug.lines_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Hough lines')
overlays.push({ src: debug.lines_overlay_b64, label: 'Hough lines', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
}
const trackOverlay = editorOverlays.value.find(o => o.label === 'Motion tracks')
if (trackOverlay) overlays.push(trackOverlay)
editorOverlays.value = overlays
}
if (Object.keys(allFrameRegions.value).length >= 2 && frameDimensions.value) {
updateTrackOverlay(seq)
}
}
async function updateTrackOverlay(currentSeq: number) {
const dims = frameDimensions.value
if (!dims || Object.keys(allFrameRegions.value).length < 2) return
const tracks = matchTracks(allFrameRegions.value)
const imageData = renderTracksToImageData(tracks, dims.w, dims.h, currentSeq)
const b64 = await imageDataToPngB64(imageData)
const existing = editorOverlays.value.find(o => o.label === 'Motion tracks')
const trackOverlay: FrameOverlay = {
src: b64,
label: 'Motion tracks',
visible: existing?.visible ?? true,
opacity: existing?.opacity ?? 0.9,
srcFormat: 'png',
}
editorOverlays.value = [
...editorOverlays.value.filter(o => o.label !== 'Motion tracks'),
trackOverlay,
]
}
function onReplayResult(result: {
regions_by_frame?: Record<string, RegionBox[]>
debug?: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
frameWidth?: number
frameHeight?: number
}) {
if (result.frameWidth && result.frameHeight) {
frameDimensions.value = { w: result.frameWidth, h: result.frameHeight }
}
if (result.regions_by_frame) {
for (const [seqStr, regions] of Object.entries(result.regions_by_frame)) {
allFrameRegions.value[Number(seqStr)] = regions
}
}
if (result.debug) {
for (const [seqStr, dbg] of Object.entries(result.debug)) {
allFrameDebug.value[Number(seqStr)] = {
edge_overlay_b64: dbg.edge_overlay_b64,
lines_overlay_b64: dbg.lines_overlay_b64,
}
}
}
const currentSeq = currentFrameRef.value ?? 0
updateDisplayForFrame(currentSeq)
}
function resetEditorState() {
allFrameRegions.value = {}
allFrameDebug.value = {}
frameDimensions.value = null
editorOverlays.value = []
editorBoxes.value = []
}
return {
editorOverlays,
editorBoxes,
allFrameRegions,
allFrameDebug,
frameDimensions,
updateDisplayForFrame,
onReplayResult,
resetEditorState,
}
}

106
ui/detection-app/src/composables/useSSEConnection.ts Normal file
View File

@@ -0,0 +1,106 @@
import { ref, watch } from 'vue'
import { SSEDataSource } from 'mpr-ui-framework'
import type { DataSource } from 'mpr-ui-framework'
import type { StatsUpdate, RunContext } from '../types/sse-contract'
import { usePipelineStore } from '../stores/pipeline'
type AppStatus = 'idle' | 'live' | 'processing' | 'error'
const STATUS_MAP: Record<string, AppStatus> = {
idle: 'idle',
connecting: 'processing',
live: 'live',
error: 'error',
}
export function useSSEConnection() {
const pipeline = usePipelineStore()
const jobParam = new URLSearchParams(window.location.search).get('job')
const jobId = ref(jobParam || '')
const stats = ref<StatsUpdate | null>(null)
const runContext = ref<RunContext | null>(null)
const status = ref<AppStatus>('idle')
const sseConnected = ref(false)
// No job selected and no hash route → open source selector
if (!jobParam && !window.location.hash.replace(/^#\/?/, '')) {
pipeline.openSourceSelector()
}
const source = new SSEDataSource({
id: 'detect-stream',
url: jobId.value ? `/api/detect/stream/${jobId.value}` : '',
eventTypes: ['graph_update', 'stats_update', 'frame_update', 'detection', 'log', 'job_complete', 'waiting'],
})
source.on<StatsUpdate>('stats_update', (e) => {
stats.value = e
if (!runContext.value && e.run_id) {
runContext.value = {
run_id: e.run_id!,
parent_job_id: e.parent_job_id!,
run_type: e.run_type ?? 'initial',
}
}
})
source.on<{ report?: { status?: string; error?: string } }>('job_complete', (e) => {
if (e.report?.status === 'failed') {
status.value = 'error'
}
})
// Reactive status sync — replaces setInterval polling
watch(
() => source.status.value,
(sourceStatus) => {
if (sseConnected.value) {
status.value = STATUS_MAP[sourceStatus] ?? 'idle'
}
},
)
// Only connect SSE for live pipeline runs
const isScenarioMode = pipeline.isEditing || pipeline.layoutMode !== 'normal'
if (jobId.value && !isScenarioMode) {
source.connect()
sseConnected.value = true
}
async function stopPipeline() {
if (!jobId.value) return
try {
await fetch(`/api/detect/stop/${jobId.value}`, { method: 'POST' })
} catch { /* ignore — UI will see the cancel event via SSE */ }
}
function onJobStarted(newJobId: string) {
jobId.value = newJobId
stats.value = null
runContext.value = null
status.value = 'processing'
pipeline.reset()
pipeline.setStatus('running')
// Update URL without reload
const url = new URL(window.location.href)
url.searchParams.set('job', newJobId)
window.history.pushState({}, '', url.toString())
// Connect SSE to new job
source.disconnect()
source.setUrl(`/api/detect/stream/${newJobId}`)
source.connect()
sseConnected.value = true
}
return {
jobId,
stats,
runContext,
status,
sseConnected,
source: source as DataSource,
stopPipeline,
onJobStarted,
}
}