timeline

2026-03-27 07:20:27 -03:00
parent 51ce14a812
commit a3b51c458d
10 changed files with 653 additions and 67 deletions
--- a/ui/detection-app/src/App.vue
+++ b/ui/detection-app/src/App.vue
@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { ref } from 'vue'
+import { ref, computed } from 'vue'
 import { SSEDataSource, Panel, ResizeHandle } from 'mpr-ui-framework'
 import 'mpr-ui-framework/src/tokens.css'
 import LogPanel from './panels/LogPanel.vue'
@@ -11,8 +11,10 @@ import TimelinePanel from './panels/TimelinePanel.vue'
 import CostStatsPanel from './panels/CostStatsPanel.vue'
 import SourceSelector from './panels/SourceSelector.vue'
 import StageConfigSliders from './components/StageConfigSliders.vue'
+import FrameStrip from './components/FrameStrip.vue'
 import type { StatsUpdate, RunContext } from './types/sse-contract'
 import type { FrameOverlay } from 'mpr-ui-framework/src/renderers/FrameRenderer.vue'
+import { matchTracks, renderTracksToImageData, imageDataToPngB64 } from 'mpr-ui-framework/src/cv'
 import { usePipelineStore } from './stores/pipeline'

 const pipeline = usePipelineStore()
@@ -127,6 +129,18 @@ const checkpointFrames = ref<{ seq: number; timestamp: number; jpeg_b64: string
 const checkpointFrameIndex = ref(0)
 const checkpointStage = ref<string | null>(null)  // which stage the checkpoint is at

+// Frame strip selection (indices into checkpointFrames)
+const stripSelStart = ref(0)
+const stripSelEnd = computed(() =>
+  stripSelEndOverride.value ?? Math.max(0, checkpointFrames.value.length - 1),
+)
+const stripSelEndOverride = ref<number | null>(null)
+
+// Per-frame CV results — accumulated across all processed frames
+const allFrameRegions = ref<Record<number, Array<{ x: number; y: number; w: number; h: number; confidence: number; label: string }>>>({})
+const allFrameDebug = ref<Record<number, { edge_overlay_b64: string; lines_overlay_b64: string }>>({})
+const frameDimensions = ref<{ w: number; h: number } | null>(null)
+

 source.on<{ frame_ref: number; jpeg_b64: string }>('frame_update', (e) => {
  currentFrameImage.value = e.jpeg_b64
@@ -151,6 +165,13 @@ async function loadCheckpoint(job: string, stage: string) {
      currentFrameRef.value = first.seq
    }

+    // Reset accumulated CV results and strip selection for this checkpoint
+    allFrameRegions.value = {}
+    allFrameDebug.value = {}
+    frameDimensions.value = null
+    stripSelStart.value = 0
+    stripSelEndOverride.value = null
+
    status.value = 'idle'
  } catch (e) {
    console.error('Failed to load checkpoint:', e)
@@ -163,6 +184,61 @@ function setCheckpointFrame(index: number) {
  const frame = checkpointFrames.value[index]
  currentFrameImage.value = frame.jpeg_b64
  currentFrameRef.value = frame.seq
+  updateDisplayForFrame(frame.seq)
+}
+
+function updateDisplayForFrame(seq: number) {
+  // Update boxes for this frame from accumulated results
+  const regions = allFrameRegions.value[seq] ?? []
+  editorBoxes.value = regions.map(r => ({
+    x: r.x, y: r.y, w: r.w, h: r.h,
+    confidence: r.confidence,
+    label: r.label ?? 'edge_region',
+    stage: 'detect_edges',
+  }))
+
+  // Update Canny/Hough overlays for this frame (preserving visibility/opacity)
+  const debug = allFrameDebug.value[seq]
+  if (debug) {
+    const overlays: FrameOverlay[] = []
+    if (debug.edge_overlay_b64) {
+      const existing = editorOverlays.value.find(o => o.label === 'Canny edges')
+      overlays.push({ src: debug.edge_overlay_b64, label: 'Canny edges', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
+    }
+    if (debug.lines_overlay_b64) {
+      const existing = editorOverlays.value.find(o => o.label === 'Hough lines')
+      overlays.push({ src: debug.lines_overlay_b64, label: 'Hough lines', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
+    }
+    // Re-append track overlay if it exists
+    const trackOverlay = editorOverlays.value.find(o => o.label === 'Motion tracks')
+    if (trackOverlay) overlays.push(trackOverlay)
+    editorOverlays.value = overlays
+  }
+
+  // Re-render track overlay with updated currentSeq
+  if (Object.keys(allFrameRegions.value).length >= 2 && frameDimensions.value) {
+    updateTrackOverlay(seq)
+  }
+}
+
+async function updateTrackOverlay(currentSeq: number) {
+  const dims = frameDimensions.value
+  if (!dims || Object.keys(allFrameRegions.value).length < 2) return
+  const tracks = matchTracks(allFrameRegions.value)
+  const imageData = renderTracksToImageData(tracks, dims.w, dims.h, currentSeq)
+  const b64 = await imageDataToPngB64(imageData)
+  const existing = editorOverlays.value.find(o => o.label === 'Motion tracks')
+  const trackOverlay: FrameOverlay = {
+    src: b64,
+    label: 'Motion tracks',
+    visible: existing?.visible ?? true,
+    opacity: existing?.opacity ?? 0.9,
+    srcFormat: 'png',
+  }
+  editorOverlays.value = [
+    ...editorOverlays.value.filter(o => o.label !== 'Motion tracks'),
+    trackOverlay,
+  ]
 }

 // Load checkpoint when in editor mode with a job (scenario URL)
@@ -194,35 +270,34 @@ const editorBoxes = ref<import('mpr-ui-framework/src/renderers/FrameRenderer.vue
 function onReplayResult(result: {
  regions_by_frame?: Record<string, unknown[]>
  debug?: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
+  frameWidth?: number
+  frameHeight?: number
 }) {
-  // Update boxes
-  if (result.regions_by_frame) {
-    const firstRegions = Object.values(result.regions_by_frame)[0] as any[] ?? []
-    editorBoxes.value = firstRegions.map((r: any) => ({
-      x: r.x, y: r.y, w: r.w, h: r.h,
-      confidence: r.confidence,
-      label: r.label ?? 'edge_region',
-      stage: 'detect_edges',
-    }))
+  // Store frame dimensions for track overlay rendering
+  if (result.frameWidth && result.frameHeight) {
+    frameDimensions.value = { w: result.frameWidth, h: result.frameHeight }
  }

-  // Update overlays — only when debug data is present, preserve existing otherwise
-  if (result.debug) {
-    const firstDebug = Object.values(result.debug)[0]
-    if (firstDebug) {
-      const overlays: FrameOverlay[] = []
-      if (firstDebug.edge_overlay_b64) {
-        // Preserve visibility/opacity from existing overlays if they exist
-        const existing = editorOverlays.value.find(o => o.label === 'Canny edges')
-        overlays.push({ src: firstDebug.edge_overlay_b64, label: 'Canny edges', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
-      }
-      if (firstDebug.lines_overlay_b64) {
-        const existing = editorOverlays.value.find(o => o.label === 'Hough lines')
-        overlays.push({ src: firstDebug.lines_overlay_b64, label: 'Hough lines', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
-      }
-      editorOverlays.value = overlays
+  // Merge incoming per-frame regions into accumulated store
+  if (result.regions_by_frame) {
+    for (const [seqStr, regions] of Object.entries(result.regions_by_frame)) {
+      allFrameRegions.value[Number(seqStr)] = regions as any[]
    }
  }
+
+  // Merge incoming per-frame debug overlays into accumulated store
+  if (result.debug) {
+    for (const [seqStr, dbg] of Object.entries(result.debug)) {
+      allFrameDebug.value[Number(seqStr)] = {
+        edge_overlay_b64: dbg.edge_overlay_b64,
+        lines_overlay_b64: dbg.lines_overlay_b64,
+      }
+    }
+  }
+
+  // Update display for the currently shown frame
+  const currentSeq = currentFrameRef.value ?? 0
+  updateDisplayForFrame(currentSeq)
 }

 function onJobStarted(newJobId: string) {
@@ -338,10 +413,25 @@ function onJobStarted(newJobId: string) {
                  :job-id="jobId"
                  :frame-image="currentFrameImage"
                  :frame-ref="currentFrameRef"
+                  :frames="checkpointFrames"
+                  :selection-start="stripSelStart"
+                  :selection-end="stripSelEnd"
                  @replay-result="onReplayResult"
                />
              </div>
            </div>
+
+            <!-- Frame strip: thumbnails + selection handles -->
+            <FrameStrip
+              v-if="checkpointFrames.length > 0"
+              :frames="checkpointFrames"
+              :current-index="checkpointFrameIndex"
+              :selection-start="stripSelStart"
+              :selection-end="stripSelEnd"
+              @frame-click="setCheckpointFrame"
+              @selection-change="(s, e) => { stripSelStart.value = s; stripSelEndOverride.value = e }"
+            />
+
            <!-- Bottom: debug overlays + close -->
            <div class="editor-bottom">
              <div class="overlay-controls">
--- a/ui/detection-app/src/components/FrameStrip.vue
+++ b/ui/detection-app/src/components/FrameStrip.vue
@@ -0,0 +1,263 @@
+<script setup lang="ts">
+import { ref, computed, onMounted, onUnmounted } from 'vue'
+
+const props = defineProps<{
+  frames: Array<{ seq: number; timestamp: number; jpeg_b64: string }>
+  currentIndex: number
+  selectionStart: number
+  selectionEnd: number
+}>()
+
+const emit = defineEmits<{
+  'frame-click': [index: number]
+  'selection-change': [start: number, end: number]
+}>()
+
+const stripEl = ref<HTMLElement | null>(null)
+
+// --- Drag handle logic ---
+type DragTarget = 'start' | 'end' | null
+const dragging = ref<DragTarget>(null)
+
+function onHandleMousedown(target: DragTarget, e: MouseEvent) {
+  dragging.value = target
+  e.preventDefault()
+}
+
+function onMousemove(e: MouseEvent) {
+  if (!dragging.value || !stripEl.value) return
+  const rect = stripEl.value.getBoundingClientRect()
+  const x = e.clientX - rect.left
+  const ratio = Math.max(0, Math.min(1, x / rect.width))
+  const idx = Math.round(ratio * (props.frames.length - 1))
+
+  if (dragging.value === 'start') {
+    const newStart = Math.min(idx, props.selectionEnd)
+    emit('selection-change', newStart, props.selectionEnd)
+  } else {
+    const newEnd = Math.max(idx, props.selectionStart)
+    emit('selection-change', props.selectionStart, newEnd)
+  }
+}
+
+function onMouseup() {
+  dragging.value = null
+}
+
+onMounted(() => {
+  window.addEventListener('mousemove', onMousemove)
+  window.addEventListener('mouseup', onMouseup)
+})
+onUnmounted(() => {
+  window.removeEventListener('mousemove', onMousemove)
+  window.removeEventListener('mouseup', onMouseup)
+})
+
+// Handle positions as % of strip width
+const startPct = computed(() => {
+  if (props.frames.length <= 1) return 0
+  return (props.selectionStart / (props.frames.length - 1)) * 100
+})
+const endPct = computed(() => {
+  if (props.frames.length <= 1) return 100
+  return (props.selectionEnd / (props.frames.length - 1)) * 100
+})
+
+function isInSelection(idx: number) {
+  return idx >= props.selectionStart && idx <= props.selectionEnd
+}
+
+// --- Scrub slider ---
+function onScrub(e: Event) {
+  const val = Number((e.target as HTMLInputElement).value)
+  emit('frame-click', val)
+}
+</script>
+
+<template>
+  <div class="frame-strip" ref="stripEl">
+    <!-- Thumbnails -->
+    <div class="thumbs">
+      <div
+        v-for="(frame, idx) in frames"
+        :key="frame.seq"
+        class="thumb"
+        :class="{
+          current: idx === currentIndex,
+          dimmed: !isInSelection(idx),
+        }"
+        @click="emit('frame-click', idx)"
+        :title="`Frame ${frame.seq} · ${frame.timestamp.toFixed(2)}s`"
+      >
+        <img :src="`data:image/jpeg;base64,${frame.jpeg_b64}`" />
+      </div>
+    </div>
+
+    <!-- Scrub slider -->
+    <div class="scrub-row">
+      <input
+        type="range"
+        class="scrub-slider"
+        :min="0"
+        :max="frames.length - 1"
+        :value="currentIndex"
+        @input="onScrub"
+      />
+      <span class="scrub-label">{{ currentIndex + 1 }}/{{ frames.length }}</span>
+    </div>
+
+    <!-- Selection handles -->
+    <div class="handles">
+      <div
+        class="handle handle-start"
+        :style="{ left: startPct + '%' }"
+        @mousedown="onHandleMousedown('start', $event)"
+        title="Drag to set selection start"
+      />
+      <div
+        class="handle handle-end"
+        :style="{ left: endPct + '%' }"
+        @mousedown="onHandleMousedown('end', $event)"
+        title="Drag to set selection end"
+      />
+      <div
+        class="selection-range"
+        :style="{ left: startPct + '%', width: (endPct - startPct) + '%' }"
+      />
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.frame-strip {
+  position: relative;
+  display: flex;
+  flex-direction: column;
+  background: var(--surface-1);
+  border-top: var(--panel-border);
+  user-select: none;
+}
+
+.thumbs {
+  display: flex;
+  flex-direction: row;
+  gap: 2px;
+  padding: 4px 6px 0;
+  overflow-x: auto;
+  scrollbar-width: none;
+}
+.thumbs::-webkit-scrollbar { display: none; }
+
+.thumb {
+  flex-shrink: 0;
+  width: 80px;
+  height: 50px;
+  border: 2px solid transparent;
+  border-radius: 2px;
+  overflow: hidden;
+  cursor: pointer;
+  transition: border-color 0.1s, opacity 0.15s;
+}
+
+.thumb img {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+  display: block;
+}
+
+.thumb.current {
+  border-color: #00bcd4;
+}
+
+.thumb.dimmed {
+  opacity: 0.35;
+}
+
+.thumb:hover:not(.current) {
+  border-color: var(--surface-3);
+}
+
+/* Scrub slider row */
+.scrub-row {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  padding: 2px 6px;
+}
+
+.scrub-slider {
+  -webkit-appearance: none;
+  appearance: none;
+  flex: 1;
+  height: 4px;
+  background: var(--surface-3);
+  border-radius: 2px;
+  outline: none;
+}
+
+.scrub-slider::-webkit-slider-thumb {
+  -webkit-appearance: none;
+  appearance: none;
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  background: #00bcd4;
+  cursor: pointer;
+}
+
+.scrub-slider::-moz-range-thumb {
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  background: #00bcd4;
+  cursor: pointer;
+  border: none;
+}
+
+.scrub-label {
+  font-size: 9px;
+  color: var(--text-dim);
+  font-family: var(--font-mono);
+  min-width: 30px;
+  text-align: right;
+}
+
+/* Handle track area */
+.handles {
+  position: relative;
+  height: 14px;
+  margin: 0 6px;
+}
+
+.selection-range {
+  position: absolute;
+  top: 5px;
+  height: 4px;
+  background: #00bcd4;
+  opacity: 0.35;
+  pointer-events: none;
+}
+
+.handle {
+  position: absolute;
+  top: 0;
+  width: 10px;
+  height: 14px;
+  transform: translateX(-50%);
+  cursor: ew-resize;
+  z-index: 2;
+}
+
+.handle::after {
+  content: '';
+  position: absolute;
+  left: 50%;
+  top: 2px;
+  transform: translateX(-50%);
+  width: 3px;
+  height: 10px;
+  background: #00bcd4;
+  border-radius: 2px;
+}
+</style>
--- a/ui/detection-app/src/components/StageConfigSliders.vue
+++ b/ui/detection-app/src/components/StageConfigSliders.vue
@@ -27,12 +27,20 @@ const props = defineProps<{
  frameImage?: string | null
  /** Currently displayed frame sequence number */
  frameRef?: number | null
+  /** All checkpoint frames — when provided, Apply runs on the selection range */
+  frames?: Array<{ seq: number; jpeg_b64: string }>
+  /** Index into frames[] for selection start (default 0) */
+  selectionStart?: number
+  /** Index into frames[] for selection end (default frames.length - 1) */
+  selectionEnd?: number
 }>()

 const emit = defineEmits<{
  'replay-result': [result: {
    regions_by_frame: Record<string, unknown[]>
    debug: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
+    frameWidth?: number
+    frameHeight?: number
  }]
 }>()

@@ -45,6 +53,7 @@ const debugEnabled = ref(true)
 const autoApply = ref(true)  // auto-run on slider change (fast CV); uncheck for heavy stages
 const execMode = ref<'local' | 'server'>('local')
 const execTimeMs = ref<number | null>(null)
+const processingIndex = ref<number | null>(null)  // current frame index during multi-frame run

 // Config field defaults for detect_edges (used when API is unavailable)
 const EDGE_DEFAULTS: ConfigField[] = [
@@ -89,6 +98,11 @@ watch(() => props.frameImage, (newVal, oldVal) => {
  }
 })

+// Auto-run when selection range changes (strip handle drag)
+watch([() => props.selectionStart, () => props.selectionEnd], () => {
+  if (autoApply.value) onSliderChange()
+})
+
 const numericFields = computed(() => fields.value.filter(f => f.type === 'int' || f.type === 'float'))
 const boolFields = computed(() => fields.value.filter(f => f.type === 'bool'))

@@ -108,7 +122,8 @@ function onSliderChange() {
 }

 async function applyDetection() {
-  if (!props.frameImage) {
+  const hasFrames = props.frames && props.frames.length > 0
+  if (!props.frameImage && !hasFrames) {
    error.value = 'No frame available'
    return
  }
@@ -130,14 +145,10 @@ async function applyDetection() {
  }
 }

-/** Browser-side CV — no network, instant */
+/** Browser-side CV — runs on current frame or selection range */
 async function runLocal() {
  const t0 = performance.now()

-  // Decode base64 JPEG → ImageData
-  const imageData = await b64ToImageData(props.frameImage!)
-
-  // Build params from slider values
  const params: Partial<EdgeDetectionParams> = {
    cannyLow: values.value['edge_canny_low'] as number,
    cannyHigh: values.value['edge_canny_high'] as number,
@@ -148,38 +159,49 @@ async function runLocal() {
    pairMinDistance: values.value['edge_pair_min_distance'] as number,
  }

-  const frameKey = String(props.frameRef ?? 0)
+  // Determine which frames to process
+  const targetFrames = props.frames && props.frames.length > 0
+    ? props.frames.slice(props.selectionStart ?? 0, (props.selectionEnd ?? props.frames.length - 1) + 1)
+    : [{ seq: props.frameRef ?? 0, jpeg_b64: props.frameImage! }]

-  if (debugEnabled.value) {
-    const result = await runEdgeDetectionDebug(imageData, params)
-    execTimeMs.value = Math.round(performance.now() - t0)
-    regionCount.value = result.regions.length
+  const regions_by_frame: Record<string, unknown[]> = {}
+  const debug: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }> = {}
+  let totalRegions = 0
+  let frameWidth = 0
+  let frameHeight = 0

-    // Convert ImageData overlays to base64 for FrameRenderer
-    const edgeB64 = await imageDataToB64(result.edgeImageData)
-    const linesB64 = await imageDataToB64(result.linesImageData)
+  for (let i = 0; i < targetFrames.length; i++) {
+    processingIndex.value = i
+    const frame = targetFrames[i]
+    const imageData = await b64ToImageData(frame.jpeg_b64)
+    frameWidth = imageData.width
+    frameHeight = imageData.height
+    const frameKey = String(frame.seq)

-    emit('replay-result', {
-      regions_by_frame: { [frameKey]: result.regions },
-      debug: {
-        [frameKey]: {
-          edge_overlay_b64: edgeB64,
-          lines_overlay_b64: linesB64,
-          horizontal_count: result.horizontalCount,
-          pair_count: result.pairCount,
-        },
-      },
-    })
-  } else {
-    const result = await runEdgeDetection(imageData, params)
-    execTimeMs.value = Math.round(performance.now() - t0)
-    regionCount.value = result.regions.length
-
-    emit('replay-result', {
-      regions_by_frame: { [frameKey]: result.regions },
-      debug: {},
-    })
+    if (debugEnabled.value) {
+      const result = await runEdgeDetectionDebug(imageData, params)
+      totalRegions += result.regions.length
+      const edgeB64 = await imageDataToB64(result.edgeImageData)
+      const linesB64 = await imageDataToB64(result.linesImageData)
+      regions_by_frame[frameKey] = result.regions
+      debug[frameKey] = {
+        edge_overlay_b64: edgeB64,
+        lines_overlay_b64: linesB64,
+        horizontal_count: result.horizontalCount,
+        pair_count: result.pairCount,
+      }
+    } else {
+      const result = await runEdgeDetection(imageData, params)
+      totalRegions += result.regions.length
+      regions_by_frame[frameKey] = result.regions
+    }
  }
+
+  processingIndex.value = null
+  execTimeMs.value = Math.round(performance.now() - t0)
+  regionCount.value = totalRegions
+
+  emit('replay-result', { regions_by_frame, debug, frameWidth, frameHeight })
 }

 /** Server-side CV — calls GPU box via proxy */
@@ -297,7 +319,10 @@ async function runServer() {
        <input type="checkbox" v-model="autoApply" />
        <span>Auto</span>
      </label>
-      <span v-if="regionCount != null" class="region-count">
+      <span v-if="processingIndex != null && frames && frames.length > 1" class="region-count">
+        {{ processingIndex + 1 }}/{{ frames.length }}
+      </span>
+      <span v-else-if="regionCount != null" class="region-count">
        {{ regionCount }} regions
      </span>
      <span v-if="execTimeMs != null" class="exec-time">