mariano/mediaproc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase 4

Browse Source
This commit is contained in:
buenosairesam
2026-03-30 07:22:14 -03:00
parent d0707333fd
commit 4220b0418e
182 changed files with 3668 additions and 5231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ui/common/types/generated.ts
View File

@@ -57,6 +57,7 @@ export interface Job {
source_asset_id: string;
video_path: string;
profile_name: string;
timeline_id: string | null;
parent_id: string | null;
run_type: RunType;
config_overrides: Record<string, unknown>;
@@ -68,7 +69,6 @@ export interface Job {
brands_found: number;
cloud_llm_calls: number;
estimated_cost_usd: number;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
@@ -90,6 +90,7 @@ export interface Timeline {
export interface Checkpoint {
id: string;
timeline_id: string;
job_id: string | null;
parent_id: string | null;
stage_outputs: Record<string, unknown>;
config_overrides: Record<string, unknown>;
@@ -111,6 +112,13 @@ export interface Brand {
updated_at: string | null;
}
export interface Profile {
id: string;
name: string;
pipeline: Record<string, unknown>;
configs: Record<string, unknown>;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;

4
ui/detection-app/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
node_modules/
dist/
public/opencv.js
public/opencv_js.wasm

7
ui/detection-app/package.json
View File

@@ -10,14 +10,15 @@
"typecheck": "vue-tsc --noEmit"
},
"dependencies": {
"vue": "^3.5",
"@techstark/opencv-js": "4.12.0-release.1",
"mpr-ui-framework": "link:../framework",
"pinia": "^2.2",
"mpr-ui-framework": "link:../framework"
"vue": "^3.5"
},
"devDependencies": {
"@vitejs/plugin-vue": "^5",
"typescript": "^5.6",
"vite": "^6",
"@vitejs/plugin-vue": "^5",
"vue-tsc": "^2"
}
}

8
ui/detection-app/pnpm-lock.yaml generated
View File

@@ -8,6 +8,9 @@ importers:
.:
dependencies:
'@techstark/opencv-js':
specifier: 4.12.0-release.1
version: 4.12.0-release.1
mpr-ui-framework:
specifier: link:../framework
version: link:../framework
@@ -334,6 +337,9 @@ packages:
cpu: [x64]
os: [win32]
'@techstark/opencv-js@4.12.0-release.1':
resolution: {integrity: sha512-LtTaph9v/HqLPXEg3m1xs2h7QJh10pUpuDT0nj8g77lelWnTwwQrehtd+fXElLOdrkqc4Fea6Z/sJBvEJLYPfw==}
'@types/estree@1.0.8':
resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
@@ -735,6 +741,8 @@ snapshots:
'@rollup/rollup-win32-x64-msvc@4.60.0':
optional: true
'@techstark/opencv-js@4.12.0-release.1': {}
'@types/estree@1.0.8': {}
'@vitejs/plugin-vue@5.2.4(vite@6.4.1)(vue@3.5.30(typescript@5.9.3))':

39
ui/detection-app/src/App.vue
View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { ref } from 'vue'
import { ref, watch } from 'vue'
import { Panel, ResizeHandle, SplitPane } from 'mpr-ui-framework'
import 'mpr-ui-framework/src/tokens.css'
import LogPanel from './panels/LogPanel.vue'
@@ -10,7 +10,7 @@ import BrandTablePanel from './panels/BrandTablePanel.vue'
import TimelinePanel from './panels/TimelinePanel.vue'
import CostStatsPanel from './panels/CostStatsPanel.vue'
import SourceSelector from './panels/SourceSelector.vue'
import StageConfigSliders from './components/StageConfigSliders.vue'
import StageConfig from './components/StageConfig.vue'
import FrameStrip from './components/FrameStrip.vue'
import { usePipelineStore } from './stores/pipeline'
import { useSSEConnection } from './composables/useSSEConnection'
@@ -41,9 +41,14 @@ const {
// Editor overlays + CV result accumulation
const {
editorOverlays, editorBoxes,
updateDisplayForFrame, onReplayResult,
updateDisplayForFrame, onReplayResult, setActiveStage,
} = useEditorState(currentFrameRef)
// Set active stage when editor opens
watch(() => pipeline.editorStage, (stage) => {
if (stage) setActiveStage(stage)
}, { immediate: true })
// Wire checkpoint frame change to editor display update
function setCheckpointFrame(index: number) {
cpSetFrame(index)
@@ -178,7 +183,7 @@ function onJobStarted(newJobId: string, opts?: { pauseAfterStage?: boolean }) {
</template>
<template #second>
<div class="editor-sliders">
<StageConfigSliders
<StageConfig
v-if="pipeline.editorStage"
:stage="pipeline.editorStage"
:job-id="jobId"
@@ -206,20 +211,18 @@ function onJobStarted(newJobId: string, opts?: { pauseAfterStage?: boolean }) {
<div class="editor-bottom">
<div class="overlay-controls">
<template v-if="editorOverlays.length > 0">
<label v-for="(overlay, idx) in editorOverlays" :key="idx" class="overlay-toggle">
<input type="checkbox" v-model="overlay.visible" />
<span class="overlay-label">{{ overlay.label }}</span>
<input
type="range"
min="0" max="1" step="0.05"
:value="overlay.opacity ?? 0.5"
@input="(e: Event) => overlay.opacity = Number((e.target as HTMLInputElement).value)"
class="opacity-slider"
/>
<span class="opacity-value">{{ Math.round((overlay.opacity ?? 0.5) * 100) }}%</span>
</label>
</template>
<label v-for="(overlay, idx) in editorOverlays" :key="idx" class="overlay-toggle">
<input type="checkbox" v-model="overlay.visible" />
<span class="overlay-label">{{ overlay.label }}</span>
<input
type="range"
min="0" max="1" step="0.05"
:value="overlay.opacity ?? 0.5"
@input="(e: Event) => overlay.opacity = Number((e.target as HTMLInputElement).value)"
class="opacity-slider"
/>
<span class="opacity-value">{{ Math.round((overlay.opacity ?? 0.5) * 100) }}%</span>
</label>
</div>
<button class="editor-close" @click="pipeline.closeEditor()">✕ Close</button>
</div>

634
ui/detection-app/src/components/StageConfig.vue Normal file
View File

@@ -0,0 +1,634 @@
<script setup lang="ts">
import { ref, computed, onMounted, watch } from 'vue'
import { ParameterEditor, useEditorExecution } from 'mpr-ui-framework'
import type { ConfigField } from 'mpr-ui-framework'
import {
runEdgeDetectionTs,
runEdgeDetectionTsDebug,
runEdgeDetection,
runEdgeDetectionDebug,
runSegmentationDebug,
b64ToImageData,
imageDataToB64,
} from '@/cv'
import { getCV } from '@/cv/opencv'
import type { EdgeDetectionParams, SegmentationParams } from '@/cv'
export interface StageResult {
regions_by_frame?: Record<string, unknown[]>
overlays_by_frame?: Record<string, Record<string, string>>
stats_by_frame?: Record<string, Record<string, number>>
frameWidth?: number
frameHeight?: number
}
type ExecMode = 'ts' | 'wasm' | 'server'
const props = defineProps<{
stage: string
jobId: string
frameImage?: string | null
frameRef?: number | null
frames?: Array<{ seq: number; jpeg_b64: string }>
selectionStart?: number
selectionEnd?: number
}>()
const emit = defineEmits<{
'replay-result': [result: StageResult]
}>()
const fields = ref<ConfigField[]>([])
const values = ref<Record<string, unknown>>({})
const statusText = ref<string | null>(null)
const debugEnabled = ref(true)
const processingIndex = ref<number | null>(null)
// --- Execution mode ---
// detect_edges: ts (default) → wasm (when ready) → server
// field_segmentation: wasm (when ready, else server) → server
// other: server only
const HAS_TS = ['detect_edges']
const HAS_WASM = ['detect_edges', 'field_segmentation']
const wasmReady = ref(false)
const wasmLoading = ref(false)
const defaultMode = computed<ExecMode>(() => {
if (HAS_TS.includes(props.stage)) return 'ts'
return 'server'
})
const execMode = ref<ExecMode>(defaultMode.value)
const availableModes = computed<{ mode: ExecMode; label: string; disabled: boolean }[]>(() => {
const modes: { mode: ExecMode; label: string; disabled: boolean }[] = []
if (HAS_TS.includes(props.stage)) {
modes.push({ mode: 'ts', label: 'TS', disabled: false })
}
if (HAS_WASM.includes(props.stage)) {
modes.push({
mode: 'wasm',
label: wasmLoading.value ? 'WASM...' : 'WASM',
disabled: !wasmReady.value,
})
}
modes.push({ mode: 'server', label: 'Server', disabled: false })
return modes
})
// Load WASM on demand — only when user selects WASM mode
async function loadWasmIfNeeded() {
if (wasmReady.value || wasmLoading.value) return
wasmLoading.value = true
const cv = await getCV()
wasmLoading.value = false
if (cv) wasmReady.value = true
}
function onModeClick(mode: ExecMode) {
if (mode === 'wasm' && !wasmReady.value) {
loadWasmIfNeeded().then(() => {
if (wasmReady.value) execMode.value = 'wasm'
})
return
}
execMode.value = mode
}
// --- Execution dispatch ---
type StageExecutors = {
ts?: () => Promise<void>
wasm?: () => Promise<void>
server: () => Promise<void>
}
const STAGE_EXECUTORS: Record<string, StageExecutors> = {
detect_edges: {
ts: () => runEdgesLocalTs(),
wasm: () => runEdgesLocalWasm(),
server: () => runEdgesServer(),
},
field_segmentation: {
wasm: () => runSegmentationLocalWasm(),
server: () => runSegmentationServer(),
},
}
async function executeDetection() {
const hasFrames = props.frames && props.frames.length > 0
if (!props.frameImage && !hasFrames) return
const executors = STAGE_EXECUTORS[props.stage]
if (!executors) {
await runGenericServer()
return
}
// Build fallback chain based on current mode
const chain: (() => Promise<void>)[] = []
if (execMode.value === 'ts' && executors.ts) {
chain.push(executors.ts)
}
if (execMode.value === 'wasm' && executors.wasm && wasmReady.value) {
chain.push(executors.wasm)
}
// Server is always the final fallback
chain.push(executors.server)
// If chain is empty (shouldn't happen), server
if (chain.length === 0) chain.push(executors.server)
for (let i = 0; i < chain.length; i++) {
try {
await chain[i]()
return
} catch (e) {
const isLast = i === chain.length - 1
if (isLast) throw e
console.warn(`[StageConfig] ${execMode.value} failed, trying next:`, e)
}
}
}
const {
loading, error, autoApply, execTimeMs,
apply: applyDetection, onParameterChange,
} = useEditorExecution(executeDetection)
// --- Init ---
onMounted(async () => {
try {
const resp = await fetch(`/api/detect/config/stages/${props.stage}`)
if (resp.ok) {
const data = await resp.json()
fields.value = data.config_fields ?? []
}
} catch { /* use empty fields */ }
for (const f of fields.value) {
values.value[f.name] = f.default
}
if (props.frameImage) {
applyDetection()
}
})
watch(() => props.frameImage, (newVal, oldVal) => {
if (newVal && !oldVal && fields.value.length > 0) {
applyDetection()
}
})
watch([() => props.selectionStart, () => props.selectionEnd], () => {
if (autoApply.value) onParameterChange()
})
function onFieldUpdate(name: string, value: unknown) {
values.value[name] = value
onParameterChange()
}
function resetDefaults() {
for (const f of fields.value) {
values.value[f.name] = f.default
}
applyDetection()
}
// --- Helpers ---
function buildConfigBody(): Record<string, unknown> {
const body: Record<string, unknown> = {}
for (const f of fields.value) {
if (f.name !== 'enabled') {
body[f.name] = values.value[f.name]
}
}
return body
}
function getTargetFrames() {
if (props.frames && props.frames.length > 0) {
return props.frames.slice(
props.selectionStart ?? 0,
(props.selectionEnd ?? props.frames.length - 1) + 1,
)
}
return [{ seq: props.frameRef ?? 0, jpeg_b64: props.frameImage! }]
}
function edgeParams(): Partial<EdgeDetectionParams> {
return {
cannyLow: values.value['edge_canny_low'] as number,
cannyHigh: values.value['edge_canny_high'] as number,
houghThreshold: values.value['edge_hough_threshold'] as number,
houghMinLength: values.value['edge_hough_min_length'] as number,
houghMaxGap: values.value['edge_hough_max_gap'] as number,
pairMaxDistance: values.value['edge_pair_max_distance'] as number,
pairMinDistance: values.value['edge_pair_min_distance'] as number,
}
}
function segmentationParams(): Partial<SegmentationParams> {
return {
hueLow: values.value['hue_low'] as number,
hueHigh: values.value['hue_high'] as number,
satLow: values.value['sat_low'] as number,
satHigh: values.value['sat_high'] as number,
valLow: values.value['val_low'] as number,
valHigh: values.value['val_high'] as number,
morphKernel: values.value['morph_kernel'] as number,
minAreaRatio: values.value['min_area_ratio'] as number,
}
}
// --- Edge detection: pure TS (always available) ---
async function runEdgesLocalTs() {
const params = edgeParams()
const targetFrames = getTargetFrames()
const regions_by_frame: Record<string, unknown[]> = {}
const overlays_by_frame: Record<string, Record<string, string>> = {}
let totalRegions = 0
let frameWidth = 0
let frameHeight = 0
for (let i = 0; i < targetFrames.length; i++) {
processingIndex.value = i
const frame = targetFrames[i]
const imageData = await b64ToImageData(frame.jpeg_b64)
frameWidth = imageData.width
frameHeight = imageData.height
const frameKey = String(frame.seq)
if (debugEnabled.value) {
const result = runEdgeDetectionTsDebug(imageData, params)
totalRegions += result.regions.length
const edgeB64 = await imageDataToB64(result.edgeImageData)
const linesB64 = await imageDataToB64(result.linesImageData)
regions_by_frame[frameKey] = result.regions
overlays_by_frame[frameKey] = { edge_overlay_b64: edgeB64, lines_overlay_b64: linesB64 }
} else {
const result = runEdgeDetectionTs(imageData, params)
totalRegions += result.regions.length
regions_by_frame[frameKey] = result.regions
}
emit('replay-result', {
regions_by_frame: { ...regions_by_frame },
overlays_by_frame: { ...overlays_by_frame },
frameWidth, frameHeight,
})
}
processingIndex.value = null
statusText.value = `${totalRegions} regions`
}
// --- Edge detection: WASM ---
async function runEdgesLocalWasm() {
const params = edgeParams()
const targetFrames = getTargetFrames()
const regions_by_frame: Record<string, unknown[]> = {}
const overlays_by_frame: Record<string, Record<string, string>> = {}
let totalRegions = 0
let frameWidth = 0
let frameHeight = 0
for (let i = 0; i < targetFrames.length; i++) {
processingIndex.value = i
const frame = targetFrames[i]
const imageData = await b64ToImageData(frame.jpeg_b64)
frameWidth = imageData.width
frameHeight = imageData.height
const frameKey = String(frame.seq)
if (debugEnabled.value) {
const result = await runEdgeDetectionDebug(imageData, params)
totalRegions += result.regions.length
const edgeB64 = await imageDataToB64(result.edgeImageData)
const linesB64 = await imageDataToB64(result.linesImageData)
regions_by_frame[frameKey] = result.regions
overlays_by_frame[frameKey] = { edge_overlay_b64: edgeB64, lines_overlay_b64: linesB64 }
} else {
const result = await runEdgeDetection(imageData, params)
totalRegions += result.regions.length
regions_by_frame[frameKey] = result.regions
}
emit('replay-result', {
regions_by_frame: { ...regions_by_frame },
overlays_by_frame: { ...overlays_by_frame },
frameWidth, frameHeight,
})
}
processingIndex.value = null
statusText.value = `${totalRegions} regions`
}
// --- Edge detection: server ---
async function runEdgesServer() {
const body = buildConfigBody()
body.image = props.frameImage
const endpoint = debugEnabled.value
? '/api/detect/gpu/detect_edges/debug'
: '/api/detect/gpu/detect_edges'
const resp = await fetch(endpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
})
if (!resp.ok) throw new Error(await resp.text())
const data = await resp.json()
const frameKey = String(props.frameRef ?? 0)
const result: StageResult = {
regions_by_frame: { [frameKey]: data.regions ?? [] },
overlays_by_frame: {},
}
if (data.edge_overlay_b64 || data.lines_overlay_b64) {
result.overlays_by_frame = {
[frameKey]: {
edge_overlay_b64: data.edge_overlay_b64 ?? '',
lines_overlay_b64: data.lines_overlay_b64 ?? '',
},
}
}
statusText.value = `${data.regions?.length ?? 0} regions`
emit('replay-result', result)
}
// --- Field segmentation: WASM ---
async function runSegmentationLocalWasm() {
const params = segmentationParams()
const targetFrames = getTargetFrames()
const overlays_by_frame: Record<string, Record<string, string>> = {}
const stats_by_frame: Record<string, Record<string, number>> = {}
let frameWidth = 0
let frameHeight = 0
for (let i = 0; i < targetFrames.length; i++) {
processingIndex.value = i
const frame = targetFrames[i]
const imageData = await b64ToImageData(frame.jpeg_b64)
frameWidth = imageData.width
frameHeight = imageData.height
const frameKey = String(frame.seq)
const result = await runSegmentationDebug(imageData, params)
const overlayB64 = await imageDataToB64(result.overlayImageData)
overlays_by_frame[frameKey] = { mask_overlay_b64: overlayB64 }
stats_by_frame[frameKey] = { coverage: result.coverage }
emit('replay-result', {
overlays_by_frame: { ...overlays_by_frame },
stats_by_frame: { ...stats_by_frame },
frameWidth, frameHeight,
})
}
processingIndex.value = null
const coverages = Object.values(stats_by_frame).map(s => s.coverage)
const avg = coverages.reduce((a, b) => a + b, 0) / Math.max(coverages.length, 1)
statusText.value = `${Math.round(avg * 100)}% coverage`
}
// --- Field segmentation: server ---
async function runSegmentationServer() {
const targetFrames = getTargetFrames()
const overlays_by_frame: Record<string, Record<string, string>> = {}
const stats_by_frame: Record<string, Record<string, number>> = {}
for (let i = 0; i < targetFrames.length; i++) {
processingIndex.value = i
const frame = targetFrames[i]
const frameKey = String(frame.seq)
const body = buildConfigBody()
body.image = frame.jpeg_b64
const resp = await fetch('/api/detect/gpu/segment_field/debug', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
})
if (!resp.ok) throw new Error(await resp.text())
const data = await resp.json()
if (data.mask_overlay_b64) {
overlays_by_frame[frameKey] = { mask_overlay_b64: data.mask_overlay_b64 }
}
stats_by_frame[frameKey] = { coverage: data.coverage ?? 0 }
emit('replay-result', {
overlays_by_frame: { ...overlays_by_frame },
stats_by_frame: { ...stats_by_frame },
})
}
processingIndex.value = null
const coverages = Object.values(stats_by_frame).map(s => s.coverage)
const avg = coverages.reduce((a, b) => a + b, 0) / Math.max(coverages.length, 1)
statusText.value = `${Math.round(avg * 100)}% coverage`
}
// --- Generic fallback ---
async function runGenericServer() {
statusText.value = 'No executor for this stage'
}
</script>
<template>
<div class="sliders-panel">
<div class="sliders-header">
<span class="sliders-title">{{ stage.replace(/_/g, ' ') }}</span>
<button class="sliders-reset" @click="resetDefaults" title="Reset to defaults">Reset</button>
</div>
<div class="mode-toggle">
<button
v-for="m in availableModes" :key="m.mode"
:class="['mode-btn', { active: execMode === m.mode, waiting: m.mode === 'wasm' && wasmLoading }]"
:disabled="m.disabled && m.mode !== 'wasm'"
@click="onModeClick(m.mode)"
>{{ m.label }}</button>
</div>
<div v-if="error" class="sliders-error">{{ error }}</div>
<div class="sliders-list">
<ParameterEditor
:fields="fields"
:values="values"
@update="onFieldUpdate"
@reset="resetDefaults"
/>
</div>
<div class="sliders-footer">
<button :class="['apply-btn', { waiting: loading }]" :disabled="loading || autoApply" @click="applyDetection">
{{ loading ? 'Running...' : 'Apply' }}
</button>
<label class="auto-apply-toggle">
<input type="checkbox" v-model="autoApply" />
<span>Auto</span>
</label>
<span v-if="processingIndex != null && frames && frames.length > 1" class="region-count">
{{ processingIndex + 1 }}/{{ frames.length }}
</span>
<span v-else-if="statusText" class="region-count">
{{ statusText }}
</span>
<span v-if="execTimeMs != null" class="exec-time">
{{ execTimeMs }}ms
</span>
</div>
</div>
</template>
<style scoped>
.sliders-panel {
display: flex;
flex-direction: column;
gap: var(--space-2);
height: 100%;
overflow-y: auto;
font-size: var(--font-size-sm);
}
.sliders-header {
display: flex;
justify-content: space-between;
align-items: center;
padding-bottom: var(--space-1);
border-bottom: var(--panel-border);
}
.sliders-title {
font-weight: 600;
text-transform: capitalize;
}
.sliders-reset {
background: var(--surface-3);
border: 1px solid var(--surface-3);
border-radius: 3px;
padding: 2px 6px;
color: var(--text-secondary);
font-family: var(--font-mono);
font-size: 9px;
cursor: pointer;
}
.sliders-reset:hover { background: var(--surface-2); }
.mode-toggle {
display: flex;
gap: 1px;
background: var(--surface-2);
border-radius: 3px;
overflow: hidden;
}
.mode-btn {
flex: 1;
padding: 3px 8px;
border: none;
background: var(--surface-2);
color: var(--text-dim);
font-family: var(--font-mono);
font-size: 9px;
cursor: pointer;
transition: all 0.15s;
}
.mode-btn.active {
background: var(--surface-3);
color: var(--text-primary);
font-weight: 600;
}
.mode-btn:hover:not(.active):not(:disabled) {
color: var(--text-secondary);
}
.mode-btn:disabled {
opacity: 0.35;
cursor: not-allowed;
}
.sliders-error {
color: var(--status-error);
font-size: 10px;
padding: 4px;
background: var(--surface-2);
border-radius: 3px;
}
.sliders-list {
display: flex;
flex-direction: column;
gap: var(--space-2);
flex: 1;
overflow-y: auto;
}
.sliders-footer {
display: flex;
align-items: center;
gap: var(--space-2);
padding-top: var(--space-1);
border-top: var(--panel-border);
}
.apply-btn {
background: #00bcd4;
border: none;
border-radius: 3px;
padding: 4px 12px;
color: #000;
font-family: var(--font-mono);
font-size: 10px;
font-weight: 600;
cursor: pointer;
}
.apply-btn:hover { opacity: 0.85; }
.apply-btn:disabled { opacity: 0.4; cursor: not-allowed; }
.auto-apply-toggle {
display: flex;
align-items: center;
gap: 4px;
font-size: 9px;
color: var(--text-secondary);
cursor: pointer;
}
.auto-apply-toggle input { accent-color: #00bcd4; }
.region-count {
color: var(--text-secondary);
font-size: 10px;
}
.exec-time {
color: var(--text-dim);
font-size: 9px;
margin-left: auto;
}
</style>

385
ui/detection-app/src/components/StageConfigSliders.vue
View File

@@ -1,385 +0,0 @@
<script setup lang="ts">
import { ref, onMounted, watch } from 'vue'
import { ParameterEditor, useEditorExecution } from 'mpr-ui-framework'
import type { ConfigField } from 'mpr-ui-framework'
import {
runEdgeDetection,
runEdgeDetectionDebug,
b64ToImageData,
imageDataToB64,
} from '@/cv'
import type { EdgeDetectionParams } from '@/cv'
const props = defineProps<{
stage: string
jobId: string
frameImage?: string | null
frameRef?: number | null
frames?: Array<{ seq: number; jpeg_b64: string }>
selectionStart?: number
selectionEnd?: number
}>()
const emit = defineEmits<{
'replay-result': [result: {
regions_by_frame: Record<string, unknown[]>
debug: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
frameWidth?: number
frameHeight?: number
}]
}>()
const fields = ref<ConfigField[]>([])
const values = ref<Record<string, unknown>>({})
const regionCount = ref<number | null>(null)
const debugEnabled = ref(true)
const execMode = ref<'local' | 'server'>('local')
const processingIndex = ref<number | null>(null)
// Config field defaults for detect_edges (used when API is unavailable)
const EDGE_DEFAULTS: ConfigField[] = [
{ name: 'enabled', type: 'bool', default: true, description: 'Enable edge detection', min: null, max: null, options: null },
{ name: 'edge_canny_low', type: 'int', default: 50, description: 'Canny low threshold', min: 0, max: 255, options: null },
{ name: 'edge_canny_high', type: 'int', default: 150, description: 'Canny high threshold', min: 0, max: 255, options: null },
{ name: 'edge_hough_threshold', type: 'int', default: 80, description: 'Hough accumulator threshold', min: 1, max: 500, options: null },
{ name: 'edge_hough_min_length', type: 'int', default: 100, description: 'Min line length (px)', min: 10, max: 2000, options: null },
{ name: 'edge_hough_max_gap', type: 'int', default: 10, description: 'Max line gap (px)', min: 1, max: 100, options: null },
{ name: 'edge_pair_max_distance', type: 'int', default: 200, description: 'Max pair distance (px)', min: 10, max: 500, options: null },
{ name: 'edge_pair_min_distance', type: 'int', default: 15, description: 'Min pair distance (px)', min: 5, max: 200, options: null },
]
// --- Execution logic (detection-specific) ---
async function executeDetection() {
const hasFrames = props.frames && props.frames.length > 0
if (!props.frameImage && !hasFrames) return
if (execMode.value === 'local') {
await runLocal()
} else {
await runServer()
}
}
const {
loading, error, autoApply, execTimeMs,
apply: applyDetection, onParameterChange,
} = useEditorExecution(executeDetection)
// --- Init ---
onMounted(async () => {
try {
const resp = await fetch(`/api/detect/config/stages/${props.stage}`)
if (resp.ok) {
const data = await resp.json()
fields.value = data.config_fields ?? []
} else {
fields.value = EDGE_DEFAULTS
}
} catch {
fields.value = EDGE_DEFAULTS
}
for (const f of fields.value) {
values.value[f.name] = f.default
}
if (props.frameImage) {
applyDetection()
}
})
watch(() => props.frameImage, (newVal, oldVal) => {
if (newVal && !oldVal && fields.value.length > 0) {
applyDetection()
}
})
watch([() => props.selectionStart, () => props.selectionEnd], () => {
if (autoApply.value) onParameterChange()
})
function onFieldUpdate(name: string, value: unknown) {
values.value[name] = value
onParameterChange()
}
function resetDefaults() {
for (const f of fields.value) {
values.value[f.name] = f.default
}
applyDetection()
}
// --- Local CV execution ---
async function runLocal() {
const params: Partial<EdgeDetectionParams> = {
cannyLow: values.value['edge_canny_low'] as number,
cannyHigh: values.value['edge_canny_high'] as number,
houghThreshold: values.value['edge_hough_threshold'] as number,
houghMinLength: values.value['edge_hough_min_length'] as number,
houghMaxGap: values.value['edge_hough_max_gap'] as number,
pairMaxDistance: values.value['edge_pair_max_distance'] as number,
pairMinDistance: values.value['edge_pair_min_distance'] as number,
}
const targetFrames = props.frames && props.frames.length > 0
? props.frames.slice(props.selectionStart ?? 0, (props.selectionEnd ?? props.frames.length - 1) + 1)
: [{ seq: props.frameRef ?? 0, jpeg_b64: props.frameImage! }]
const regions_by_frame: Record<string, unknown[]> = {}
const debug: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }> = {}
let totalRegions = 0
let frameWidth = 0
let frameHeight = 0
for (let i = 0; i < targetFrames.length; i++) {
processingIndex.value = i
const frame = targetFrames[i]
const imageData = await b64ToImageData(frame.jpeg_b64)
frameWidth = imageData.width
frameHeight = imageData.height
const frameKey = String(frame.seq)
if (debugEnabled.value) {
const result = await runEdgeDetectionDebug(imageData, params)
totalRegions += result.regions.length
const edgeB64 = await imageDataToB64(result.edgeImageData)
const linesB64 = await imageDataToB64(result.linesImageData)
regions_by_frame[frameKey] = result.regions
debug[frameKey] = {
edge_overlay_b64: edgeB64,
lines_overlay_b64: linesB64,
horizontal_count: result.horizontalCount,
pair_count: result.pairCount,
}
} else {
const result = await runEdgeDetection(imageData, params)
totalRegions += result.regions.length
regions_by_frame[frameKey] = result.regions
}
// Emit incrementally so overlays appear per frame, not all at once
emit('replay-result', { regions_by_frame: { ...regions_by_frame }, debug: { ...debug }, frameWidth, frameHeight })
}
processingIndex.value = null
regionCount.value = totalRegions
}
// --- Server CV execution ---
async function runServer() {
const body: Record<string, unknown> = { image: props.frameImage }
for (const f of fields.value) {
if (f.name !== 'enabled') {
body[f.name] = values.value[f.name]
}
}
const endpoint = debugEnabled.value
? '/api/detect/gpu/detect_edges/debug'
: '/api/detect/gpu/detect_edges'
const resp = await fetch(endpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
})
if (!resp.ok) {
const detail = await resp.text()
throw new Error(detail)
}
const data = await resp.json()
regionCount.value = data.regions?.length ?? 0
const frameKey = String(props.frameRef ?? 0)
const result: any = {
regions_by_frame: { [frameKey]: data.regions ?? [] },
debug: {},
}
if (data.edge_overlay_b64 || data.lines_overlay_b64) {
result.debug = {
[frameKey]: {
edge_overlay_b64: data.edge_overlay_b64 ?? '',
lines_overlay_b64: data.lines_overlay_b64 ?? '',
horizontal_count: data.horizontal_count ?? 0,
pair_count: data.pair_count ?? 0,
},
}
}
emit('replay-result', result)
}
</script>
<template>
<div class="sliders-panel">
<div class="sliders-header">
<span class="sliders-title">{{ stage.replace(/_/g, ' ') }}</span>
<button class="sliders-reset" @click="resetDefaults" title="Reset to defaults">Reset</button>
</div>
<div class="mode-toggle">
<button :class="['mode-btn', { active: execMode === 'local' }]" @click="execMode = 'local'">Local</button>
<button :class="['mode-btn', { active: execMode === 'server' }]" @click="execMode = 'server'">Server</button>
</div>
<div v-if="error" class="sliders-error">{{ error }}</div>
<div class="sliders-list">
<ParameterEditor
:fields="fields"
:values="values"
@update="onFieldUpdate"
@reset="resetDefaults"
/>
</div>
<div class="sliders-footer">
<button class="apply-btn" :disabled="loading || autoApply" @click="applyDetection">
{{ loading ? 'Running...' : 'Apply' }}
</button>
<label class="auto-apply-toggle">
<input type="checkbox" v-model="autoApply" />
<span>Auto</span>
</label>
<span v-if="processingIndex != null && frames && frames.length > 1" class="region-count">
{{ processingIndex + 1 }}/{{ frames.length }}
</span>
<span v-else-if="regionCount != null" class="region-count">
{{ regionCount }} regions
</span>
<span v-if="execTimeMs != null" class="exec-time">
{{ execTimeMs }}ms
</span>
</div>
</div>
</template>
<style scoped>
.sliders-panel {
display: flex;
flex-direction: column;
gap: var(--space-2);
height: 100%;
overflow-y: auto;
font-size: var(--font-size-sm);
}
.sliders-header {
display: flex;
justify-content: space-between;
align-items: center;
padding-bottom: var(--space-1);
border-bottom: var(--panel-border);
}
.sliders-title {
font-weight: 600;
text-transform: capitalize;
}
.sliders-reset {
background: var(--surface-3);
border: 1px solid var(--surface-3);
border-radius: 3px;
padding: 2px 6px;
color: var(--text-secondary);
font-family: var(--font-mono);
font-size: 9px;
cursor: pointer;
}
.sliders-reset:hover { background: var(--surface-2); }
.mode-toggle {
display: flex;
gap: 1px;
background: var(--surface-2);
border-radius: 3px;
overflow: hidden;
}
.mode-btn {
flex: 1;
padding: 3px 8px;
border: none;
background: var(--surface-2);
color: var(--text-dim);
font-family: var(--font-mono);
font-size: 9px;
cursor: pointer;
transition: all 0.15s;
}
.mode-btn.active {
background: var(--surface-3);
color: var(--text-primary);
font-weight: 600;
}
.mode-btn:hover:not(.active) {
color: var(--text-secondary);
}
.sliders-error {
color: var(--status-error);
font-size: 10px;
padding: 4px;
background: var(--surface-2);
border-radius: 3px;
}
.sliders-list {
display: flex;
flex-direction: column;
gap: var(--space-2);
flex: 1;
overflow-y: auto;
}
.sliders-footer {
display: flex;
align-items: center;
gap: var(--space-2);
padding-top: var(--space-1);
border-top: var(--panel-border);
}
.apply-btn {
background: #00bcd4;
border: none;
border-radius: 3px;
padding: 4px 12px;
color: #000;
font-family: var(--font-mono);
font-size: 10px;
font-weight: 600;
cursor: pointer;
}
.apply-btn:hover { opacity: 0.85; }
.apply-btn:disabled { opacity: 0.4; cursor: not-allowed; }
.auto-apply-toggle {
display: flex;
align-items: center;
gap: 4px;
font-size: 9px;
color: var(--text-secondary);
cursor: pointer;
}
.auto-apply-toggle input { accent-color: #00bcd4; }
.region-count {
color: var(--text-secondary);
font-size: 10px;
}
.exec-time {
color: var(--text-dim);
font-size: 9px;
margin-left: auto;
}
</style>

26
ui/detection-app/src/composables/useCheckpointLoader.ts
View File

@@ -29,15 +29,37 @@ export function useCheckpointLoader(
stripSelEndOverride.value ?? Math.max(0, checkpointFrames.value.length - 1),
)
// Cache job_id → timeline_id mappings
const timelineCache = new Map<string, string>()
// Track current frame from SSE
source.on<{ frame_ref: number; jpeg_b64: string }>('frame_update', (e) => {
currentFrameImage.value = e.jpeg_b64
currentFrameRef.value = e.frame_ref
})
async function resolveTimelineId(job: string): Promise<string | null> {
if (timelineCache.has(job)) return timelineCache.get(job)!
try {
const resp = await fetch(`/api/detect/timeline/${job}`)
if (!resp.ok) return null
const data = await resp.json()
const tid = data.timeline_id
if (tid) timelineCache.set(job, tid)
return tid
} catch {
return null
}
}
async function loadCheckpoint(job: string, stage: string) {
try {
const resp = await fetch(`/api/detect/checkpoints/${job}/${stage}`)
// Resolve timeline_id from job_id
const timelineId = await resolveTimelineId(job)
const lookupId = timelineId ?? job
const resp = await fetch(`/api/detect/checkpoints/${lookupId}/${stage}`)
if (!resp.ok) return
const data = await resp.json()
@@ -69,8 +91,6 @@ export function useCheckpointLoader(
const { stages, checkpointStageFor } = useStageRegistry()
// Auto-load checkpoint when entering editor mode.
// Also watches stages — the registry fetch is async, so on first load
// stages may be empty. When they arrive, re-evaluate.
watch(
() => [pipeline.layoutMode, pipeline.editorStage, jobId.value, stages.value.length] as const,
([mode, stage, job]) => {

131
ui/detection-app/src/composables/useEditorState.ts
View File

@@ -1,7 +1,7 @@
import { ref } from 'vue'
import type { Ref } from 'vue'
import type { FrameOverlay, FrameBBox } from 'mpr-ui-framework'
import { matchTracks, renderTracksToImageData, imageDataToPngB64 } from '@/cv'
import type { StageResult } from '@/components/StageConfig.vue'
export type RegionBox = {
x: number
@@ -15,83 +15,80 @@ export type RegionBox = {
export function useEditorState(currentFrameRef: Ref<number | null>) {
const editorOverlays = ref<FrameOverlay[]>([])
const editorBoxes = ref<FrameBBox[]>([])
const activeStage = ref<string | null>(null)
const allFrameRegions = ref<Record<number, RegionBox[]>>({})
const allFrameDebug = ref<Record<number, { edge_overlay_b64: string; lines_overlay_b64: string }>>({})
const allFrameOverlays = ref<Record<number, Record<string, string>>>({})
const allFrameStats = ref<Record<number, Record<string, number>>>({})
const frameDimensions = ref<{ w: number; h: number } | null>(null)
// Overlay definitions per stage — maps overlay keys to display labels
const STAGE_OVERLAYS: Record<string, { key: string; label: string; defaultOpacity: number; srcFormat?: 'jpeg' | 'png' }[]> = {
detect_edges: [
{ key: 'edge_overlay_b64', label: 'Canny edges', defaultOpacity: 0.25 },
{ key: 'lines_overlay_b64', label: 'Hough lines', defaultOpacity: 0.25 },
],
field_segmentation: [
{ key: 'mask_overlay_b64', label: 'Field mask', defaultOpacity: 0.5, srcFormat: 'png' },
],
}
function updateDisplayForFrame(seq: number) {
const stage = activeStage.value ?? 'detect_edges'
// Boxes — only for stages that produce regions
const regions = allFrameRegions.value[seq] ?? []
editorBoxes.value = regions.map(r => ({
x: r.x, y: r.y, w: r.w, h: r.h,
confidence: r.confidence,
label: r.label ?? 'edge_region',
stage: 'detect_edges',
label: r.label ?? 'region',
stage,
}))
const debug = allFrameDebug.value[seq]
if (debug) {
// Overlays — driven by stage overlay definitions
const overlayData = allFrameOverlays.value[seq]
const overlayDefs = STAGE_OVERLAYS[stage] ?? []
if (overlayData && overlayDefs.length > 0) {
const overlays: FrameOverlay[] = []
if (debug.edge_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Canny edges')
overlays.push({ src: debug.edge_overlay_b64, label: 'Canny edges', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
for (const def of overlayDefs) {
const src = overlayData[def.key]
if (!src) continue
const existing = editorOverlays.value.find(o => o.label === def.label)
overlays.push({
src,
label: def.label,
visible: existing?.visible ?? true,
opacity: existing?.opacity ?? def.defaultOpacity,
srcFormat: def.srcFormat,
})
}
if (debug.lines_overlay_b64) {
const existing = editorOverlays.value.find(o => o.label === 'Hough lines')
overlays.push({ src: debug.lines_overlay_b64, label: 'Hough lines', visible: existing?.visible ?? true, opacity: existing?.opacity ?? 0.25 })
}
const trackOverlay = editorOverlays.value.find(o => o.label === 'Motion tracks')
if (trackOverlay) overlays.push(trackOverlay)
editorOverlays.value = overlays
}
if (Object.keys(allFrameRegions.value).length >= 2 && frameDimensions.value) {
updateTrackOverlay(seq)
} else if (overlayDefs.length === 0) {
editorOverlays.value = []
}
}
async function updateTrackOverlay(currentSeq: number) {
const dims = frameDimensions.value
if (!dims || Object.keys(allFrameRegions.value).length < 2) return
const tracks = matchTracks(allFrameRegions.value)
const imageData = renderTracksToImageData(tracks, dims.w, dims.h, currentSeq)
const b64 = await imageDataToPngB64(imageData)
const existing = editorOverlays.value.find(o => o.label === 'Motion tracks')
const trackOverlay: FrameOverlay = {
src: b64,
label: 'Motion tracks',
visible: existing?.visible ?? true,
opacity: existing?.opacity ?? 0.9,
srcFormat: 'png',
}
editorOverlays.value = [
...editorOverlays.value.filter(o => o.label !== 'Motion tracks'),
trackOverlay,
]
}
function onReplayResult(result: {
regions_by_frame?: Record<string, RegionBox[]>
debug?: Record<string, { edge_overlay_b64: string; lines_overlay_b64: string; horizontal_count: number; pair_count: number }>
frameWidth?: number
frameHeight?: number
}) {
function onReplayResult(result: StageResult) {
if (result.frameWidth && result.frameHeight) {
frameDimensions.value = { w: result.frameWidth, h: result.frameHeight }
}
if (result.regions_by_frame) {
for (const [seqStr, regions] of Object.entries(result.regions_by_frame)) {
allFrameRegions.value[Number(seqStr)] = regions
allFrameRegions.value[Number(seqStr)] = regions as RegionBox[]
}
}
if (result.debug) {
for (const [seqStr, dbg] of Object.entries(result.debug)) {
allFrameDebug.value[Number(seqStr)] = {
edge_overlay_b64: dbg.edge_overlay_b64,
lines_overlay_b64: dbg.lines_overlay_b64,
}
if (result.overlays_by_frame) {
for (const [seqStr, overlayMap] of Object.entries(result.overlays_by_frame)) {
allFrameOverlays.value[Number(seqStr)] = overlayMap
}
}
if (result.stats_by_frame) {
for (const [seqStr, stats] of Object.entries(result.stats_by_frame)) {
allFrameStats.value[Number(seqStr)] = stats
}
}
@@ -99,9 +96,30 @@ export function useEditorState(currentFrameRef: Ref<number | null>) {
updateDisplayForFrame(currentSeq)
}
function resetEditorState() {
function setActiveStage(stage: string) {
activeStage.value = stage
// Clear accumulated state when switching stages
allFrameRegions.value = {}
allFrameDebug.value = {}
allFrameOverlays.value = {}
allFrameStats.value = {}
frameDimensions.value = null
editorBoxes.value = []
// Initialize overlay controls from stage definitions (visible before any results)
const defs = STAGE_OVERLAYS[stage] ?? []
editorOverlays.value = defs.map(def => ({
src: '',
label: def.label,
visible: true,
opacity: def.defaultOpacity,
}))
}
function resetEditorState() {
activeStage.value = null
allFrameRegions.value = {}
allFrameOverlays.value = {}
allFrameStats.value = {}
frameDimensions.value = null
editorOverlays.value = []
editorBoxes.value = []
@@ -110,11 +128,14 @@ export function useEditorState(currentFrameRef: Ref<number | null>) {
return {
editorOverlays,
editorBoxes,
activeStage,
allFrameRegions,
allFrameDebug,
allFrameOverlays,
allFrameStats,
frameDimensions,
updateDisplayForFrame,
onReplayResult,
setActiveStage,
resetEditorState,
}
}

443
ui/detection-app/src/cv/edges.ts
View File

@@ -1,12 +1,11 @@
/**
* Edge detection — TypeScript port of gpu/models/cv/edges.py
* Edge detection — OpenCV WASM version.
*
* 1:1 with the Python version. Same algorithm, same parameters,
* same output format. Runs in the browser, no network.
* 1:1 with gpu/models/cv/edges.py. Same algorithm, same parameters,
* same output format. Uses cv.Canny() and cv.HoughLinesP() via WASM.
*/
import { toGrayscale, canny } from './imageOps'
import { houghLinesP, type LineSegment } from './hough'
import { getCV } from './opencv'
export interface EdgeRegion {
x: number
@@ -32,15 +31,243 @@ export interface EdgeDetectionResult {
}
export interface EdgeDetectionDebugResult extends EdgeDetectionResult {
edgeImageData: ImageData // Canny output for overlay
linesImageData: ImageData // Frame with Hough lines drawn
edgeImageData: ImageData
linesImageData: ImageData
horizontalCount: number
pairCount: number
}
type HLine = { xMin: number; xMax: number; yMid: number; length: number }
/** Set a pixel on ImageData with bounds check */
const DEFAULT_PARAMS: EdgeDetectionParams = {
cannyLow: 50,
cannyHigh: 150,
houghThreshold: 80,
houghMinLength: 100,
houghMaxGap: 10,
pairMaxDistance: 200,
pairMinDistance: 15,
}
/**
* Detect edges in an RGBA ImageData using OpenCV WASM.
*/
export async function detectEdges(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): Promise<EdgeDetectionResult> {
const cv = await getCV()
if (!cv) throw new Error('OpenCV WASM not available')
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height } = imageData
const src = cv.matFromImageData(imageData)
const gray = new cv.Mat()
const edges = new cv.Mat()
try {
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY)
cv.Canny(gray, edges, p.cannyLow, p.cannyHigh)
const lines = new cv.Mat()
try {
cv.HoughLinesP(edges, lines, 1, Math.PI / 180, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(lines)
if (horizontals.length < 2) return { regions: [] }
const pairs = findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
const regions = pairsToBoxes(pairs, width, height)
return { regions }
} finally {
lines.delete()
}
} finally {
src.delete()
gray.delete()
edges.delete()
}
}
/**
* Detect edges with debug visualizations using OpenCV WASM.
*/
export async function detectEdgesDebug(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): Promise<EdgeDetectionDebugResult> {
const cv = await getCV()
if (!cv) throw new Error('OpenCV WASM not available')
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height, data } = imageData
const src = cv.matFromImageData(imageData)
const gray = new cv.Mat()
const edges = new cv.Mat()
try {
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY)
cv.Canny(gray, edges, p.cannyLow, p.cannyHigh)
// Edge overlay — white edges on black
const edgeRgba = new cv.Mat()
try {
cv.cvtColor(edges, edgeRgba, cv.COLOR_GRAY2RGBA)
const edgeImageData = new ImageData(
new Uint8ClampedArray(edgeRgba.data),
width,
height,
)
const lines = new cv.Mat()
try {
cv.HoughLinesP(edges, lines, 1, Math.PI / 180, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(lines)
// Lines overlay — darken original, draw lines
const linesImageData = new ImageData(new Uint8ClampedArray(data), width, height)
for (let i = 0; i < linesImageData.data.length; i += 4) {
linesImageData.data[i] = Math.round(linesImageData.data[i] * 0.3)
linesImageData.data[i + 1] = Math.round(linesImageData.data[i + 1] * 0.3)
linesImageData.data[i + 2] = Math.round(linesImageData.data[i + 2] * 0.3)
}
// All Hough lines in red
for (let i = 0; i < lines.rows; i++) {
const x1 = lines.data32S[i * 4]
const y1 = lines.data32S[i * 4 + 1]
const x2 = lines.data32S[i * 4 + 2]
const y2 = lines.data32S[i * 4 + 3]
drawLineThick(linesImageData, x1, y1, x2, y2, 255, 50, 50, 2)
}
// Horizontal lines in cyan
for (const h of horizontals) {
drawLineThick(linesImageData, Math.round(h.xMin), Math.round(h.yMid), Math.round(h.xMax), Math.round(h.yMid), 0, 255, 255, 3)
}
const pairs = horizontals.length >= 2
? findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
: []
// Paired lines in green
for (const [top, bottom] of pairs) {
drawLineThick(linesImageData, Math.round(top.xMin), Math.round(top.yMid), Math.round(top.xMax), Math.round(top.yMid), 0, 255, 0, 4)
drawLineThick(linesImageData, Math.round(bottom.xMin), Math.round(bottom.yMid), Math.round(bottom.xMax), Math.round(bottom.yMid), 0, 255, 0, 4)
}
const regions = pairsToBoxes(pairs, width, height)
return {
regions,
edgeImageData,
linesImageData,
horizontalCount: horizontals.length,
pairCount: pairs.length,
}
} finally {
lines.delete()
}
} finally {
edgeRgba.delete()
}
} finally {
src.delete()
gray.delete()
edges.delete()
}
}
// --- Line analysis (same logic as Python version) ---
function filterHorizontal(lines: any, maxAngleDeg: number = 10): HLine[] {
const maxSlope = Math.tan((maxAngleDeg * Math.PI) / 180)
const result: HLine[] = []
for (let i = 0; i < lines.rows; i++) {
const x1 = lines.data32S[i * 4]
const y1 = lines.data32S[i * 4 + 1]
const x2 = lines.data32S[i * 4 + 2]
const y2 = lines.data32S[i * 4 + 3]
const dx = x2 - x1
if (dx === 0) continue
const slope = Math.abs((y2 - y1) / dx)
if (slope <= maxSlope) {
const yMid = (y1 + y2) / 2
const xMin = Math.min(x1, x2)
const xMax = Math.max(x1, x2)
const length = Math.sqrt(dx * dx + (y2 - y1) ** 2)
result.push({ xMin, xMax, yMid, length })
}
}
return result
}
function findLinePairs(
horizontals: HLine[],
minDistance: number,
maxDistance: number,
): [HLine, HLine][] {
const sorted = [...horizontals].sort((a, b) => a.yMid - b.yMid)
const pairs: [HLine, HLine][] = []
const used = new Set<number>()
for (let i = 0; i < sorted.length; i++) {
if (used.has(i)) continue
const top = sorted[i]
for (let j = i + 1; j < sorted.length; j++) {
if (used.has(j)) continue
const bottom = sorted[j]
const yGap = bottom.yMid - top.yMid
if (yGap < minDistance) continue
if (yGap > maxDistance) break
const overlapStart = Math.max(top.xMin, bottom.xMin)
const overlapEnd = Math.min(top.xMax, bottom.xMax)
const overlap = overlapEnd - overlapStart
const shorterLength = Math.min(top.xMax - top.xMin, bottom.xMax - bottom.xMin)
if (shorterLength > 0 && overlap / shorterLength >= 0.5) {
pairs.push([top, bottom])
used.add(i)
used.add(j)
break
}
}
}
return pairs
}
function pairsToBoxes(pairs: [HLine, HLine][], frameWidth: number, frameHeight: number): EdgeRegion[] {
const regions: EdgeRegion[] = []
for (const [top, bottom] of pairs) {
const x = Math.max(0, Math.min(top.xMin, bottom.xMin))
const y = Math.max(0, top.yMid)
const x2 = Math.min(frameWidth, Math.max(top.xMax, bottom.xMax))
const y2 = Math.min(frameHeight, bottom.yMid)
const w = x2 - x
const h = y2 - y
if (w < 20 || h < 5) continue
const avgLineLength = (top.length + bottom.length) / 2
const coverage = Math.min(1.0, avgLineLength / Math.max(w, 1))
regions.push({
x: Math.round(x),
y: Math.round(y),
w: Math.round(w),
h: Math.round(h),
confidence: Math.round(coverage * 1000) / 1000,
label: 'edge_region',
})
}
return regions
}
// --- Drawing helpers ---
function setPixel(img: ImageData, x: number, y: number, r: number, g: number, b: number) {
if (x >= 0 && x < img.width && y >= 0 && y < img.height) {
const p = (y * img.width + x) * 4
@@ -51,7 +278,6 @@ function setPixel(img: ImageData, x: number, y: number, r: number, g: number, b:
}
}
/** Bresenham line drawing with thickness */
function drawLineThick(
img: ImageData,
x0: number, y0: number, x1: number, y1: number,
@@ -77,202 +303,3 @@ function drawLineThick(
if (e2 < dx) { err += dx; y0 += sy }
}
}
const DEFAULT_PARAMS: EdgeDetectionParams = {
cannyLow: 50,
cannyHigh: 150,
houghThreshold: 80,
houghMinLength: 100,
houghMaxGap: 10,
pairMaxDistance: 200,
pairMinDistance: 15,
}
/** Filter to near-horizontal lines (within 10 degrees) */
function filterHorizontal(lines: LineSegment[], maxAngleDeg: number = 10): HLine[] {
const maxSlope = Math.tan((maxAngleDeg * Math.PI) / 180)
const result: HLine[] = []
for (const line of lines) {
const dx = line.x2 - line.x1
if (dx === 0) continue
const slope = Math.abs((line.y2 - line.y1) / dx)
if (slope <= maxSlope) {
const yMid = (line.y1 + line.y2) / 2
const xMin = Math.min(line.x1, line.x2)
const xMax = Math.max(line.x1, line.x2)
const length = Math.sqrt(dx * dx + (line.y2 - line.y1) ** 2)
result.push({ xMin, xMax, yMid, length })
}
}
return result
}
/** Find pairs of horizontal lines that could be top/bottom of a hoarding */
function findLinePairs(
horizontals: HLine[],
minDistance: number,
maxDistance: number,
): [HLine, HLine][] {
const sorted = [...horizontals].sort((a, b) => a.yMid - b.yMid)
const pairs: [HLine, HLine][] = []
const used = new Set<number>()
for (let i = 0; i < sorted.length; i++) {
if (used.has(i)) continue
const top = sorted[i]
for (let j = i + 1; j < sorted.length; j++) {
if (used.has(j)) continue
const bottom = sorted[j]
const yGap = bottom.yMid - top.yMid
if (yGap < minDistance) continue
if (yGap > maxDistance) break
// Check horizontal overlap (50% of shorter line)
const overlapStart = Math.max(top.xMin, bottom.xMin)
const overlapEnd = Math.min(top.xMax, bottom.xMax)
const overlap = overlapEnd - overlapStart
const shorterLength = Math.min(top.xMax - top.xMin, bottom.xMax - bottom.xMin)
if (shorterLength > 0 && overlap / shorterLength >= 0.5) {
pairs.push([top, bottom])
used.add(i)
used.add(j)
break
}
}
}
return pairs
}
/** Convert a line pair to a bounding box */
function pairToBox(
top: HLine,
bottom: HLine,
frameWidth: number,
frameHeight: number,
): EdgeRegion | null {
const x = Math.max(0, Math.min(top.xMin, bottom.xMin))
const y = Math.max(0, top.yMid)
const x2 = Math.min(frameWidth, Math.max(top.xMax, bottom.xMax))
const y2 = Math.min(frameHeight, bottom.yMid)
const w = x2 - x
const h = y2 - y
if (w < 20 || h < 5) return null
const avgLineLength = (top.length + bottom.length) / 2
const coverage = Math.min(1.0, avgLineLength / Math.max(w, 1))
return {
x: Math.round(x),
y: Math.round(y),
w: Math.round(w),
h: Math.round(h),
confidence: Math.round(coverage * 1000) / 1000,
label: 'edge_region',
}
}
/**
* Detect edges in an RGBA ImageData.
*
* Equivalent to gpu/models/cv/edges.py detect_edges()
*/
export function detectEdges(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): EdgeDetectionResult {
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height } = imageData
const gray = toGrayscale(imageData.data, width, height)
const edges = canny(gray, width, height, p.cannyLow, p.cannyHigh)
const rawLines = houghLinesP(edges, width, height, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(rawLines)
if (horizontals.length < 2) return { regions: [] }
const pairs = findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
const regions: EdgeRegion[] = []
for (const [top, bottom] of pairs) {
const box = pairToBox(top, bottom, width, height)
if (box) regions.push(box)
}
return { regions }
}
/**
* Detect edges with debug visualizations.
*
* Equivalent to gpu/models/cv/edges.py detect_edges_debug()
*/
export function detectEdgesDebug(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): EdgeDetectionDebugResult {
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height, data } = imageData
const gray = toGrayscale(data, width, height)
const edges = canny(gray, width, height, p.cannyLow, p.cannyHigh)
// Edge overlay — white edges on black
const edgeImageData = new ImageData(width, height)
for (let i = 0; i < edges.length; i++) {
const px = i * 4
edgeImageData.data[px] = edges[i]
edgeImageData.data[px + 1] = edges[i]
edgeImageData.data[px + 2] = edges[i]
edgeImageData.data[px + 3] = 255
}
const rawLines = houghLinesP(edges, width, height, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(rawLines)
// Lines overlay — darken original frame so lines pop, then draw
const linesImageData = new ImageData(new Uint8ClampedArray(data), width, height)
for (let i = 0; i < linesImageData.data.length; i += 4) {
linesImageData.data[i] = Math.round(linesImageData.data[i] * 0.3)
linesImageData.data[i + 1] = Math.round(linesImageData.data[i + 1] * 0.3)
linesImageData.data[i + 2] = Math.round(linesImageData.data[i + 2] * 0.3)
}
// Draw all Hough lines in red (3px thick)
for (const line of rawLines) {
drawLineThick(linesImageData, line.x1, line.y1, line.x2, line.y2, 255, 50, 50, 2)
}
// Draw horizontal lines in cyan (3px thick)
for (const h of horizontals) {
drawLineThick(linesImageData, Math.round(h.xMin), Math.round(h.yMid), Math.round(h.xMax), Math.round(h.yMid), 0, 255, 255, 3)
}
const pairs = horizontals.length >= 2
? findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
: []
// Draw paired lines in bright green (4px thick)
for (const [top, bottom] of pairs) {
drawLineThick(linesImageData, Math.round(top.xMin), Math.round(top.yMid), Math.round(top.xMax), Math.round(top.yMid), 0, 255, 0, 4)
drawLineThick(linesImageData, Math.round(bottom.xMin), Math.round(bottom.yMid), Math.round(bottom.xMax), Math.round(bottom.yMid), 0, 255, 0, 4)
}
const regions: EdgeRegion[] = []
for (const [top, bottom] of pairs) {
const box = pairToBox(top, bottom, width, height)
if (box) regions.push(box)
}
return {
regions,
edgeImageData,
linesImageData,
horizontalCount: horizontals.length,
pairCount: pairs.length,
}
}

278
ui/detection-app/src/cv/edgesTs.ts Normal file
View File

@@ -0,0 +1,278 @@
/**
* Edge detection — TypeScript port of gpu/models/cv/edges.py
*
* 1:1 with the Python version. Same algorithm, same parameters,
* same output format. Runs in the browser, no network.
*/
import { toGrayscale, canny } from './imageOps'
import { houghLinesP, type LineSegment } from './hough'
export interface EdgeRegion {
x: number
y: number
w: number
h: number
confidence: number
label: string
}
export interface EdgeDetectionParams {
cannyLow: number
cannyHigh: number
houghThreshold: number
houghMinLength: number
houghMaxGap: number
pairMaxDistance: number
pairMinDistance: number
}
export interface EdgeDetectionResult {
regions: EdgeRegion[]
}
export interface EdgeDetectionDebugResult extends EdgeDetectionResult {
edgeImageData: ImageData // Canny output for overlay
linesImageData: ImageData // Frame with Hough lines drawn
horizontalCount: number
pairCount: number
}
type HLine = { xMin: number; xMax: number; yMid: number; length: number }
/** Set a pixel on ImageData with bounds check */
function setPixel(img: ImageData, x: number, y: number, r: number, g: number, b: number) {
if (x >= 0 && x < img.width && y >= 0 && y < img.height) {
const p = (y * img.width + x) * 4
img.data[p] = r
img.data[p + 1] = g
img.data[p + 2] = b
img.data[p + 3] = 255
}
}
/** Bresenham line drawing with thickness */
function drawLineThick(
img: ImageData,
x0: number, y0: number, x1: number, y1: number,
r: number, g: number, b: number,
thickness: number = 1,
) {
const dx = Math.abs(x1 - x0)
const dy = Math.abs(y1 - y0)
const sx = x0 < x1 ? 1 : -1
const sy = y0 < y1 ? 1 : -1
let err = dx - dy
const half = Math.floor(thickness / 2)
while (true) {
for (let oy = -half; oy <= half; oy++) {
for (let ox = -half; ox <= half; ox++) {
setPixel(img, x0 + ox, y0 + oy, r, g, b)
}
}
if (x0 === x1 && y0 === y1) break
const e2 = 2 * err
if (e2 > -dy) { err -= dy; x0 += sx }
if (e2 < dx) { err += dx; y0 += sy }
}
}
const DEFAULT_PARAMS: EdgeDetectionParams = {
cannyLow: 50,
cannyHigh: 150,
houghThreshold: 80,
houghMinLength: 100,
houghMaxGap: 10,
pairMaxDistance: 200,
pairMinDistance: 15,
}
/** Filter to near-horizontal lines (within 10 degrees) */
function filterHorizontal(lines: LineSegment[], maxAngleDeg: number = 10): HLine[] {
const maxSlope = Math.tan((maxAngleDeg * Math.PI) / 180)
const result: HLine[] = []
for (const line of lines) {
const dx = line.x2 - line.x1
if (dx === 0) continue
const slope = Math.abs((line.y2 - line.y1) / dx)
if (slope <= maxSlope) {
const yMid = (line.y1 + line.y2) / 2
const xMin = Math.min(line.x1, line.x2)
const xMax = Math.max(line.x1, line.x2)
const length = Math.sqrt(dx * dx + (line.y2 - line.y1) ** 2)
result.push({ xMin, xMax, yMid, length })
}
}
return result
}
/** Find pairs of horizontal lines that could be top/bottom of a hoarding */
function findLinePairs(
horizontals: HLine[],
minDistance: number,
maxDistance: number,
): [HLine, HLine][] {
const sorted = [...horizontals].sort((a, b) => a.yMid - b.yMid)
const pairs: [HLine, HLine][] = []
const used = new Set<number>()
for (let i = 0; i < sorted.length; i++) {
if (used.has(i)) continue
const top = sorted[i]
for (let j = i + 1; j < sorted.length; j++) {
if (used.has(j)) continue
const bottom = sorted[j]
const yGap = bottom.yMid - top.yMid
if (yGap < minDistance) continue
if (yGap > maxDistance) break
// Check horizontal overlap (50% of shorter line)
const overlapStart = Math.max(top.xMin, bottom.xMin)
const overlapEnd = Math.min(top.xMax, bottom.xMax)
const overlap = overlapEnd - overlapStart
const shorterLength = Math.min(top.xMax - top.xMin, bottom.xMax - bottom.xMin)
if (shorterLength > 0 && overlap / shorterLength >= 0.5) {
pairs.push([top, bottom])
used.add(i)
used.add(j)
break
}
}
}
return pairs
}
/** Convert a line pair to a bounding box */
function pairToBox(
top: HLine,
bottom: HLine,
frameWidth: number,
frameHeight: number,
): EdgeRegion | null {
const x = Math.max(0, Math.min(top.xMin, bottom.xMin))
const y = Math.max(0, top.yMid)
const x2 = Math.min(frameWidth, Math.max(top.xMax, bottom.xMax))
const y2 = Math.min(frameHeight, bottom.yMid)
const w = x2 - x
const h = y2 - y
if (w < 20 || h < 5) return null
const avgLineLength = (top.length + bottom.length) / 2
const coverage = Math.min(1.0, avgLineLength / Math.max(w, 1))
return {
x: Math.round(x),
y: Math.round(y),
w: Math.round(w),
h: Math.round(h),
confidence: Math.round(coverage * 1000) / 1000,
label: 'edge_region',
}
}
/**
* Detect edges in an RGBA ImageData.
*
* Equivalent to gpu/models/cv/edges.py detect_edges()
*/
export function detectEdges(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): EdgeDetectionResult {
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height } = imageData
const gray = toGrayscale(imageData.data, width, height)
const edges = canny(gray, width, height, p.cannyLow, p.cannyHigh)
const rawLines = houghLinesP(edges, width, height, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(rawLines)
if (horizontals.length < 2) return { regions: [] }
const pairs = findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
const regions: EdgeRegion[] = []
for (const [top, bottom] of pairs) {
const box = pairToBox(top, bottom, width, height)
if (box) regions.push(box)
}
return { regions }
}
/**
* Detect edges with debug visualizations.
*
* Equivalent to gpu/models/cv/edges.py detect_edges_debug()
*/
export function detectEdgesDebug(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): EdgeDetectionDebugResult {
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height, data } = imageData
const gray = toGrayscale(data, width, height)
const edges = canny(gray, width, height, p.cannyLow, p.cannyHigh)
// Edge overlay — white edges on black
const edgeImageData = new ImageData(width, height)
for (let i = 0; i < edges.length; i++) {
const px = i * 4
edgeImageData.data[px] = edges[i]
edgeImageData.data[px + 1] = edges[i]
edgeImageData.data[px + 2] = edges[i]
edgeImageData.data[px + 3] = 255
}
const rawLines = houghLinesP(edges, width, height, p.houghThreshold, p.houghMinLength, p.houghMaxGap)
const horizontals = filterHorizontal(rawLines)
// Lines overlay — darken original frame so lines pop, then draw
const linesImageData = new ImageData(new Uint8ClampedArray(data), width, height)
for (let i = 0; i < linesImageData.data.length; i += 4) {
linesImageData.data[i] = Math.round(linesImageData.data[i] * 0.3)
linesImageData.data[i + 1] = Math.round(linesImageData.data[i + 1] * 0.3)
linesImageData.data[i + 2] = Math.round(linesImageData.data[i + 2] * 0.3)
}
// Draw all Hough lines in red (3px thick)
for (const line of rawLines) {
drawLineThick(linesImageData, line.x1, line.y1, line.x2, line.y2, 255, 50, 50, 2)
}
// Draw horizontal lines in cyan (3px thick)
for (const h of horizontals) {
drawLineThick(linesImageData, Math.round(h.xMin), Math.round(h.yMid), Math.round(h.xMax), Math.round(h.yMid), 0, 255, 255, 3)
}
const pairs = horizontals.length >= 2
? findLinePairs(horizontals, p.pairMinDistance, p.pairMaxDistance)
: []
// Draw paired lines in bright green (4px thick)
for (const [top, bottom] of pairs) {
drawLineThick(linesImageData, Math.round(top.xMin), Math.round(top.yMid), Math.round(top.xMax), Math.round(top.yMid), 0, 255, 0, 4)
drawLineThick(linesImageData, Math.round(bottom.xMin), Math.round(bottom.yMid), Math.round(bottom.xMax), Math.round(bottom.yMid), 0, 255, 0, 4)
}
const regions: EdgeRegion[] = []
for (const [top, bottom] of pairs) {
const box = pairToBox(top, bottom, width, height)
if (box) regions.push(box)
}
return {
regions,
edgeImageData,
linesImageData,
horizontalCount: horizontals.length,
pairCount: pairs.length,
}
}

95
ui/detection-app/src/cv/index.ts
View File

@@ -1,52 +1,77 @@
/**
* Browser-side CV — public API.
*
* Runs edge detection directly on the main thread.
* Pure TypeScript, no WASM, no dependencies.
* ~10-50ms per 1080p frame — fast enough for slider feedback.
* Three execution backends for edge detection:
* - TS: pure TypeScript (always available, ~10-50ms per 1080p frame)
* - WASM: OpenCV.js via WebAssembly (async load, same perf, same API as GPU server)
* - Server: GPU box over HTTP
*
* TODO: Move to Web Worker when processing larger batches.
*
* Usage:
* import { runEdgeDetection, runEdgeDetectionDebug } from '@/cv'
* const result = await runEdgeDetection(imageData, params)
* Field segmentation only has WASM + Server (no TS port).
*/
import { detectEdges, detectEdgesDebug, type EdgeRegion, type EdgeDetectionParams } from './edges'
// WASM versions (edges + segmentation)
import { detectEdges, detectEdgesDebug } from './edges'
import { segmentField, segmentFieldDebug } from './segmentation'
export type { EdgeRegion, EdgeDetectionParams } from './edges'
export type { EdgeDetectionResult, EdgeDetectionDebugResult } from './edges'
// Pure TS versions (edges only — the original fallback)
import { detectEdges as detectEdgesTs, detectEdgesDebug as detectEdgesTsDebug } from './edgesTs'
export type { EdgeRegion, EdgeDetectionParams } from './edgesTs'
export type { EdgeDetectionResult, EdgeDetectionDebugResult } from './edgesTs'
export type { SegmentationParams, SegmentationResult, SegmentationDebugResult } from './segmentation'
export { matchTracks, renderTracksToImageData } from './tracks'
export type { Track, TrackPoint } from './tracks'
/** Run edge detection. Returns bounding boxes. */
// --- Edge detection: pure TS (always works, no async load) ---
export function runEdgeDetectionTs(
imageData: ImageData,
params: Partial<import('./edgesTs').EdgeDetectionParams> = {},
) {
return detectEdgesTs(imageData, params)
}
export function runEdgeDetectionTsDebug(
imageData: ImageData,
params: Partial<import('./edgesTs').EdgeDetectionParams> = {},
) {
return detectEdgesTsDebug(imageData, params)
}
// --- Edge detection: WASM (needs async opencv load) ---
export async function runEdgeDetection(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): Promise<{ regions: EdgeRegion[] }> {
params: Partial<import('./edges').EdgeDetectionParams> = {},
) {
return detectEdges(imageData, params)
}
/** Run edge detection with debug overlays. Returns boxes + visualization ImageData. */
export async function runEdgeDetectionDebug(
imageData: ImageData,
params: Partial<EdgeDetectionParams> = {},
): Promise<{
regions: EdgeRegion[]
edgeImageData: ImageData
linesImageData: ImageData
horizontalCount: number
pairCount: number
}> {
params: Partial<import('./edges').EdgeDetectionParams> = {},
) {
return detectEdgesDebug(imageData, params)
}
/**
* Decode a base64 JPEG string to ImageData.
*
* Used to convert the checkpoint frame (base64) into ImageData
* that the CV functions can process.
*/
// --- Segmentation: WASM only ---
export async function runSegmentation(
imageData: ImageData,
params: Partial<import('./segmentation').SegmentationParams> = {},
) {
return segmentField(imageData, params)
}
export async function runSegmentationDebug(
imageData: ImageData,
params: Partial<import('./segmentation').SegmentationParams> = {},
) {
return segmentFieldDebug(imageData, params)
}
// --- Utilities ---
export function b64ToImageData(b64: string): Promise<ImageData> {
return new Promise((resolve, reject) => {
const img = new Image()
@@ -61,12 +86,6 @@ export function b64ToImageData(b64: string): Promise<ImageData> {
})
}
/**
* Encode ImageData to base64 PNG string (preserves transparency).
*
* Used for overlays that need a transparent background (e.g. motion tracks).
* Pair with srcFormat: 'png' on the FrameOverlay.
*/
export async function imageDataToPngB64(imageData: ImageData): Promise<string> {
const canvas = new OffscreenCanvas(imageData.width, imageData.height)
const ctx = canvas.getContext('2d')!
@@ -81,12 +100,6 @@ export async function imageDataToPngB64(imageData: ImageData): Promise<string> {
return btoa(binary)
}
/**
* Encode ImageData to base64 JPEG string.
*
* Used to convert debug overlay ImageData back to base64
* for the FrameRenderer overlays prop.
*/
export async function imageDataToB64(imageData: ImageData): Promise<string> {
const canvas = new OffscreenCanvas(imageData.width, imageData.height)
const ctx = canvas.getContext('2d')!

84
ui/detection-app/src/cv/opencv.ts Normal file
View File

@@ -0,0 +1,84 @@
/**
* OpenCV WASM loader — split build.
*
* public/opencv.js — 146KB JS loader (stripped of embedded WASM)
* public/opencv_js.wasm — 3.1MB WASM binary (custom build: imgproc + core only)
*
* The JS parses instantly. The WASM compiles asynchronously in the
* background — no main thread freeze.
*/
let cvInstance: any = null
let cvPromise: Promise<any | null> | null = null
let loadFailed = false
let loading = false
export async function getCV(): Promise<any> {
if (cvInstance) return cvInstance
if (loadFailed) return null
if (!cvPromise) {
cvPromise = loadCV()
}
return cvPromise
}
export function isCVLoaded(): boolean {
return cvInstance != null
}
export function isCVLoading(): boolean {
return loading
}
export function isCVFailed(): boolean {
return loadFailed
}
async function loadCV(): Promise<any | null> {
loading = true
try {
const base = import.meta.env.BASE_URL ?? '/'
// Module config must exist before script loads — the UMD wrapper reads it
;(globalThis as any).Module = {
locateFile: (path: string) => `${base}${path}`,
}
await new Promise<void>((resolve, reject) => {
const script = document.createElement('script')
script.src = `${base}opencv.js`
script.async = true
const timeout = setTimeout(() => {
reject(new Error('opencv.js load timed out'))
}, 10000)
script.onload = () => {
clearTimeout(timeout)
resolve()
}
script.onerror = () => {
clearTimeout(timeout)
reject(new Error('Failed to load opencv.js'))
}
document.head.appendChild(script)
})
// OpenCV 4.12 UMD wrapper calls the factory and assigns the readyPromise to window.cv
const cvReady = (globalThis as any).cv
if (!cvReady) throw new Error('cv not on globalThis after script load')
cvInstance = await cvReady
loading = false
return cvInstance
} catch (e) {
loading = false
loadFailed = true
cvPromise = null
console.warn('[opencv] Load failed:', e)
return null
}
}

212
ui/detection-app/src/cv/segmentation.ts Normal file
View File

@@ -0,0 +1,212 @@
/**
* Field segmentation — OpenCV WASM version.
*
* 1:1 with gpu/models/cv/segmentation.py. HSV green mask +
* morphology + contour → pitch boundary detection.
*/
import { getCV } from './opencv'
export interface SegmentationParams {
hueLow: number
hueHigh: number
satLow: number
satHigh: number
valLow: number
valHigh: number
morphKernel: number
minAreaRatio: number
}
export interface SegmentationResult {
boundary: [number, number][]
coverage: number
maskImageData: ImageData
}
export interface SegmentationDebugResult extends SegmentationResult {
overlayImageData: ImageData
}
const DEFAULT_PARAMS: SegmentationParams = {
hueLow: 30,
hueHigh: 85,
satLow: 30,
satHigh: 255,
valLow: 30,
valHigh: 255,
morphKernel: 15,
minAreaRatio: 0.05,
}
/**
* Detect the pitch area using HSV green thresholding.
*/
export async function segmentField(
imageData: ImageData,
params: Partial<SegmentationParams> = {},
): Promise<SegmentationResult> {
const cv = await getCV()
if (!cv) throw new Error('OpenCV WASM not available')
const p = { ...DEFAULT_PARAMS, ...params }
const { width, height } = imageData
const src = cv.matFromImageData(imageData)
const rgb = new cv.Mat()
const hsv = new cv.Mat()
const mask = new cv.Mat()
try {
// ImageData is RGBA, convert to RGB then HSV
cv.cvtColor(src, rgb, cv.COLOR_RGBA2RGB)
cv.cvtColor(rgb, hsv, cv.COLOR_RGB2HSV)
const lower = new cv.Mat(1, 1, cv.CV_8UC3, new cv.Scalar(p.hueLow, p.satLow, p.valLow))
const upper = new cv.Mat(1, 1, cv.CV_8UC3, new cv.Scalar(p.hueHigh, p.satHigh, p.valHigh))
try {
cv.inRange(hsv, lower, upper, mask)
} finally {
lower.delete()
upper.delete()
}
// Morphology — close then open
const k = p.morphKernel
const kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(k, k))
try {
cv.morphologyEx(mask, mask, cv.MORPH_CLOSE, kernel)
cv.morphologyEx(mask, mask, cv.MORPH_OPEN, kernel)
} finally {
kernel.delete()
}
// Find contours
const contours = new cv.MatVector()
const hierarchy = new cv.Mat()
try {
cv.findContours(mask, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
const minArea = p.minAreaRatio * height * width
let boundary: [number, number][] = []
let coverage = 0
// Find the largest contour above min area
let bestContour: any = null
let bestArea = 0
for (let i = 0; i < contours.size(); i++) {
const c = contours.get(i)
const area = cv.contourArea(c)
if (area >= minArea && area > bestArea) {
bestContour = c
bestArea = area
}
}
if (bestContour) {
coverage = bestArea / (height * width)
// Extract boundary points
for (let j = 0; j < bestContour.rows; j++) {
boundary.push([bestContour.data32S[j * 2], bestContour.data32S[j * 2 + 1]])
}
// Refine mask to just the largest contour
mask.setTo(new cv.Scalar(0))
const contourVec = new cv.MatVector()
contourVec.push_back(bestContour)
try {
cv.drawContours(mask, contourVec, -1, new cv.Scalar(255), cv.FILLED)
} finally {
contourVec.delete()
}
}
// Convert mask to RGBA ImageData
const maskRgba = new cv.Mat()
try {
cv.cvtColor(mask, maskRgba, cv.COLOR_GRAY2RGBA)
const maskImageData = new ImageData(
new Uint8ClampedArray(maskRgba.data),
width,
height,
)
return { boundary, coverage, maskImageData }
} finally {
maskRgba.delete()
}
} finally {
contours.delete()
hierarchy.delete()
}
} finally {
src.delete()
rgb.delete()
hsv.delete()
mask.delete()
}
}
/**
* Same as segmentField but includes a blended overlay for the editor.
*/
export async function segmentFieldDebug(
imageData: ImageData,
params: Partial<SegmentationParams> = {},
): Promise<SegmentationDebugResult> {
const cv = await getCV()
if (!cv) throw new Error('OpenCV WASM not available')
const result = await segmentField(imageData, params)
const { width, height } = imageData
// Build green overlay blended with original frame
const src = cv.matFromImageData(imageData)
const rgb = new cv.Mat()
const overlay = new cv.Mat(height, width, cv.CV_8UC3, new cv.Scalar(0, 0, 0))
const maskGray = new cv.Mat()
try {
cv.cvtColor(src, rgb, cv.COLOR_RGBA2RGB)
// Recreate mask from result maskImageData
const maskSrc = cv.matFromImageData(result.maskImageData)
try {
cv.cvtColor(maskSrc, maskGray, cv.COLOR_RGBA2GRAY)
} finally {
maskSrc.delete()
}
// Green overlay where mask > 0
overlay.setTo(new cv.Scalar(0, 255, 0), maskGray)
// Blend: 70% original + 30% overlay
const blended = new cv.Mat()
try {
cv.addWeighted(rgb, 0.7, overlay, 0.3, 0, blended)
// Convert to RGBA ImageData
const blendedRgba = new cv.Mat()
try {
cv.cvtColor(blended, blendedRgba, cv.COLOR_RGB2RGBA)
const overlayImageData = new ImageData(
new Uint8ClampedArray(blendedRgba.data),
width,
height,
)
return { ...result, overlayImageData }
} finally {
blendedRgba.delete()
}
} finally {
blended.delete()
}
} finally {
src.delete()
rgb.delete()
overlay.delete()
maskGray.delete()
}
}

121
ui/detection-app/src/cv/wasmBridge.ts Normal file
View File

@@ -0,0 +1,121 @@
/**
* WASM Worker Bridge — runs OpenCV operations in a Web Worker.
*
* The worker loads opencv.js (~10MB) in its own thread.
* Main thread stays responsive during WASM compilation.
*
* Lazy-creates the worker on first call. Sends ImageData +
* params, gets back results via transferable buffers.
*/
import type { EdgeDetectionParams, EdgeDetectionResult, EdgeDetectionDebugResult } from './edges'
import type { SegmentationParams, SegmentationDebugResult } from './segmentation'
let worker: Worker | null = null
let messageId = 0
const pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>()
let initPromise: Promise<boolean> | null = null
let ready = false
let failed = false
function getWorker(): Worker {
if (!worker) {
worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' })
worker.onmessage = (event) => {
const { id, type, error: errMsg, ...data } = event.data
const handler = pending.get(id)
if (!handler) return
pending.delete(id)
if (type === 'error') {
handler.reject(new Error(errMsg ?? 'Worker error'))
} else {
handler.resolve(data)
}
}
worker.onerror = (event) => {
// Reject all pending
for (const [, handler] of pending) {
handler.reject(new Error(event.message ?? 'Worker crashed'))
}
pending.clear()
}
}
return worker
}
function postMessage(type: string, imageData: ImageData, params: Record<string, unknown>): Promise<any> {
return new Promise((resolve, reject) => {
const id = ++messageId
pending.set(id, { resolve, reject })
getWorker().postMessage({ id, type, imageData, params }, [imageData.data.buffer])
})
}
/** Initialize the worker + load WASM. Returns true if ready. */
export async function initWasm(): Promise<boolean> {
if (ready) return true
if (failed) return false
if (initPromise) return initPromise
initPromise = (async () => {
try {
// Send a ping — the worker will load opencv.js on first message
const result = await postMessage('ping', new ImageData(1, 1), {})
ready = true
return true
} catch {
failed = true
return false
}
})()
return initPromise
}
export function isWasmReady(): boolean {
return ready
}
export function isWasmFailed(): boolean {
return failed
}
// --- Edge detection ---
export async function detectEdgesWasm(
imageData: ImageData,
params: Partial<EdgeDetectionParams>,
): Promise<EdgeDetectionResult> {
const data = await postMessage('detect_edges', imageData, params as Record<string, unknown>)
return { regions: data.regions }
}
export async function detectEdgesWasmDebug(
imageData: ImageData,
params: Partial<EdgeDetectionParams>,
): Promise<EdgeDetectionDebugResult> {
const data = await postMessage('detect_edges_debug', imageData, params as Record<string, unknown>)
return {
regions: data.regions,
edgeImageData: data.edgeImageData,
linesImageData: data.linesImageData,
horizontalCount: data.horizontalCount,
pairCount: data.pairCount,
}
}
// --- Segmentation ---
export async function segmentFieldWasmDebug(
imageData: ImageData,
params: Partial<SegmentationParams>,
): Promise<SegmentationDebugResult> {
const data = await postMessage('segment_field_debug', imageData, params as Record<string, unknown>)
return {
boundary: data.boundary,
coverage: data.coverage,
maskImageData: data.maskImageData,
overlayImageData: data.overlayImageData,
}
}

45
ui/detection-app/src/cv/worker.ts
View File

@@ -1,25 +1,27 @@
/**
* CV Web Worker — runs edge detection off the main thread.
* CV Web Worker — runs CV operations off the main thread.
*
* Message protocol:
* Main → Worker: { type: 'detect_edges', imageData: ImageData, params: {...} }
* Main → Worker: { type: 'detect_edges_debug', imageData: ImageData, params: {...} }
* Worker → Main: { type: 'result', regions: [...] }
* Worker → Main: { type: 'debug_result', regions: [...], edgeImageData, linesImageData, horizontalCount, pairCount }
* Worker → Main: { type: 'error', message: string }
* Main → Worker: { type: 'detect_edges', imageData, params }
* Main → Worker: { type: 'detect_edges_debug', imageData, params }
* Main → Worker: { type: 'segment_field', imageData, params }
* Main → Worker: { type: 'segment_field_debug', imageData, params }
* Worker → Main: { type: 'result', ... }
* Worker → Main: { type: 'error', message }
*/
import { detectEdges, detectEdgesDebug, type EdgeDetectionParams } from './edges'
import { detectEdges, detectEdgesDebug } from './edges'
import { segmentField, segmentFieldDebug } from './segmentation'
self.onmessage = (event: MessageEvent) => {
self.onmessage = async (event: MessageEvent) => {
const { type, imageData, params } = event.data
try {
if (type === 'detect_edges') {
const result = detectEdges(imageData, params)
const result = await detectEdges(imageData, params)
self.postMessage({ type: 'result', regions: result.regions })
} else if (type === 'detect_edges_debug') {
const result = detectEdgesDebug(imageData, params)
const result = await detectEdgesDebug(imageData, params)
self.postMessage({
type: 'debug_result',
regions: result.regions,
@@ -28,10 +30,31 @@ self.onmessage = (event: MessageEvent) => {
horizontalCount: result.horizontalCount,
pairCount: result.pairCount,
}, [
// Transfer ownership of the backing buffers for zero-copy
result.edgeImageData.data.buffer,
result.linesImageData.data.buffer,
])
} else if (type === 'segment_field') {
const result = await segmentField(imageData, params)
self.postMessage({
type: 'result',
boundary: result.boundary,
coverage: result.coverage,
maskImageData: result.maskImageData,
}, [
result.maskImageData.data.buffer,
])
} else if (type === 'segment_field_debug') {
const result = await segmentFieldDebug(imageData, params)
self.postMessage({
type: 'debug_result',
boundary: result.boundary,
coverage: result.coverage,
maskImageData: result.maskImageData,
overlayImageData: result.overlayImageData,
}, [
result.maskImageData.data.buffer,
result.overlayImageData.data.buffer,
])
} else {
self.postMessage({ type: 'error', message: `Unknown message type: ${type}` })
}

16
ui/detection-app/src/types/sse-contract.ts
View File

@@ -139,3 +139,19 @@ export interface RetryResponse {
task_id: string;
job_id: string;
}
export interface RunRequest {
video_path: string;
profile_name: string;
source_asset_id: string;
checkpoint: boolean;
skip_vlm: boolean;
skip_cloud: boolean;
log_level: string;
}
export interface RunResponse {
status: string;
job_id: string;
video_path: string;
}

1
ui/detection-app/src/vite-env.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
/// <reference types="vite/client" />

14
ui/framework/src/tokens.css
View File

@@ -43,3 +43,17 @@
--panel-border: 1px solid var(--border);
--panel-header-height: 36px;
}
/* Animated gradient outline for buttons in a waiting state.
Usage: add class="waiting" to any button/element. */
@keyframes waiting-glow {
0% { box-shadow: 0 0 3px 1px var(--status-processing); }
33% { box-shadow: 0 0 3px 1px var(--status-live); }
66% { box-shadow: 0 0 3px 1px var(--status-escalating); }
100% { box-shadow: 0 0 3px 1px var(--status-processing); }
}
.waiting {
animation: waiting-glow 2s linear infinite;
outline: 1px solid transparent;
}