scan

.gitignore

@@ -9,6 +9,9 @@ output/*
 # doocus document extraction output (textified docs + metadata; local-only)
 docs-output/
 
+# doocus managed collections (UI-scanned sources → one subfolder per source)
+doocus-data/
+
 # Python cache
 __pycache__
 *.pyc

ui/doocus-app/server/doocusApi.ts

@@ -19,12 +19,15 @@
  */
 import type { Plugin } from 'vite'
 import type { IncomingMessage, ServerResponse } from 'node:http'
+import { spawn } from 'node:child_process'
 import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import path from 'node:path'
 
 interface Options {
-  outputDirs: string[]
+  outputDirs: string[]    // explicit output dirs (env)
+  outputsRoot: string     // managed root: UI-scanned sources land here
+  repoRoot: string        // where process_tree.py lives (run via uv)
 }
 
 interface Collection {
@@ -79,8 +82,37 @@ function buildCollections(dirs: string[]): Collection[] {
 }
 
 export function doocusApi(opts: Options): Plugin {
-  const collections = buildCollections(opts.outputDirs)
+  /** Collections discovered under the managed root (one subdir per source). */
-  const byId = new Map(collections.map((c) => [c.id, c]))
+  function discoverInRoot(): Collection[] {
+    let subs: fs.Dirent[] = []
+    try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] }
+    return subs
+      .filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json')))
+      .map((d) => ({
+        id: d.name, name: d.name,
+        dir: path.join(opts.outputsRoot, d.name),
+        indexPath: path.join(opts.outputsRoot, d.name, 'index.json'),
+      }))
+  }
+
+  /** Current collections = explicit env dirs + managed-root subdirs (deduped,
+   *  recomputed each request so UI scans appear without a restart). */
+  function currentCollections(): Collection[] {
+    const out: Collection[] = []
+    const seen = new Set<string>()
+    for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) {
+      if (out.some((o) => o.dir === c.dir)) continue
+      let id = c.id, n = 1
+      while (seen.has(id)) { n++; id = `${c.id}-${n}` }
+      seen.add(id)
+      out.push({ ...c, id })
+    }
+    return out
+  }
+
+  function collectionById(id: string): Collection | undefined {
+    return currentCollections().find((c) => c.id === id)
+  }
 
   function readIndexFor(c: Collection): Index | null {
     try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null }
@@ -91,7 +123,7 @@ export function doocusApi(opts: Options): Plugin {
     const i = p.indexOf('/')
     const id = i < 0 ? p : p.slice(0, i)
     const rel = i < 0 ? '' : p.slice(i + 1)
-    const col = byId.get(id)
+    const col = collectionById(id)
     return col ? { col, rel } : null
   }
 
@@ -138,8 +170,9 @@ export function doocusApi(opts: Options): Plugin {
   return {
     name: 'doocus-api',
     configureServer(server) {
+      const cols = currentCollections()
       server.config.logger.info(
-        `[doocus-api] ${collections.length} collection(s): ${collections.map((c) => `${c.id} → ${c.dir}`).join(', ')}`)
+        `[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`)
       server.middlewares.use('/api', (req, res, next) => {
         const url = new URL(req.url ?? '/', 'http://localhost')
         const parts = url.pathname.split('/').filter(Boolean)
@@ -153,6 +186,8 @@ export function doocusApi(opts: Options): Plugin {
 
   async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
     if (parts[0] === 'collections') { listCollections(res); return true }
+    if (parts[0] === 'scan' && req.method === 'POST') { await runScan(req, res, false); return true }
+    if (parts[0] === 'rescan' && req.method === 'POST') { await runScan(req, res, true); return true }
     if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true }
     if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true }
     if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true }
@@ -168,16 +203,58 @@ export function doocusApi(opts: Options): Plugin {
   }
 
   function listCollections(res: ServerResponse): void {
-    const out = collections.map((c) => {
+    const out = currentCollections().map((c) => {
       const idx = readIndexFor(c)
       return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 }
     })
     sendJson(res, 200, { collections: out })
   }
 
+  /** Run process_tree.py over a source folder (new scan) or an existing
+   *  collection's recorded source (rescan), writing to the managed root. */
+  async function runScan(req: IncomingMessage, res: ServerResponse, rerun: boolean): Promise<void> {
+    let body: { source?: string; id?: string }
+    try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return }
+
+    let source: string
+    let dir: string
+    if (rerun) {
+      const col = body.id ? collectionById(body.id) : undefined
+      const idx = col ? readIndexFor(col) : null
+      if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
+      source = idx.root
+      dir = col.dir
+    } else {
+      source = (body.source ?? '').trim()
+      if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
+      const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
+      dir = path.join(opts.outputsRoot, slug)
+      fs.mkdirSync(opts.outputsRoot, { recursive: true })
+    }
+
+    try {
+      await runProcessTree(source, dir)
+      sendJson(res, 200, { ok: true, id: path.basename(dir) })
+    } catch (e) {
+      sendJson(res, 500, { error: String(e) })
+    }
+  }
+
+  /** Spawn `uv run python process_tree.py <source> --output <dir>` at repoRoot. */
+  function runProcessTree(source: string, dir: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir],
+        { cwd: opts.repoRoot })
+      let err = ''
+      child.stderr.on('data', (d) => { err += d })
+      child.on('error', reject)
+      child.on('close', (code) => code === 0 ? resolve() : reject(new Error(err || `exit ${code}`)))
+    })
+  }
+
   function getTree(res: ServerResponse, activeCsv: string | null): void {
     const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
-    const cols = collections.filter((c) => !active || active.has(c.id))
+    const cols = currentCollections().filter((c) => !active || active.has(c.id))
     const files: Array<Node & { collection: string }> = []
     const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: 0 }
     for (const c of cols) {
@@ -190,7 +267,7 @@ export function doocusApi(opts: Options): Plugin {
       }
     }
     sendJson(res, 200, {
-      collections: collections.map((c) => ({ id: c.id, name: c.name })),
+      collections: currentCollections().map((c) => ({ id: c.id, name: c.name })),
       files,
       counts,
     })
@@ -363,7 +440,7 @@ export function doocusApi(opts: Options): Plugin {
   async function search(res: ServerResponse, query: string, activeCsv: string | null): Promise<void> {
     const qq = query.trim().toLowerCase()
     const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
-    const cols = collections.filter((c) => !active || active.has(c.id))
+    const cols = currentCollections().filter((c) => !active || active.has(c.id))
     let matches: string[] = []
     let bytes = 0
     if (qq) {
@@ -393,6 +470,10 @@ function isFile(p: string): boolean {
   try { return fs.statSync(p).isFile() } catch { return false }
 }
 
+function isDir(p: string): boolean {
+  try { return fs.statSync(p).isDirectory() } catch { return false }
+}
+
 function readBody(req: IncomingMessage): Promise<string> {
   return new Promise((resolve, reject) => {
     let data = ''

ui/doocus-app/src/App.vue

@@ -6,11 +6,19 @@ import FileTree from '@/components/FileTree.vue'
 import DocDetail from '@/components/DocDetail.vue'
 import ExportBar from '@/components/ExportBar.vue'
 import { ref } from 'vue'
-import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection } from '@/store'
+import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection, scanFolder, rescanCollection } from '@/store'
 
 onMounted(loadTree)
 
 const showCols = ref(false)
+const scanPath = ref('')
+
+async function doScan() {
+  const p = scanPath.value
+  if (!p.trim()) return
+  await scanFolder(p)
+  if (!store.scanError) scanPath.value = ''
+}
 
 const detailTitle = computed(() =>
   store.detail?.node.mode === 'meeting' ? 'Meeting' : 'File')
@@ -32,18 +40,35 @@ function onSearch(e: Event) {
           :title="`Sources — ${store.activeCollections.size}/${store.collections.length} active`"
           @click="showCols = !showCols"
         >⧉</button>
-        <div v-if="showCols" class="col-menu" @mouseleave="showCols = false">
+        <div v-if="showCols" class="col-menu">
-          <div class="col-title">Output sources</div>
+          <div class="col-title">Sources</div>
-          <label v-for="c in store.collections" :key="c.id" :class="{ off: !c.available }">
+          <div v-for="c in store.collections" :key="c.id" class="col-row">
+            <label :class="{ off: !c.available }">
+              <input
+                type="checkbox"
+                :checked="store.activeCollections.has(c.id)"
+                :disabled="!c.available"
+                @change="toggleCollection(c.id)"
+              />
+              <span class="col-name">{{ c.name }}</span>
+              <span class="dim">{{ c.available ? c.count : 'no index' }}</span>
+            </label>
+            <button class="rescan" title="Re-scan this source" :disabled="store.scanning" @click="rescanCollection(c.id)">⟳</button>
+          </div>
+          <p v-if="!store.collections.length" class="dim">No sources yet.</p>
+
+          <div class="col-scan">
             <input
-              type="checkbox"
+              v-model="scanPath"
-              :checked="store.activeCollections.has(c.id)"
+              class="scan-input"
-              :disabled="!c.available"
+              placeholder="/path/to/source folder"
-              @change="toggleCollection(c.id)"
+              @keyup.enter="doScan"
             />
-            <span class="col-name">{{ c.name }}</span>
+            <button :disabled="store.scanning || !scanPath.trim()" @click="doScan">
-            <span class="dim">{{ c.available ? c.count : 'no index' }}</span>
+              {{ store.scanning ? 'Scanning…' : 'Scan' }}
-          </label>
+            </button>
+          </div>
+          <p v-if="store.scanError" class="scan-err">{{ store.scanError }}</p>
         </div>
       </div>
 
@@ -125,14 +150,19 @@ function onSearch(e: Event) {
   padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35);
 }
 .col-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); }
+.col-row { display: flex; align-items: center; gap: var(--space-1); }
 .col-menu label {
-  display: flex; align-items: center; gap: var(--space-2);
+  flex: 1; display: flex; align-items: center; gap: var(--space-2);
-  padding: var(--space-1); border-radius: 3px; cursor: pointer;
+  padding: var(--space-1); border-radius: 3px; cursor: pointer; min-width: 0;
 }
 .col-menu label:hover { background: var(--surface-2); }
 .col-menu label.off { opacity: 0.5; }
 .col-name { flex: 1; overflow: hidden; text-overflow: ellipsis; }
+.rescan { padding: 0 var(--space-2); background: transparent; border: 0; color: var(--text-secondary); cursor: pointer; }
 .dim { color: var(--text-secondary); font-size: var(--font-size-sm); }
+.col-scan { display: flex; gap: var(--space-1); margin-top: var(--space-2); border-top: 1px solid var(--surface-2); padding-top: var(--space-2); }
+.scan-input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); background: var(--surface-2); border: var(--panel-border); border-radius: var(--panel-radius); color: var(--text-primary); }
+.scan-err { color: var(--status-error, #e0a030); font-size: var(--font-size-sm); margin: var(--space-1) 0 0; }
 .search { flex: 1; display: flex; align-items: center; justify-content: center; gap: var(--space-2); }
 .search-input {
   width: 100%; max-width: 520px;

ui/doocus-app/src/store.ts

@@ -80,6 +80,9 @@ interface State {
   matches: Set<string> | null   // null = not searching; else the matched paths
   searching: boolean
   searchStats: { count: number; bytes: number }
+
+  scanning: boolean
+  scanError: string
 }
 
 export const store = reactive<State>({
@@ -102,6 +105,9 @@ export const store = reactive<State>({
   matches: null,
   searching: false,
   searchStats: { count: 0, bytes: 0 },
+
+  scanning: false,
+  scanError: '',
 })
 
 let searchTimer: ReturnType<typeof setTimeout> | null = null
@@ -216,6 +222,49 @@ export async function toggleCollection(id: string): Promise<void> {
   if (store.query) runSearch(store.query)
 }
 
+/** Scan a source folder (runs process_tree.py server-side) → new collection. */
+export async function scanFolder(source: string): Promise<void> {
+  if (!source.trim()) return
+  store.scanning = true
+  store.scanError = ''
+  try {
+    const res = await fetch('/api/scan', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ source: source.trim() }),
+    })
+    const data = await res.json()
+    if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return }
+    await loadCollections()
+    if (data.id) store.activeCollections.add(data.id)   // show the new source
+    await loadTree()
+  } catch (e) {
+    store.scanError = String(e)
+  } finally {
+    store.scanning = false
+  }
+}
+
+/** Re-run the scan for an existing collection (its recorded source). */
+export async function rescanCollection(id: string): Promise<void> {
+  store.scanning = true
+  store.scanError = ''
+  try {
+    const res = await fetch('/api/rescan', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ id }),
+    })
+    const data = await res.json()
+    if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return }
+    await loadCollections()
+    await loadTree()
+    if (store.query) runSearch(store.query)
+  } catch (e) {
+    store.scanError = String(e)
+  } finally {
+    store.scanning = false
+  }
+}
+
 export async function select(path: string): Promise<void> {
   store.selectedPath = path
   store.detailCollapsed = false   // opening a file expands the viewer

ui/doocus-app/vite.config.ts

@@ -3,16 +3,23 @@ import vue from '@vitejs/plugin-vue'
 import { fileURLToPath } from 'node:url'
 import { doocusApi } from './server/doocusApi'
 
-// Output directories produced by the doocus pipeline. One or many, so several
+// Explicit output dirs (env). DOOCUS_OUTPUTS is comma-separated; DOOCUS_OUTPUT
-// sources can be joined in the tree/search. DOOCUS_OUTPUTS is comma-separated;
+// (single) is honored too. Defaults to the repo's docs-output for back-compat.
-// DOOCUS_OUTPUT (single) is still honored for back-compat.
 const outputEnv = process.env.DOOCUS_OUTPUTS ?? process.env.DOOCUS_OUTPUT
 const outputDirs = (outputEnv
   ? outputEnv.split(',').map((s) => s.trim()).filter(Boolean)
   : [fileURLToPath(new URL('../../docs-output', import.meta.url))])
 
+// Managed collections root (gitignored): UI-scanned sources land here, one
+// subfolder per source, discovered automatically. Override with DOOCUS_DATA.
+const outputsRoot = process.env.DOOCUS_DATA
+  ?? fileURLToPath(new URL('../../doocus-data', import.meta.url))
+
+// Repo root — where process_tree.py lives (run via `uv run` for the scan).
+const repoRoot = fileURLToPath(new URL('../../', import.meta.url))
+
 export default defineConfig({
-  plugins: [vue(), doocusApi({ outputDirs })],
+  plugins: [vue(), doocusApi({ outputDirs, outputsRoot, repoRoot })],
   resolve: {
     alias: {
       '@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),

ui/meetus-app/server/meetusApi.ts

@@ -45,7 +45,8 @@ const VIDEO_MIME: Record<string, string> = {
 }
 
 export function meetusApi(opts: Options): Plugin {
-  const outputDir = path.resolve(opts.outputDir)
+  // Mutable so the UI can re-point the scan at another folder at runtime.
+  let outputDir = path.resolve(opts.outputDir)
   const videoDir = opts.videoDir ? path.resolve(opts.videoDir) : null
 
   /** Locate a run's source video. The manifest records an absolute path from
@@ -139,6 +140,22 @@ export function meetusApi(opts: Options): Plugin {
     parts: string[],
     logger: { warn: (m: string) => void },
   ): Promise<boolean> {
+    // GET/PUT the folder scanned (recursively) for meeting runs.
+    if (parts[0] === 'output') {
+      if (req.method === 'GET') { sendJson(res, 200, { dir: outputDir }); return true }
+      if (req.method === 'PUT') {
+        let body: { dir?: string }
+        try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return true }
+        const dir = (body.dir ?? '').trim()
+        try {
+          if (!dir || !fs.statSync(dir).isDirectory()) throw new Error('not a directory')
+        } catch { sendJson(res, 400, { error: `not a directory: ${dir}` }); return true }
+        outputDir = path.resolve(dir)
+        sendJson(res, 200, { dir: outputDir })
+        return true
+      }
+    }
+
     if (parts[0] !== 'runs') return false
 
     // GET /api/runs

ui/meetus-app/src/components/RunPicker.vue

@@ -1,15 +1,23 @@
 <script setup lang="ts">
-import { onMounted } from 'vue'
+import { onMounted, ref } from 'vue'
 import { useStore } from '@/store'
 
-const { state, loadRuns, openRun } = useStore()
+const { state, loadRuns, openRun, setOutputDir } = useStore()
 
 onMounted(loadRuns)
 
+const showFolder = ref(false)
+const folder = ref('')
+
 function onChange(e: Event) {
   const id = (e.target as HTMLSelectElement).value
   if (id) openRun(id)
 }
+
+async function scanFolder() {
+  await setOutputDir(folder.value)
+  if (!state.error) showFolder.value = false
+}
 </script>
 
 <template>
@@ -21,11 +29,21 @@ function onChange(e: Event) {
         {{ r.name }} · {{ r.frameCount }} frames{{ r.hasVideo ? '' : ' · no video' }}
       </option>
     </select>
+    <button class="folder-btn" :title="`Scan folder — ${state.outputDir ?? ''}`" @click="showFolder = !showFolder">📁</button>
     <span v-if="state.loading" class="hint">loading…</span>
     <span v-else-if="state.error" class="hint err">{{ state.error }}</span>
     <span v-else-if="!state.runs.length" class="hint" :title="state.outputDir ?? ''">
       no runs found in {{ state.outputDir ?? '(unknown)' }}
     </span>
+
+    <div v-if="showFolder" class="folder-pop">
+      <div class="pop-title">Scan folder (recursive) for meetings</div>
+      <div class="pop-row">
+        <input v-model="folder" :placeholder="state.outputDir ?? '/path/to/meetings'" @keyup.enter="scanFolder" />
+        <button :disabled="state.loading || !folder.trim()" @click="scanFolder">Scan</button>
+      </div>
+      <div class="pop-cur">current: {{ state.outputDir ?? '(unset)' }}</div>
+    </div>
   </div>
 </template>
 
@@ -34,7 +52,18 @@ function onChange(e: Event) {
   display: flex;
   align-items: center;
   gap: var(--space-2);
+  position: relative;
 }
+.folder-btn { padding: var(--space-1) var(--space-2); line-height: 1; }
+.folder-pop {
+  position: absolute; top: 110%; left: 0; z-index: 20; min-width: 340px;
+  background: var(--surface-1); border: var(--panel-border); border-radius: var(--panel-radius);
+  padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35);
+}
+.pop-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); }
+.pop-row { display: flex; gap: var(--space-1); }
+.pop-row input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); }
+.pop-cur { font-size: var(--font-size-sm); color: var(--text-secondary); margin-top: var(--space-1); overflow: hidden; text-overflow: ellipsis; }
 .lbl {
   color: var(--text-secondary);
   font-size: var(--font-size-sm);