scan

2026-07-05 22:59:27 -03:00
parent 8cfb4c2cbe
commit bb84558526
8 changed files with 245 additions and 29 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,9 @@ output/*
 # doocus document extraction output (textified docs + metadata; local-only)
 docs-output/

+# doocus managed collections (UI-scanned sources → one subfolder per source)
+doocus-data/
+
 # Python cache
 __pycache__
 *.pyc
--- a/ui/doocus-app/server/doocusApi.ts
+++ b/ui/doocus-app/server/doocusApi.ts
@@ -19,12 +19,15 @@
 */
 import type { Plugin } from 'vite'
 import type { IncomingMessage, ServerResponse } from 'node:http'
+import { spawn } from 'node:child_process'
 import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import path from 'node:path'

 interface Options {
-  outputDirs: string[]
+  outputDirs: string[]    // explicit output dirs (env)
+  outputsRoot: string     // managed root: UI-scanned sources land here
+  repoRoot: string        // where process_tree.py lives (run via uv)
 }

 interface Collection {
@@ -79,8 +82,37 @@ function buildCollections(dirs: string[]): Collection[] {
 }

 export function doocusApi(opts: Options): Plugin {
-  const collections = buildCollections(opts.outputDirs)
-  const byId = new Map(collections.map((c) => [c.id, c]))
+  /** Collections discovered under the managed root (one subdir per source). */
+  function discoverInRoot(): Collection[] {
+    let subs: fs.Dirent[] = []
+    try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] }
+    return subs
+      .filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json')))
+      .map((d) => ({
+        id: d.name, name: d.name,
+        dir: path.join(opts.outputsRoot, d.name),
+        indexPath: path.join(opts.outputsRoot, d.name, 'index.json'),
+      }))
+  }
+
+  /** Current collections = explicit env dirs + managed-root subdirs (deduped,
+   *  recomputed each request so UI scans appear without a restart). */
+  function currentCollections(): Collection[] {
+    const out: Collection[] = []
+    const seen = new Set<string>()
+    for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) {
+      if (out.some((o) => o.dir === c.dir)) continue
+      let id = c.id, n = 1
+      while (seen.has(id)) { n++; id = `${c.id}-${n}` }
+      seen.add(id)
+      out.push({ ...c, id })
+    }
+    return out
+  }
+
+  function collectionById(id: string): Collection | undefined {
+    return currentCollections().find((c) => c.id === id)
+  }

  function readIndexFor(c: Collection): Index | null {
    try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null }
@@ -91,7 +123,7 @@ export function doocusApi(opts: Options): Plugin {
    const i = p.indexOf('/')
    const id = i < 0 ? p : p.slice(0, i)
    const rel = i < 0 ? '' : p.slice(i + 1)
-    const col = byId.get(id)
+    const col = collectionById(id)
    return col ? { col, rel } : null
  }

@@ -138,8 +170,9 @@ export function doocusApi(opts: Options): Plugin {
  return {
    name: 'doocus-api',
    configureServer(server) {
+      const cols = currentCollections()
      server.config.logger.info(
-        `[doocus-api] ${collections.length} collection(s): ${collections.map((c) => `${c.id} → ${c.dir}`).join(', ')}`)
+        `[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`)
      server.middlewares.use('/api', (req, res, next) => {
        const url = new URL(req.url ?? '/', 'http://localhost')
        const parts = url.pathname.split('/').filter(Boolean)
@@ -153,6 +186,8 @@ export function doocusApi(opts: Options): Plugin {

  async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
    if (parts[0] === 'collections') { listCollections(res); return true }
+    if (parts[0] === 'scan' && req.method === 'POST') { await runScan(req, res, false); return true }
+    if (parts[0] === 'rescan' && req.method === 'POST') { await runScan(req, res, true); return true }
    if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true }
    if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true }
    if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true }
@@ -168,16 +203,58 @@ export function doocusApi(opts: Options): Plugin {
  }

  function listCollections(res: ServerResponse): void {
-    const out = collections.map((c) => {
+    const out = currentCollections().map((c) => {
      const idx = readIndexFor(c)
      return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 }
    })
    sendJson(res, 200, { collections: out })
  }

+  /** Run process_tree.py over a source folder (new scan) or an existing
+   *  collection's recorded source (rescan), writing to the managed root. */
+  async function runScan(req: IncomingMessage, res: ServerResponse, rerun: boolean): Promise<void> {
+    let body: { source?: string; id?: string }
+    try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return }
+
+    let source: string
+    let dir: string
+    if (rerun) {
+      const col = body.id ? collectionById(body.id) : undefined
+      const idx = col ? readIndexFor(col) : null
+      if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
+      source = idx.root
+      dir = col.dir
+    } else {
+      source = (body.source ?? '').trim()
+      if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
+      const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
+      dir = path.join(opts.outputsRoot, slug)
+      fs.mkdirSync(opts.outputsRoot, { recursive: true })
+    }
+
+    try {
+      await runProcessTree(source, dir)
+      sendJson(res, 200, { ok: true, id: path.basename(dir) })
+    } catch (e) {
+      sendJson(res, 500, { error: String(e) })
+    }
+  }
+
+  /** Spawn `uv run python process_tree.py <source> --output <dir>` at repoRoot. */
+  function runProcessTree(source: string, dir: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir],
+        { cwd: opts.repoRoot })
+      let err = ''
+      child.stderr.on('data', (d) => { err += d })
+      child.on('error', reject)
+      child.on('close', (code) => code === 0 ? resolve() : reject(new Error(err || `exit ${code}`)))
+    })
+  }
+
  function getTree(res: ServerResponse, activeCsv: string | null): void {
    const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
-    const cols = collections.filter((c) => !active || active.has(c.id))
+    const cols = currentCollections().filter((c) => !active || active.has(c.id))
    const files: Array<Node & { collection: string }> = []
    const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: 0 }
    for (const c of cols) {
@@ -190,7 +267,7 @@ export function doocusApi(opts: Options): Plugin {
      }
    }
    sendJson(res, 200, {
-      collections: collections.map((c) => ({ id: c.id, name: c.name })),
+      collections: currentCollections().map((c) => ({ id: c.id, name: c.name })),
      files,
      counts,
    })
@@ -363,7 +440,7 @@ export function doocusApi(opts: Options): Plugin {
  async function search(res: ServerResponse, query: string, activeCsv: string | null): Promise<void> {
    const qq = query.trim().toLowerCase()
    const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
-    const cols = collections.filter((c) => !active || active.has(c.id))
+    const cols = currentCollections().filter((c) => !active || active.has(c.id))
    let matches: string[] = []
    let bytes = 0
    if (qq) {
@@ -393,6 +470,10 @@ function isFile(p: string): boolean {
  try { return fs.statSync(p).isFile() } catch { return false }
 }

+function isDir(p: string): boolean {
+  try { return fs.statSync(p).isDirectory() } catch { return false }
+}
+
 function readBody(req: IncomingMessage): Promise<string> {
  return new Promise((resolve, reject) => {
    let data = ''
--- a/ui/doocus-app/src/App.vue
+++ b/ui/doocus-app/src/App.vue
@@ -6,11 +6,19 @@ import FileTree from '@/components/FileTree.vue'
 import DocDetail from '@/components/DocDetail.vue'
 import ExportBar from '@/components/ExportBar.vue'
 import { ref } from 'vue'
-import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection } from '@/store'
+import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection, scanFolder, rescanCollection } from '@/store'

 onMounted(loadTree)

 const showCols = ref(false)
+const scanPath = ref('')
+
+async function doScan() {
+  const p = scanPath.value
+  if (!p.trim()) return
+  await scanFolder(p)
+  if (!store.scanError) scanPath.value = ''
+}

 const detailTitle = computed(() =>
  store.detail?.node.mode === 'meeting' ? 'Meeting' : 'File')
@@ -32,9 +40,10 @@ function onSearch(e: Event) {
          :title="`Sources — ${store.activeCollections.size}/${store.collections.length} active`"
          @click="showCols = !showCols"
        >⧉</button>
-        <div v-if="showCols" class="col-menu" @mouseleave="showCols = false">
-          <div class="col-title">Output sources</div>
-          <label v-for="c in store.collections" :key="c.id" :class="{ off: !c.available }">
+        <div v-if="showCols" class="col-menu">
+          <div class="col-title">Sources</div>
+          <div v-for="c in store.collections" :key="c.id" class="col-row">
+            <label :class="{ off: !c.available }">
              <input
                type="checkbox"
                :checked="store.activeCollections.has(c.id)"
@@ -44,6 +53,22 @@ function onSearch(e: Event) {
              <span class="col-name">{{ c.name }}</span>
              <span class="dim">{{ c.available ? c.count : 'no index' }}</span>
            </label>
+            <button class="rescan" title="Re-scan this source" :disabled="store.scanning" @click="rescanCollection(c.id)">⟳</button>
+          </div>
+          <p v-if="!store.collections.length" class="dim">No sources yet.</p>
+
+          <div class="col-scan">
+            <input
+              v-model="scanPath"
+              class="scan-input"
+              placeholder="/path/to/source folder"
+              @keyup.enter="doScan"
+            />
+            <button :disabled="store.scanning || !scanPath.trim()" @click="doScan">
+              {{ store.scanning ? 'Scanning…' : 'Scan' }}
+            </button>
+          </div>
+          <p v-if="store.scanError" class="scan-err">{{ store.scanError }}</p>
        </div>
      </div>

@@ -125,14 +150,19 @@ function onSearch(e: Event) {
  padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35);
 }
 .col-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); }
+.col-row { display: flex; align-items: center; gap: var(--space-1); }
 .col-menu label {
-  display: flex; align-items: center; gap: var(--space-2);
-  padding: var(--space-1); border-radius: 3px; cursor: pointer;
+  flex: 1; display: flex; align-items: center; gap: var(--space-2);
+  padding: var(--space-1); border-radius: 3px; cursor: pointer; min-width: 0;
 }
 .col-menu label:hover { background: var(--surface-2); }
 .col-menu label.off { opacity: 0.5; }
 .col-name { flex: 1; overflow: hidden; text-overflow: ellipsis; }
+.rescan { padding: 0 var(--space-2); background: transparent; border: 0; color: var(--text-secondary); cursor: pointer; }
 .dim { color: var(--text-secondary); font-size: var(--font-size-sm); }
+.col-scan { display: flex; gap: var(--space-1); margin-top: var(--space-2); border-top: 1px solid var(--surface-2); padding-top: var(--space-2); }
+.scan-input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); background: var(--surface-2); border: var(--panel-border); border-radius: var(--panel-radius); color: var(--text-primary); }
+.scan-err { color: var(--status-error, #e0a030); font-size: var(--font-size-sm); margin: var(--space-1) 0 0; }
 .search { flex: 1; display: flex; align-items: center; justify-content: center; gap: var(--space-2); }
 .search-input {
  width: 100%; max-width: 520px;
--- a/ui/doocus-app/src/store.ts
+++ b/ui/doocus-app/src/store.ts
@@ -80,6 +80,9 @@ interface State {
  matches: Set<string> | null   // null = not searching; else the matched paths
  searching: boolean
  searchStats: { count: number; bytes: number }
+
+  scanning: boolean
+  scanError: string
 }

 export const store = reactive<State>({
@@ -102,6 +105,9 @@ export const store = reactive<State>({
  matches: null,
  searching: false,
  searchStats: { count: 0, bytes: 0 },
+
+  scanning: false,
+  scanError: '',
 })

 let searchTimer: ReturnType<typeof setTimeout> | null = null
@@ -216,6 +222,49 @@ export async function toggleCollection(id: string): Promise<void> {
  if (store.query) runSearch(store.query)
 }

+/** Scan a source folder (runs process_tree.py server-side) → new collection. */
+export async function scanFolder(source: string): Promise<void> {
+  if (!source.trim()) return
+  store.scanning = true
+  store.scanError = ''
+  try {
+    const res = await fetch('/api/scan', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ source: source.trim() }),
+    })
+    const data = await res.json()
+    if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return }
+    await loadCollections()
+    if (data.id) store.activeCollections.add(data.id)   // show the new source
+    await loadTree()
+  } catch (e) {
+    store.scanError = String(e)
+  } finally {
+    store.scanning = false
+  }
+}
+
+/** Re-run the scan for an existing collection (its recorded source). */
+export async function rescanCollection(id: string): Promise<void> {
+  store.scanning = true
+  store.scanError = ''
+  try {
+    const res = await fetch('/api/rescan', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ id }),
+    })
+    const data = await res.json()
+    if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return }
+    await loadCollections()
+    await loadTree()
+    if (store.query) runSearch(store.query)
+  } catch (e) {
+    store.scanError = String(e)
+  } finally {
+    store.scanning = false
+  }
+}
+
 export async function select(path: string): Promise<void> {
  store.selectedPath = path
  store.detailCollapsed = false   // opening a file expands the viewer
--- a/ui/doocus-app/vite.config.ts
+++ b/ui/doocus-app/vite.config.ts
@@ -3,16 +3,23 @@ import vue from '@vitejs/plugin-vue'
 import { fileURLToPath } from 'node:url'
 import { doocusApi } from './server/doocusApi'

-// Output directories produced by the doocus pipeline. One or many, so several
-// sources can be joined in the tree/search. DOOCUS_OUTPUTS is comma-separated;
-// DOOCUS_OUTPUT (single) is still honored for back-compat.
+// Explicit output dirs (env). DOOCUS_OUTPUTS is comma-separated; DOOCUS_OUTPUT
+// (single) is honored too. Defaults to the repo's docs-output for back-compat.
 const outputEnv = process.env.DOOCUS_OUTPUTS ?? process.env.DOOCUS_OUTPUT
 const outputDirs = (outputEnv
  ? outputEnv.split(',').map((s) => s.trim()).filter(Boolean)
  : [fileURLToPath(new URL('../../docs-output', import.meta.url))])

+// Managed collections root (gitignored): UI-scanned sources land here, one
+// subfolder per source, discovered automatically. Override with DOOCUS_DATA.
+const outputsRoot = process.env.DOOCUS_DATA
+  ?? fileURLToPath(new URL('../../doocus-data', import.meta.url))
+
+// Repo root — where process_tree.py lives (run via `uv run` for the scan).
+const repoRoot = fileURLToPath(new URL('../../', import.meta.url))
+
 export default defineConfig({
-  plugins: [vue(), doocusApi({ outputDirs })],
+  plugins: [vue(), doocusApi({ outputDirs, outputsRoot, repoRoot })],
  resolve: {
    alias: {
      '@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),
--- a/ui/meetus-app/server/meetusApi.ts
+++ b/ui/meetus-app/server/meetusApi.ts
@@ -45,7 +45,8 @@ const VIDEO_MIME: Record<string, string> = {
 }

 export function meetusApi(opts: Options): Plugin {
-  const outputDir = path.resolve(opts.outputDir)
+  // Mutable so the UI can re-point the scan at another folder at runtime.
+  let outputDir = path.resolve(opts.outputDir)
  const videoDir = opts.videoDir ? path.resolve(opts.videoDir) : null

  /** Locate a run's source video. The manifest records an absolute path from
@@ -139,6 +140,22 @@ export function meetusApi(opts: Options): Plugin {
    parts: string[],
    logger: { warn: (m: string) => void },
  ): Promise<boolean> {
+    // GET/PUT the folder scanned (recursively) for meeting runs.
+    if (parts[0] === 'output') {
+      if (req.method === 'GET') { sendJson(res, 200, { dir: outputDir }); return true }
+      if (req.method === 'PUT') {
+        let body: { dir?: string }
+        try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return true }
+        const dir = (body.dir ?? '').trim()
+        try {
+          if (!dir || !fs.statSync(dir).isDirectory()) throw new Error('not a directory')
+        } catch { sendJson(res, 400, { error: `not a directory: ${dir}` }); return true }
+        outputDir = path.resolve(dir)
+        sendJson(res, 200, { dir: outputDir })
+        return true
+      }
+    }
+
    if (parts[0] !== 'runs') return false

    // GET /api/runs
--- a/ui/meetus-app/src/components/RunPicker.vue
+++ b/ui/meetus-app/src/components/RunPicker.vue
@@ -1,15 +1,23 @@
 <script setup lang="ts">
-import { onMounted } from 'vue'
+import { onMounted, ref } from 'vue'
 import { useStore } from '@/store'

-const { state, loadRuns, openRun } = useStore()
+const { state, loadRuns, openRun, setOutputDir } = useStore()

 onMounted(loadRuns)

+const showFolder = ref(false)
+const folder = ref('')
+
 function onChange(e: Event) {
  const id = (e.target as HTMLSelectElement).value
  if (id) openRun(id)
 }
+
+async function scanFolder() {
+  await setOutputDir(folder.value)
+  if (!state.error) showFolder.value = false
+}
 </script>

 <template>
@@ -21,11 +29,21 @@ function onChange(e: Event) {
        {{ r.name }} · {{ r.frameCount }} frames{{ r.hasVideo ? '' : ' · no video' }}
      </option>
    </select>
+    <button class="folder-btn" :title="`Scan folder — ${state.outputDir ?? ''}`" @click="showFolder = !showFolder">📁</button>
    <span v-if="state.loading" class="hint">loading…</span>
    <span v-else-if="state.error" class="hint err">{{ state.error }}</span>
    <span v-else-if="!state.runs.length" class="hint" :title="state.outputDir ?? ''">
      no runs found in {{ state.outputDir ?? '(unknown)' }}
    </span>
+
+    <div v-if="showFolder" class="folder-pop">
+      <div class="pop-title">Scan folder (recursive) for meetings</div>
+      <div class="pop-row">
+        <input v-model="folder" :placeholder="state.outputDir ?? '/path/to/meetings'" @keyup.enter="scanFolder" />
+        <button :disabled="state.loading || !folder.trim()" @click="scanFolder">Scan</button>
+      </div>
+      <div class="pop-cur">current: {{ state.outputDir ?? '(unset)' }}</div>
+    </div>
  </div>
 </template>

@@ -34,7 +52,18 @@ function onChange(e: Event) {
  display: flex;
  align-items: center;
  gap: var(--space-2);
+  position: relative;
 }
+.folder-btn { padding: var(--space-1) var(--space-2); line-height: 1; }
+.folder-pop {
+  position: absolute; top: 110%; left: 0; z-index: 20; min-width: 340px;
+  background: var(--surface-1); border: var(--panel-border); border-radius: var(--panel-radius);
+  padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35);
+}
+.pop-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); }
+.pop-row { display: flex; gap: var(--space-1); }
+.pop-row input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); }
+.pop-cur { font-size: var(--font-size-sm); color: var(--text-secondary); margin-top: var(--space-1); overflow: hidden; text-overflow: ellipsis; }
 .lbl {
  color: var(--text-secondary);
  font-size: var(--font-size-sm);
--- a/ui/meetus-app/src/store.ts
+++ b/ui/meetus-app/src/store.ts