From bb8455852607b952337e7eb92d516d41e09898c2 Mon Sep 17 00:00:00 2001 From: Mariano Gabriel Date: Sun, 5 Jul 2026 22:59:27 -0300 Subject: [PATCH] scan --- .gitignore | 3 + ui/doocus-app/server/doocusApi.ts | 99 +++++++++++++++++++-- ui/doocus-app/src/App.vue | 56 +++++++++--- ui/doocus-app/src/store.ts | 49 ++++++++++ ui/doocus-app/vite.config.ts | 15 +++- ui/meetus-app/server/meetusApi.ts | 19 +++- ui/meetus-app/src/components/RunPicker.vue | 33 ++++++- ui/meetus-app/src/store.ts | Bin 8504 -> 9116 bytes 8 files changed, 245 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 2581b31..4ce00a6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ output/* # doocus document extraction output (textified docs + metadata; local-only) docs-output/ +# doocus managed collections (UI-scanned sources → one subfolder per source) +doocus-data/ + # Python cache __pycache__ *.pyc diff --git a/ui/doocus-app/server/doocusApi.ts b/ui/doocus-app/server/doocusApi.ts index 2e225f3..f01e0ec 100644 --- a/ui/doocus-app/server/doocusApi.ts +++ b/ui/doocus-app/server/doocusApi.ts @@ -19,12 +19,15 @@ */ import type { Plugin } from 'vite' import type { IncomingMessage, ServerResponse } from 'node:http' +import { spawn } from 'node:child_process' import fs from 'node:fs' import fsp from 'node:fs/promises' import path from 'node:path' interface Options { - outputDirs: string[] + outputDirs: string[] // explicit output dirs (env) + outputsRoot: string // managed root: UI-scanned sources land here + repoRoot: string // where process_tree.py lives (run via uv) } interface Collection { @@ -79,8 +82,37 @@ function buildCollections(dirs: string[]): Collection[] { } export function doocusApi(opts: Options): Plugin { - const collections = buildCollections(opts.outputDirs) - const byId = new Map(collections.map((c) => [c.id, c])) + /** Collections discovered under the managed root (one subdir per source). */ + function discoverInRoot(): Collection[] { + let subs: fs.Dirent[] = [] + try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] } + return subs + .filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json'))) + .map((d) => ({ + id: d.name, name: d.name, + dir: path.join(opts.outputsRoot, d.name), + indexPath: path.join(opts.outputsRoot, d.name, 'index.json'), + })) + } + + /** Current collections = explicit env dirs + managed-root subdirs (deduped, + * recomputed each request so UI scans appear without a restart). */ + function currentCollections(): Collection[] { + const out: Collection[] = [] + const seen = new Set() + for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) { + if (out.some((o) => o.dir === c.dir)) continue + let id = c.id, n = 1 + while (seen.has(id)) { n++; id = `${c.id}-${n}` } + seen.add(id) + out.push({ ...c, id }) + } + return out + } + + function collectionById(id: string): Collection | undefined { + return currentCollections().find((c) => c.id === id) + } function readIndexFor(c: Collection): Index | null { try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null } @@ -91,7 +123,7 @@ export function doocusApi(opts: Options): Plugin { const i = p.indexOf('/') const id = i < 0 ? p : p.slice(0, i) const rel = i < 0 ? '' : p.slice(i + 1) - const col = byId.get(id) + const col = collectionById(id) return col ? { col, rel } : null } @@ -138,8 +170,9 @@ export function doocusApi(opts: Options): Plugin { return { name: 'doocus-api', configureServer(server) { + const cols = currentCollections() server.config.logger.info( - `[doocus-api] ${collections.length} collection(s): ${collections.map((c) => `${c.id} → ${c.dir}`).join(', ')}`) + `[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`) server.middlewares.use('/api', (req, res, next) => { const url = new URL(req.url ?? '/', 'http://localhost') const parts = url.pathname.split('/').filter(Boolean) @@ -153,6 +186,8 @@ export function doocusApi(opts: Options): Plugin { async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise { if (parts[0] === 'collections') { listCollections(res); return true } + if (parts[0] === 'scan' && req.method === 'POST') { await runScan(req, res, false); return true } + if (parts[0] === 'rescan' && req.method === 'POST') { await runScan(req, res, true); return true } if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true } if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true } if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true } @@ -168,16 +203,58 @@ export function doocusApi(opts: Options): Plugin { } function listCollections(res: ServerResponse): void { - const out = collections.map((c) => { + const out = currentCollections().map((c) => { const idx = readIndexFor(c) return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 } }) sendJson(res, 200, { collections: out }) } + /** Run process_tree.py over a source folder (new scan) or an existing + * collection's recorded source (rescan), writing to the managed root. */ + async function runScan(req: IncomingMessage, res: ServerResponse, rerun: boolean): Promise { + let body: { source?: string; id?: string } + try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return } + + let source: string + let dir: string + if (rerun) { + const col = body.id ? collectionById(body.id) : undefined + const idx = col ? readIndexFor(col) : null + if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return } + source = idx.root + dir = col.dir + } else { + source = (body.source ?? '').trim() + if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return } + const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source' + dir = path.join(opts.outputsRoot, slug) + fs.mkdirSync(opts.outputsRoot, { recursive: true }) + } + + try { + await runProcessTree(source, dir) + sendJson(res, 200, { ok: true, id: path.basename(dir) }) + } catch (e) { + sendJson(res, 500, { error: String(e) }) + } + } + + /** Spawn `uv run python process_tree.py --output ` at repoRoot. */ + function runProcessTree(source: string, dir: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir], + { cwd: opts.repoRoot }) + let err = '' + child.stderr.on('data', (d) => { err += d }) + child.on('error', reject) + child.on('close', (code) => code === 0 ? resolve() : reject(new Error(err || `exit ${code}`))) + }) + } + function getTree(res: ServerResponse, activeCsv: string | null): void { const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null - const cols = collections.filter((c) => !active || active.has(c.id)) + const cols = currentCollections().filter((c) => !active || active.has(c.id)) const files: Array = [] const counts: Record = { extracted: 0, meeting: 0, link: 0, total: 0 } for (const c of cols) { @@ -190,7 +267,7 @@ export function doocusApi(opts: Options): Plugin { } } sendJson(res, 200, { - collections: collections.map((c) => ({ id: c.id, name: c.name })), + collections: currentCollections().map((c) => ({ id: c.id, name: c.name })), files, counts, }) @@ -363,7 +440,7 @@ export function doocusApi(opts: Options): Plugin { async function search(res: ServerResponse, query: string, activeCsv: string | null): Promise { const qq = query.trim().toLowerCase() const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null - const cols = collections.filter((c) => !active || active.has(c.id)) + const cols = currentCollections().filter((c) => !active || active.has(c.id)) let matches: string[] = [] let bytes = 0 if (qq) { @@ -393,6 +470,10 @@ function isFile(p: string): boolean { try { return fs.statSync(p).isFile() } catch { return false } } +function isDir(p: string): boolean { + try { return fs.statSync(p).isDirectory() } catch { return false } +} + function readBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { let data = '' diff --git a/ui/doocus-app/src/App.vue b/ui/doocus-app/src/App.vue index e1dae63..9e53f5c 100644 --- a/ui/doocus-app/src/App.vue +++ b/ui/doocus-app/src/App.vue @@ -6,11 +6,19 @@ import FileTree from '@/components/FileTree.vue' import DocDetail from '@/components/DocDetail.vue' import ExportBar from '@/components/ExportBar.vue' import { ref } from 'vue' -import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection } from '@/store' +import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection, scanFolder, rescanCollection } from '@/store' onMounted(loadTree) const showCols = ref(false) +const scanPath = ref('') + +async function doScan() { + const p = scanPath.value + if (!p.trim()) return + await scanFolder(p) + if (!store.scanError) scanPath.value = '' +} const detailTitle = computed(() => store.detail?.node.mode === 'meeting' ? 'Meeting' : 'File') @@ -32,18 +40,35 @@ function onSearch(e: Event) { :title="`Sources — ${store.activeCollections.size}/${store.collections.length} active`" @click="showCols = !showCols" >⧉ -
-
Output sources
-
@@ -125,14 +150,19 @@ function onSearch(e: Event) { padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35); } .col-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); } +.col-row { display: flex; align-items: center; gap: var(--space-1); } .col-menu label { - display: flex; align-items: center; gap: var(--space-2); - padding: var(--space-1); border-radius: 3px; cursor: pointer; + flex: 1; display: flex; align-items: center; gap: var(--space-2); + padding: var(--space-1); border-radius: 3px; cursor: pointer; min-width: 0; } .col-menu label:hover { background: var(--surface-2); } .col-menu label.off { opacity: 0.5; } .col-name { flex: 1; overflow: hidden; text-overflow: ellipsis; } +.rescan { padding: 0 var(--space-2); background: transparent; border: 0; color: var(--text-secondary); cursor: pointer; } .dim { color: var(--text-secondary); font-size: var(--font-size-sm); } +.col-scan { display: flex; gap: var(--space-1); margin-top: var(--space-2); border-top: 1px solid var(--surface-2); padding-top: var(--space-2); } +.scan-input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); background: var(--surface-2); border: var(--panel-border); border-radius: var(--panel-radius); color: var(--text-primary); } +.scan-err { color: var(--status-error, #e0a030); font-size: var(--font-size-sm); margin: var(--space-1) 0 0; } .search { flex: 1; display: flex; align-items: center; justify-content: center; gap: var(--space-2); } .search-input { width: 100%; max-width: 520px; diff --git a/ui/doocus-app/src/store.ts b/ui/doocus-app/src/store.ts index ccfece1..03cd5f7 100644 --- a/ui/doocus-app/src/store.ts +++ b/ui/doocus-app/src/store.ts @@ -80,6 +80,9 @@ interface State { matches: Set | null // null = not searching; else the matched paths searching: boolean searchStats: { count: number; bytes: number } + + scanning: boolean + scanError: string } export const store = reactive({ @@ -102,6 +105,9 @@ export const store = reactive({ matches: null, searching: false, searchStats: { count: 0, bytes: 0 }, + + scanning: false, + scanError: '', }) let searchTimer: ReturnType | null = null @@ -216,6 +222,49 @@ export async function toggleCollection(id: string): Promise { if (store.query) runSearch(store.query) } +/** Scan a source folder (runs process_tree.py server-side) → new collection. */ +export async function scanFolder(source: string): Promise { + if (!source.trim()) return + store.scanning = true + store.scanError = '' + try { + const res = await fetch('/api/scan', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ source: source.trim() }), + }) + const data = await res.json() + if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return } + await loadCollections() + if (data.id) store.activeCollections.add(data.id) // show the new source + await loadTree() + } catch (e) { + store.scanError = String(e) + } finally { + store.scanning = false + } +} + +/** Re-run the scan for an existing collection (its recorded source). */ +export async function rescanCollection(id: string): Promise { + store.scanning = true + store.scanError = '' + try { + const res = await fetch('/api/rescan', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id }), + }) + const data = await res.json() + if (!res.ok) { store.scanError = data.error ?? `HTTP ${res.status}`; return } + await loadCollections() + await loadTree() + if (store.query) runSearch(store.query) + } catch (e) { + store.scanError = String(e) + } finally { + store.scanning = false + } +} + export async function select(path: string): Promise { store.selectedPath = path store.detailCollapsed = false // opening a file expands the viewer diff --git a/ui/doocus-app/vite.config.ts b/ui/doocus-app/vite.config.ts index 83b6c1a..dc751d4 100644 --- a/ui/doocus-app/vite.config.ts +++ b/ui/doocus-app/vite.config.ts @@ -3,16 +3,23 @@ import vue from '@vitejs/plugin-vue' import { fileURLToPath } from 'node:url' import { doocusApi } from './server/doocusApi' -// Output directories produced by the doocus pipeline. One or many, so several -// sources can be joined in the tree/search. DOOCUS_OUTPUTS is comma-separated; -// DOOCUS_OUTPUT (single) is still honored for back-compat. +// Explicit output dirs (env). DOOCUS_OUTPUTS is comma-separated; DOOCUS_OUTPUT +// (single) is honored too. Defaults to the repo's docs-output for back-compat. const outputEnv = process.env.DOOCUS_OUTPUTS ?? process.env.DOOCUS_OUTPUT const outputDirs = (outputEnv ? outputEnv.split(',').map((s) => s.trim()).filter(Boolean) : [fileURLToPath(new URL('../../docs-output', import.meta.url))]) +// Managed collections root (gitignored): UI-scanned sources land here, one +// subfolder per source, discovered automatically. Override with DOOCUS_DATA. +const outputsRoot = process.env.DOOCUS_DATA + ?? fileURLToPath(new URL('../../doocus-data', import.meta.url)) + +// Repo root — where process_tree.py lives (run via `uv run` for the scan). +const repoRoot = fileURLToPath(new URL('../../', import.meta.url)) + export default defineConfig({ - plugins: [vue(), doocusApi({ outputDirs })], + plugins: [vue(), doocusApi({ outputDirs, outputsRoot, repoRoot })], resolve: { alias: { '@framework': fileURLToPath(new URL('../framework/src', import.meta.url)), diff --git a/ui/meetus-app/server/meetusApi.ts b/ui/meetus-app/server/meetusApi.ts index ce0e115..f1820cc 100644 --- a/ui/meetus-app/server/meetusApi.ts +++ b/ui/meetus-app/server/meetusApi.ts @@ -45,7 +45,8 @@ const VIDEO_MIME: Record = { } export function meetusApi(opts: Options): Plugin { - const outputDir = path.resolve(opts.outputDir) + // Mutable so the UI can re-point the scan at another folder at runtime. + let outputDir = path.resolve(opts.outputDir) const videoDir = opts.videoDir ? path.resolve(opts.videoDir) : null /** Locate a run's source video. The manifest records an absolute path from @@ -139,6 +140,22 @@ export function meetusApi(opts: Options): Plugin { parts: string[], logger: { warn: (m: string) => void }, ): Promise { + // GET/PUT the folder scanned (recursively) for meeting runs. + if (parts[0] === 'output') { + if (req.method === 'GET') { sendJson(res, 200, { dir: outputDir }); return true } + if (req.method === 'PUT') { + let body: { dir?: string } + try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return true } + const dir = (body.dir ?? '').trim() + try { + if (!dir || !fs.statSync(dir).isDirectory()) throw new Error('not a directory') + } catch { sendJson(res, 400, { error: `not a directory: ${dir}` }); return true } + outputDir = path.resolve(dir) + sendJson(res, 200, { dir: outputDir }) + return true + } + } + if (parts[0] !== 'runs') return false // GET /api/runs diff --git a/ui/meetus-app/src/components/RunPicker.vue b/ui/meetus-app/src/components/RunPicker.vue index 718d35c..b17a20a 100644 --- a/ui/meetus-app/src/components/RunPicker.vue +++ b/ui/meetus-app/src/components/RunPicker.vue @@ -1,15 +1,23 @@ @@ -34,7 +52,18 @@ function onChange(e: Event) { display: flex; align-items: center; gap: var(--space-2); + position: relative; } +.folder-btn { padding: var(--space-1) var(--space-2); line-height: 1; } +.folder-pop { + position: absolute; top: 110%; left: 0; z-index: 20; min-width: 340px; + background: var(--surface-1); border: var(--panel-border); border-radius: var(--panel-radius); + padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35); +} +.pop-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); } +.pop-row { display: flex; gap: var(--space-1); } +.pop-row input { flex: 1; min-width: 0; padding: var(--space-1) var(--space-2); } +.pop-cur { font-size: var(--font-size-sm); color: var(--text-secondary); margin-top: var(--space-1); overflow: hidden; text-overflow: ellipsis; } .lbl { color: var(--text-secondary); font-size: var(--font-size-sm); diff --git a/ui/meetus-app/src/store.ts b/ui/meetus-app/src/store.ts index 8530537ccc31289a9df96cfbb619644ad84dc983..4e61b93eb6bb373e755b966d6e2c26f6566b98da 100644 GIT binary patch delta 351 zcmYL^%}T>S6opqJb|E@$0?pzs3QnRXF5Fm=f{3~iOWU<{NTxcP&XoDFln{J~+4>NQ z8=t~=aNl>(n^@>2BE_|-5`MUI2k zd&zUqJagKJ7hWt-(=g@S3i$xqNfRXyr!8g{GJ}benH5Tc;r7ninX{Kd(_H8-7^{(p zy3p7DitPB~aD$+b;Cf1<2qUc1(ho`MPj>01YUq9c9R_|G-nL6&Q#;G)d5g7rvG%ISU$jt c)ISHEtzpxE^2mP#2cK=S+-|K)`d>Kx15pxqlK=n! delta 16 XcmbQ^zQbvQg3#t7k>`Aqvy}}2Ino9|