From 8cfb4c2cbeba2fe394ec12f5aa4d236a1da163c3 Mon Sep 17 00:00:00 2001 From: Mariano Gabriel Date: Sun, 5 Jul 2026 22:37:46 -0300 Subject: [PATCH] merging --- ui/doocus-app/server/doocusApi.ts | 369 ++++++++++++++++-------------- ui/doocus-app/src/App.vue | 42 +++- ui/doocus-app/src/store.ts | 47 +++- ui/doocus-app/vite.config.ts | 17 +- ui/meetus-app/src/store.ts | Bin 8366 -> 8504 bytes 5 files changed, 289 insertions(+), 186 deletions(-) diff --git a/ui/doocus-app/server/doocusApi.ts b/ui/doocus-app/server/doocusApi.ts index 0b6f892..2e225f3 100644 --- a/ui/doocus-app/server/doocusApi.ts +++ b/ui/doocus-app/server/doocusApi.ts @@ -1,15 +1,21 @@ /** - * Vite dev middleware exposing the local doocus tree index to the app. + * Vite dev middleware exposing one or more local doocus output trees. * - * Reads `docs-output/index.json` (produced by process_tree.py) — a replica of - * the whole local drive tree — plus the extracted `.doocus/content.md` - * sidecars, and streams the ORIGINAL files by relative path (resolved against - * the index's recorded root). All reads are local; nothing leaves the machine. + * Each output dir is a "collection" (its own index.json + sidecars + source + * root). The client can turn collections on/off; file paths are composite — + * "/" — so a single path identifies both the collection + * and the file within it. All reads are local; nothing leaves the machine. * - * Routes: - * GET /api/tree the index.json tree - * GET /api/detail?path= node + extracted content + meta + inline text - * GET /api/original?path= stream the original file (native viewer / packaging) + * Routes (path params are composite, query is ?path=/?q=): + * GET /api/collections available collections + * GET /api/tree?collections=a,b merged tree of the active collections + * GET /api/detail?path= node + extracted content + meta + text + * GET /api/original?path= stream the original (Range-enabled) + * GET /api/meeting?path= meetus review data for a meeting + * GET /api/meeting/frame?path=&file= a meeting frame + * GET /api/runs/:cid (cid=c/rel) meetus-shaped run (for the embed) + * PUT /api/runs/:cid/review write the meeting's review.json + * GET /api/search?q=&collections=a,b search cached text across active cols */ import type { Plugin } from 'vite' import type { IncomingMessage, ServerResponse } from 'node:http' @@ -18,7 +24,14 @@ import fsp from 'node:fs/promises' import path from 'node:path' interface Options { - outputDir: string + outputDirs: string[] +} + +interface Collection { + id: string + name: string + dir: string + indexPath: string } interface Node { @@ -31,16 +44,16 @@ interface Node { modified: string url: string | null out?: string + title?: string + warnings?: string[] } interface Index { root: string - generatedAt: string - counts: Record files: Node[] + counts?: Record } -// Text-native files: readable as-is, so the "original" doubles as its own text. const TEXT_EXTS = new Set(['md', 'markdown', 'txt', 'json', 'yaml', 'yml', 'csv', 'html', 'htm']) const MIME: Record = { @@ -53,19 +66,36 @@ const MIME: Record = { '.mp4': 'video/mp4', } -export function doocusApi(opts: Options): Plugin { - const outputDir = path.resolve(opts.outputDir) - const indexPath = path.join(outputDir, 'index.json') +function buildCollections(dirs: string[]): Collection[] { + const seen = new Map() + return dirs.map((d) => { + const dir = path.resolve(d) + let id = path.basename(dir) || 'output' + const n = seen.get(id) ?? 0 + seen.set(id, n + 1) + if (n > 0) id = `${id}-${n + 1}` + return { id, name: id, dir, indexPath: path.join(dir, 'index.json') } + }) +} - function readIndex(): Index | null { - try { - return JSON.parse(fs.readFileSync(indexPath, 'utf-8')) - } catch { - return null - } +export function doocusApi(opts: Options): Plugin { + const collections = buildCollections(opts.outputDirs) + const byId = new Map(collections.map((c) => [c.id, c])) + + function readIndexFor(c: Collection): Index | null { + try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null } } - /** Resolve a node's original file, rejecting traversal outside the index root. */ + /** Split a composite "/" into its collection + rel path. */ + function splitPath(p: string): { col: Collection; rel: string } | null { + const i = p.indexOf('/') + const id = i < 0 ? p : p.slice(0, i) + const rel = i < 0 ? '' : p.slice(i + 1) + const col = byId.get(id) + return col ? { col, rel } : null + } + + /** Resolve a file within its collection's source root, rejecting traversal. */ function originalAbs(index: Index, rel: string): string | null { const root = path.resolve(index.root) const abs = path.resolve(root, rel) @@ -73,57 +103,43 @@ export function doocusApi(opts: Options): Plugin { return abs } - // Lazy search index over the cached small text — rebuilt when index.json changes. - let searchCache: { mtime: number; entries: Array<{ path: string; hay: string; bytes: number }> } | null = null + function meetusDir(col: Collection, rel: string): { dir: string; stem: string } { + return { dir: path.join(col.dir, rel + '.meetus'), stem: path.parse(rel).name } + } + + // Per-collection search index over cached text — rebuilt on index.json change. + const searchCache = new Map }>() - // Read full text for reliable search (local files; speed is secondary). The - // generous cap only guards against a pathologically huge file. async function readCap(p: string | null, cap = 20_000_000): Promise { if (!p) return '' try { return (await fsp.readFile(p, 'utf-8')).slice(0, cap) } catch { return '' } } - async function buildSearchIndex(): Promise { - const index = readIndex() - if (!index) { searchCache = { mtime: 0, entries: [] }; return } + async function buildSearchIndex(c: Collection): Promise { + const index = readIndexFor(c) let mtime = 0 - try { mtime = fs.statSync(indexPath).mtimeMs } catch { /* */ } + try { mtime = fs.statSync(c.indexPath).mtimeMs } catch { /* */ } const entries: Array<{ path: string; hay: string; bytes: number }> = [] - for (const f of index.files) { - let text = `${f.path} ${(f as any).title ?? ''}` + for (const f of index?.files ?? []) { + let text = `${f.path} ${f.title ?? ''}` if (f.mode === 'extracted' && f.out) { - text += ' ' + await readCap(path.join(outputDir, f.out, 'content.md')) + text += ' ' + await readCap(path.join(c.dir, f.out, 'content.md')) } else if (f.mode === 'meeting') { - const { dir, stem } = meetusDirFor(f.path) + const { dir, stem } = meetusDir(c, f.path) text += ' ' + await readCap(path.join(dir, `${stem}_enhanced.txt`)) - } else if (TEXT_EXTS.has(f.ext)) { + } else if (TEXT_EXTS.has(f.ext) && index) { text += ' ' + await readCap(originalAbs(index, f.path)) } - entries.push({ path: f.path, hay: text.toLowerCase(), bytes: f.bytes }) + entries.push({ path: `${c.id}/${f.path}`, hay: text.toLowerCase(), bytes: f.bytes }) } - searchCache = { mtime, entries } - } - - async function search(res: ServerResponse, query: string): Promise { - const q = query.trim().toLowerCase() - let curMtime = 0 - try { curMtime = fs.statSync(indexPath).mtimeMs } catch { /* */ } - if (!searchCache || searchCache.mtime !== curMtime) await buildSearchIndex() - const entries = searchCache?.entries ?? [] - // Empty query matches nothing (the UI shows the full tree when not searching). - const hits = q ? entries.filter((e) => e.hay.includes(q)) : [] - sendJson(res, 200, { - query: q, - matches: hits.map((e) => e.path), - count: hits.length, - bytes: hits.reduce((s, e) => s + e.bytes, 0), - }) + searchCache.set(c.id, { mtime, entries }) } return { name: 'doocus-api', configureServer(server) { - server.config.logger.info(`[doocus-api] serving tree from: ${indexPath}`) + server.config.logger.info( + `[doocus-api] ${collections.length} collection(s): ${collections.map((c) => `${c.id} → ${c.dir}`).join(', ')}`) server.middlewares.use('/api', (req, res, next) => { const url = new URL(req.url ?? '/', 'http://localhost') const parts = url.pathname.split('/').filter(Boolean) @@ -136,54 +152,100 @@ export function doocusApi(opts: Options): Plugin { } async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise { - const range = req.headers.range - if (parts[0] === 'tree') { - const index = readIndex() - if (!index) { sendJson(res, 404, { error: `index.json not found in ${outputDir}` }); return true } - sendJson(res, 200, index) - return true - } - - if (parts[0] === 'detail') { - await getDetail(res, q.get('path') ?? '') - return true - } - - if (parts[0] === 'original') { - await serveOriginal(res, q.get('path') ?? '', range) - return true - } - - // GET /api/meeting?path= meetus review data for a meeting - // GET /api/meeting/frame?path=&file= stream a frame from its .meetus/ - if (parts[0] === 'meeting' && parts[1] === 'frame') { - serveMeetingFrame(res, q.get('path') ?? '', q.get('file') ?? '') - return true - } - if (parts[0] === 'meeting') { - await getMeeting(res, q.get('path') ?? '') - return true - } - - if (parts[0] === 'search') { - await search(res, q.get('q') ?? '') - return true - } - - // meetus-shaped run API (id = meeting rel path), so the composed meetus - // review store works unchanged when embedded in doocus. + if (parts[0] === 'collections') { listCollections(res); return true } + if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true } + if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true } + if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true } + if (parts[0] === 'meeting' && parts[1] === 'frame') { serveMeetingFrame(res, q.get('path') ?? '', q.get('file') ?? ''); return true } + if (parts[0] === 'meeting') { await getMeeting(res, q.get('path') ?? ''); return true } + if (parts[0] === 'search') { await search(res, q.get('q') ?? '', q.get('collections')); return true } if (parts[0] === 'runs' && parts.length >= 2) { const id = decodeURIComponent(parts[1]) if (parts.length === 2 && req.method === 'GET') { await getRunShaped(res, id); return true } if (parts.length === 3 && parts[2] === 'review' && req.method === 'PUT') { await saveRunReview(req, res, id); return true } } - return false } - /** The .meetus sidecar dir + the video stem for a meeting node's rel path. */ - function meetusDirFor(rel: string): { dir: string; stem: string } { - return { dir: path.join(outputDir, rel + '.meetus'), stem: path.parse(rel).name } + function listCollections(res: ServerResponse): void { + const out = collections.map((c) => { + const idx = readIndexFor(c) + return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 } + }) + sendJson(res, 200, { collections: out }) + } + + function getTree(res: ServerResponse, activeCsv: string | null): void { + const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null + const cols = collections.filter((c) => !active || active.has(c.id)) + const files: Array = [] + const counts: Record = { extracted: 0, meeting: 0, link: 0, total: 0 } + for (const c of cols) { + const idx = readIndexFor(c) + if (!idx) continue + for (const f of idx.files) { + files.push({ ...f, collection: c.id, path: `${c.id}/${f.path}` }) + counts[f.mode] = (counts[f.mode] ?? 0) + 1 + counts.total += 1 + } + } + sendJson(res, 200, { + collections: collections.map((c) => ({ id: c.id, name: c.name })), + files, + counts, + }) + } + + async function getDetail(res: ServerResponse, composite: string): Promise { + const sp = splitPath(composite) + if (!sp) { sendJson(res, 404, { error: 'collection not found' }); return } + const idx = readIndexFor(sp.col) + const node = idx?.files.find((f) => f.path === sp.rel) + if (!idx || !node) { sendJson(res, 404, { error: 'node not found' }); return } + + let content: string | null = null + let meta: unknown = null + let text: string | null = null + if (node.mode === 'extracted' && node.out) { + try { content = await fsp.readFile(path.join(sp.col.dir, node.out, 'content.md'), 'utf-8') } catch { /* */ } + try { meta = JSON.parse(await fsp.readFile(path.join(sp.col.dir, node.out, 'meta.json'), 'utf-8')) } catch { /* */ } + } else if (node.mode === 'link' && TEXT_EXTS.has(node.ext)) { + const abs = originalAbs(idx, sp.rel) + if (abs) { try { text = await fsp.readFile(abs, 'utf-8') } catch { /* */ } } + } + + sendJson(res, 200, { + node: { ...node, collection: sp.col.id, path: composite }, + content, meta, text, + mime: MIME[path.extname(node.name).toLowerCase()] ?? 'application/octet-stream', + originalUrl: `/api/original?path=${encodeURIComponent(composite)}`, + }) + } + + async function serveOriginal(res: ServerResponse, composite: string, range?: string): Promise { + const sp = splitPath(composite) + const idx = sp ? readIndexFor(sp.col) : null + const abs = sp && idx ? originalAbs(idx, sp.rel) : null + if (!abs || !isFile(abs)) { sendJson(res, 404, { error: 'original not reachable' }); return } + const mime = MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' + const size = fs.statSync(abs).size + res.setHeader('Content-Type', mime) + res.setHeader('Accept-Ranges', 'bytes') + const m = range?.match(/bytes=(\d*)-(\d*)/) + if (m) { + const start = m[1] ? parseInt(m[1], 10) : 0 + const end = m[2] ? parseInt(m[2], 10) : size - 1 + if (start >= size || end >= size || start > end) { + res.statusCode = 416; res.setHeader('Content-Range', `bytes */${size}`); res.end(); return + } + res.statusCode = 206 + res.setHeader('Content-Range', `bytes ${start}-${end}/${size}`) + res.setHeader('Content-Length', String(end - start + 1)) + fs.createReadStream(abs, { start, end }).pipe(res) + return + } + res.setHeader('Content-Length', String(size)) + fs.createReadStream(abs).pipe(res) } interface MeetingData { @@ -192,9 +254,10 @@ export function doocusApi(opts: Options): Plugin { enhancedAvailable: boolean } - /** Read a meeting's .meetus output into the shared review shape. */ - async function meetingData(rel: string): Promise { - const { dir, stem } = meetusDirFor(rel) + async function meetingData(composite: string): Promise { + const sp = splitPath(composite) + if (!sp) return null + const { dir, stem } = meetusDir(sp.col, sp.rel) const segments: MeetingData['segments'] = [] const whisper = path.join(dir, `${stem}.json`) if (fs.existsSync(whisper)) { @@ -208,8 +271,6 @@ export function doocusApi(opts: Options): Plugin { } } catch { /* none */ } } - - // Frame → time from enhanced.txt ([MM:SS] before each `Frame: frames/…`). const enhanced = path.join(dir, `${stem}_enhanced.txt`) const times = new Map() let enhancedAvailable = false @@ -225,7 +286,6 @@ export function doocusApi(opts: Options): Plugin { } } catch { /* none */ } } - const frames: MeetingData['frames'] = [] try { const framesDir = path.join(dir, 'frames') @@ -235,36 +295,37 @@ export function doocusApi(opts: Options): Plugin { try { size = (await fsp.stat(path.join(framesDir, file))).size } catch { /* */ } frames.push({ file, - url: `/api/meeting/frame?path=${encodeURIComponent(rel)}&file=${encodeURIComponent(file)}`, + url: `/api/meeting/frame?path=${encodeURIComponent(composite)}&file=${encodeURIComponent(file)}`, time: fny ? Number(fny[1]) : times.get(file) ?? null, size, }) } } catch { /* no frames */ } - return { segments, frames, enhancedAvailable } } - async function getMeeting(res: ServerResponse, rel: string): Promise { - const { dir } = meetusDirFor(rel) - const data = await meetingData(rel) + async function getMeeting(res: ServerResponse, composite: string): Promise { + const sp = splitPath(composite) + const data = await meetingData(composite) + if (!sp || !data) { sendJson(res, 404, { error: 'meeting not found' }); return } sendJson(res, 200, { - path: rel, - hasOutput: fs.existsSync(dir), + path: composite, + hasOutput: fs.existsSync(meetusDir(sp.col, sp.rel).dir), ...data, - videoUrl: `/api/original?path=${encodeURIComponent(rel)}`, + videoUrl: `/api/original?path=${encodeURIComponent(composite)}`, }) } - /** meetus store's /api/runs/:id shape (id = meeting rel path). */ async function getRunShaped(res: ServerResponse, id: string): Promise { - const { dir } = meetusDirFor(id) + const sp = splitPath(id) const data = await meetingData(id) + if (!sp || !data) { sendJson(res, 404, { error: 'run not found' }); return } + const { dir } = meetusDir(sp.col, sp.rel) let review: unknown = null - try { review = JSON.parse(await fsp.readFile(path.join(dir, 'review.json'), 'utf-8')) } catch { /* none */ } + try { review = JSON.parse(await fsp.readFile(path.join(dir, 'review.json'), 'utf-8')) } catch { /* */ } sendJson(res, 200, { id, - manifest: { video: { name: path.basename(id) } }, + manifest: { video: { name: path.basename(sp.rel) } }, segments: data.segments, frames: data.frames, enhancedAvailable: data.enhancedAvailable, @@ -274,9 +335,10 @@ export function doocusApi(opts: Options): Plugin { }) } - /** Persist the review sidecar next to the meeting's .meetus output (atomic). */ async function saveRunReview(req: IncomingMessage, res: ServerResponse, id: string): Promise { - const { dir } = meetusDirFor(id) + const sp = splitPath(id) + if (!sp) { sendJson(res, 404, { error: 'run not found' }); return } + const { dir } = meetusDir(sp.col, sp.rel) if (!fs.existsSync(dir)) { sendJson(res, 404, { error: 'no meetus output for this meeting' }); return } let parsed: unknown try { parsed = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return } @@ -286,8 +348,10 @@ export function doocusApi(opts: Options): Plugin { sendJson(res, 200, { ok: true }) } - function serveMeetingFrame(res: ServerResponse, rel: string, file: string): void { - const { dir } = meetusDirFor(rel) + function serveMeetingFrame(res: ServerResponse, composite: string, file: string): void { + const sp = splitPath(composite) + if (!sp) { sendJson(res, 404, { error: 'not found' }); return } + const { dir } = meetusDir(sp.col, sp.rel) const safe = path.basename(file) const full = path.join(dir, 'frames', safe) if (safe !== file || !fs.existsSync(full)) { sendJson(res, 404, { error: 'frame not found' }); return } @@ -296,63 +360,24 @@ export function doocusApi(opts: Options): Plugin { fs.createReadStream(full).pipe(res) } - async function getDetail(res: ServerResponse, rel: string): Promise { - const index = readIndex() - if (!index) { sendJson(res, 404, { error: 'no index' }); return } - const node = index.files.find((f) => f.path === rel) - if (!node) { sendJson(res, 404, { error: 'node not found' }); return } - - let content: string | null = null // extracted search text (heavy formats) - let meta: unknown = null - let text: string | null = null // inline text for text-native originals - - if (node.mode === 'extracted' && node.out) { - try { content = await fsp.readFile(path.join(outputDir, node.out, 'content.md'), 'utf-8') } catch { /* */ } - try { meta = JSON.parse(await fsp.readFile(path.join(outputDir, node.out, 'meta.json'), 'utf-8')) } catch { /* */ } - } else if (node.mode === 'link' && TEXT_EXTS.has(node.ext)) { - const abs = originalAbs(index, rel) - if (abs) { try { text = await fsp.readFile(abs, 'utf-8') } catch { /* */ } } - } - - sendJson(res, 200, { - node, - content, - meta, - text, - mime: MIME[path.extname(node.name).toLowerCase()] ?? 'application/octet-stream', - originalUrl: `/api/original?path=${encodeURIComponent(rel)}`, - }) - } - - async function serveOriginal(res: ServerResponse, rel: string, range?: string): Promise { - const index = readIndex() - if (!index) { sendJson(res, 404, { error: 'no index' }); return } - const abs = originalAbs(index, rel) - if (!abs || !isFile(abs)) { sendJson(res, 404, { error: 'original not reachable' }); return } - const mime = MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' - const size = fs.statSync(abs).size - res.setHeader('Content-Type', mime) - res.setHeader('Accept-Ranges', 'bytes') - - // Range support — needed so large meeting videos can seek/stream. - const m = range?.match(/bytes=(\d*)-(\d*)/) - if (m) { - const start = m[1] ? parseInt(m[1], 10) : 0 - const end = m[2] ? parseInt(m[2], 10) : size - 1 - if (start >= size || end >= size || start > end) { - res.statusCode = 416 - res.setHeader('Content-Range', `bytes */${size}`) - res.end() - return + async function search(res: ServerResponse, query: string, activeCsv: string | null): Promise { + const qq = query.trim().toLowerCase() + const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null + const cols = collections.filter((c) => !active || active.has(c.id)) + let matches: string[] = [] + let bytes = 0 + if (qq) { + for (const c of cols) { + let cur = 0 + try { cur = fs.statSync(c.indexPath).mtimeMs } catch { /* */ } + const cached = searchCache.get(c.id) + if (!cached || cached.mtime !== cur) await buildSearchIndex(c) + for (const e of searchCache.get(c.id)?.entries ?? []) { + if (e.hay.includes(qq)) { matches.push(e.path); bytes += e.bytes } + } } - res.statusCode = 206 - res.setHeader('Content-Range', `bytes ${start}-${end}/${size}`) - res.setHeader('Content-Length', String(end - start + 1)) - fs.createReadStream(abs, { start, end }).pipe(res) - return } - res.setHeader('Content-Length', String(size)) - fs.createReadStream(abs).pipe(res) + sendJson(res, 200, { query: qq, matches, count: matches.length, bytes }) } } diff --git a/ui/doocus-app/src/App.vue b/ui/doocus-app/src/App.vue index 53cbda2..e1dae63 100644 --- a/ui/doocus-app/src/App.vue +++ b/ui/doocus-app/src/App.vue @@ -5,10 +5,13 @@ import SplitPane from '@framework/components/SplitPane.vue' import FileTree from '@/components/FileTree.vue' import DocDetail from '@/components/DocDetail.vue' import ExportBar from '@/components/ExportBar.vue' -import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes } from '@/store' +import { ref } from 'vue' +import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection } from '@/store' onMounted(loadTree) +const showCols = ref(false) + const detailTitle = computed(() => store.detail?.node.mode === 'meeting' ? 'Meeting' : 'File') const barLabel = computed(() => store.detail?.node.name ?? 'viewer') @@ -23,6 +26,27 @@ function onSearch(e: Event) {
doocus +
+ +
+
Output sources
+ +
+
+