diff --git a/.gitignore b/.gitignore index 4ce00a6..84222a2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,10 @@ output/* # doocus document extraction output (textified docs + metadata; local-only) docs-output/ -# doocus managed collections (UI-scanned sources → one subfolder per source) +# Managed collection roots (one subfolder per source; index.json + sidecars). +# doocus-data = document collections, meetus-data = meeting collections. doocus-data/ +meetus-data/ # Python cache __pycache__ @@ -21,4 +23,4 @@ __pycache__ .venv/ local-run.sh -meetings.txt +meetings diff --git a/doocus/tree.py b/doocus/tree.py index f9e7a4f..5697736 100644 --- a/doocus/tree.py +++ b/doocus/tree.py @@ -45,7 +45,9 @@ def classify(ext: str) -> str: return "link" -def build_index(source_root, output_dir, options=None, dry_run=False) -> dict: +def build_index(source_root, output_dir, options=None, dry_run=False, only="all") -> dict: + """Index a source tree. `only` scopes the collection so outputs stay separate: + 'all' (default), 'docs' (skip meetings — no bloat), 'meetings' (only videos).""" options = options or {} root = Path(source_root).resolve() if not root.is_dir(): @@ -63,6 +65,9 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict: rel = p.relative_to(root).as_posix() ext = p.suffix.lower().lstrip(".") mode = classify(ext) + # Scope the collection: docs exclude meetings; meetings exclude the rest. + if (only == "docs" and mode == "meeting") or (only == "meetings" and mode != "meeting"): + continue counts[mode] += 1 stat = p.stat() birth = getattr(stat, "st_birthtime", None) # not on every filesystem @@ -90,6 +95,7 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict: index = { "root": str(root), + "only": only, "generatedAt": datetime.now().isoformat(), "counts": {**counts, "total": len(files)}, "files": files, diff --git a/process_tree.py b/process_tree.py index 24286ab..8f5b0fd 100644 --- a/process_tree.py +++ b/process_tree.py @@ -23,6 +23,9 @@ def main() -> int: parser.add_argument("source", help="Local drive-download root to index") parser.add_argument("--output", default="docs-output", help="Output directory for index.json + sidecars (default: docs-output)") + parser.add_argument("--only", choices=["all", "docs", "meetings"], default="all", + help="Scope the collection: 'docs' skips meetings (no bloat), " + "'meetings' indexes only videos (co-locate with .meetus). Default: all") parser.add_argument("--render", action="store_true", help="Render page/slide images where supported (needs extra deps)") parser.add_argument("--ocr", action="store_true", @@ -41,6 +44,7 @@ def main() -> int: args.output, options={"render": args.render, "ocr": args.ocr}, dry_run=args.dry_run, + only=args.only, ) except NotADirectoryError as e: print(f"ERROR: {e}", file=sys.stderr) diff --git a/ui/doocus-app/server/doocusApi.ts b/ui/doocus-app/server/doocusApi.ts index f01e0ec..c879939 100644 --- a/ui/doocus-app/server/doocusApi.ts +++ b/ui/doocus-app/server/doocusApi.ts @@ -26,7 +26,7 @@ import path from 'node:path' interface Options { outputDirs: string[] // explicit output dirs (env) - outputsRoot: string // managed root: UI-scanned sources land here + outputsRoots: string[] // managed roots (doocus-data, meetus-data): // repoRoot: string // where process_tree.py lives (run via uv) } @@ -82,17 +82,23 @@ function buildCollections(dirs: string[]): Collection[] { } export function doocusApi(opts: Options): Plugin { - /** Collections discovered under the managed root (one subdir per source). */ - function discoverInRoot(): Collection[] { - let subs: fs.Dirent[] = [] - try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] } - return subs - .filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json'))) - .map((d) => ({ - id: d.name, name: d.name, - dir: path.join(opts.outputsRoot, d.name), - indexPath: path.join(opts.outputsRoot, d.name, 'index.json'), - })) + /** Collections discovered across the managed roots (each is //). */ + function discoverInRoots(): Collection[] { + const found: Collection[] = [] + for (const root of opts.outputsRoots) { + let subs: fs.Dirent[] = [] + try { subs = fs.readdirSync(root, { withFileTypes: true }) } catch { continue } + for (const d of subs) { + if (d.isDirectory() && fs.existsSync(path.join(root, d.name, 'index.json'))) { + found.push({ + id: d.name, name: d.name, + dir: path.join(root, d.name), + indexPath: path.join(root, d.name, 'index.json'), + }) + } + } + } + return found } /** Current collections = explicit env dirs + managed-root subdirs (deduped, @@ -100,7 +106,7 @@ export function doocusApi(opts: Options): Plugin { function currentCollections(): Collection[] { const out: Collection[] = [] const seen = new Set() - for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) { + for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoots()]) { if (out.some((o) => o.dir === c.dir)) continue let id = c.id, n = 1 while (seen.has(id)) { n++; id = `${c.id}-${n}` } @@ -172,7 +178,7 @@ export function doocusApi(opts: Options): Plugin { configureServer(server) { const cols = currentCollections() server.config.logger.info( - `[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`) + `[doocus-api] ${cols.length} collection(s); managed roots: ${opts.outputsRoots.join(', ')}`) server.middlewares.use('/api', (req, res, next) => { const url = new URL(req.url ?? '/', 'http://localhost') const parts = url.pathname.split('/').filter(Boolean) @@ -205,7 +211,14 @@ export function doocusApi(opts: Options): Plugin { function listCollections(res: ServerResponse): void { const out = currentCollections().map((c) => { const idx = readIndexFor(c) - return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 } + const meetings = idx?.files.filter((f) => f.mode === 'meeting').length ?? 0 + return { + id: c.id, name: c.name, available: !!idx, + count: idx?.files.length ?? 0, + root: idx?.root ?? null, // shared root → same source + only: (idx as any)?.only ?? 'all', + meetings, + } }) sendJson(res, 200, { collections: out }) } @@ -218,32 +231,35 @@ export function doocusApi(opts: Options): Plugin { let source: string let dir: string + let only = 'docs' // UI scans create document collections (meetings come from the batch) if (rerun) { const col = body.id ? collectionById(body.id) : undefined const idx = col ? readIndexFor(col) : null if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return } source = idx.root dir = col.dir + only = (idx as any).only ?? 'all' // preserve the collection's kind on rescan } else { source = (body.source ?? '').trim() if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return } const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source' - dir = path.join(opts.outputsRoot, slug) - fs.mkdirSync(opts.outputsRoot, { recursive: true }) + dir = path.join(opts.outputsRoots[0], slug) // docs land in the first root (doocus-data) + fs.mkdirSync(opts.outputsRoots[0], { recursive: true }) } try { - await runProcessTree(source, dir) + await runProcessTree(source, dir, only) sendJson(res, 200, { ok: true, id: path.basename(dir) }) } catch (e) { sendJson(res, 500, { error: String(e) }) } } - /** Spawn `uv run python process_tree.py --output ` at repoRoot. */ - function runProcessTree(source: string, dir: string): Promise { + /** Spawn `uv run python process_tree.py --output --only `. */ + function runProcessTree(source: string, dir: string, only: string): Promise { return new Promise((resolve, reject) => { - const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir], + const child = spawn('uv', + ['run', 'python', 'process_tree.py', source, '--output', dir, '--only', only], { cwd: opts.repoRoot }) let err = '' child.stderr.on('data', (d) => { err += d }) @@ -252,20 +268,42 @@ export function doocusApi(opts: Options): Plugin { }) } + /** Higher = "richer" version of a file — the one whose sidecar resolves wins, + * so a meetus-data meeting (with .meetus) overrides a doocus copy without it. */ + function scoreNode(c: Collection, f: Node): number { + if (f.mode === 'meeting') return fs.existsSync(meetusDir(c, f.path).dir) ? 4 : 2 + if (f.mode === 'extracted') { + return f.out && fs.existsSync(path.join(c.dir, f.out, 'content.md')) ? 3 : 1.5 + } + return 1 + } + function getTree(res: ServerResponse, activeCsv: string | null): void { const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null const cols = currentCollections().filter((c) => !active || active.has(c.id)) - const files: Array = [] - const counts: Record = { extracted: 0, meeting: 0, link: 0, total: 0 } + + // Dedup across collections by shared root + rel; the resolving copy wins. + const best = new Map() for (const c of cols) { const idx = readIndexFor(c) if (!idx) continue for (const f of idx.files) { - files.push({ ...f, collection: c.id, path: `${c.id}/${f.path}` }) - counts[f.mode] = (counts[f.mode] ?? 0) + 1 - counts.total += 1 + const key = `${idx.root}${f.path}` + const score = scoreNode(c, f) + const cur = best.get(key) + if (!cur || score > cur.score) { + best.set(key, { + file: { ...f, collection: c.id, root: idx.root, rel: f.path, path: `${c.id}/${f.path}` }, + score, + }) + } } } + + const files = [...best.values()].map((v) => v.file) + const counts: Record = { extracted: 0, meeting: 0, link: 0, total: files.length } + for (const f of files) counts[f.mode] = (counts[f.mode] ?? 0) + 1 + sendJson(res, 200, { collections: currentCollections().map((c) => ({ id: c.id, name: c.name })), files, diff --git a/ui/doocus-app/src/App.vue b/ui/doocus-app/src/App.vue index 9e53f5c..4866445 100644 --- a/ui/doocus-app/src/App.vue +++ b/ui/doocus-app/src/App.vue @@ -6,7 +6,7 @@ import FileTree from '@/components/FileTree.vue' import DocDetail from '@/components/DocDetail.vue' import ExportBar from '@/components/ExportBar.vue' import { ref } from 'vue' -import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection, scanFolder, rescanCollection } from '@/store' +import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, sources, sourceActive, toggleSource, scanFolder, refreshSources } from '@/store' onMounted(loadTree) @@ -37,25 +37,26 @@ function onSearch(e: Event) {
-
Sources
-
-