attempt to use separate folder for docs and meetings
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -9,8 +9,10 @@ output/*
|
||||
# doocus document extraction output (textified docs + metadata; local-only)
|
||||
docs-output/
|
||||
|
||||
# doocus managed collections (UI-scanned sources → one subfolder per source)
|
||||
# Managed collection roots (one subfolder per source; index.json + sidecars).
|
||||
# doocus-data = document collections, meetus-data = meeting collections.
|
||||
doocus-data/
|
||||
meetus-data/
|
||||
|
||||
# Python cache
|
||||
__pycache__
|
||||
@@ -21,4 +23,4 @@ __pycache__
|
||||
.venv/
|
||||
|
||||
local-run.sh
|
||||
meetings.txt
|
||||
meetings
|
||||
|
||||
@@ -45,7 +45,9 @@ def classify(ext: str) -> str:
|
||||
return "link"
|
||||
|
||||
|
||||
def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
def build_index(source_root, output_dir, options=None, dry_run=False, only="all") -> dict:
|
||||
"""Index a source tree. `only` scopes the collection so outputs stay separate:
|
||||
'all' (default), 'docs' (skip meetings — no bloat), 'meetings' (only videos)."""
|
||||
options = options or {}
|
||||
root = Path(source_root).resolve()
|
||||
if not root.is_dir():
|
||||
@@ -63,6 +65,9 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
rel = p.relative_to(root).as_posix()
|
||||
ext = p.suffix.lower().lstrip(".")
|
||||
mode = classify(ext)
|
||||
# Scope the collection: docs exclude meetings; meetings exclude the rest.
|
||||
if (only == "docs" and mode == "meeting") or (only == "meetings" and mode != "meeting"):
|
||||
continue
|
||||
counts[mode] += 1
|
||||
stat = p.stat()
|
||||
birth = getattr(stat, "st_birthtime", None) # not on every filesystem
|
||||
@@ -90,6 +95,7 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
|
||||
index = {
|
||||
"root": str(root),
|
||||
"only": only,
|
||||
"generatedAt": datetime.now().isoformat(),
|
||||
"counts": {**counts, "total": len(files)},
|
||||
"files": files,
|
||||
|
||||
@@ -23,6 +23,9 @@ def main() -> int:
|
||||
parser.add_argument("source", help="Local drive-download root to index")
|
||||
parser.add_argument("--output", default="docs-output",
|
||||
help="Output directory for index.json + sidecars (default: docs-output)")
|
||||
parser.add_argument("--only", choices=["all", "docs", "meetings"], default="all",
|
||||
help="Scope the collection: 'docs' skips meetings (no bloat), "
|
||||
"'meetings' indexes only videos (co-locate with .meetus). Default: all")
|
||||
parser.add_argument("--render", action="store_true",
|
||||
help="Render page/slide images where supported (needs extra deps)")
|
||||
parser.add_argument("--ocr", action="store_true",
|
||||
@@ -41,6 +44,7 @@ def main() -> int:
|
||||
args.output,
|
||||
options={"render": args.render, "ocr": args.ocr},
|
||||
dry_run=args.dry_run,
|
||||
only=args.only,
|
||||
)
|
||||
except NotADirectoryError as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
|
||||
@@ -26,7 +26,7 @@ import path from 'node:path'
|
||||
|
||||
interface Options {
|
||||
outputDirs: string[] // explicit output dirs (env)
|
||||
outputsRoot: string // managed root: UI-scanned sources land here
|
||||
outputsRoots: string[] // managed roots (doocus-data, meetus-data): <root>/<source>/
|
||||
repoRoot: string // where process_tree.py lives (run via uv)
|
||||
}
|
||||
|
||||
@@ -82,17 +82,23 @@ function buildCollections(dirs: string[]): Collection[] {
|
||||
}
|
||||
|
||||
export function doocusApi(opts: Options): Plugin {
|
||||
/** Collections discovered under the managed root (one subdir per source). */
|
||||
function discoverInRoot(): Collection[] {
|
||||
let subs: fs.Dirent[] = []
|
||||
try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] }
|
||||
return subs
|
||||
.filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json')))
|
||||
.map((d) => ({
|
||||
id: d.name, name: d.name,
|
||||
dir: path.join(opts.outputsRoot, d.name),
|
||||
indexPath: path.join(opts.outputsRoot, d.name, 'index.json'),
|
||||
}))
|
||||
/** Collections discovered across the managed roots (each is <root>/<source>/). */
|
||||
function discoverInRoots(): Collection[] {
|
||||
const found: Collection[] = []
|
||||
for (const root of opts.outputsRoots) {
|
||||
let subs: fs.Dirent[] = []
|
||||
try { subs = fs.readdirSync(root, { withFileTypes: true }) } catch { continue }
|
||||
for (const d of subs) {
|
||||
if (d.isDirectory() && fs.existsSync(path.join(root, d.name, 'index.json'))) {
|
||||
found.push({
|
||||
id: d.name, name: d.name,
|
||||
dir: path.join(root, d.name),
|
||||
indexPath: path.join(root, d.name, 'index.json'),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
/** Current collections = explicit env dirs + managed-root subdirs (deduped,
|
||||
@@ -100,7 +106,7 @@ export function doocusApi(opts: Options): Plugin {
|
||||
function currentCollections(): Collection[] {
|
||||
const out: Collection[] = []
|
||||
const seen = new Set<string>()
|
||||
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) {
|
||||
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoots()]) {
|
||||
if (out.some((o) => o.dir === c.dir)) continue
|
||||
let id = c.id, n = 1
|
||||
while (seen.has(id)) { n++; id = `${c.id}-${n}` }
|
||||
@@ -172,7 +178,7 @@ export function doocusApi(opts: Options): Plugin {
|
||||
configureServer(server) {
|
||||
const cols = currentCollections()
|
||||
server.config.logger.info(
|
||||
`[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`)
|
||||
`[doocus-api] ${cols.length} collection(s); managed roots: ${opts.outputsRoots.join(', ')}`)
|
||||
server.middlewares.use('/api', (req, res, next) => {
|
||||
const url = new URL(req.url ?? '/', 'http://localhost')
|
||||
const parts = url.pathname.split('/').filter(Boolean)
|
||||
@@ -205,7 +211,14 @@ export function doocusApi(opts: Options): Plugin {
|
||||
function listCollections(res: ServerResponse): void {
|
||||
const out = currentCollections().map((c) => {
|
||||
const idx = readIndexFor(c)
|
||||
return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 }
|
||||
const meetings = idx?.files.filter((f) => f.mode === 'meeting').length ?? 0
|
||||
return {
|
||||
id: c.id, name: c.name, available: !!idx,
|
||||
count: idx?.files.length ?? 0,
|
||||
root: idx?.root ?? null, // shared root → same source
|
||||
only: (idx as any)?.only ?? 'all',
|
||||
meetings,
|
||||
}
|
||||
})
|
||||
sendJson(res, 200, { collections: out })
|
||||
}
|
||||
@@ -218,32 +231,35 @@ export function doocusApi(opts: Options): Plugin {
|
||||
|
||||
let source: string
|
||||
let dir: string
|
||||
let only = 'docs' // UI scans create document collections (meetings come from the batch)
|
||||
if (rerun) {
|
||||
const col = body.id ? collectionById(body.id) : undefined
|
||||
const idx = col ? readIndexFor(col) : null
|
||||
if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
|
||||
source = idx.root
|
||||
dir = col.dir
|
||||
only = (idx as any).only ?? 'all' // preserve the collection's kind on rescan
|
||||
} else {
|
||||
source = (body.source ?? '').trim()
|
||||
if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
|
||||
const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
|
||||
dir = path.join(opts.outputsRoot, slug)
|
||||
fs.mkdirSync(opts.outputsRoot, { recursive: true })
|
||||
dir = path.join(opts.outputsRoots[0], slug) // docs land in the first root (doocus-data)
|
||||
fs.mkdirSync(opts.outputsRoots[0], { recursive: true })
|
||||
}
|
||||
|
||||
try {
|
||||
await runProcessTree(source, dir)
|
||||
await runProcessTree(source, dir, only)
|
||||
sendJson(res, 200, { ok: true, id: path.basename(dir) })
|
||||
} catch (e) {
|
||||
sendJson(res, 500, { error: String(e) })
|
||||
}
|
||||
}
|
||||
|
||||
/** Spawn `uv run python process_tree.py <source> --output <dir>` at repoRoot. */
|
||||
function runProcessTree(source: string, dir: string): Promise<void> {
|
||||
/** Spawn `uv run python process_tree.py <source> --output <dir> --only <only>`. */
|
||||
function runProcessTree(source: string, dir: string, only: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir],
|
||||
const child = spawn('uv',
|
||||
['run', 'python', 'process_tree.py', source, '--output', dir, '--only', only],
|
||||
{ cwd: opts.repoRoot })
|
||||
let err = ''
|
||||
child.stderr.on('data', (d) => { err += d })
|
||||
@@ -252,20 +268,42 @@ export function doocusApi(opts: Options): Plugin {
|
||||
})
|
||||
}
|
||||
|
||||
/** Higher = "richer" version of a file — the one whose sidecar resolves wins,
|
||||
* so a meetus-data meeting (with .meetus) overrides a doocus copy without it. */
|
||||
function scoreNode(c: Collection, f: Node): number {
|
||||
if (f.mode === 'meeting') return fs.existsSync(meetusDir(c, f.path).dir) ? 4 : 2
|
||||
if (f.mode === 'extracted') {
|
||||
return f.out && fs.existsSync(path.join(c.dir, f.out, 'content.md')) ? 3 : 1.5
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
function getTree(res: ServerResponse, activeCsv: string | null): void {
|
||||
const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
|
||||
const cols = currentCollections().filter((c) => !active || active.has(c.id))
|
||||
const files: Array<Node & { collection: string }> = []
|
||||
const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: 0 }
|
||||
|
||||
// Dedup across collections by shared root + rel; the resolving copy wins.
|
||||
const best = new Map<string, { file: Node & { collection: string; root: string; rel: string }; score: number }>()
|
||||
for (const c of cols) {
|
||||
const idx = readIndexFor(c)
|
||||
if (!idx) continue
|
||||
for (const f of idx.files) {
|
||||
files.push({ ...f, collection: c.id, path: `${c.id}/${f.path}` })
|
||||
counts[f.mode] = (counts[f.mode] ?? 0) + 1
|
||||
counts.total += 1
|
||||
const key = `${idx.root} | ||||