mariano/meetus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scan

Browse Source
This commit is contained in:
Mariano Gabriel
2026-07-05 22:59:27 -03:00
parent 8cfb4c2cbe
commit bb84558526
8 changed files with 245 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
ui/doocus-app/server/doocusApi.ts
View File

@@ -19,12 +19,15 @@
*/
import type { Plugin } from 'vite'
import type { IncomingMessage, ServerResponse } from 'node:http'
import { spawn } from 'node:child_process'
import fs from 'node:fs'
import fsp from 'node:fs/promises'
import path from 'node:path'
interface Options {
outputDirs: string[]
outputDirs: string[] // explicit output dirs (env)
outputsRoot: string // managed root: UI-scanned sources land here
repoRoot: string // where process_tree.py lives (run via uv)
}
interface Collection {
@@ -79,8 +82,37 @@ function buildCollections(dirs: string[]): Collection[] {
}
export function doocusApi(opts: Options): Plugin {
const collections = buildCollections(opts.outputDirs)
const byId = new Map(collections.map((c) => [c.id, c]))
/** Collections discovered under the managed root (one subdir per source). */
function discoverInRoot(): Collection[] {
let subs: fs.Dirent[] = []
try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] }
return subs
.filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json')))
.map((d) => ({
id: d.name, name: d.name,
dir: path.join(opts.outputsRoot, d.name),
indexPath: path.join(opts.outputsRoot, d.name, 'index.json'),
}))
}
/** Current collections = explicit env dirs + managed-root subdirs (deduped,
* recomputed each request so UI scans appear without a restart). */
function currentCollections(): Collection[] {
const out: Collection[] = []
const seen = new Set<string>()
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) {
if (out.some((o) => o.dir === c.dir)) continue
let id = c.id, n = 1
while (seen.has(id)) { n++; id = `${c.id}-${n}` }
seen.add(id)
out.push({ ...c, id })
}
return out
}
function collectionById(id: string): Collection | undefined {
return currentCollections().find((c) => c.id === id)
}
function readIndexFor(c: Collection): Index | null {
try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null }
@@ -91,7 +123,7 @@ export function doocusApi(opts: Options): Plugin {
const i = p.indexOf('/')
const id = i < 0 ? p : p.slice(0, i)
const rel = i < 0 ? '' : p.slice(i + 1)
const col = byId.get(id)
const col = collectionById(id)
return col ? { col, rel } : null
}
@@ -138,8 +170,9 @@ export function doocusApi(opts: Options): Plugin {
return {
name: 'doocus-api',
configureServer(server) {
const cols = currentCollections()
server.config.logger.info(
`[doocus-api] ${collections.length} collection(s): ${collections.map((c) => `${c.id} → ${c.dir}`).join(', ')}`)
`[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`)
server.middlewares.use('/api', (req, res, next) => {
const url = new URL(req.url ?? '/', 'http://localhost')
const parts = url.pathname.split('/').filter(Boolean)
@@ -153,6 +186,8 @@ export function doocusApi(opts: Options): Plugin {
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
if (parts[0] === 'collections') { listCollections(res); return true }
if (parts[0] === 'scan' && req.method === 'POST') { await runScan(req, res, false); return true }
if (parts[0] === 'rescan' && req.method === 'POST') { await runScan(req, res, true); return true }
if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true }
if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true }
if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true }
@@ -168,16 +203,58 @@ export function doocusApi(opts: Options): Plugin {
}
function listCollections(res: ServerResponse): void {
const out = collections.map((c) => {
const out = currentCollections().map((c) => {
const idx = readIndexFor(c)
return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 }
})
sendJson(res, 200, { collections: out })
}
/** Run process_tree.py over a source folder (new scan) or an existing
* collection's recorded source (rescan), writing to the managed root. */
async function runScan(req: IncomingMessage, res: ServerResponse, rerun: boolean): Promise<void> {
let body: { source?: string; id?: string }
try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return }
let source: string
let dir: string
if (rerun) {
const col = body.id ? collectionById(body.id) : undefined
const idx = col ? readIndexFor(col) : null
if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
source = idx.root
dir = col.dir
} else {
source = (body.source ?? '').trim()
if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
dir = path.join(opts.outputsRoot, slug)
fs.mkdirSync(opts.outputsRoot, { recursive: true })
}
try {
await runProcessTree(source, dir)
sendJson(res, 200, { ok: true, id: path.basename(dir) })
} catch (e) {
sendJson(res, 500, { error: String(e) })
}
}
/** Spawn `uv run python process_tree.py <source> --output <dir>` at repoRoot. */
function runProcessTree(source: string, dir: string): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir],
{ cwd: opts.repoRoot })
let err = ''
child.stderr.on('data', (d) => { err += d })
child.on('error', reject)
child.on('close', (code) => code === 0 ? resolve() : reject(new Error(err || `exit ${code}`)))
})
}
function getTree(res: ServerResponse, activeCsv: string | null): void {
const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
const cols = collections.filter((c) => !active || active.has(c.id))
const cols = currentCollections().filter((c) => !active || active.has(c.id))
const files: Array<Node & { collection: string }> = []
const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: 0 }
for (const c of cols) {
@@ -190,7 +267,7 @@ export function doocusApi(opts: Options): Plugin {
}
}
sendJson(res, 200, {
collections: collections.map((c) => ({ id: c.id, name: c.name })),
collections: currentCollections().map((c) => ({ id: c.id, name: c.name })),
files,
counts,
})
@@ -363,7 +440,7 @@ export function doocusApi(opts: Options): Plugin {
async function search(res: ServerResponse, query: string, activeCsv: string | null): Promise<void> {
const qq = query.trim().toLowerCase()
const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
const cols = collections.filter((c) => !active || active.has(c.id))
const cols = currentCollections().filter((c) => !active || active.has(c.id))
let matches: string[] = []
let bytes = 0
if (qq) {
@@ -393,6 +470,10 @@ function isFile(p: string): boolean {
try { return fs.statSync(p).isFile() } catch { return false }
}
function isDir(p: string): boolean {
try { return fs.statSync(p).isDirectory() } catch { return false }
}
function readBody(req: IncomingMessage): Promise<string> {
return new Promise((resolve, reject) => {
let data = ''