379 lines
15 KiB
TypeScript
379 lines
15 KiB
TypeScript
/**
|
|
* Vite dev middleware exposing the local doocus tree index to the app.
|
|
*
|
|
* Reads `docs-output/index.json` (produced by process_tree.py) — a replica of
|
|
* the whole local drive tree — plus the extracted `<file>.doocus/content.md`
|
|
* sidecars, and streams the ORIGINAL files by relative path (resolved against
|
|
* the index's recorded root). All reads are local; nothing leaves the machine.
|
|
*
|
|
* Routes:
|
|
* GET /api/tree the index.json tree
|
|
* GET /api/detail?path=<rel> node + extracted content + meta + inline text
|
|
* GET /api/original?path=<rel> stream the original file (native viewer / packaging)
|
|
*/
|
|
import type { Plugin } from 'vite'
|
|
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
import fs from 'node:fs'
|
|
import fsp from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
|
|
interface Options {
|
|
outputDir: string
|
|
}
|
|
|
|
interface Node {
|
|
path: string
|
|
name: string
|
|
ext: string
|
|
family: string
|
|
mode: 'extracted' | 'meeting' | 'link'
|
|
bytes: number
|
|
modified: string
|
|
url: string | null
|
|
out?: string
|
|
}
|
|
|
|
interface Index {
|
|
root: string
|
|
generatedAt: string
|
|
counts: Record<string, number>
|
|
files: Node[]
|
|
}
|
|
|
|
// Text-native files: readable as-is, so the "original" doubles as its own text.
|
|
const TEXT_EXTS = new Set(['md', 'markdown', 'txt', 'json', 'yaml', 'yml', 'csv', 'html', 'htm'])
|
|
|
|
const MIME: Record<string, string> = {
|
|
'.pdf': 'application/pdf', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
|
'.html': 'text/html', '.htm': 'text/html', '.md': 'text/markdown', '.txt': 'text/plain',
|
|
'.csv': 'text/csv', '.json': 'application/json', '.yaml': 'text/yaml', '.yml': 'text/yaml',
|
|
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
'.mp4': 'video/mp4',
|
|
}
|
|
|
|
export function doocusApi(opts: Options): Plugin {
|
|
const outputDir = path.resolve(opts.outputDir)
|
|
const indexPath = path.join(outputDir, 'index.json')
|
|
|
|
function readIndex(): Index | null {
|
|
try {
|
|
return JSON.parse(fs.readFileSync(indexPath, 'utf-8'))
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
/** Resolve a node's original file, rejecting traversal outside the index root. */
|
|
function originalAbs(index: Index, rel: string): string | null {
|
|
const root = path.resolve(index.root)
|
|
const abs = path.resolve(root, rel)
|
|
if (abs !== root && !abs.startsWith(root + path.sep)) return null
|
|
return abs
|
|
}
|
|
|
|
// Lazy search index over the cached small text — rebuilt when index.json changes.
|
|
let searchCache: { mtime: number; entries: Array<{ path: string; hay: string; bytes: number }> } | null = null
|
|
|
|
// Read full text for reliable search (local files; speed is secondary). The
|
|
// generous cap only guards against a pathologically huge file.
|
|
async function readCap(p: string | null, cap = 20_000_000): Promise<string> {
|
|
if (!p) return ''
|
|
try { return (await fsp.readFile(p, 'utf-8')).slice(0, cap) } catch { return '' }
|
|
}
|
|
|
|
async function buildSearchIndex(): Promise<void> {
|
|
const index = readIndex()
|
|
if (!index) { searchCache = { mtime: 0, entries: [] }; return }
|
|
let mtime = 0
|
|
try { mtime = fs.statSync(indexPath).mtimeMs } catch { /* */ }
|
|
const entries: Array<{ path: string; hay: string; bytes: number }> = []
|
|
for (const f of index.files) {
|
|
let text = `${f.path} ${(f as any).title ?? ''}`
|
|
if (f.mode === 'extracted' && f.out) {
|
|
text += ' ' + await readCap(path.join(outputDir, f.out, 'content.md'))
|
|
} else if (f.mode === 'meeting') {
|
|
const { dir, stem } = meetusDirFor(f.path)
|
|
text += ' ' + await readCap(path.join(dir, `${stem}_enhanced.txt`))
|
|
} else if (TEXT_EXTS.has(f.ext)) {
|
|
text += ' ' + await readCap(originalAbs(index, f.path))
|
|
}
|
|
entries.push({ path: f.path, hay: text.toLowerCase(), bytes: f.bytes })
|
|
}
|
|
searchCache = { mtime, entries }
|
|
}
|
|
|
|
async function search(res: ServerResponse, query: string): Promise<void> {
|
|
const q = query.trim().toLowerCase()
|
|
let curMtime = 0
|
|
try { curMtime = fs.statSync(indexPath).mtimeMs } catch { /* */ }
|
|
if (!searchCache || searchCache.mtime !== curMtime) await buildSearchIndex()
|
|
const entries = searchCache?.entries ?? []
|
|
// Empty query matches nothing (the UI shows the full tree when not searching).
|
|
const hits = q ? entries.filter((e) => e.hay.includes(q)) : []
|
|
sendJson(res, 200, {
|
|
query: q,
|
|
matches: hits.map((e) => e.path),
|
|
count: hits.length,
|
|
bytes: hits.reduce((s, e) => s + e.bytes, 0),
|
|
})
|
|
}
|
|
|
|
return {
|
|
name: 'doocus-api',
|
|
configureServer(server) {
|
|
server.config.logger.info(`[doocus-api] serving tree from: ${indexPath}`)
|
|
server.middlewares.use('/api', (req, res, next) => {
|
|
const url = new URL(req.url ?? '/', 'http://localhost')
|
|
const parts = url.pathname.split('/').filter(Boolean)
|
|
handle(req, res, parts, url.searchParams).catch((err) => {
|
|
server.config.logger.error(`[doocus-api] ${String(err)}`)
|
|
sendJson(res, 500, { error: String(err) })
|
|
}).then((handled) => { if (!handled) next() })
|
|
})
|
|
},
|
|
}
|
|
|
|
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
|
|
const range = req.headers.range
|
|
if (parts[0] === 'tree') {
|
|
const index = readIndex()
|
|
if (!index) { sendJson(res, 404, { error: `index.json not found in ${outputDir}` }); return true }
|
|
sendJson(res, 200, index)
|
|
return true
|
|
}
|
|
|
|
if (parts[0] === 'detail') {
|
|
await getDetail(res, q.get('path') ?? '')
|
|
return true
|
|
}
|
|
|
|
if (parts[0] === 'original') {
|
|
await serveOriginal(res, q.get('path') ?? '', range)
|
|
return true
|
|
}
|
|
|
|
// GET /api/meeting?path=<rel> meetus review data for a meeting
|
|
// GET /api/meeting/frame?path=<rel>&file=<f> stream a frame from its .meetus/
|
|
if (parts[0] === 'meeting' && parts[1] === 'frame') {
|
|
serveMeetingFrame(res, q.get('path') ?? '', q.get('file') ?? '')
|
|
return true
|
|
}
|
|
if (parts[0] === 'meeting') {
|
|
await getMeeting(res, q.get('path') ?? '')
|
|
return true
|
|
}
|
|
|
|
if (parts[0] === 'search') {
|
|
await search(res, q.get('q') ?? '')
|
|
return true
|
|
}
|
|
|
|
// meetus-shaped run API (id = meeting rel path), so the composed meetus
|
|
// review store works unchanged when embedded in doocus.
|
|
if (parts[0] === 'runs' && parts.length >= 2) {
|
|
const id = decodeURIComponent(parts[1])
|
|
if (parts.length === 2 && req.method === 'GET') { await getRunShaped(res, id); return true }
|
|
if (parts.length === 3 && parts[2] === 'review' && req.method === 'PUT') { await saveRunReview(req, res, id); return true }
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
/** The .meetus sidecar dir + the video stem for a meeting node's rel path. */
|
|
function meetusDirFor(rel: string): { dir: string; stem: string } {
|
|
return { dir: path.join(outputDir, rel + '.meetus'), stem: path.parse(rel).name }
|
|
}
|
|
|
|
interface MeetingData {
|
|
segments: Array<{ start: number; end: number; text: string; speaker: string | null }>
|
|
frames: Array<{ file: string; url: string; time: number | null; size: number }>
|
|
enhancedAvailable: boolean
|
|
}
|
|
|
|
/** Read a meeting's .meetus output into the shared review shape. */
|
|
async function meetingData(rel: string): Promise<MeetingData> {
|
|
const { dir, stem } = meetusDirFor(rel)
|
|
const segments: MeetingData['segments'] = []
|
|
const whisper = path.join(dir, `${stem}.json`)
|
|
if (fs.existsSync(whisper)) {
|
|
try {
|
|
const data = JSON.parse(await fsp.readFile(whisper, 'utf-8'))
|
|
for (const s of (Array.isArray(data) ? data : data.segments ?? [])) {
|
|
segments.push({
|
|
start: Number(s.start ?? 0), end: Number(s.end ?? s.start ?? 0),
|
|
text: String(s.text ?? '').trim(), speaker: s.speaker ?? null,
|
|
})
|
|
}
|
|
} catch { /* none */ }
|
|
}
|
|
|
|
// Frame → time from enhanced.txt ([MM:SS] before each `Frame: frames/…`).
|
|
const enhanced = path.join(dir, `${stem}_enhanced.txt`)
|
|
const times = new Map<string, number>()
|
|
let enhancedAvailable = false
|
|
if (fs.existsSync(enhanced)) {
|
|
try {
|
|
enhancedAvailable = true
|
|
let last: number | null = null
|
|
for (const line of (await fsp.readFile(enhanced, 'utf-8')).split('\n')) {
|
|
const ts = line.match(/^\[(\d+):(\d+)\]/)
|
|
if (ts) last = Number(ts[1]) * 60 + Number(ts[2])
|
|
const fm = line.match(/Frame:\s*\S*?([^/\\\s]+\.jpg)/i)
|
|
if (fm && last != null) times.set(fm[1], last)
|
|
}
|
|
} catch { /* none */ }
|
|
}
|
|
|
|
const frames: MeetingData['frames'] = []
|
|
try {
|
|
const framesDir = path.join(dir, 'frames')
|
|
for (const file of (await fsp.readdir(framesDir)).filter((f) => f.toLowerCase().endsWith('.jpg')).sort()) {
|
|
const fny = file.match(/_(\d+(?:\.\d+)?)s\.jpg$/i)
|
|
let size = 0
|
|
try { size = (await fsp.stat(path.join(framesDir, file))).size } catch { /* */ }
|
|
frames.push({
|
|
file,
|
|
url: `/api/meeting/frame?path=${encodeURIComponent(rel)}&file=${encodeURIComponent(file)}`,
|
|
time: fny ? Number(fny[1]) : times.get(file) ?? null,
|
|
size,
|
|
})
|
|
}
|
|
} catch { /* no frames */ }
|
|
|
|
return { segments, frames, enhancedAvailable }
|
|
}
|
|
|
|
async function getMeeting(res: ServerResponse, rel: string): Promise<void> {
|
|
const { dir } = meetusDirFor(rel)
|
|
const data = await meetingData(rel)
|
|
sendJson(res, 200, {
|
|
path: rel,
|
|
hasOutput: fs.existsSync(dir),
|
|
...data,
|
|
videoUrl: `/api/original?path=${encodeURIComponent(rel)}`,
|
|
})
|
|
}
|
|
|
|
/** meetus store's /api/runs/:id shape (id = meeting rel path). */
|
|
async function getRunShaped(res: ServerResponse, id: string): Promise<void> {
|
|
const { dir } = meetusDirFor(id)
|
|
const data = await meetingData(id)
|
|
let review: unknown = null
|
|
try { review = JSON.parse(await fsp.readFile(path.join(dir, 'review.json'), 'utf-8')) } catch { /* none */ }
|
|
sendJson(res, 200, {
|
|
id,
|
|
manifest: { video: { name: path.basename(id) } },
|
|
segments: data.segments,
|
|
frames: data.frames,
|
|
enhancedAvailable: data.enhancedAvailable,
|
|
hasVideo: true,
|
|
videoUrl: `/api/original?path=${encodeURIComponent(id)}`,
|
|
review,
|
|
})
|
|
}
|
|
|
|
/** Persist the review sidecar next to the meeting's .meetus output (atomic). */
|
|
async function saveRunReview(req: IncomingMessage, res: ServerResponse, id: string): Promise<void> {
|
|
const { dir } = meetusDirFor(id)
|
|
if (!fs.existsSync(dir)) { sendJson(res, 404, { error: 'no meetus output for this meeting' }); return }
|
|
let parsed: unknown
|
|
try { parsed = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return }
|
|
const tmp = path.join(dir, `.review.${process.pid}.tmp`)
|
|
await fsp.writeFile(tmp, JSON.stringify(parsed, null, 2), 'utf-8')
|
|
await fsp.rename(tmp, path.join(dir, 'review.json'))
|
|
sendJson(res, 200, { ok: true })
|
|
}
|
|
|
|
function serveMeetingFrame(res: ServerResponse, rel: string, file: string): void {
|
|
const { dir } = meetusDirFor(rel)
|
|
const safe = path.basename(file)
|
|
const full = path.join(dir, 'frames', safe)
|
|
if (safe !== file || !fs.existsSync(full)) { sendJson(res, 404, { error: 'frame not found' }); return }
|
|
res.setHeader('Content-Type', 'image/jpeg')
|
|
res.setHeader('Cache-Control', 'no-cache')
|
|
fs.createReadStream(full).pipe(res)
|
|
}
|
|
|
|
async function getDetail(res: ServerResponse, rel: string): Promise<void> {
|
|
const index = readIndex()
|
|
if (!index) { sendJson(res, 404, { error: 'no index' }); return }
|
|
const node = index.files.find((f) => f.path === rel)
|
|
if (!node) { sendJson(res, 404, { error: 'node not found' }); return }
|
|
|
|
let content: string | null = null // extracted search text (heavy formats)
|
|
let meta: unknown = null
|
|
let text: string | null = null // inline text for text-native originals
|
|
|
|
if (node.mode === 'extracted' && node.out) {
|
|
try { content = await fsp.readFile(path.join(outputDir, node.out, 'content.md'), 'utf-8') } catch { /* */ }
|
|
try { meta = JSON.parse(await fsp.readFile(path.join(outputDir, node.out, 'meta.json'), 'utf-8')) } catch { /* */ }
|
|
} else if (node.mode === 'link' && TEXT_EXTS.has(node.ext)) {
|
|
const abs = originalAbs(index, rel)
|
|
if (abs) { try { text = await fsp.readFile(abs, 'utf-8') } catch { /* */ } }
|
|
}
|
|
|
|
sendJson(res, 200, {
|
|
node,
|
|
content,
|
|
meta,
|
|
text,
|
|
mime: MIME[path.extname(node.name).toLowerCase()] ?? 'application/octet-stream',
|
|
originalUrl: `/api/original?path=${encodeURIComponent(rel)}`,
|
|
})
|
|
}
|
|
|
|
async function serveOriginal(res: ServerResponse, rel: string, range?: string): Promise<void> {
|
|
const index = readIndex()
|
|
if (!index) { sendJson(res, 404, { error: 'no index' }); return }
|
|
const abs = originalAbs(index, rel)
|
|
if (!abs || !isFile(abs)) { sendJson(res, 404, { error: 'original not reachable' }); return }
|
|
const mime = MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream'
|
|
const size = fs.statSync(abs).size
|
|
res.setHeader('Content-Type', mime)
|
|
res.setHeader('Accept-Ranges', 'bytes')
|
|
|
|
// Range support — needed so large meeting videos can seek/stream.
|
|
const m = range?.match(/bytes=(\d*)-(\d*)/)
|
|
if (m) {
|
|
const start = m[1] ? parseInt(m[1], 10) : 0
|
|
const end = m[2] ? parseInt(m[2], 10) : size - 1
|
|
if (start >= size || end >= size || start > end) {
|
|
res.statusCode = 416
|
|
res.setHeader('Content-Range', `bytes */${size}`)
|
|
res.end()
|
|
return
|
|
}
|
|
res.statusCode = 206
|
|
res.setHeader('Content-Range', `bytes ${start}-${end}/${size}`)
|
|
res.setHeader('Content-Length', String(end - start + 1))
|
|
fs.createReadStream(abs, { start, end }).pipe(res)
|
|
return
|
|
}
|
|
res.setHeader('Content-Length', String(size))
|
|
fs.createReadStream(abs).pipe(res)
|
|
}
|
|
}
|
|
|
|
function sendJson(res: ServerResponse, status: number, obj: unknown): void {
|
|
const body = JSON.stringify(obj)
|
|
res.statusCode = status
|
|
res.setHeader('Content-Type', 'application/json')
|
|
res.setHeader('Content-Length', Buffer.byteLength(body))
|
|
res.end(body)
|
|
}
|
|
|
|
function isFile(p: string): boolean {
|
|
try { return fs.statSync(p).isFile() } catch { return false }
|
|
}
|
|
|
|
function readBody(req: IncomingMessage): Promise<string> {
|
|
return new Promise((resolve, reject) => {
|
|
let data = ''
|
|
req.on('data', (c) => { data += c; if (data.length > 5_000_000) reject(new Error('body too large')) })
|
|
req.on('end', () => resolve(data))
|
|
req.on('error', reject)
|
|
})
|
|
}
|