doocus update and meeting list export
This commit is contained in:
Mariano Gabriel
@@ -1,19 +1,18 @@
|
||||
/**
|
||||
* Vite dev middleware exposing the local doocus docs-output tree to the app.
|
||||
* Vite dev middleware exposing the local doocus tree index to the app.
|
||||
*
|
||||
* Runs in the dev server process (Node), so it can reach the original source
|
||||
* file by the absolute path stored in each doc's manifest.json — needed for the
|
||||
* package builder, which bundles the original alongside content.md + meta.json.
|
||||
* All reads are local; nothing leaves the machine.
|
||||
* Reads `docs-output/index.json` (produced by process_tree.py) — a replica of
|
||||
* the whole local drive tree — plus the extracted `<file>.doocus/content.md`
|
||||
* sidecars, and streams the ORIGINAL files by relative path (resolved against
|
||||
* the index's recorded root). All reads are local; nothing leaves the machine.
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/docs list extracted docs
|
||||
* GET /api/docs/:id meta.json + content.md + thumb/original flags
|
||||
* GET /api/docs/:id/thumb stream thumb.jpg
|
||||
* GET /api/docs/:id/original stream the original source file
|
||||
* GET /api/tree the index.json tree
|
||||
* GET /api/detail?path=<rel> node + extracted content + meta + inline text
|
||||
* GET /api/original?path=<rel> stream the original file (native viewer / packaging)
|
||||
*/
|
||||
import type { Plugin } from 'vite'
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http'
|
||||
import type { ServerResponse } from 'node:http'
|
||||
import fs from 'node:fs'
|
||||
import fsp from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
@@ -22,196 +21,130 @@ interface Options {
|
||||
outputDir: string
|
||||
}
|
||||
|
||||
interface Manifest {
|
||||
source?: { name?: string; path?: string; sha256?: string }
|
||||
processed_at?: string
|
||||
outputs?: { content?: string; meta?: string; thumb?: string | null }
|
||||
interface Node {
|
||||
path: string
|
||||
name: string
|
||||
ext: string
|
||||
family: string
|
||||
mode: 'extracted' | 'meeting' | 'link'
|
||||
bytes: number
|
||||
modified: string
|
||||
url: string | null
|
||||
out?: string
|
||||
}
|
||||
|
||||
const ORIGINAL_MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
interface Index {
|
||||
root: string
|
||||
generatedAt: string
|
||||
counts: Record<string, number>
|
||||
files: Node[]
|
||||
}
|
||||
|
||||
// Text-native files: readable as-is, so the "original" doubles as its own text.
|
||||
const TEXT_EXTS = new Set(['md', 'markdown', 'txt', 'json', 'yaml', 'yml', 'csv', 'html', 'htm'])
|
||||
|
||||
const MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||
'.html': 'text/html', '.htm': 'text/html', '.md': 'text/markdown', '.txt': 'text/plain',
|
||||
'.csv': 'text/csv', '.json': 'application/json', '.yaml': 'text/yaml', '.yml': 'text/yaml',
|
||||
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'.csv': 'text/csv',
|
||||
'.html': 'text/html',
|
||||
'.htm': 'text/html',
|
||||
'.json': 'application/json',
|
||||
'.md': 'text/markdown',
|
||||
'.txt': 'text/plain',
|
||||
'.yaml': 'text/yaml',
|
||||
'.yml': 'text/yaml',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.mp4': 'video/mp4',
|
||||
}
|
||||
|
||||
export function doocusApi(opts: Options): Plugin {
|
||||
const outputDir = path.resolve(opts.outputDir)
|
||||
const toPosix = (p: string) => p.split(path.sep).join('/')
|
||||
const indexPath = path.join(outputDir, 'index.json')
|
||||
|
||||
/** Resolve a doc id (possibly nested) to its directory, rejecting traversal. */
|
||||
function docDir(id: string): string | null {
|
||||
const dir = path.resolve(outputDir, id)
|
||||
if (dir !== outputDir && !dir.startsWith(outputDir + path.sep)) return null
|
||||
function readIndex(): Index | null {
|
||||
try {
|
||||
if (fs.statSync(dir).isDirectory()) return dir
|
||||
} catch { /* missing */ }
|
||||
return null
|
||||
}
|
||||
|
||||
/** Find every doc run directory (one containing meta.json) under `base`. */
|
||||
async function findDocDirs(base: string, maxDepth = 8): Promise<string[]> {
|
||||
const found: string[] = []
|
||||
const walk = async (dir: string, depth: number): Promise<void> => {
|
||||
if (depth > maxDepth) return
|
||||
let ents
|
||||
try {
|
||||
ents = await fsp.readdir(dir, { withFileTypes: true })
|
||||
} catch { return }
|
||||
if (ents.some((e) => e.isFile() && e.name === 'meta.json')) {
|
||||
found.push(dir)
|
||||
return // a doc dir; don't descend into assets/
|
||||
}
|
||||
for (const e of ents) {
|
||||
if (!e.isDirectory()) continue
|
||||
if (e.name === 'assets' || e.name === 'node_modules' || e.name.startsWith('.')) continue
|
||||
await walk(path.join(dir, e.name), depth + 1)
|
||||
}
|
||||
}
|
||||
await walk(base, 0)
|
||||
return found
|
||||
}
|
||||
|
||||
function readJson<T>(file: string): T | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf-8')) as T
|
||||
return JSON.parse(fs.readFileSync(indexPath, 'utf-8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve a node's original file, rejecting traversal outside the index root. */
|
||||
function originalAbs(index: Index, rel: string): string | null {
|
||||
const root = path.resolve(index.root)
|
||||
const abs = path.resolve(root, rel)
|
||||
if (abs !== root && !abs.startsWith(root + path.sep)) return null
|
||||
return abs
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'doocus-api',
|
||||
configureServer(server) {
|
||||
server.config.logger.info(`[doocus-api] serving docs from: ${outputDir}`)
|
||||
server.config.logger.info(`[doocus-api] serving tree from: ${indexPath}`)
|
||||
server.middlewares.use('/api', (req, res, next) => {
|
||||
const url = new URL(req.url ?? '/', 'http://localhost')
|
||||
const parts = url.pathname.split('/').filter(Boolean)
|
||||
handle(req, res, parts).catch((err) => {
|
||||
handle(res, parts, url.searchParams).catch((err) => {
|
||||
server.config.logger.error(`[doocus-api] ${String(err)}`)
|
||||
sendJson(res, 500, { error: String(err) })
|
||||
}).then((handled) => {
|
||||
if (!handled) next()
|
||||
})
|
||||
}).then((handled) => { if (!handled) next() })
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[]): Promise<boolean> {
|
||||
if (parts[0] !== 'docs') return false
|
||||
|
||||
// GET /api/docs
|
||||
if (parts.length === 1 && req.method === 'GET') {
|
||||
await listDocs(res)
|
||||
async function handle(res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
|
||||
if (parts[0] === 'tree') {
|
||||
const index = readIndex()
|
||||
if (!index) { sendJson(res, 404, { error: `index.json not found in ${outputDir}` }); return true }
|
||||
sendJson(res, 200, index)
|
||||
return true
|
||||
}
|
||||
|
||||
const id = parts[1] ? decodeURIComponent(parts[1]) : ''
|
||||
const dir = id ? docDir(id) : null
|
||||
if (!dir) { sendJson(res, 404, { error: 'doc not found' }); return true }
|
||||
|
||||
// GET /api/docs/:id
|
||||
if (parts.length === 2 && req.method === 'GET') {
|
||||
await getDoc(res, id, dir)
|
||||
return true
|
||||
}
|
||||
// GET /api/docs/:id/thumb
|
||||
if (parts.length === 3 && parts[2] === 'thumb' && req.method === 'GET') {
|
||||
serveThumb(res, dir)
|
||||
return true
|
||||
}
|
||||
// GET /api/docs/:id/original
|
||||
if (parts.length === 3 && parts[2] === 'original' && req.method === 'GET') {
|
||||
serveOriginal(res, dir)
|
||||
if (parts[0] === 'detail') {
|
||||
await getDetail(res, q.get('path') ?? '')
|
||||
return true
|
||||
}
|
||||
|
||||
sendJson(res, 404, { error: 'not found' })
|
||||
return true
|
||||
if (parts[0] === 'original') {
|
||||
await serveOriginal(res, q.get('path') ?? '')
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
async function listDocs(res: ServerResponse): Promise<void> {
|
||||
const dirs = await findDocDirs(outputDir)
|
||||
const docs = []
|
||||
for (const dir of dirs) {
|
||||
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||
if (!meta) continue
|
||||
const id = toPosix(path.relative(outputDir, dir))
|
||||
const source = (meta.source ?? {}) as { name?: string; bytes?: number }
|
||||
docs.push({
|
||||
id,
|
||||
name: source.name ?? id,
|
||||
type: meta.type ?? null,
|
||||
family: meta.family ?? null,
|
||||
title: meta.title ?? null,
|
||||
extractedAt: meta.extracted_at ?? null,
|
||||
wordCount: meta.word_count ?? 0,
|
||||
bytes: source.bytes ?? 0,
|
||||
hasThumb: fs.existsSync(path.join(dir, 'thumb.jpg')),
|
||||
warnings: Array.isArray(meta.warnings) ? meta.warnings.length : 0,
|
||||
})
|
||||
async function getDetail(res: ServerResponse, rel: string): Promise<void> {
|
||||
const index = readIndex()
|
||||
if (!index) { sendJson(res, 404, { error: 'no index' }); return }
|
||||
const node = index.files.find((f) => f.path === rel)
|
||||
if (!node) { sendJson(res, 404, { error: 'node not found' }); return }
|
||||
|
||||
let content: string | null = null // extracted search text (heavy formats)
|
||||
let meta: unknown = null
|
||||
let text: string | null = null // inline text for text-native originals
|
||||
|
||||
if (node.mode === 'extracted' && node.out) {
|
||||
try { content = await fsp.readFile(path.join(outputDir, node.out, 'content.md'), 'utf-8') } catch { /* */ }
|
||||
try { meta = JSON.parse(await fsp.readFile(path.join(outputDir, node.out, 'meta.json'), 'utf-8')) } catch { /* */ }
|
||||
} else if (node.mode === 'link' && TEXT_EXTS.has(node.ext)) {
|
||||
const abs = originalAbs(index, rel)
|
||||
if (abs) { try { text = await fsp.readFile(abs, 'utf-8') } catch { /* */ } }
|
||||
}
|
||||
docs.sort((a, b) =>
|
||||
String(b.extractedAt ?? '').localeCompare(String(a.extractedAt ?? '')) || a.id.localeCompare(b.id))
|
||||
sendJson(res, 200, { docs, outputDir })
|
||||
}
|
||||
|
||||
async function getDoc(res: ServerResponse, id: string, dir: string): Promise<void> {
|
||||
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||
if (!meta) { sendJson(res, 404, { error: 'meta missing' }); return }
|
||||
const manifest = readJson<Manifest>(path.join(dir, 'manifest.json'))
|
||||
|
||||
let content = ''
|
||||
try {
|
||||
content = await fsp.readFile(path.join(dir, 'content.md'), 'utf-8')
|
||||
} catch { /* none */ }
|
||||
|
||||
const hasThumb = fs.existsSync(path.join(dir, 'thumb.jpg'))
|
||||
const originalPath = manifest?.source?.path
|
||||
const hasOriginal = !!originalPath && isFile(originalPath)
|
||||
|
||||
sendJson(res, 200, {
|
||||
id,
|
||||
meta,
|
||||
node,
|
||||
content,
|
||||
hasThumb,
|
||||
thumbUrl: hasThumb ? `/api/docs/${encodeURIComponent(id)}/thumb` : null,
|
||||
hasOriginal,
|
||||
originalUrl: hasOriginal ? `/api/docs/${encodeURIComponent(id)}/original` : null,
|
||||
meta,
|
||||
text,
|
||||
mime: MIME[path.extname(node.name).toLowerCase()] ?? 'application/octet-stream',
|
||||
originalUrl: `/api/original?path=${encodeURIComponent(rel)}`,
|
||||
})
|
||||
}
|
||||
|
||||
function serveThumb(res: ServerResponse, dir: string): void {
|
||||
const full = path.join(dir, 'thumb.jpg')
|
||||
if (!fs.existsSync(full)) { sendJson(res, 404, { error: 'no thumb' }); return }
|
||||
res.setHeader('Content-Type', 'image/jpeg')
|
||||
async function serveOriginal(res: ServerResponse, rel: string): Promise<void> {
|
||||
const index = readIndex()
|
||||
if (!index) { sendJson(res, 404, { error: 'no index' }); return }
|
||||
const abs = originalAbs(index, rel)
|
||||
if (!abs || !isFile(abs)) { sendJson(res, 404, { error: 'original not reachable' }); return }
|
||||
res.setHeader('Content-Type', MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream')
|
||||
res.setHeader('Cache-Control', 'no-cache')
|
||||
fs.createReadStream(full).pipe(res)
|
||||
}
|
||||
|
||||
function serveOriginal(res: ServerResponse, dir: string): void {
|
||||
const manifest = readJson<Manifest>(path.join(dir, 'manifest.json'))
|
||||
const original = manifest?.source?.path
|
||||
if (!original || !isFile(original)) {
|
||||
sendJson(res, 404, { error: 'original source not reachable on this machine' })
|
||||
return
|
||||
}
|
||||
const mime = ORIGINAL_MIME[path.extname(original).toLowerCase()] ?? 'application/octet-stream'
|
||||
res.setHeader('Content-Type', mime)
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${path.basename(original)}"`)
|
||||
fs.createReadStream(original).pipe(res)
|
||||
fs.createReadStream(abs).pipe(res)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user