attempt to use separate folder for docs and meetings

This commit is contained in:
Mariano Gabriel
2026-07-06 01:18:17 -03:00
parent bb84558526
commit 31ad8e5928
10 changed files with 261 additions and 62 deletions

View File

@@ -26,7 +26,7 @@ import path from 'node:path'
interface Options {
outputDirs: string[] // explicit output dirs (env)
outputsRoot: string // managed root: UI-scanned sources land here
outputsRoots: string[] // managed roots (doocus-data, meetus-data): <root>/<source>/
repoRoot: string // where process_tree.py lives (run via uv)
}
@@ -82,17 +82,23 @@ function buildCollections(dirs: string[]): Collection[] {
}
export function doocusApi(opts: Options): Plugin {
/** Collections discovered under the managed root (one subdir per source). */
function discoverInRoot(): Collection[] {
let subs: fs.Dirent[] = []
try { subs = fs.readdirSync(opts.outputsRoot, { withFileTypes: true }) } catch { return [] }
return subs
.filter((d) => d.isDirectory() && fs.existsSync(path.join(opts.outputsRoot, d.name, 'index.json')))
.map((d) => ({
id: d.name, name: d.name,
dir: path.join(opts.outputsRoot, d.name),
indexPath: path.join(opts.outputsRoot, d.name, 'index.json'),
}))
/** Collections discovered across the managed roots (each is <root>/<source>/). */
function discoverInRoots(): Collection[] {
const found: Collection[] = []
for (const root of opts.outputsRoots) {
let subs: fs.Dirent[] = []
try { subs = fs.readdirSync(root, { withFileTypes: true }) } catch { continue }
for (const d of subs) {
if (d.isDirectory() && fs.existsSync(path.join(root, d.name, 'index.json'))) {
found.push({
id: d.name, name: d.name,
dir: path.join(root, d.name),
indexPath: path.join(root, d.name, 'index.json'),
})
}
}
}
return found
}
/** Current collections = explicit env dirs + managed-root subdirs (deduped,
@@ -100,7 +106,7 @@ export function doocusApi(opts: Options): Plugin {
function currentCollections(): Collection[] {
const out: Collection[] = []
const seen = new Set<string>()
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoot()]) {
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoots()]) {
if (out.some((o) => o.dir === c.dir)) continue
let id = c.id, n = 1
while (seen.has(id)) { n++; id = `${c.id}-${n}` }
@@ -172,7 +178,7 @@ export function doocusApi(opts: Options): Plugin {
configureServer(server) {
const cols = currentCollections()
server.config.logger.info(
`[doocus-api] ${cols.length} collection(s); managed root: ${opts.outputsRoot}`)
`[doocus-api] ${cols.length} collection(s); managed roots: ${opts.outputsRoots.join(', ')}`)
server.middlewares.use('/api', (req, res, next) => {
const url = new URL(req.url ?? '/', 'http://localhost')
const parts = url.pathname.split('/').filter(Boolean)
@@ -205,7 +211,14 @@ export function doocusApi(opts: Options): Plugin {
function listCollections(res: ServerResponse): void {
const out = currentCollections().map((c) => {
const idx = readIndexFor(c)
return { id: c.id, name: c.name, available: !!idx, count: idx?.files.length ?? 0 }
const meetings = idx?.files.filter((f) => f.mode === 'meeting').length ?? 0
return {
id: c.id, name: c.name, available: !!idx,
count: idx?.files.length ?? 0,
root: idx?.root ?? null, // shared root → same source
only: (idx as any)?.only ?? 'all',
meetings,
}
})
sendJson(res, 200, { collections: out })
}
@@ -218,32 +231,35 @@ export function doocusApi(opts: Options): Plugin {
let source: string
let dir: string
let only = 'docs' // UI scans create document collections (meetings come from the batch)
if (rerun) {
const col = body.id ? collectionById(body.id) : undefined
const idx = col ? readIndexFor(col) : null
if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
source = idx.root
dir = col.dir
only = (idx as any).only ?? 'all' // preserve the collection's kind on rescan
} else {
source = (body.source ?? '').trim()
if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
dir = path.join(opts.outputsRoot, slug)
fs.mkdirSync(opts.outputsRoot, { recursive: true })
dir = path.join(opts.outputsRoots[0], slug) // docs land in the first root (doocus-data)
fs.mkdirSync(opts.outputsRoots[0], { recursive: true })
}
try {
await runProcessTree(source, dir)
await runProcessTree(source, dir, only)
sendJson(res, 200, { ok: true, id: path.basename(dir) })
} catch (e) {
sendJson(res, 500, { error: String(e) })
}
}
/** Spawn `uv run python process_tree.py <source> --output <dir>` at repoRoot. */
function runProcessTree(source: string, dir: string): Promise<void> {
/** Spawn `uv run python process_tree.py <source> --output <dir> --only <only>`. */
function runProcessTree(source: string, dir: string, only: string): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn('uv', ['run', 'python', 'process_tree.py', source, '--output', dir],
const child = spawn('uv',
['run', 'python', 'process_tree.py', source, '--output', dir, '--only', only],
{ cwd: opts.repoRoot })
let err = ''
child.stderr.on('data', (d) => { err += d })
@@ -252,20 +268,42 @@ export function doocusApi(opts: Options): Plugin {
})
}
/** Higher = "richer" version of a file — the one whose sidecar resolves wins,
* so a meetus-data meeting (with .meetus) overrides a doocus copy without it. */
function scoreNode(c: Collection, f: Node): number {
if (f.mode === 'meeting') return fs.existsSync(meetusDir(c, f.path).dir) ? 4 : 2
if (f.mode === 'extracted') {
return f.out && fs.existsSync(path.join(c.dir, f.out, 'content.md')) ? 3 : 1.5
}
return 1
}
function getTree(res: ServerResponse, activeCsv: string | null): void {
const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
const cols = currentCollections().filter((c) => !active || active.has(c.id))
const files: Array<Node & { collection: string }> = []
const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: 0 }
// Dedup across collections by shared root + rel; the resolving copy wins.
const best = new Map<string, { file: Node & { collection: string; root: string; rel: string }; score: number }>()
for (const c of cols) {
const idx = readIndexFor(c)
if (!idx) continue
for (const f of idx.files) {
files.push({ ...f, collection: c.id, path: `${c.id}/${f.path}` })
counts[f.mode] = (counts[f.mode] ?? 0) + 1
counts.total += 1
const key = `${idx.root}${f.path}`
const score = scoreNode(c, f)
const cur = best.get(key)
if (!cur || score > cur.score) {
best.set(key, {
file: { ...f, collection: c.id, root: idx.root, rel: f.path, path: `${c.id}/${f.path}` },
score,
})
}
}
}
const files = [...best.values()].map((v) => v.file)
const counts: Record<string, number> = { extracted: 0, meeting: 0, link: 0, total: files.length }
for (const f of files) counts[f.mode] = (counts[f.mode] ?? 0) + 1
sendJson(res, 200, {
collections: currentCollections().map((c) => ({ id: c.id, name: c.name })),
files,

View File

@@ -6,7 +6,7 @@ import FileTree from '@/components/FileTree.vue'
import DocDetail from '@/components/DocDetail.vue'
import ExportBar from '@/components/ExportBar.vue'
import { ref } from 'vue'
import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, toggleCollection, scanFolder, rescanCollection } from '@/store'
import { store, loadTree, runSearch, visibleTree, selectMatches, humanBytes, sources, sourceActive, toggleSource, scanFolder, refreshSources } from '@/store'
onMounted(loadTree)
@@ -37,25 +37,26 @@ function onSearch(e: Event) {
<div class="collections">
<button
class="col-btn"
:title="`Sources — ${store.activeCollections.size}/${store.collections.length} active`"
:title="`Sources — ${sources.length}`"
@click="showCols = !showCols"
></button>
<div v-if="showCols" class="col-menu">
<div class="col-title">Sources</div>
<div v-for="c in store.collections" :key="c.id" class="col-row">
<label :class="{ off: !c.available }">
<div class="col-title">
Sources (docs + meetings share a source)
<button class="refresh" title="Re-discover newly-scanned sources" @click="refreshSources"></button>
</div>
<div v-for="s in sources" :key="s.root" class="col-row">
<label>
<input
type="checkbox"
:checked="store.activeCollections.has(c.id)"
:disabled="!c.available"
@change="toggleCollection(c.id)"
:checked="sourceActive(s.root)"
@change="toggleSource(s.root)"
/>
<span class="col-name">{{ c.name }}</span>
<span class="dim">{{ c.available ? c.count : 'no index' }}</span>
<span class="col-name" :title="s.root">{{ s.label }}</span>
<span class="dim">{{ s.docs }} docs · {{ s.meetings }} mtg</span>
</label>
<button class="rescan" title="Re-scan this source" :disabled="store.scanning" @click="rescanCollection(c.id)"></button>
</div>
<p v-if="!store.collections.length" class="dim">No sources yet.</p>
<p v-if="!sources.length" class="dim">No sources yet.</p>
<div class="col-scan">
<input
@@ -149,7 +150,8 @@ function onSearch(e: Event) {
background: var(--surface-1); border: var(--panel-border); border-radius: var(--panel-radius);
padding: var(--space-2); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35);
}
.col-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); }
.col-title { font-size: 11px; color: var(--text-secondary); margin-bottom: var(--space-1); display: flex; align-items: center; justify-content: space-between; }
.refresh { padding: 0 var(--space-1); background: transparent; border: 0; color: var(--text-secondary); cursor: pointer; }
.col-row { display: flex; align-items: center; gap: var(--space-1); }
.col-menu label {
flex: 1; display: flex; align-items: center; gap: var(--space-2);

View File

@@ -29,15 +29,15 @@ function folderCollapsed(path: string): boolean {
<template v-else>
<div
class="row file"
:class="{ active: it.path === store.selectedPath, meeting: it.node?.mode === 'meeting' }"
:class="{ active: it.node?.path === store.selectedPath, meeting: it.node?.mode === 'meeting' }"
:style="{ paddingLeft: (depth ?? 0) * 14 + 8 + 'px' }"
@click="select(it.path)"
@click="it.node && select(it.node.path)"
>
<input
type="checkbox"
:checked="store.selected.has(it.path)"
:checked="!!it.node && store.selected.has(it.node.path)"
title="Add to package"
@click.stop="toggleSelect(it.path)"
@click.stop="it.node && toggleSelect(it.node.path)"
/>
<span class="badge" :data-mode="it.node?.mode">{{ MODE_BADGE[it.node?.mode ?? ''] ?? it.node?.mode }}</span>
<span class="ext">{{ it.node?.ext }}</span>

View File

@@ -8,8 +8,10 @@
import { reactive, computed } from 'vue'
export interface FileNode {
path: string // composite: "<collectionId>/<relpath>"
path: string // composite: "<collectionId>/<relpath>" (API resolution)
collection?: string
root?: string // shared source path (groups docs + meetings of a source)
rel?: string // path within the source (for the merged tree)
name: string
ext: string
family: string
@@ -28,6 +30,19 @@ export interface CollectionInfo {
name: string
available: boolean
count: number
root: string | null
only: string
meetings: number
}
/** A source = one downloaded drive, possibly split into a docs collection and a
* meetings collection that share the same `root`. */
export interface SourceInfo {
root: string
label: string
collectionIds: string[]
docs: number
meetings: number
}
export interface Detail {
@@ -152,11 +167,31 @@ export function toggleFolder(path: string): void {
else store.collapsed.add(path)
}
/** Build a nested folder tree from flat posix paths, folders before files. */
/** Unique display label per source root (top-level tree group). */
function sourceLabels(files: FileNode[]): Map<string, string> {
const roots = [...new Set(files.map((f) => f.root).filter(Boolean))] as string[]
const labels = new Map<string, string>()
const seen = new Set<string>()
for (const r of roots.sort()) {
const base = r.split('/').filter(Boolean).pop() || r
let label = base, n = 1
while (seen.has(label)) { n++; label = `${base}-${n}` }
seen.add(label)
labels.set(r, label)
}
return labels
}
/** Build the tree grouped by source root — so a source's docs + meetings
* interleave under one folder — while each file keeps its composite `node.path`
* for API calls. Folder items are keyed by their display path. */
function buildTree(files: FileNode[]): TreeItem[] {
const labels = sourceLabels(files)
const root: TreeItem = { name: '', path: '', dir: true, children: [] }
for (const f of files) {
const parts = f.path.split('/')
const src = (f.root && labels.get(f.root)) || f.collection || 'source'
const display = `${src}/${f.rel ?? f.path}`
const parts = display.split('/')
let cur = root
for (let i = 0; i < parts.length - 1; i++) {
const seg = parts[i]
@@ -168,7 +203,7 @@ function buildTree(files: FileNode[]): TreeItem[] {
}
cur = next
}
cur.children!.push({ name: f.name, path: f.path, dir: false, node: f })
cur.children!.push({ name: f.name, path: display, dir: false, node: f })
}
const sort = (items: TreeItem[]): TreeItem[] => {
items.sort((a, b) =>
@@ -196,6 +231,21 @@ export async function loadCollections(): Promise<void> {
}
}
/** Re-discover sources (picks up newly-scanned collections) without a page
* reload; preserves current toggles and turns on any brand-new source. */
export async function refreshSources(): Promise<void> {
const prevActive = new Set(store.activeCollections)
const known = new Set(store.collections.map((c) => c.id))
await loadCollections() // sets active = all available
store.activeCollections = new Set(
store.collections
.filter((c) => c.available && (prevActive.has(c.id) || !known.has(c.id)))
.map((c) => c.id),
)
await loadTree()
if (store.query) runSearch(store.query)
}
export async function loadTree(): Promise<void> {
store.loading = true
store.error = ''
@@ -214,10 +264,42 @@ export async function loadTree(): Promise<void> {
}
}
/** Toggle a collection on/off and re-merge the tree (and re-run any search). */
export async function toggleCollection(id: string): Promise<void> {
if (store.activeCollections.has(id)) store.activeCollections.delete(id)
else store.activeCollections.add(id)
/** Collections grouped by shared root — the selector unit. */
export const sources = computed<SourceInfo[]>(() => {
const byRoot = new Map<string, SourceInfo>()
const seen = new Set<string>()
for (const c of store.collections) {
if (!c.root) continue
let s = byRoot.get(c.root)
if (!s) {
const base = c.root.split('/').filter(Boolean).pop() || c.root
let label = base, n = 1
while (seen.has(label)) { n++; label = `${base}-${n}` }
seen.add(label)
s = { root: c.root, label, collectionIds: [], docs: 0, meetings: 0 }
byRoot.set(c.root, s)
}
s.collectionIds.push(c.id)
s.docs += c.count - c.meetings
s.meetings += c.meetings
}
return [...byRoot.values()]
})
export function sourceActive(root: string): boolean {
const s = sources.value.find((x) => x.root === root)
return !!s && s.collectionIds.some((id) => store.activeCollections.has(id))
}
/** Toggle a whole source (its docs + meetings collections) and re-merge. */
export async function toggleSource(root: string): Promise<void> {
const s = sources.value.find((x) => x.root === root)
if (!s) return
const allOn = s.collectionIds.every((id) => store.activeCollections.has(id))
for (const id of s.collectionIds) {
if (allOn) store.activeCollections.delete(id)
else store.activeCollections.add(id)
}
await loadTree()
if (store.query) runSearch(store.query)
}

View File

@@ -10,16 +10,23 @@ const outputDirs = (outputEnv
? outputEnv.split(',').map((s) => s.trim()).filter(Boolean)
: [fileURLToPath(new URL('../../docs-output', import.meta.url))])
// Managed collections root (gitignored): UI-scanned sources land here, one
// subfolder per source, discovered automatically. Override with DOOCUS_DATA.
const outputsRoot = process.env.DOOCUS_DATA
?? fileURLToPath(new URL('../../doocus-data', import.meta.url))
// Managed collection roots (gitignored), discovered automatically:
// doocus-data/<source>/ → document collections (UI "Scan" writes here)
// meetus-data/<source>/ → meeting collections (from the meetus batch)
// Collections that share the same index.json `root` are the same source and are
// grouped together in the UI. Override with DOOCUS_DATA (comma-separated).
const outputsRoots = process.env.DOOCUS_DATA
? process.env.DOOCUS_DATA.split(',').map((s) => s.trim()).filter(Boolean)
: [
fileURLToPath(new URL('../../doocus-data', import.meta.url)),
fileURLToPath(new URL('../../meetus-data', import.meta.url)),
]
// Repo root — where process_tree.py lives (run via `uv run` for the scan).
const repoRoot = fileURLToPath(new URL('../../', import.meta.url))
export default defineConfig({
plugins: [vue(), doocusApi({ outputDirs, outputsRoot, repoRoot })],
plugins: [vue(), doocusApi({ outputDirs, outputsRoots, repoRoot })],
resolve: {
alias: {
'@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),
@@ -36,6 +43,7 @@ export default defineConfig({
fileURLToPath(new URL('../framework', import.meta.url)),
fileURLToPath(new URL('../meetus-app', import.meta.url)),
...outputDirs,
...outputsRoots,
],
},
},

View File

@@ -5,7 +5,21 @@ import { useStore } from '../store'
const props = withDefaults(defineProps<{ allowEdit?: boolean }>(), { allowEdit: true })
const { state, activeSegmentIndex, seek } = useStore()
const { state, activeSegmentIndex, seek, setSelectMode, setSegmentIncluded } = useStore()
// Turn-level include (a turn = a speaker's continuous run of segments).
function turnIncluded(items: Array<{ i: number }>): boolean {
return items.length > 0 && items.every((it) => state.segments[it.i]?.included)
}
function toggleTurn(items: Array<{ i: number }>): void {
const val = !turnIncluded(items)
for (const it of items) setSegmentIncluded(it.i, val)
}
function toggleSelect(): void {
const on = !state.selectMode
setSelectMode(on)
if (on) viewMode.value = 'read' // selection lives in the read view
}
// View preferences (persisted). When edit is disallowed (e.g. embedded review),
// the transcript is read-only regardless of the saved preference.
@@ -93,6 +107,7 @@ watch(activeSegmentIndex, async (i) => {
</div>
<button v-if="allowEdit && effectiveMode === 'edit'" :class="{ on: compact }" @click="compact = !compact">Compact</button>
<button v-else :class="{ on: showTimes }" @click="showTimes = !showTimes">Times</button>
<button :class="{ on: state.selectMode }" title="Pick segments (+ their frames) for the package" @click="toggleSelect">Select</button>
<span class="count">{{ state.segments.length }} segments</span>
</template>
@@ -127,6 +142,14 @@ watch(activeSegmentIndex, async (i) => {
<p v-if="!state.segments.length" class="empty">No transcript for this run.</p>
<div v-for="(turn, ti) in readingTurns" :key="ti" class="turn">
<div class="turn-head">
<input
v-if="state.selectMode"
type="checkbox"
class="turn-pick"
:checked="turnIncluded(turn.items)"
title="Include this part (+ its frames) in the package"
@change="toggleTurn(turn.items)"
/>
<button class="ts" title="Seek here" @click="seek(turn.start)">{{ fmt(turn.start) }}</button>
<span class="speaker-lbl">{{ turn.speaker }}</span>
</div>
@@ -141,7 +164,7 @@ watch(activeSegmentIndex, async (i) => {
><span
:ref="(el) => setSeg(it.i, el as Element | null)"
class="rseg"
:class="{ active: it.i === activeSegmentIndex }"
:class="{ active: it.i === activeSegmentIndex, included: state.selectMode && state.segments[it.i]?.included }"
@click="seek(it.start)"
>{{ it.text.trim() + ' ' }}</span>
</template>
@@ -274,6 +297,10 @@ watch(activeSegmentIndex, async (i) => {
background: var(--surface-2);
box-shadow: 0 0 0 1px var(--status-processing);
}
.rseg.included {
background: color-mix(in srgb, var(--status-live, #2a6) 22%, transparent);
}
.turn-pick { margin-right: var(--space-1); }
.inline-ts {
font-family: var(--font-mono);
font-size: 10px;

View File

@@ -23,6 +23,7 @@ export interface SegmentState {
origSpeaker: string | null
text: string
speaker: string | null
included: boolean // in "select" mode, whether this segment is in the package
}
export interface FrameState {
@@ -56,6 +57,8 @@ interface State {
duration: number
playing: boolean
selectMode: boolean // package = only the included segments + their frames
saveStatus: SaveStatus
}
@@ -78,6 +81,8 @@ const state = reactive<State>({
duration: 0,
playing: false,
selectMode: false,
saveStatus: 'idle',
})
@@ -140,6 +145,7 @@ export function buildTranscript(): string {
const items: Item[] = []
for (const s of state.segments) {
if (!s.text.trim()) continue
if (state.selectMode && !s.included) continue // select mode: only chosen segments
items.push({ type: 'audio', time: s.start, speaker: s.speaker || 'SPEAKER', text: s.text.trim() })
}
for (const f of state.frames) {
@@ -221,6 +227,8 @@ async function openRun(id: string): Promise<void> {
const review = data.review && typeof data.review === 'object' ? data.review : null
const segOverrides = review?.segments ?? {}
const includedSet = new Set<number>(review?.selection?.segments ?? [])
state.selectMode = includedSet.size > 0
state.segments = (data.segments ?? []).map((s: any, i: number): SegmentState => {
const ov = segOverrides[i] ?? segOverrides[String(i)] ?? {}
return {
@@ -230,6 +238,7 @@ async function openRun(id: string): Promise<void> {
origSpeaker: s.speaker ?? null,
text: ov.text ?? s.text,
speaker: ov.speaker ?? s.speaker ?? null,
included: includedSet.has(i),
}
})
@@ -268,6 +277,8 @@ function buildReview(): unknown {
updatedAt: new Date().toISOString(),
segments,
frames: { selected: state.frames.filter((f) => f.selected).map((f) => f.file) },
// Select-mode: which segments are included in the package (indices).
selection: { segments: state.segments.map((s, i) => (s.included ? i : -1)).filter((i) => i >= 0) },
}
}
@@ -291,7 +302,7 @@ async function flushSave(): Promise<void> {
// Watch the editable surface and the frame selection; debounce writes.
watch(
() => [
state.segments.map((s) => `${s.text}${s.speaker ?? ''}`).join(''),
state.segments.map((s) => `${s.text} ${s.speaker ?? ''} ${s.included ? '1' : '0'}`).join(''),
state.frames.map((f) => (f.selected ? f.file : '')).join(''),
],
() => {
@@ -312,6 +323,23 @@ function setSelectAll(selected: boolean): void {
state.frames.forEach((f) => { f.selected = selected })
}
/** Include/exclude a segment in the package, and the frames captured in its time
* window along with it. */
function setSegmentIncluded(i: number, included: boolean): void {
const s = state.segments[i]
if (!s) return
s.included = included
state.frames.forEach((f) => {
if (f.time != null && f.time >= s.start && f.time < s.end) f.selected = included
})
}
/** Toggle the whole select-mode; leaving it clears any partial selection. */
function setSelectMode(on: boolean): void {
state.selectMode = on
if (!on) state.segments.forEach((s) => { s.included = false })
}
export function useStore() {
return {
state,
@@ -324,6 +352,8 @@ export function useStore() {
setOutputDir,
seek,
setSelectAll,
setSegmentIncluded,
setSelectMode,
buildTranscript,
}
}