Compare commits
10 Commits
fa3a0e3d1a
...
ef72d06bfe
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef72d06bfe | ||
|
|
31ad8e5928 | ||
|
|
bb84558526 | ||
|
|
8cfb4c2cbe | ||
|
|
b65d6075be | ||
|
|
8ff29fc776 | ||
|
|
a92615d6bc | ||
|
|
8606520ef2 | ||
|
|
7b276e8f3d | ||
|
|
300806b936 |
6
.gitignore
vendored
6
.gitignore
vendored
@@ -9,6 +9,11 @@ output/*
|
||||
# doocus document extraction output (textified docs + metadata; local-only)
|
||||
docs-output/
|
||||
|
||||
# Managed collection roots (one subfolder per source; index.json + sidecars).
|
||||
# doocus-data = document collections, meetus-data = meeting collections.
|
||||
doocus-data/
|
||||
meetus-data/
|
||||
|
||||
# Python cache
|
||||
__pycache__
|
||||
*.pyc
|
||||
@@ -18,3 +23,4 @@ __pycache__
|
||||
.venv/
|
||||
|
||||
local-run.sh
|
||||
meetings
|
||||
|
||||
34
INDEX.md
34
INDEX.md
@@ -51,9 +51,16 @@ docs-output/ whole tree replicated, no flatt
|
||||
|
||||
Node modes: **extracted** (docx/pdf/pptx/xlsx → text sidecar), **meeting**
|
||||
(mp4/... → belongs to meetus), **link** (everything else → original is the
|
||||
artifact, not duplicated). Each node has a `url` slot for the eventual Drive link.
|
||||
`make docs` (→ `process_tree.py`) indexes a whole tree; `process_doc.py` still does
|
||||
one-off single-file extraction. Full doocus docs: [`doocus/README.md`](doocus/README.md).
|
||||
artifact, not duplicated). Each node has a `url` slot for the eventual Drive link
|
||||
and a `created` date. `make docs` (→ `process_tree.py`) indexes a whole tree;
|
||||
`process_doc.py` still does one-off single-file extraction.
|
||||
|
||||
Meetings stay coherent with docs: `docs-output/meetings.txt` lists the meeting
|
||||
paths, and `make batch … LIST=… OUT_FORMAT="{name}.meetus"` runs meetus into a
|
||||
`<file>.meetus/` sidecar beside the docs — the exact path the indexer already
|
||||
pointed each `meeting` node at (`out`/`transcript`). meetus's `--out-format`
|
||||
(tokens `{date} {run} {stem} {name}`) makes the run-folder name configurable;
|
||||
default is unchanged. Full doocus docs: [`doocus/README.md`](doocus/README.md).
|
||||
Cross-search (docs frame + meetings frame, over cached text) is a later phase. 🚧
|
||||
|
||||
## Status legend
|
||||
@@ -96,15 +103,30 @@ ctrl/ control plane / operational scripts
|
||||
└─ interleave_cht_frames.py whisperx JSON + cht frames/index.json → enhanced.txt
|
||||
ui/ local browser UIs (Vue 3 + Vite), shared framework
|
||||
├─ framework/ ✅ shared component/renderer library + design tokens
|
||||
├─ meetus-app/ ✅ review meeting runs (was ui/app/)
|
||||
└─ doocus-app/ ✅ browse extracted docs + build packages (Gemini/NotebookLM)
|
||||
├─ meetus-app/ ✅ review meeting runs (video · transcript · frames; @review source)
|
||||
└─ doocus-app/ ✅ browse tree · search · package; embeds the meetus review
|
||||
doocus-data/ meetus-data/ ✅ managed collection roots (gitignored): <root>/<source>/
|
||||
docs/graphs/ 📄 architecture diagram (.dot → .svg, `make graphs`)
|
||||
def/ 📄 design/decision notes, in order (the feature history)
|
||||
README.md 📄 manual for process_meeting.py
|
||||
README.md 📄 project overview (meetus + doocus) + architecture diagram
|
||||
INDEX.md 📄 this file
|
||||
MARIAN.md 📄 genesis brainstorm (explains why the deprecated OCR path exists)
|
||||
local-run.sh 📄 personal scratch invocations (gitignored)
|
||||
```
|
||||
|
||||
## Collections & UIs
|
||||
|
||||
- **Managed roots** (gitignored): `doocus-data/<source>/` (docs) and
|
||||
`meetus-data/<source>/` (meetings), each `<root>/<source>/` mirroring the source.
|
||||
`process_tree.py --only {all|docs|meetings}` scopes a collection.
|
||||
- **doocus-app** discovers both roots (`DOOCUS_DATA` overrides, comma-separated),
|
||||
**merges collections that share `index.json.root`** into one source, searches the
|
||||
cached text, and can **scan a folder** (runs `process_tree` via `uv`) from the UI.
|
||||
- **meetus-app** is a thin shell over `@review` (`ui/meetus-app/src/components/ReviewBody.vue`),
|
||||
the review composition doocus embeds — meetus stays the single source.
|
||||
- **Diagram**: `docs/graphs/architecture.dot` → `docs/graphs/architecture.svg`
|
||||
(`make graphs`; needs system graphviz). Both committed.
|
||||
|
||||
## Notes
|
||||
|
||||
- **Default flow** uses `--embed-images` (frames referenced for the LLM to read) +
|
||||
|
||||
57
Makefile
57
Makefile
@@ -9,6 +9,13 @@
|
||||
# make batch IN="..." EXT="mp4 mov" # limit extensions
|
||||
# make batch IN="..." FLAGS="--embed-images" # replace the base flags
|
||||
#
|
||||
# Meetings flow (process only the videos doocus found, coherent with the docs):
|
||||
# 1. uv run make docs IN="/mnt/drive" # → docs-output/{index.json, meetings.txt}
|
||||
# 2. make batch IN="/mnt/drive" OUT=docs-output \ # meetus over just the meetings…
|
||||
# LIST=docs-output/meetings.txt \ # …only the listed relative paths
|
||||
# OUT_FORMAT="{name}.meetus" # …into <file>.meetus/ sidecars
|
||||
# (matches doocus's index pointer, so meetings sit beside the <file>.doocus/ docs.)
|
||||
#
|
||||
# IN is required. OUT defaults to IN (write in place); otherwise the input
|
||||
# folder structure is mirrored under OUT.
|
||||
|
||||
@@ -33,6 +40,16 @@ EXT ?=
|
||||
PYTHON ?=
|
||||
export PYTHON
|
||||
|
||||
# Optional: process only a newline list of RELATIVE paths (re-rooted at IN),
|
||||
# e.g. the doocus meeting export. Used to run meetus over just the meetings:
|
||||
# make batch IN="/mnt/drive" OUT=docs-output LIST=docs-output/meetings.txt
|
||||
LIST ?=
|
||||
|
||||
# Optional: run-folder name template forwarded to process_meeting.py --out-format.
|
||||
# Tokens: {date} {run} {stem} {name}. Default (empty) → "{date}-{run:03d}-{stem}".
|
||||
# Use "{name}.meetus" so meetings land in a docs-coherent <file>.meetus/ sidecar.
|
||||
OUT_FORMAT ?=
|
||||
|
||||
# doocus (tree indexing) knobs. DOCS_OUT defaults to docs-output; the whole input
|
||||
# tree is replicated into index.json under it. DOC_EXTRA forwards one-off flags to
|
||||
# process_tree.py (e.g. --render for pdf page thumbnails, --ocr for scanned pdfs).
|
||||
@@ -42,14 +59,28 @@ DOC_EXTRA ?=
|
||||
# Python interpreter for doocus (override PYTHON, else python3 — use `uv run make`).
|
||||
PY := $(if $(PYTHON),$(PYTHON),python3)
|
||||
|
||||
.PHONY: batch dry docs docs-dry help
|
||||
# Merge a separate output tree into another (e.g. a meetings run done in its own
|
||||
# folder → docs-output). Both mirror the same source-relative structure, so this
|
||||
# just unions the sidecars beside the docs. Always dry-run first.
|
||||
# make merge-dry SRC=meetings-output # preview
|
||||
# make merge SRC=meetings-output # apply (DST defaults to docs-output)
|
||||
MERGE_SRC ?=
|
||||
MERGE_DST ?= docs-output
|
||||
|
||||
# Diagrams: graphviz .dot → .svg (both committed). Requires system graphviz
|
||||
# (apt install graphviz). `make graphs` renders all docs/graphs/*.dot.
|
||||
DOT_SRC := $(wildcard docs/graphs/*.dot)
|
||||
SVG_OUT := $(DOT_SRC:.dot=.svg)
|
||||
|
||||
.PHONY: batch dry docs docs-dry merge merge-dry graphs help
|
||||
|
||||
batch:
|
||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -- \
|
||||
$(FLAGS) $(if $(AUDIO_LANG),--language $(AUDIO_LANG)) $(EXTRA)
|
||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") $(if $(LIST),-l "$(LIST)") -- \
|
||||
$(FLAGS) $(if $(AUDIO_LANG),--language $(AUDIO_LANG)) \
|
||||
$(if $(OUT_FORMAT),--out-format "$(OUT_FORMAT)") $(EXTRA)
|
||||
|
||||
dry:
|
||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -n
|
||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") $(if $(LIST),-l "$(LIST)") -n
|
||||
|
||||
# Document indexing (doocus): replicate the whole IN tree into DOCS_OUT/index.json,
|
||||
# extracting text sidecars only for docx/pdf/pptx/xlsx.
|
||||
@@ -62,5 +93,23 @@ docs:
|
||||
docs-dry:
|
||||
@$(PY) process_tree.py "$(IN)" --output "$(DOCS_OUT)" --dry-run
|
||||
|
||||
# index.json/meetings.txt are excluded so a merge of meeting dumps can never
|
||||
# clobber the canonical index that lives only in docs-output.
|
||||
MERGE_EXCLUDES = --exclude index.json --exclude meetings.txt
|
||||
|
||||
merge:
|
||||
@test -n "$(MERGE_SRC)" || { echo "ERROR: MERGE_SRC is required" >&2; exit 1; }
|
||||
@rsync -a $(MERGE_EXCLUDES) "$(MERGE_SRC)/" "$(MERGE_DST)/" && echo "merged $(MERGE_SRC) → $(MERGE_DST)"
|
||||
|
||||
merge-dry:
|
||||
@test -n "$(MERGE_SRC)" || { echo "ERROR: MERGE_SRC is required" >&2; exit 1; }
|
||||
@rsync -avn $(MERGE_EXCLUDES) "$(MERGE_SRC)/" "$(MERGE_DST)/"
|
||||
|
||||
docs/graphs/%.svg: docs/graphs/%.dot
|
||||
dot -Tsvg $< -o $@
|
||||
|
||||
graphs: $(SVG_OUT)
|
||||
@echo "rendered $(words $(SVG_OUT)) svg(s) from $(words $(DOT_SRC)) dot file(s)"
|
||||
|
||||
help:
|
||||
@sed -n '3,14p' Makefile
|
||||
|
||||
340
README.md
340
README.md
@@ -1,293 +1,115 @@
|
||||
# Meeting Processor
|
||||
# meetus + doocus — a local, offline context extractor
|
||||
|
||||
Extract screen content from meeting recordings and merge with Whisper/WhisperX transcripts for better AI summarization.
|
||||
Turn a downloaded Google Drive (meetings **and** documents) into a **searchable,
|
||||
browsable, packageable** local knowledge base — then hand focused subsets to the
|
||||
AI services you're allowed to use (Gemini web, NotebookLM). Every extraction step
|
||||
is **deterministic and offline**; no cloud AI ever touches the raw source.
|
||||
|
||||
## Overview
|
||||
Two pipelines feed two local UIs over a shared component framework:
|
||||
|
||||
This tool enhances meeting transcripts by combining:
|
||||
- **Audio transcription** (Whisper or WhisperX with speaker diarization)
|
||||
- **Screen content extraction** via FFmpeg scene detection
|
||||
- **Frame embedding** for direct LLM analysis
|
||||
- **meetus** — meeting recordings → enhanced transcripts (WhisperX + screen frames).
|
||||
- **doocus** — documents → textified content + metadata, indexed into a tree.
|
||||
|
||||
The result is a rich, timestamped transcript with embedded screen frames that provides full context for AI summarization.
|
||||

|
||||
|
||||
## Installation
|
||||
> Diagram source: [`docs/graphs/architecture.dot`](docs/graphs/architecture.dot)
|
||||
> (regenerate with `make graphs`). Repo map: [`INDEX.md`](INDEX.md).
|
||||
|
||||
### 1. System Dependencies
|
||||
## Install
|
||||
|
||||
**FFmpeg** (required for scene detection and frame extraction):
|
||||
### System dependencies
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo apt-get install ffmpeg
|
||||
|
||||
# macOS
|
||||
brew install ffmpeg
|
||||
sudo apt-get install ffmpeg graphviz # ffmpeg: frames/thumbnails · graphviz: docs diagrams
|
||||
# optional: tesseract-ocr (doocus --ocr), poppler-utils (pdf page render)
|
||||
# macOS: brew install ffmpeg graphviz
|
||||
```
|
||||
|
||||
### 2. Python Dependencies (uv)
|
||||
|
||||
Dependencies live in `pyproject.toml` as [uv](https://docs.astral.sh/uv/)
|
||||
feature groups — install only what you need:
|
||||
|
||||
### Python (uv feature groups)
|
||||
Dependencies live in `pyproject.toml` as [uv](https://docs.astral.sh/uv/) groups —
|
||||
install only what you need:
|
||||
```bash
|
||||
uv sync --group meetus # meeting pipeline (frame extraction)
|
||||
uv sync --group doocus # document extraction (see doocus/README.md)
|
||||
uv sync --group meetus # meeting pipeline (opencv, ffmpeg-python)
|
||||
uv sync --group doocus # document extraction (docx/pdf/pptx/xlsx/…)
|
||||
uv sync --group doocus --group ocr # + OCR for images / scanned pdfs
|
||||
uv run process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize
|
||||
```
|
||||
Groups: `meetus`, `doocus`, `ocr`, `pdf-render`, `deprecated`. **whisper/whisperx**
|
||||
are external CLI tools (like ffmpeg) — install separately (often a GPU env):
|
||||
`pip install whisperx` (diarization needs a HuggingFace token with pyannote access).
|
||||
|
||||
Groups: `meetus`, `doocus`, `ocr`, `pdf-render`, `deprecated` (the last is the
|
||||
unwired OCR/vision path — the only user of `ollama`, kept out of every default
|
||||
install).
|
||||
|
||||
### 3. Whisper or WhisperX (for audio transcription)
|
||||
|
||||
meetus calls these as **external CLI tools** (like ffmpeg), so they are *not*
|
||||
uv-managed — install them however suits your machine (often a separate GPU env):
|
||||
|
||||
### UIs (Node)
|
||||
```bash
|
||||
# standard whisper
|
||||
pip install openai-whisper
|
||||
# or WhisperX (recommended - adds speaker diarization)
|
||||
pip install whisperx
|
||||
cd ui/meetus-app && npm install
|
||||
cd ui/doocus-app && npm install
|
||||
```
|
||||
|
||||
For speaker diarization, you'll need a HuggingFace token with access to pyannote models.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Recommended Usage
|
||||
## The two pipelines
|
||||
|
||||
### meetus — meetings
|
||||
```bash
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 10 --diarize
|
||||
uv run process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 10 --diarize
|
||||
```
|
||||
Runs WhisperX (speaker diarization) + FFmpeg scene-detection frames → an enhanced
|
||||
transcript with frame references, in `output/<YYYYMMDD-NNN-stem>/`. `--out-format`
|
||||
customizes the run-folder name (e.g. `{name}.meetus` for a docs-coherent sidecar).
|
||||
Batch a tree with `make batch IN=<dir>`. `process_meeting.py --help` is the flag
|
||||
source of truth; caching (`--no-cache`, `--skip-cache-frames/-whisper`) lets you
|
||||
iterate on scene thresholds without re-running Whisper.
|
||||
|
||||
This will:
|
||||
1. Run WhisperX transcription with speaker diarization
|
||||
2. Extract frames at scene changes (threshold 10 = moderately sensitive)
|
||||
3. Create an enhanced transcript with frame file references
|
||||
4. Save everything to `output/` folder
|
||||
|
||||
The `--embed-images` flag adds frame paths to the transcript (e.g., `Frame: frames/video_00257.jpg`), keeping the transcript small while frames stay in `frames/` folder for LLM access.
|
||||
|
||||
### Re-run with Cached Results
|
||||
|
||||
Already ran it once? Re-run instantly using cached results:
|
||||
### doocus — documents
|
||||
```bash
|
||||
# Uses cached transcript and frames
|
||||
python process_meeting.py samples/meeting.mkv --embed-images
|
||||
|
||||
# Skip only specific cached items
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --skip-cache-frames
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --skip-cache-whisper
|
||||
|
||||
# Force complete reprocessing
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize --no-cache
|
||||
uv run process_tree.py "/path/to/drive" --output "doocus-data/<source>" --only docs
|
||||
```
|
||||
Replicates the whole tree into `index.json` (every file a node), extracting text
|
||||
sidecars (`content.md` + `meta.json`) only for `docx/pdf/pptx/xlsx`; other files
|
||||
are linked, not duplicated. `--only {all|docs|meetings}` scopes a collection.
|
||||
Full manual: [`doocus/README.md`](doocus/README.md).
|
||||
|
||||
## Usage Examples
|
||||
## Outputs & collections
|
||||
|
||||
### Scene Detection Options
|
||||
Each app writes to a **managed root** (gitignored), one subfolder per source,
|
||||
mirroring the source structure:
|
||||
|
||||
- `doocus-data/<source>/` — document collections (`index.json` + `.doocus/`)
|
||||
- `meetus-data/<source>/` — meeting collections (`index.json` + `.meetus/`)
|
||||
|
||||
Collections whose `index.json.root` matches are the **same source**: the doocus UI
|
||||
merges their docs + meetings into one tree, toggled as one source. A meeting's
|
||||
`.meetus` must sit in the **same folder** as its collection's `index.json`.
|
||||
|
||||
## The UIs
|
||||
|
||||
Local Vue 3 + Vite apps over `ui/framework/` (shared components + design tokens):
|
||||
|
||||
- **`ui/meetus-app`** — review a meeting: video + transcript (edit/read, speaker
|
||||
merge, select mode) + frame selector, time-synced.
|
||||
- **`ui/doocus-app`** — browse the merged tree, **search** cached text (transcripts,
|
||||
extracted docs, raw files), per-type viewers, and **package** selected files.
|
||||
Meetings **embed the meetus review** (`@review`, composed — not duplicated).
|
||||
```bash
|
||||
# Default threshold (15)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize
|
||||
|
||||
# More sensitive (more frames, threshold: 5)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 5 --diarize
|
||||
|
||||
# Less sensitive (fewer frames, threshold: 30)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 30 --diarize
|
||||
cd ui/doocus-app && DOOCUS_DATA="$PWD/../../doocus-data,$PWD/../../meetus-data" npm run dev
|
||||
cd ui/meetus-app && MEETUS_OUTPUT=/abs/path/to/output npm run dev
|
||||
```
|
||||
|
||||
### Fixed Interval Extraction (alternative to scene detection)
|
||||
```bash
|
||||
# Every 10 seconds
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --interval 10 --diarize
|
||||
## Packaging (the point)
|
||||
|
||||
# Every 3 seconds (more detailed)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --interval 3 --diarize
|
||||
The doocus package builder zips selected **originals** + their `content.md` for a
|
||||
target (Gemini web ≤10 files/≤100 MB, or NotebookLM). Originals are what the
|
||||
permitted services consume; the extracted text is the local **search index** and is
|
||||
never a replacement for the document.
|
||||
|
||||
## Project layout
|
||||
|
||||
See [`INDEX.md`](INDEX.md) for the full map. In brief:
|
||||
```
|
||||
process_meeting.py meetus CLI process_tree.py / process_doc.py doocus CLIs
|
||||
meetus/ meetus core doocus/ doocus core
|
||||
ctrl/ batch + ops scripts ui/{meetus-app,doocus-app,framework}
|
||||
Makefile batch · docs · merge · graphs docs/graphs/ architecture diagram
|
||||
def/ design notes samples/ output/ (gitignored)
|
||||
```
|
||||
|
||||
### Caching Examples
|
||||
```bash
|
||||
# First run - processes everything
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 10 --diarize
|
||||
|
||||
# Iterate on scene threshold (reuse whisper transcript)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 5 --skip-cache-frames
|
||||
|
||||
# Re-run whisper only
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --skip-cache-whisper
|
||||
|
||||
# Force complete reprocessing
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize --no-cache
|
||||
```
|
||||
|
||||
### Custom output location
|
||||
```bash
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize --output-dir my_outputs/
|
||||
```
|
||||
|
||||
### Enable verbose logging
|
||||
```bash
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize --verbose
|
||||
```
|
||||
|
||||
## Output Files
|
||||
|
||||
Each video gets its own timestamped output directory:
|
||||
|
||||
```
|
||||
output/
|
||||
└── 20241019_143022-meeting/
|
||||
├── manifest.json # Processing configuration
|
||||
├── meeting_enhanced.txt # Enhanced transcript for AI
|
||||
├── meeting.json # Whisper/WhisperX transcript
|
||||
└── frames/ # Extracted video frames
|
||||
├── frame_00001_5.00s.jpg
|
||||
├── frame_00002_10.00s.jpg
|
||||
└── ...
|
||||
```
|
||||
|
||||
### Caching Behavior
|
||||
|
||||
The tool automatically reuses the most recent output directory for the same video:
|
||||
- **First run**: Creates new timestamped directory (e.g., `20241019_143022-meeting/`)
|
||||
- **Subsequent runs**: Reuses the same directory and cached results
|
||||
- **Cached items**: Whisper transcript, extracted frames, analysis results
|
||||
|
||||
**Fine-grained cache control:**
|
||||
- `--no-cache`: Force complete reprocessing
|
||||
- `--skip-cache-frames`: Re-extract frames only
|
||||
- `--skip-cache-whisper`: Re-run transcription only
|
||||
|
||||
This allows you to iterate on scene detection thresholds without re-running Whisper!
|
||||
|
||||
## Workflow for Meeting Analysis
|
||||
|
||||
### Complete Workflow (One Command!)
|
||||
|
||||
```bash
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 10 --diarize
|
||||
```
|
||||
|
||||
### Typical Iterative Workflow
|
||||
|
||||
```bash
|
||||
# First run - full processing
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 10 --diarize
|
||||
|
||||
# Adjust scene threshold (keeps cached whisper transcript)
|
||||
python process_meeting.py samples/meeting.mkv --embed-images --scene-detection --scene-threshold 5 --skip-cache-frames
|
||||
```
|
||||
|
||||
### Example Prompt for Claude
|
||||
|
||||
```
|
||||
Please summarize this meeting transcript. Pay special attention to:
|
||||
1. Key decisions made
|
||||
2. Action items
|
||||
3. Technical details shown on screen
|
||||
4. Any metrics or data presented
|
||||
|
||||
[Paste enhanced transcript here]
|
||||
```
|
||||
|
||||
## Command Reference
|
||||
|
||||
`process_meeting.py --help` is the source of truth for flags — run it rather than
|
||||
relying on a copy here. The essentials:
|
||||
|
||||
- `--diarize` — WhisperX with speaker diarization (needs a HuggingFace token)
|
||||
- `--embed-images` — reference frames in the transcript for the LLM (default behavior)
|
||||
- `--scene-detection` / `--scene-threshold N` — frame extraction (lower = more frames)
|
||||
- `--interval N` — fixed-interval extraction instead of scene detection
|
||||
- `--transcript-formats srt,vtt,…` — extra transcript formats alongside JSON
|
||||
- `--no-cache` / `--skip-cache-frames` / `--skip-cache-whisper` — cache control
|
||||
|
||||
For batches, prefer `make batch IN=<dir>` (see [`INDEX.md`](INDEX.md)).
|
||||
|
||||
## Tips for Best Results
|
||||
|
||||
### Scene Detection vs Interval
|
||||
- **Scene detection** (`--scene-detection`): Recommended. Captures frames when content changes. More efficient.
|
||||
- **Interval extraction** (`--interval N`): Alternative for continuous content. Captures every N seconds.
|
||||
|
||||
### Scene Detection Threshold
|
||||
- Lower values (5-10): More sensitive, captures more frames
|
||||
- Default (15): Good balance for most meetings
|
||||
- Higher values (20-30): Less sensitive, fewer frames
|
||||
|
||||
### Whisper vs WhisperX
|
||||
- **Whisper** (`--run-whisper`): Standard transcription, fast
|
||||
- **WhisperX** (`--run-whisper --diarize`): Adds speaker identification, requires HuggingFace token
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Frame Extraction Issues
|
||||
|
||||
**"No frames extracted"**
|
||||
- Check video file is valid: `ffmpeg -i video.mkv`
|
||||
- Try lower scene threshold: `--scene-threshold 5`
|
||||
- Try interval extraction: `--interval 3`
|
||||
- Check disk space in output directory
|
||||
|
||||
**Scene detection not working**
|
||||
- Ensure FFmpeg is installed
|
||||
- Falls back to interval extraction automatically
|
||||
- Try manual interval: `--interval 5`
|
||||
|
||||
### Whisper/WhisperX Issues
|
||||
|
||||
**WhisperX diarization not working**
|
||||
- Ensure you have a HuggingFace token set
|
||||
- Token needs access to pyannote models
|
||||
- Fall back to standard Whisper without `--diarize`
|
||||
|
||||
### Cache Issues
|
||||
|
||||
**Cache not being used**
|
||||
- Ensure you're using the same video filename
|
||||
- Check that output directory contains cached files
|
||||
- Use `--verbose` to see what's being cached/loaded
|
||||
|
||||
**Want to re-run specific steps**
|
||||
- `--skip-cache-frames`: Re-extract frames
|
||||
- `--skip-cache-whisper`: Re-run transcription
|
||||
- `--no-cache`: Force complete reprocessing
|
||||
|
||||
## Deprecated Features (kept for reference)
|
||||
|
||||
### OCR and Vision Analysis
|
||||
|
||||
OCR, Vision and Hybrid screen-text analysis were the original approach but went nowhere. They have been **removed from the CLI** (the `--ocr-engine` / `--use-vision` / `--use-hybrid` flags no longer exist) and now live, unwired, in `meetus/deprecated/` for reference only. The tool always references frames (`--embed-images`) so your LLM reads them directly. The realtime continuation of the idea is the separate `cht` project. See [`INDEX.md`](INDEX.md).
|
||||
|
||||
## Project Structure
|
||||
|
||||
See [`INDEX.md`](INDEX.md) for the full repo map. In brief:
|
||||
|
||||
```
|
||||
meetus/
|
||||
├── process_meeting.py # Main CLI script (entry point)
|
||||
├── Makefile # `make batch` convenience wrapper
|
||||
├── meetus/ # Core package
|
||||
│ ├── workflow.py # Processing orchestrator
|
||||
│ ├── frame_extractor.py # Frame extraction (FFmpeg scene detection)
|
||||
│ ├── transcript_merger.py # Transcript + frame-ref merging
|
||||
│ ├── output_manager.py # Run dirs (YYYYMMDD-NNN-<stem>) & manifest
|
||||
│ ├── cache_manager.py # Per-step caching
|
||||
│ └── deprecated/ # Old OCR/vision/hybrid analysis (reference only)
|
||||
├── ctrl/ # Control plane / operational scripts
|
||||
│ ├── batch.sh # Recursive batch runner
|
||||
│ ├── transcribe_oneoff.sh # High-quality re-transcription
|
||||
│ ├── summarize/ # Local-LLM summarization (WIP, on hold)
|
||||
│ └── cht/ # Bridge to the realtime `cht` project
|
||||
├── def/ # Design/decision notes
|
||||
├── output/ # Run directories (gitignored)
|
||||
├── samples/ # Sample inputs (gitignored)
|
||||
└── README.md # This file
|
||||
```
|
||||
Deprecated OCR/vision code lives unwired in `meetus/deprecated/` (the only user of
|
||||
`ollama`, kept out of every default install). See [`INDEX.md`](INDEX.md).
|
||||
|
||||
## License
|
||||
|
||||
For personal use.
|
||||
|
||||
@@ -7,14 +7,19 @@
|
||||
# the mirrored output subfolder.
|
||||
#
|
||||
# Usage:
|
||||
# ctrl/batch.sh -i <input-dir> -o <output-dir> [-e "mkv mp4 ..."] [-n] \
|
||||
# [-- <process_meeting.py flags>]
|
||||
# ctrl/batch.sh -i <input-dir> -o <output-dir> [-e "mkv mp4 ..."] \
|
||||
# [-l <list-file>] [-n] [-- <process_meeting.py flags>]
|
||||
#
|
||||
# Examples:
|
||||
# # Everything under a mounted share, default extraction flags forwarded
|
||||
# ctrl/batch.sh -i "/mnt/win/trainings" -o output/batch \
|
||||
# -- --embed-images --scene-detection --diarize
|
||||
#
|
||||
# # Only the meetings doocus found (relative paths), re-rooted at a new mount,
|
||||
# # output into the doocus folder for later packaging:
|
||||
# ctrl/batch.sh -i "/mnt/drive" -o docs-output -l docs-output/meetings.txt \
|
||||
# -- --embed-images --scene-detection --diarize
|
||||
#
|
||||
# # Dry run: just show which videos map to which output folders
|
||||
# ctrl/batch.sh -i "/mnt/win/trainings" -o output/batch -n
|
||||
#
|
||||
@@ -42,6 +47,7 @@ INPUT=""
|
||||
OUTPUT=""
|
||||
EXTS="mkv mp4 mov avi m4v webm wmv ogg mp3 wav m4a opus flac aac"
|
||||
DRY=false
|
||||
LIST="" # optional newline list of RELATIVE paths (re-rooted at INPUT)
|
||||
FORWARD=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -49,6 +55,7 @@ while [[ $# -gt 0 ]]; do
|
||||
-i|--input) INPUT="$2"; shift 2 ;;
|
||||
-o|--output) OUTPUT="$2"; shift 2 ;;
|
||||
-e|--ext) EXTS="$2"; shift 2 ;;
|
||||
-l|--list) LIST="$2"; shift 2 ;;
|
||||
-n|--dry-run) DRY=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--) shift; FORWARD=("$@"); break ;;
|
||||
@@ -59,6 +66,7 @@ done
|
||||
[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; }
|
||||
[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; }
|
||||
[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; }
|
||||
[ -z "$LIST" ] || [ -f "$LIST" ] || { echo "ERROR: list file not found: $LIST" >&2; exit 1; }
|
||||
|
||||
# Absolute paths so relative-path math is stable regardless of where we cd'd.
|
||||
INPUT="$(realpath "$INPUT")"
|
||||
@@ -74,16 +82,35 @@ unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o
|
||||
|
||||
echo "Input : $INPUT"
|
||||
echo "Output: $OUTPUT"
|
||||
echo "Exts : $EXTS"
|
||||
[ -n "$LIST" ] && echo "List : $LIST (relative paths, re-rooted at input)" || echo "Exts : $EXTS"
|
||||
[ "$DRY" = true ] && echo "(dry run — nothing will be processed)"
|
||||
echo
|
||||
|
||||
# Producer: either the given list of relative paths (re-rooted at INPUT) or a
|
||||
# recursive find by extension. Both emit null-delimited absolute paths.
|
||||
producer() {
|
||||
if [ -n "$LIST" ]; then
|
||||
while IFS= read -r rel || [ -n "$rel" ]; do
|
||||
[ -z "$rel" ] && continue
|
||||
printf '%s\0' "$INPUT/$rel"
|
||||
done < "$LIST"
|
||||
else
|
||||
find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z
|
||||
fi
|
||||
}
|
||||
|
||||
total=0 ok=0 fail=0
|
||||
# Process substitution (not a pipe) so counters survive into the summary.
|
||||
while IFS= read -r -d '' video; do
|
||||
total=$((total + 1))
|
||||
|
||||
rel="${video#"$INPUT"/}" # path relative to the input root
|
||||
|
||||
if [ ! -f "$video" ]; then # list may name a path missing under this root
|
||||
echo "[$total] $rel !! not found under input (skipped)" >&2
|
||||
fail=$((fail + 1))
|
||||
continue
|
||||
fi
|
||||
reldir="$(dirname "$rel")"
|
||||
if [ "$reldir" = "." ]; then
|
||||
outdir="$OUTPUT" # video sat directly in the input root
|
||||
@@ -110,7 +137,7 @@ while IFS= read -r -d '' video; do
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
echo
|
||||
done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z)
|
||||
done < <(producer)
|
||||
|
||||
echo "----------------------------------------"
|
||||
if [ "$DRY" = true ]; then
|
||||
|
||||
53
docs/graphs/architecture.dot
Normal file
53
docs/graphs/architecture.dot
Normal file
@@ -0,0 +1,53 @@
|
||||
// Architecture of the meetus + doocus toolkit.
|
||||
// Regenerate the SVG with: make graphs (or: dot -Tsvg docs/graphs/architecture.dot -o docs/graphs/architecture.svg)
|
||||
digraph meetus_doocus {
|
||||
rankdir=LR
|
||||
splines=ortho
|
||||
fontname="Helvetica"
|
||||
node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=11, fillcolor="#f5f5f5", color="#cccccc"]
|
||||
edge [fontname="Helvetica", fontsize=9, color="#888888"]
|
||||
|
||||
drive [label="Downloaded Drive\n(local · sensitive)", fillcolor="#e8eef7"]
|
||||
|
||||
subgraph cluster_pipe {
|
||||
label="Deterministic, offline pipelines — no cloud AI touches the source"
|
||||
labeljust=l ; style=dashed ; color="#bbbbbb" ; fontsize=10
|
||||
meetus_cli [label="meetus\nprocess_meeting.py · make batch\nwhisperx + ffmpeg frames"]
|
||||
doocus_cli [label="doocus\nprocess_tree.py · process_doc.py\nlocal text extraction"]
|
||||
}
|
||||
|
||||
subgraph cluster_out {
|
||||
label="Managed collection roots (gitignored) — <root>/<source>/, mirrored"
|
||||
labeljust=l ; style=dashed ; color="#bbbbbb" ; fontsize=10
|
||||
meetus_data [label="meetus-data/<source>/\nindex.json + <file>.meetus/\ntranscript · frames", fillcolor="#fdf0e3"]
|
||||
doocus_data [label="doocus-data/<source>/\nindex.json + <file>.doocus/\ncontent.md · meta.json", fillcolor="#eaf6ea"]
|
||||
}
|
||||
|
||||
subgraph cluster_ui {
|
||||
label="Local UIs (Vue 3 + Vite)"
|
||||
labeljust=l ; style=dashed ; color="#bbbbbb" ; fontsize=10
|
||||
doocus_app [label="doocus-app\ntree · search · package\nembeds the meeting review"]
|
||||
meetus_app [label="meetus-app\nmeeting review\nvideo · transcript · frames"]
|
||||
review [label="@review\n(meetus ReviewBody — composed by both)", fillcolor="#f3ecfa"]
|
||||
framework [label="ui/framework\nshared components + tokens", fillcolor="#f3ecfa"]
|
||||
}
|
||||
|
||||
package [label="Package (zip)\noriginals + content.md", fillcolor="#e8eef7"]
|
||||
services [label="Gemini web · NotebookLM\n(permitted services)", shape=ellipse, fillcolor="#e8eef7"]
|
||||
|
||||
drive -> meetus_cli
|
||||
drive -> doocus_cli
|
||||
meetus_cli -> meetus_data
|
||||
doocus_cli -> doocus_data
|
||||
|
||||
doocus_data -> doocus_app
|
||||
meetus_data -> doocus_app [xlabel="merge by\nshared root"]
|
||||
meetus_data -> meetus_app
|
||||
|
||||
framework -> review [style=dashed, arrowhead=none]
|
||||
review -> meetus_app [style=dashed, xlabel="standalone"]
|
||||
review -> doocus_app [style=dashed, xlabel="embedded"]
|
||||
|
||||
doocus_app -> package
|
||||
package -> services
|
||||
}
|
||||
186
docs/graphs/architecture.svg
Normal file
186
docs/graphs/architecture.svg
Normal file
@@ -0,0 +1,186 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.42.4 (0)
|
||||
-->
|
||||
<!-- Title: meetus_doocus Pages: 1 -->
|
||||
<svg width="1517pt" height="314pt"
|
||||
viewBox="0.00 0.00 1516.69 314.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 310)">
|
||||
<title>meetus_doocus</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-310 1512.69,-310 1512.69,4 -4,4"/>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_pipe</title>
|
||||
<polygon fill="none" stroke="#bbbbbb" stroke-dasharray="5,2" points="145,-157 145,-298 477,-298 477,-157 145,-157"/>
|
||||
<text text-anchor="middle" x="311" y="-286" font-family="Helvetica,sans-Serif" font-size="10.00">Deterministic, offline pipelines — no cloud AI touches the source</text>
|
||||
</g>
|
||||
<g id="clust2" class="cluster">
|
||||
<title>cluster_out</title>
|
||||
<polygon fill="none" stroke="#bbbbbb" stroke-dasharray="5,2" points="505,-157 505,-298 859,-298 859,-157 505,-157"/>
|
||||
<text text-anchor="middle" x="682" y="-286" font-family="Helvetica,sans-Serif" font-size="10.00">Managed collection roots (gitignored) — <root>/<source>/, mirrored</text>
|
||||
</g>
|
||||
<g id="clust3" class="cluster">
|
||||
<title>cluster_ui</title>
|
||||
<polygon fill="none" stroke="#bbbbbb" stroke-dasharray="5,2" points="213,-8 213,-149 1074,-149 1074,-8 213,-8"/>
|
||||
<text text-anchor="middle" x="279" y="-137" font-family="Helvetica,sans-Serif" font-size="10.00">Local UIs (Vue 3 + Vite)</text>
|
||||
</g>
|
||||
<!-- drive -->
|
||||
<g id="node1" class="node">
|
||||
<title>drive</title>
|
||||
<path fill="#e8eef7" stroke="#cccccc" d="M105,-236C105,-236 12,-236 12,-236 6,-236 0,-230 0,-224 0,-224 0,-212 0,-212 0,-206 6,-200 12,-200 12,-200 105,-200 105,-200 111,-200 117,-206 117,-212 117,-212 117,-224 117,-224 117,-230 111,-236 105,-236"/>
|
||||
<text text-anchor="middle" x="58.5" y="-221.2" font-family="Helvetica,sans-Serif" font-size="11.00">Downloaded Drive</text>
|
||||
<text text-anchor="middle" x="58.5" y="-209.2" font-family="Helvetica,sans-Serif" font-size="11.00">(local · sensitive)</text>
|
||||
</g>
|
||||
<!-- meetus_cli -->
|
||||
<g id="node2" class="node">
|
||||
<title>meetus_cli</title>
|
||||
<path fill="#f5f5f5" stroke="#cccccc" d="M400,-209C400,-209 221,-209 221,-209 215,-209 209,-203 209,-197 209,-197 209,-177 209,-177 209,-171 215,-165 221,-165 221,-165 400,-165 400,-165 406,-165 412,-171 412,-177 412,-177 412,-197 412,-197 412,-203 406,-209 400,-209"/>
|
||||
<text text-anchor="middle" x="310.5" y="-196.2" font-family="Helvetica,sans-Serif" font-size="11.00">meetus</text>
|
||||
<text text-anchor="middle" x="310.5" y="-184.2" font-family="Helvetica,sans-Serif" font-size="11.00">process_meeting.py · make batch</text>
|
||||
<text text-anchor="middle" x="310.5" y="-172.2" font-family="Helvetica,sans-Serif" font-size="11.00">whisperx + ffmpeg frames</text>
|
||||
</g>
|
||||
<!-- drive->meetus_cli -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>drive->meetus_cli</title>
|
||||
<path fill="none" stroke="#888888" d="M117.26,-204.5C117.26,-204.5 198.64,-204.5 198.64,-204.5"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="198.64,-208 208.64,-204.5 198.64,-201 198.64,-208"/>
|
||||
</g>
|
||||
<!-- doocus_cli -->
|
||||
<g id="node3" class="node">
|
||||
<title>doocus_cli</title>
|
||||
<path fill="#f5f5f5" stroke="#cccccc" d="M398.5,-271C398.5,-271 222.5,-271 222.5,-271 216.5,-271 210.5,-265 210.5,-259 210.5,-259 210.5,-239 210.5,-239 210.5,-233 216.5,-227 222.5,-227 222.5,-227 398.5,-227 398.5,-227 404.5,-227 410.5,-233 410.5,-239 410.5,-239 410.5,-259 410.5,-259 410.5,-265 404.5,-271 398.5,-271"/>
|
||||
<text text-anchor="middle" x="310.5" y="-258.2" font-family="Helvetica,sans-Serif" font-size="11.00">doocus</text>
|
||||
<text text-anchor="middle" x="310.5" y="-246.2" font-family="Helvetica,sans-Serif" font-size="11.00">process_tree.py · process_doc.py</text>
|
||||
<text text-anchor="middle" x="310.5" y="-234.2" font-family="Helvetica,sans-Serif" font-size="11.00">local text extraction</text>
|
||||
</g>
|
||||
<!-- drive->doocus_cli -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>drive->doocus_cli</title>
|
||||
<path fill="none" stroke="#888888" d="M117.26,-231.5C117.26,-231.5 200.13,-231.5 200.13,-231.5"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="200.14,-235 210.13,-231.5 200.13,-228 200.14,-235"/>
|
||||
</g>
|
||||
<!-- meetus_data -->
|
||||
<g id="node4" class="node">
|
||||
<title>meetus_data</title>
|
||||
<path fill="#fdf0e3" stroke="#cccccc" d="M756,-209C756,-209 607,-209 607,-209 601,-209 595,-203 595,-197 595,-197 595,-177 595,-177 595,-171 601,-165 607,-165 607,-165 756,-165 756,-165 762,-165 768,-171 768,-177 768,-177 768,-197 768,-197 768,-203 762,-209 756,-209"/>
|
||||
<text text-anchor="middle" x="681.5" y="-196.2" font-family="Helvetica,sans-Serif" font-size="11.00">meetus-data/<source>/</text>
|
||||
<text text-anchor="middle" x="681.5" y="-184.2" font-family="Helvetica,sans-Serif" font-size="11.00">index.json + <file>.meetus/</text>
|
||||
<text text-anchor="middle" x="681.5" y="-172.2" font-family="Helvetica,sans-Serif" font-size="11.00">transcript · frames</text>
|
||||
</g>
|
||||
<!-- meetus_cli->meetus_data -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>meetus_cli->meetus_data</title>
|
||||
<path fill="none" stroke="#888888" d="M412.37,-187C412.37,-187 584.95,-187 584.95,-187"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="584.95,-190.5 594.95,-187 584.95,-183.5 584.95,-190.5"/>
|
||||
</g>
|
||||
<!-- doocus_data -->
|
||||
<g id="node5" class="node">
|
||||
<title>doocus_data</title>
|
||||
<path fill="#eaf6ea" stroke="#cccccc" d="M754.5,-271C754.5,-271 608.5,-271 608.5,-271 602.5,-271 596.5,-265 596.5,-259 596.5,-259 596.5,-239 596.5,-239 596.5,-233 602.5,-227 608.5,-227 608.5,-227 754.5,-227 754.5,-227 760.5,-227 766.5,-233 766.5,-239 766.5,-239 766.5,-259 766.5,-259 766.5,-265 760.5,-271 754.5,-271"/>
|
||||
<text text-anchor="middle" x="681.5" y="-258.2" font-family="Helvetica,sans-Serif" font-size="11.00">doocus-data/<source>/</text>
|
||||
<text text-anchor="middle" x="681.5" y="-246.2" font-family="Helvetica,sans-Serif" font-size="11.00">index.json + <file>.doocus/</text>
|
||||
<text text-anchor="middle" x="681.5" y="-234.2" font-family="Helvetica,sans-Serif" font-size="11.00">content.md · meta.json</text>
|
||||
</g>
|
||||
<!-- doocus_cli->doocus_data -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>doocus_cli->doocus_data</title>
|
||||
<path fill="none" stroke="#888888" d="M410.9,-249C410.9,-249 586.43,-249 586.43,-249"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="586.43,-252.5 596.43,-249 586.43,-245.5 586.43,-252.5"/>
|
||||
</g>
|
||||
<!-- doocus_app -->
|
||||
<g id="node6" class="node">
|
||||
<title>doocus_app</title>
|
||||
<path fill="#f5f5f5" stroke="#cccccc" d="M1054,-122C1054,-122 907,-122 907,-122 901,-122 895,-116 895,-110 895,-110 895,-90 895,-90 895,-84 901,-78 907,-78 907,-78 1054,-78 1054,-78 1060,-78 1066,-84 1066,-90 1066,-90 1066,-110 1066,-110 1066,-116 1060,-122 1054,-122"/>
|
||||
<text text-anchor="middle" x="980.5" y="-109.2" font-family="Helvetica,sans-Serif" font-size="11.00">doocus-app</text>
|
||||
<text text-anchor="middle" x="980.5" y="-97.2" font-family="Helvetica,sans-Serif" font-size="11.00">tree · search · package</text>
|
||||
<text text-anchor="middle" x="980.5" y="-85.2" font-family="Helvetica,sans-Serif" font-size="11.00">embeds the meeting review</text>
|
||||
</g>
|
||||
<!-- meetus_data->doocus_app -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>meetus_data->doocus_app</title>
|
||||
<path fill="none" stroke="#888888" d="M681.69,-164.89C681.69,-140.45 681.69,-104.5 681.69,-104.5 681.69,-104.5 884.87,-104.5 884.87,-104.5"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="884.87,-108 894.87,-104.5 884.87,-101 884.87,-108"/>
|
||||
<text text-anchor="middle" x="727.08" y="-117.3" font-family="Helvetica,sans-Serif" font-size="9.00">merge by</text>
|
||||
<text text-anchor="middle" x="727.08" y="-107.3" font-family="Helvetica,sans-Serif" font-size="9.00">shared root</text>
|
||||
</g>
|
||||
<!-- meetus_app -->
|
||||
<g id="node7" class="node">
|
||||
<title>meetus_app</title>
|
||||
<path fill="#f5f5f5" stroke="#cccccc" d="M1049,-60C1049,-60 912,-60 912,-60 906,-60 900,-54 900,-48 900,-48 900,-28 900,-28 900,-22 906,-16 912,-16 912,-16 1049,-16 1049,-16 1055,-16 1061,-22 1061,-28 1061,-28 1061,-48 1061,-48 1061,-54 1055,-60 1049,-60"/>
|
||||
<text text-anchor="middle" x="980.5" y="-47.2" font-family="Helvetica,sans-Serif" font-size="11.00">meetus-app</text>
|
||||
<text text-anchor="middle" x="980.5" y="-35.2" font-family="Helvetica,sans-Serif" font-size="11.00">meeting review</text>
|
||||
<text text-anchor="middle" x="980.5" y="-23.2" font-family="Helvetica,sans-Serif" font-size="11.00">video · transcript · frames</text>
|
||||
</g>
|
||||
<!-- meetus_data->meetus_app -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>meetus_data->meetus_app</title>
|
||||
<path fill="none" stroke="#888888" d="M768.1,-187C810.98,-187 853.69,-187 853.69,-187 853.69,-187 853.69,-57 853.69,-57 853.69,-57 889.83,-57 889.83,-57"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="889.83,-60.5 899.83,-57 889.83,-53.5 889.83,-60.5"/>
|
||||
</g>
|
||||
<!-- doocus_data->doocus_app -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>doocus_data->doocus_app</title>
|
||||
<path fill="none" stroke="#888888" d="M766.83,-249C855.27,-249 980.69,-249 980.69,-249 980.69,-249 980.69,-132.31 980.69,-132.31"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="984.19,-132.31 980.69,-122.31 977.19,-132.31 984.19,-132.31"/>
|
||||
</g>
|
||||
<!-- package -->
|
||||
<g id="node10" class="node">
|
||||
<title>package</title>
|
||||
<path fill="#e8eef7" stroke="#cccccc" d="M1233,-118C1233,-118 1114,-118 1114,-118 1108,-118 1102,-112 1102,-106 1102,-106 1102,-94 1102,-94 1102,-88 1108,-82 1114,-82 1114,-82 1233,-82 1233,-82 1239,-82 1245,-88 1245,-94 1245,-94 1245,-106 1245,-106 1245,-112 1239,-118 1233,-118"/>
|
||||
<text text-anchor="middle" x="1173.5" y="-103.2" font-family="Helvetica,sans-Serif" font-size="11.00">Package (zip)</text>
|
||||
<text text-anchor="middle" x="1173.5" y="-91.2" font-family="Helvetica,sans-Serif" font-size="11.00">originals + content.md</text>
|
||||
</g>
|
||||
<!-- doocus_app->package -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>doocus_app->package</title>
|
||||
<path fill="none" stroke="#888888" d="M1066.28,-100C1066.28,-100 1091.84,-100 1091.84,-100"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="1091.84,-103.5 1101.84,-100 1091.84,-96.5 1091.84,-103.5"/>
|
||||
</g>
|
||||
<!-- review -->
|
||||
<g id="node8" class="node">
|
||||
<title>review</title>
|
||||
<path fill="#f3ecfa" stroke="#cccccc" d="M799,-87C799,-87 564,-87 564,-87 558,-87 552,-81 552,-75 552,-75 552,-63 552,-63 552,-57 558,-51 564,-51 564,-51 799,-51 799,-51 805,-51 811,-57 811,-63 811,-63 811,-75 811,-75 811,-81 805,-87 799,-87"/>
|
||||
<text text-anchor="middle" x="681.5" y="-72.2" font-family="Helvetica,sans-Serif" font-size="11.00">@review</text>
|
||||
<text text-anchor="middle" x="681.5" y="-60.2" font-family="Helvetica,sans-Serif" font-size="11.00">(meetus ReviewBody — composed by both)</text>
|
||||
</g>
|
||||
<!-- review->doocus_app -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>review->doocus_app</title>
|
||||
<path fill="none" stroke="#888888" stroke-dasharray="5,2" d="M811.34,-82.5C811.34,-82.5 884.71,-82.5 884.71,-82.5"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="884.71,-86 894.71,-82.5 884.71,-79 884.71,-86"/>
|
||||
<text text-anchor="middle" x="872.53" y="-75.3" font-family="Helvetica,sans-Serif" font-size="9.00">embedded</text>
|
||||
</g>
|
||||
<!-- review->meetus_app -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>review->meetus_app</title>
|
||||
<path fill="none" stroke="#888888" stroke-dasharray="5,2" d="M811.34,-54C811.34,-54 889.9,-54 889.9,-54"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="889.9,-57.5 899.9,-54 889.9,-50.5 889.9,-57.5"/>
|
||||
<text text-anchor="middle" x="875.62" y="-46.8" font-family="Helvetica,sans-Serif" font-size="9.00">standalone</text>
|
||||
</g>
|
||||
<!-- framework -->
|
||||
<g id="node9" class="node">
|
||||
<title>framework</title>
|
||||
<path fill="#f3ecfa" stroke="#cccccc" d="M388,-87C388,-87 233,-87 233,-87 227,-87 221,-81 221,-75 221,-75 221,-63 221,-63 221,-57 227,-51 233,-51 233,-51 388,-51 388,-51 394,-51 400,-57 400,-63 400,-63 400,-75 400,-75 400,-81 394,-87 388,-87"/>
|
||||
<text text-anchor="middle" x="310.5" y="-72.2" font-family="Helvetica,sans-Serif" font-size="11.00">ui/framework</text>
|
||||
<text text-anchor="middle" x="310.5" y="-60.2" font-family="Helvetica,sans-Serif" font-size="11.00">shared components + tokens</text>
|
||||
</g>
|
||||
<!-- framework->review -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>framework->review</title>
|
||||
<path fill="none" stroke="#888888" stroke-dasharray="5,2" d="M400.31,-69C445.77,-69 501.8,-69 551.84,-69"/>
|
||||
</g>
|
||||
<!-- services -->
|
||||
<g id="node11" class="node">
|
||||
<title>services</title>
|
||||
<ellipse fill="#e8eef7" stroke="#cccccc" cx="1394.84" cy="-100" rx="113.69" ry="22.76"/>
|
||||
<text text-anchor="middle" x="1394.84" y="-103.2" font-family="Helvetica,sans-Serif" font-size="11.00">Gemini web · NotebookLM</text>
|
||||
<text text-anchor="middle" x="1394.84" y="-91.2" font-family="Helvetica,sans-Serif" font-size="11.00">(permitted services)</text>
|
||||
</g>
|
||||
<!-- package->services -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>package->services</title>
|
||||
<path fill="none" stroke="#888888" d="M1245.06,-100C1245.06,-100 1270.89,-100 1270.89,-100"/>
|
||||
<polygon fill="#888888" stroke="#888888" points="1270.89,-103.5 1280.89,-100 1270.89,-96.5 1270.89,-103.5"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 13 KiB |
109
doocus/README.md
109
doocus/README.md
@@ -1,37 +1,74 @@
|
||||
# doocus — local document extraction
|
||||
# doocus — local drive tree indexer
|
||||
|
||||
The document twin of meetus. Turns locally-downloaded files (Drive exports,
|
||||
PDFs, images, ...) into a shareable **textified** product (`content.md`) plus
|
||||
structured `meta.json`, mirroring the meetus `output/<run>/` contract.
|
||||
The document twin of meetus. Replicates a **whole local drive tree** into a
|
||||
single `index.json`, preserving the folder hierarchy (no flattening). Every file
|
||||
is a node; the **original file is the artifact** (what a QA/PM opens, what a
|
||||
package links to). doocus only *extracts text* for the heavy formats
|
||||
(docx/pdf/pptx/xlsx) — and that text is a **search index only, never a
|
||||
replacement** for the document.
|
||||
|
||||
Extraction is **deterministic and offline** — no cloud AI touches the raw source.
|
||||
The textified output is what feeds permitted services (Gemini web, NotebookLM)
|
||||
and the local search index; the raw source stays on disk.
|
||||
The raw source stays on disk; permitted services (Gemini web, NotebookLM) get the
|
||||
original (as a link once we have Drive URLs, or re-uploaded).
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# one document
|
||||
python process_doc.py notes.docx # → docs-output/<YYYYMMDD-NNN-notes>/
|
||||
python process_doc.py report.pdf --render # also render page 1 → thumb.jpg
|
||||
python process_doc.py scan.png --ocr # OCR the image into content.md
|
||||
# index a whole downloaded drive tree → docs-output/index.json
|
||||
uv run make docs IN="/mnt/win/drive"
|
||||
uv run make docs IN="..." DOC_EXTRA="--ocr" # OCR scanned pdfs
|
||||
make docs-dry IN="..." # counts only, no writes
|
||||
|
||||
# a whole downloaded folder (mirrors the input tree)
|
||||
make docs IN="/mnt/win/drive" # → docs-output/...
|
||||
make docs IN="..." DOC_EXTRA="--render --ocr"
|
||||
make docs-dry IN="..." # list what would run
|
||||
# or directly
|
||||
uv run process_tree.py /mnt/win/drive --output docs-output
|
||||
|
||||
# one-off single file (older per-file model, still handy)
|
||||
uv run process_doc.py notes.docx
|
||||
```
|
||||
|
||||
## Output contract
|
||||
|
||||
```
|
||||
docs-output/<YYYYMMDD-NNN-stem>/
|
||||
content.md textified document — the shareable product
|
||||
meta.json source info, sha256, dates, author, title, pages/sheets/slides, word_count, warnings
|
||||
thumb.jpg optional single preview (image / rendered page / video frame)
|
||||
manifest.json run config + outputs inventory + source pointer (path, not a copy)
|
||||
docs-output/
|
||||
index.json whole tree; every file a node:
|
||||
{ path, name, ext, family, mode, bytes,
|
||||
modified, url:null } ◀ url = Drive link (later)
|
||||
<mirrored path>/<file>.doocus/ sidecar — ONLY for extracted formats:
|
||||
├── content.md extracted text — SEARCH INDEX ONLY
|
||||
└── meta.json source/sha/dates/author/pages/... metadata
|
||||
```
|
||||
|
||||
Node **modes**: `extracted` (docx/pdf/pptx/xlsx → text sidecar) · `meeting`
|
||||
(mp4/mkv/... → belongs to meetus, not a doc) · `link` (everything else → original
|
||||
is the artifact, nothing duplicated). Every node also carries `created` (birth
|
||||
time, falling back to mtime) so docs and meetings sort by creation date coherently.
|
||||
|
||||
## Meetings
|
||||
|
||||
Videos are `meeting` nodes — doocus does **not** transcribe them; meetus does. To
|
||||
keep everything in one coherent, searchable tree, run meetus over **only** the
|
||||
meetings doocus found and write each into a `<file>.meetus/` sidecar right next to
|
||||
its `<file>.doocus/`-style neighbours:
|
||||
|
||||
```bash
|
||||
# 1. index the drive → docs-output/{index.json, meetings.txt}
|
||||
uv run make docs IN="/mnt/drive"
|
||||
|
||||
# 2. meetus over ONLY the listed meetings, re-rooted at the mount, into
|
||||
# <file>.meetus/ sidecars (matches the pointer doocus already wrote):
|
||||
make batch IN="/mnt/drive" OUT=docs-output \
|
||||
LIST=docs-output/meetings.txt \
|
||||
OUT_FORMAT="{name}.meetus"
|
||||
```
|
||||
|
||||
- `meetings.txt` holds **relative** paths only — mount the drive anywhere and the
|
||||
`-l/--list` re-roots them under `-i`, so this machine reads only the meetings.
|
||||
- `OUT_FORMAT="{name}.meetus"` is meetus's new `--out-format` (tokens `{date}
|
||||
{run} {stem} {name}`; default keeps `YYYYMMDD-NNN-<stem>`). Omitting `{run}`
|
||||
makes the folder deterministic, so it matches the `out`/`transcript` pointer the
|
||||
indexer set on each `meeting` node — no relink step.
|
||||
- Frames land in `<file>.meetus/frames/` for now (unchanged).
|
||||
|
||||
## Supported types
|
||||
|
||||
| Family | Extensions | Notes |
|
||||
@@ -63,14 +100,38 @@ uv sync --group doocus --group ocr # + pytesseract (needs system `tesseract`
|
||||
(`--render`) additionally needs the `pdf-render` group (`pdf2image`) + system
|
||||
`poppler`.
|
||||
|
||||
## Collections (multiple sources)
|
||||
|
||||
doocus-app discovers collections from **managed roots** (gitignored),
|
||||
`<root>/<source>/` per source:
|
||||
|
||||
- `doocus-data/<source>/` — document collections
|
||||
- `meetus-data/<source>/` — meeting collections (`.meetus` + a `--only meetings` index)
|
||||
|
||||
Override the roots with `DOOCUS_DATA` (comma-separated; `DOOCUS_OUTPUT` still works
|
||||
for a single dir). **Collections whose `index.json.root` match are the same source**
|
||||
— the UI merges their docs + meetings into one interleaved tree, toggled together
|
||||
from the `⧉` sources menu. When the same file exists in two collections, the copy
|
||||
whose sidecar resolves wins (a `meetus-data` meeting with its `.meetus` overrides a
|
||||
bare `doocus` copy).
|
||||
|
||||
## Browser UI
|
||||
|
||||
`ui/doocus-app/` (sibling of `ui/meetus-app/`, shares `ui/framework/`) browses the
|
||||
extracted docs with per-type viewers, a metadata panel, and a package builder
|
||||
that zips the **original file + content.md + meta.json** per doc for a chosen
|
||||
target (Gemini web / NotebookLM):
|
||||
`ui/doocus-app/` (sibling of `ui/meetus-app/`, shares `ui/framework/`):
|
||||
|
||||
- **Tree** (folders preserved, source-grouped) with a **search bar** — server-side
|
||||
search over the cached text (transcripts, extracted `content.md`, raw files);
|
||||
matches **prune the tree** and show a file count + size.
|
||||
- **Detail**: for extracted docs, markdown-rendered **extracted text** (labelled
|
||||
"search index, not the document") beside the **native original** (pdf inline;
|
||||
docx/xlsx rendered via mammoth/SheetJS); linked files render directly; **meetings
|
||||
embed the meetus review** (video + transcript + frames, from `@review`).
|
||||
- **Scan** a folder from the UI (`⧉` menu → Scan) — runs `process_tree` via `uv`
|
||||
into `doocus-data/`; **⟳** re-discovers newly-scanned sources.
|
||||
- **Package** builder zips selected **originals** (+ `content.md` for extracted) for
|
||||
a target (Gemini web / NotebookLM); Drive **links** once nodes carry a `url`.
|
||||
|
||||
```bash
|
||||
cd ui/doocus-app && npm install
|
||||
DOOCUS_OUTPUT=/path/to/docs-output npm run dev
|
||||
DOOCUS_DATA="/abs/doocus-data,/abs/meetus-data" npm run dev
|
||||
```
|
||||
|
||||
@@ -11,9 +11,29 @@ from .base import ExtractionResult
|
||||
|
||||
MAX_SHEET_ROWS = 200
|
||||
|
||||
# Generic core-property titles Office writes by default — worse than useless as a
|
||||
# display title (they hide the real one), so we drop them and derive from content.
|
||||
PLACEHOLDER_TITLES = {
|
||||
"word document", "powerpoint presentation", "excel workbook",
|
||||
"microsoft word document", "microsoft powerpoint presentation",
|
||||
"microsoft excel worksheet", "presentation", "workbook", "document",
|
||||
}
|
||||
|
||||
|
||||
def _good_title(t) -> bool:
|
||||
return bool(t) and str(t).strip().lower() not in PLACEHOLDER_TITLES
|
||||
|
||||
|
||||
def _first_line(text: str, limit: int = 120) -> str:
|
||||
for ln in text.splitlines():
|
||||
s = ln.lstrip("#").strip()
|
||||
if s:
|
||||
return s[:limit]
|
||||
return ""
|
||||
|
||||
|
||||
def _core_props(props) -> dict:
|
||||
"""Common OOXML core properties → metadata (only non-empty ones).
|
||||
"""Common OOXML core properties → metadata (only non-empty, non-placeholder).
|
||||
|
||||
python-docx/pptx expose `.author`/`.last_modified_by`; openpyxl uses
|
||||
`.creator`/`.lastModifiedBy`. Each target checks both spellings.
|
||||
@@ -31,6 +51,8 @@ def _core_props(props) -> dict:
|
||||
for src in sources:
|
||||
val = getattr(props, src, None)
|
||||
if val:
|
||||
if dst == "title" and not _good_title(val):
|
||||
continue # skip "Word Document" & friends; derive from content
|
||||
out[dst] = str(val)
|
||||
break
|
||||
return out
|
||||
@@ -43,6 +65,10 @@ def _extract_docx(source: Path) -> ExtractionResult:
|
||||
content = "\n\n".join(p for p in paras if p.strip()) + "\n"
|
||||
metadata = _core_props(doc.core_properties)
|
||||
metadata["paragraphs"] = len([p for p in paras if p.strip()])
|
||||
if "title" not in metadata:
|
||||
t = _first_line(content)
|
||||
if t:
|
||||
metadata["title"] = t
|
||||
return ExtractionResult(content=content, metadata=metadata, extractor="office/python-docx@1")
|
||||
|
||||
|
||||
@@ -62,6 +88,12 @@ def _extract_pptx(source: Path) -> ExtractionResult:
|
||||
content = "\n\n".join(blocks) + "\n"
|
||||
metadata = _core_props(prs.core_properties)
|
||||
metadata["slides"] = len(prs.slides)
|
||||
if "title" not in metadata and prs.slides:
|
||||
# first slide's title placeholder, if any
|
||||
for shape in prs.slides[0].shapes:
|
||||
if shape.has_text_frame and shape.text_frame.text.strip():
|
||||
metadata["title"] = shape.text_frame.text.strip()[:120]
|
||||
break
|
||||
return ExtractionResult(content=content, metadata=metadata, extractor="office/python-pptx@1")
|
||||
|
||||
|
||||
|
||||
@@ -31,6 +31,11 @@ EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"}
|
||||
# Videos are meetings → meetus territory.
|
||||
MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"}
|
||||
|
||||
# Sidecar suffix meetus writes a meeting's output into. Must match the batch
|
||||
# invocation `make batch ... OUT_FORMAT="{name}.meetus"` so the index pointer set
|
||||
# here (deterministically, at index time) resolves once meetus has run.
|
||||
MEETING_SIDECAR_SUFFIX = ".meetus"
|
||||
|
||||
|
||||
def classify(ext: str) -> str:
|
||||
if ext in EXTRACT_EXTS:
|
||||
@@ -40,7 +45,9 @@ def classify(ext: str) -> str:
|
||||
return "link"
|
||||
|
||||
|
||||
def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
def build_index(source_root, output_dir, options=None, dry_run=False, only="all") -> dict:
|
||||
"""Index a source tree. `only` scopes the collection so outputs stay separate:
|
||||
'all' (default), 'docs' (skip meetings — no bloat), 'meetings' (only videos)."""
|
||||
options = options or {}
|
||||
root = Path(source_root).resolve()
|
||||
if not root.is_dir():
|
||||
@@ -58,8 +65,12 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
rel = p.relative_to(root).as_posix()
|
||||
ext = p.suffix.lower().lstrip(".")
|
||||
mode = classify(ext)
|
||||
# Scope the collection: docs exclude meetings; meetings exclude the rest.
|
||||
if (only == "docs" and mode == "meeting") or (only == "meetings" and mode != "meeting"):
|
||||
continue
|
||||
counts[mode] += 1
|
||||
stat = p.stat()
|
||||
birth = getattr(stat, "st_birthtime", None) # not on every filesystem
|
||||
node = {
|
||||
"path": rel,
|
||||
"name": p.name,
|
||||
@@ -68,16 +79,23 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
"mode": mode,
|
||||
"bytes": stat.st_size,
|
||||
"modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
||||
"created": datetime.fromtimestamp(birth or stat.st_mtime).isoformat(),
|
||||
"url": None, # Drive link — filled later from a URL source
|
||||
}
|
||||
|
||||
if mode == "extracted" and not dry_run:
|
||||
_extract_sidecar(p, rel, out, options, node, counts)
|
||||
elif mode == "meeting":
|
||||
# Deterministic pointer to meetus's output sidecar (produced later by
|
||||
# `make batch ... OUT_FORMAT="{name}.meetus"`). No relink needed.
|
||||
node["out"] = rel + MEETING_SIDECAR_SUFFIX
|
||||
node["transcript"] = f"{rel}{MEETING_SIDECAR_SUFFIX}/{p.stem}_enhanced.txt"
|
||||
|
||||
files.append(node)
|
||||
|
||||
index = {
|
||||
"root": str(root),
|
||||
"only": only,
|
||||
"generatedAt": datetime.now().isoformat(),
|
||||
"counts": {**counts, "total": len(files)},
|
||||
"files": files,
|
||||
@@ -87,6 +105,14 @@ def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
logger.info("Wrote index: %s (%d files)", out / "index.json", len(files))
|
||||
# Export ONLY the meeting paths (relative), so a meetus run elsewhere reads
|
||||
# just what it needs: mount the drive at a new root and feed this list to
|
||||
# `ctrl/batch.sh -l`. Relative paths re-root cleanly under the new mount.
|
||||
meetings = [f["path"] for f in files if f["mode"] == "meeting"]
|
||||
(out / "meetings.txt").write_text(
|
||||
"".join(m + "\n" for m in meetings), encoding="utf-8"
|
||||
)
|
||||
logger.info("Wrote meetings list: %s (%d meetings)", out / "meetings.txt", len(meetings))
|
||||
return index
|
||||
|
||||
|
||||
|
||||
@@ -11,10 +11,16 @@ from typing import Dict, Any, Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Default run-folder name format. Tokens: {date} (YYYYMMDD), {run} (per-day
|
||||
# counter), {stem} (filename w/o ext), {name} (full filename incl. ext).
|
||||
DEFAULT_FOLDER_FORMAT = "{date}-{run:03d}-{stem}"
|
||||
|
||||
|
||||
class OutputManager:
|
||||
"""Manage output directories and manifest files for video processing."""
|
||||
|
||||
def __init__(self, video_path: Path, base_output_dir: str = "output", use_cache: bool = True):
|
||||
def __init__(self, video_path: Path, base_output_dir: str = "output",
|
||||
use_cache: bool = True, folder_format: str = DEFAULT_FOLDER_FORMAT):
|
||||
"""
|
||||
Initialize output manager.
|
||||
|
||||
@@ -22,10 +28,15 @@ class OutputManager:
|
||||
video_path: Path to the video file being processed
|
||||
base_output_dir: Base directory for all outputs
|
||||
use_cache: Whether to use existing directories if found
|
||||
folder_format: Run-folder name template. Tokens: {date}, {run},
|
||||
{stem}, {name}. When {run} is absent the counter is skipped and
|
||||
the expanded name is used directly (reuse-by-name = caching), which
|
||||
is how meetings land in a stable `<file>.meetus/` sidecar.
|
||||
"""
|
||||
self.video_path = video_path
|
||||
self.base_output_dir = Path(base_output_dir)
|
||||
self.use_cache = use_cache
|
||||
self.folder_format = folder_format or DEFAULT_FOLDER_FORMAT
|
||||
|
||||
# Find or create output directory
|
||||
self.output_dir = self._get_or_create_output_dir()
|
||||
@@ -34,16 +45,45 @@ class OutputManager:
|
||||
|
||||
logger.info(f"Output directory: {self.output_dir}")
|
||||
|
||||
def _has_run(self) -> bool:
|
||||
"""Whether the folder format uses the per-day {run} counter."""
|
||||
return "{run" in self.folder_format
|
||||
|
||||
def _expand(self, run: int = 1) -> str:
|
||||
"""Expand the folder-format template to a directory name."""
|
||||
return self.folder_format.format(
|
||||
date=datetime.now().strftime("%Y%m%d"),
|
||||
run=run,
|
||||
stem=self.video_path.stem,
|
||||
name=self.video_path.name,
|
||||
)
|
||||
|
||||
def _get_or_create_output_dir(self) -> Path:
|
||||
"""
|
||||
Get existing output directory or create a new one with incremental number.
|
||||
Get existing output directory or create one per the folder format.
|
||||
|
||||
With a {run} counter (the default) this reuses the most recent run for
|
||||
this video when caching, else creates the next-numbered folder. Without
|
||||
{run} (e.g. "{name}.meetus") the name is deterministic — reuse it when it
|
||||
exists (caching), else create it.
|
||||
|
||||
Returns:
|
||||
Path to output directory
|
||||
"""
|
||||
video_name = self.video_path.stem
|
||||
|
||||
# Look for existing directories if caching is enabled
|
||||
# Deterministic (no {run}) — stable name, reuse-by-name gives caching.
|
||||
if not self._has_run():
|
||||
dir_name = self._expand()
|
||||
output_dir = self.base_output_dir / dir_name
|
||||
if self.use_cache and output_dir.exists():
|
||||
logger.info(f"Found existing output: {dir_name}")
|
||||
else:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created new output directory: {dir_name}")
|
||||
return output_dir
|
||||
|
||||
# {run} counter (default): reuse most recent for this video, else next NNN.
|
||||
if self.use_cache and self.base_output_dir.exists():
|
||||
existing_dirs = sorted([
|
||||
d for d in self.base_output_dir.iterdir()
|
||||
@@ -76,7 +116,7 @@ class OutputManager:
|
||||
else:
|
||||
next_run = 1
|
||||
|
||||
dir_name = f"{date_str}-{next_run:03d}-{video_name}"
|
||||
dir_name = self._expand(run=next_run)
|
||||
output_dir = self.base_output_dir / dir_name
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created new output directory: {dir_name}")
|
||||
|
||||
@@ -9,7 +9,7 @@ import subprocess
|
||||
import shutil
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from .output_manager import OutputManager
|
||||
from .output_manager import OutputManager, DEFAULT_FOLDER_FORMAT
|
||||
from .cache_manager import CacheManager
|
||||
from .frame_extractor import FrameExtractor
|
||||
from .transcript_merger import TranscriptMerger
|
||||
@@ -31,6 +31,8 @@ class WorkflowConfig:
|
||||
self.transcript_path = kwargs.get('transcript')
|
||||
self.output_dir = kwargs.get('output_dir', 'output')
|
||||
self.custom_output = kwargs.get('output')
|
||||
# Run-folder name template (see meetus/output_manager.py). None → default.
|
||||
self.out_format = kwargs.get('out_format')
|
||||
|
||||
# Whisper options
|
||||
self.run_whisper = kwargs.get('run_whisper', False)
|
||||
@@ -124,7 +126,8 @@ class ProcessingWorkflow:
|
||||
self.output_mgr = OutputManager(
|
||||
config.video_path,
|
||||
config.output_dir,
|
||||
use_cache=not config.no_cache
|
||||
use_cache=not config.no_cache,
|
||||
folder_format=config.out_format or DEFAULT_FOLDER_FORMAT,
|
||||
)
|
||||
self.cache_mgr = CacheManager(
|
||||
self.output_mgr.output_dir,
|
||||
|
||||
@@ -97,6 +97,13 @@ Examples:
|
||||
help='Base directory for outputs (default: output/)',
|
||||
default='output'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--out-format',
|
||||
help='Run-folder name template. Tokens: {date} {run} {stem} {name}. '
|
||||
'Default: "{date}-{run:03d}-{stem}". Omit {run} for a stable folder '
|
||||
'(e.g. "{name}.meetus" → a docs-coherent sidecar for doocus).',
|
||||
default=None
|
||||
)
|
||||
|
||||
# Frame extraction options
|
||||
parser.add_argument(
|
||||
|
||||
@@ -23,6 +23,9 @@ def main() -> int:
|
||||
parser.add_argument("source", help="Local drive-download root to index")
|
||||
parser.add_argument("--output", default="docs-output",
|
||||
help="Output directory for index.json + sidecars (default: docs-output)")
|
||||
parser.add_argument("--only", choices=["all", "docs", "meetings"], default="all",
|
||||
help="Scope the collection: 'docs' skips meetings (no bloat), "
|
||||
"'meetings' indexes only videos (co-locate with .meetus). Default: all")
|
||||
parser.add_argument("--render", action="store_true",
|
||||
help="Render page/slide images where supported (needs extra deps)")
|
||||
parser.add_argument("--ocr", action="store_true",
|
||||
@@ -41,6 +44,7 @@ def main() -> int:
|
||||
args.output,
|
||||
options={"render": args.render, "ocr": args.ocr},
|
||||
dry_run=args.dry_run,
|
||||
only=args.only,
|
||||
)
|
||||
except NotADirectoryError as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
@@ -51,6 +55,9 @@ def main() -> int:
|
||||
f"{c['meeting']} meeting, {c['link']} linked"
|
||||
+ (f", {c['warnings']} with warnings" if c.get("warnings") else "")
|
||||
+ (" (dry run — nothing written)" if args.dry_run else ""))
|
||||
if not args.dry_run and c["meeting"]:
|
||||
print(f"→ {args.output}/meetings.txt lists the {c['meeting']} meeting path(s) "
|
||||
f"for meetus: `ctrl/batch.sh -i <mount> -o {args.output} -l {args.output}/meetings.txt`")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -51,6 +51,10 @@ class TestDefaults(unittest.TestCase):
|
||||
def test_to_dict_analysis_is_embed_images(self):
|
||||
self.assertEqual(cfg().to_dict()["analysis"]["method"], "embed-images")
|
||||
|
||||
def test_out_format_defaults_none_and_passes_through(self):
|
||||
self.assertIsNone(cfg().out_format)
|
||||
self.assertEqual(cfg(out_format="{name}.meetus").out_format, "{name}.meetus")
|
||||
|
||||
def test_to_dict_whisper_has_diarize_and_formats(self):
|
||||
d = cfg(diarize=True, transcript_formats="srt").to_dict()
|
||||
self.assertTrue(d["whisper"]["diarize"])
|
||||
|
||||
@@ -13,6 +13,7 @@ from pathlib import Path
|
||||
from tests import REPO_ROOT # noqa: F401 (puts repo root on sys.path)
|
||||
from doocus import registry
|
||||
from doocus.workflow import DocConfig, DocWorkflow
|
||||
from doocus.tree import build_index, classify
|
||||
|
||||
|
||||
def has(mod: str) -> bool:
|
||||
@@ -136,5 +137,71 @@ class TestExtractors(unittest.TestCase):
|
||||
self.assertEqual(meta["title"], "PDF Doc")
|
||||
|
||||
|
||||
class TestTreeIndex(unittest.TestCase):
|
||||
def test_classify(self):
|
||||
self.assertEqual(classify("docx"), "extracted")
|
||||
self.assertEqual(classify("pdf"), "extracted")
|
||||
self.assertEqual(classify("mp4"), "meeting")
|
||||
self.assertEqual(classify("mkv"), "meeting")
|
||||
self.assertEqual(classify("md"), "link")
|
||||
self.assertEqual(classify("csv"), "link")
|
||||
self.assertEqual(classify("bin"), "link")
|
||||
|
||||
def test_build_index_replicates_tree(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / "drive"
|
||||
(src / "team a" / "sub").mkdir(parents=True)
|
||||
(src / "team a" / "notes.md").write_text("# Hi\n\nbody\n")
|
||||
(src / "team a" / "sub" / "t.csv").write_text("a,b\n1,2\n")
|
||||
(src / "team a" / "clip.mp4").write_bytes(b"\x00\x00") # video → meeting
|
||||
(src / "misc.bin").write_bytes(b"\x00") # unknown → link
|
||||
out = Path(tmp) / "out"
|
||||
|
||||
index = build_index(src, out)
|
||||
|
||||
# Whole tree replicated (no flattening), paths preserved.
|
||||
paths = {f["path"] for f in index["files"]}
|
||||
self.assertEqual(paths, {
|
||||
"team a/notes.md", "team a/sub/t.csv", "team a/clip.mp4", "misc.bin"})
|
||||
modes = {f["path"]: f["mode"] for f in index["files"]}
|
||||
self.assertEqual(modes["team a/clip.mp4"], "meeting")
|
||||
self.assertEqual(modes["team a/notes.md"], "link")
|
||||
self.assertEqual(modes["misc.bin"], "link")
|
||||
# Every node carries the Drive-url slot and a created date.
|
||||
self.assertTrue(all("url" in f for f in index["files"]))
|
||||
self.assertTrue(all(f.get("created") for f in index["files"]))
|
||||
self.assertTrue((out / "index.json").exists())
|
||||
|
||||
def test_meeting_node_points_at_meetus_sidecar(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / "drive"
|
||||
(src / "team a").mkdir(parents=True)
|
||||
(src / "team a" / "standup.mp4").write_bytes(b"\x00\x00")
|
||||
index = build_index(src, Path(tmp) / "out")
|
||||
node = next(f for f in index["files"] if f["path"] == "team a/standup.mp4")
|
||||
# Deterministic pointer that `make batch ... OUT_FORMAT="{name}.meetus"` fills.
|
||||
self.assertEqual(node["out"], "team a/standup.mp4.meetus")
|
||||
self.assertEqual(node["transcript"], "team a/standup.mp4.meetus/standup_enhanced.txt")
|
||||
|
||||
@unittest.skipUnless(has("docx"), "python-docx not installed")
|
||||
def test_build_index_extracts_sidecar(self):
|
||||
import docx
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / "drive"
|
||||
src.mkdir()
|
||||
doc = docx.Document()
|
||||
doc.add_paragraph("hello world")
|
||||
doc.save(src / "report.docx")
|
||||
out = Path(tmp) / "out"
|
||||
|
||||
index = build_index(src, out)
|
||||
node = next(f for f in index["files"] if f["path"] == "report.docx")
|
||||
self.assertEqual(node["mode"], "extracted")
|
||||
sidecar = out / node["out"]
|
||||
self.assertTrue((sidecar / "content.md").exists())
|
||||
self.assertTrue((sidecar / "meta.json").exists())
|
||||
self.assertIn("hello", (sidecar / "content.md").read_text())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -50,6 +50,15 @@ class TestMakefile(unittest.TestCase):
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("--language es", r.stdout)
|
||||
|
||||
def test_out_format_forwarded(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
ind = Path(tmp) / "in"
|
||||
ind.mkdir()
|
||||
(ind / "x.mp4").touch()
|
||||
r = make("batch", IN=str(ind), PYTHON="echo", OUT_FORMAT="{name}.meetus")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("--out-format {name}.meetus", r.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
43
tests/test_output_manager.py
Normal file
43
tests/test_output_manager.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""OutputManager run-folder naming: the default {run} counter and the new
|
||||
configurable folder_format (deterministic sidecars for the doocus meeting flow)."""
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from tests import REPO_ROOT # noqa: F401
|
||||
from meetus.output_manager import OutputManager, DEFAULT_FOLDER_FORMAT
|
||||
|
||||
|
||||
class TestFolderFormat(unittest.TestCase):
|
||||
def test_default_uses_date_run_stem(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
om = OutputManager(Path("/x/meeting.mkv"), d)
|
||||
name = om.output_dir.name
|
||||
self.assertTrue(name.endswith("-meeting"))
|
||||
# YYYYMMDD-NNN-meeting → middle part is the zero-padded run number
|
||||
self.assertRegex(name, r"^\d{8}-\d{3}-meeting$")
|
||||
|
||||
def test_deterministic_sidecar_no_counter(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
om = OutputManager(Path("/x/team a/standup.mp4"), d, folder_format="{name}.meetus")
|
||||
self.assertEqual(om.output_dir.name, "standup.mp4.meetus")
|
||||
|
||||
def test_sidecar_reused_by_name(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
a = OutputManager(Path("/x/standup.mp4"), d, folder_format="{name}.meetus")
|
||||
b = OutputManager(Path("/x/standup.mp4"), d, folder_format="{name}.meetus")
|
||||
self.assertEqual(a.output_dir, b.output_dir) # caching by stable name
|
||||
|
||||
def test_stem_token(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
om = OutputManager(Path("/x/standup.mp4"), d, folder_format="{stem}")
|
||||
self.assertEqual(om.output_dir.name, "standup")
|
||||
|
||||
def test_empty_format_falls_back_to_default(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
om = OutputManager(Path("/x/m.mkv"), d, folder_format="")
|
||||
self.assertEqual(om.folder_format, DEFAULT_FOLDER_FORMAT)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
260
ui/doocus-app/package-lock.json
generated
260
ui/doocus-app/package-lock.json
generated
@@ -8,8 +8,12 @@
|
||||
"name": "doocus-browser-ui",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"highlight.js": "^11.10",
|
||||
"jszip": "^3.10",
|
||||
"vue": "^3.5"
|
||||
"mammoth": "^1.8",
|
||||
"marked": "^14",
|
||||
"vue": "^3.5",
|
||||
"xlsx": "^0.18"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22",
|
||||
@@ -1059,6 +1063,24 @@
|
||||
"integrity": "sha512-l1rrBtBfTnmxvtsvdQDXltUUy8S1Y+ZaqdfUzmAnJkTd8Z8rv5v/ytW+TKiqEOWyHPoqtPlNFSs0lhRmYVSHVA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@xmldom/xmldom": {
|
||||
"version": "0.8.13",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
|
||||
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adler-32": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
|
||||
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/alien-signals": {
|
||||
"version": "1.0.13",
|
||||
"resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-1.0.13.tgz",
|
||||
@@ -1066,6 +1088,15 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/argparse": {
|
||||
"version": "1.0.10",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
|
||||
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"sprintf-js": "~1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
@@ -1073,6 +1104,32 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/base64-js": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
|
||||
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bluebird": {
|
||||
"version": "3.4.7",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
|
||||
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz",
|
||||
@@ -1083,12 +1140,46 @@
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cfb": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
|
||||
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"adler-32": "~1.3.0",
|
||||
"crc-32": "~1.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/codepage": {
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
|
||||
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/core-util-is": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
|
||||
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/crc-32": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
|
||||
"integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"crc32": "bin/crc32.njs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
@@ -1102,6 +1193,21 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/dingbat-to-unicode": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz",
|
||||
"integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/duck": {
|
||||
"version": "0.1.12",
|
||||
"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
|
||||
"integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==",
|
||||
"license": "BSD",
|
||||
"dependencies": {
|
||||
"underscore": "^1.13.1"
|
||||
}
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
|
||||
@@ -1180,6 +1286,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/frac": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
|
||||
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
|
||||
@@ -1205,6 +1320,15 @@
|
||||
"he": "bin/he"
|
||||
}
|
||||
},
|
||||
"node_modules/highlight.js": {
|
||||
"version": "11.11.1",
|
||||
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz",
|
||||
"integrity": "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/immediate": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
|
||||
@@ -1244,6 +1368,17 @@
|
||||
"immediate": "~3.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/lop": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz",
|
||||
"integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"duck": "^0.1.12",
|
||||
"option": "~0.2.1",
|
||||
"underscore": "^1.13.1"
|
||||
}
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "0.30.21",
|
||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
||||
@@ -1253,6 +1388,42 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/mammoth": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz",
|
||||
"integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@xmldom/xmldom": "^0.8.6",
|
||||
"argparse": "~1.0.3",
|
||||
"base64-js": "^1.5.1",
|
||||
"bluebird": "~3.4.0",
|
||||
"dingbat-to-unicode": "^1.0.1",
|
||||
"jszip": "^3.7.1",
|
||||
"lop": "^0.4.2",
|
||||
"path-is-absolute": "^1.0.0",
|
||||
"underscore": "^1.13.1",
|
||||
"xmlbuilder": "^10.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"mammoth": "bin/mammoth"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/marked": {
|
||||
"version": "14.1.4",
|
||||
"resolved": "https://registry.npmjs.org/marked/-/marked-14.1.4.tgz",
|
||||
"integrity": "sha512-vkVZ8ONmUdPnjCKc5uTRvmkRbx4EAi2OkTOXmfTDhZz3OFqMNBM1oTTWwTr4HY4uAEojhzPf+Fy8F1DWa3Sndg==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"marked": "bin/marked.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "9.0.9",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz",
|
||||
@@ -1294,6 +1465,12 @@
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/option": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz",
|
||||
"integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/pako": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
|
||||
@@ -1307,6 +1484,15 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
@@ -1441,6 +1627,24 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/ssf": {
|
||||
"version": "0.11.2",
|
||||
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
|
||||
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"frac": "~1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
@@ -1481,6 +1685,12 @@
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/underscore": {
|
||||
"version": "1.13.8",
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
|
||||
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
@@ -1613,6 +1823,54 @@
|
||||
"peerDependencies": {
|
||||
"typescript": ">=5.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wmf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
|
||||
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/word": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
|
||||
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/xlsx": {
|
||||
"version": "0.18.5",
|
||||
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
|
||||
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"adler-32": "~1.3.0",
|
||||
"cfb": "~1.2.1",
|
||||
"codepage": "~1.15.0",
|
||||
"crc-32": "~1.2.1",
|
||||
"ssf": "~0.11.2",
|
||||
"wmf": "~1.0.1",
|
||||
"word": "~0.3.0"
|
||||
},
|
||||
"bin": {
|
||||
"xlsx": "bin/xlsx.njs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlbuilder": {
|
||||
"version": "10.1.1",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
|
||||
"integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,12 @@
|
||||
"typecheck": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"highlight.js": "^11.10",
|
||||
"jszip": "^3.10",
|
||||
"vue": "^3.5"
|
||||
"mammoth": "^1.8",
|
||||
"marked": "^14",
|
||||
"vue": "^3.5",
|
||||
"xlsx": "^0.18"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22",
|
||||
|
||||
@@ -1,217 +1,498 @@
|
||||
/**
|
||||
* Vite dev middleware exposing the local doocus docs-output tree to the app.
|
||||
* Vite dev middleware exposing one or more local doocus output trees.
|
||||
*
|
||||
* Runs in the dev server process (Node), so it can reach the original source
|
||||
* file by the absolute path stored in each doc's manifest.json — needed for the
|
||||
* package builder, which bundles the original alongside content.md + meta.json.
|
||||
* All reads are local; nothing leaves the machine.
|
||||
* Each output dir is a "collection" (its own index.json + sidecars + source
|
||||
* root). The client can turn collections on/off; file paths are composite —
|
||||
* "<collectionId>/<relpath>" — so a single path identifies both the collection
|
||||
* and the file within it. All reads are local; nothing leaves the machine.
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/docs list extracted docs
|
||||
* GET /api/docs/:id meta.json + content.md + thumb/original flags
|
||||
* GET /api/docs/:id/thumb stream thumb.jpg
|
||||
* GET /api/docs/:id/original stream the original source file
|
||||
* Routes (path params are composite, query is ?path=/?q=):
|
||||
* GET /api/collections available collections
|
||||
* GET /api/tree?collections=a,b merged tree of the active collections
|
||||
* GET /api/detail?path=<c/rel> node + extracted content + meta + text
|
||||
* GET /api/original?path=<c/rel> stream the original (Range-enabled)
|
||||
* GET /api/meeting?path=<c/rel> meetus review data for a meeting
|
||||
* GET /api/meeting/frame?path=&file= a meeting frame
|
||||
* GET /api/runs/:cid (cid=c/rel) meetus-shaped run (for the embed)
|
||||
* PUT /api/runs/:cid/review write the meeting's review.json
|
||||
* GET /api/search?q=&collections=a,b search cached text across active cols
|
||||
*/
|
||||
import type { Plugin } from 'vite'
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http'
|
||||
import { spawn } from 'node:child_process'
|
||||
import fs from 'node:fs'
|
||||
import fsp from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
interface Options {
|
||||
outputDir: string
|
||||
outputDirs: string[] // explicit output dirs (env)
|
||||
outputsRoots: string[] // managed roots (doocus-data, meetus-data): <root>/<source>/
|
||||
repoRoot: string // where process_tree.py lives (run via uv)
|
||||
}
|
||||
|
||||
interface Manifest {
|
||||
source?: { name?: string; path?: string; sha256?: string }
|
||||
processed_at?: string
|
||||
outputs?: { content?: string; meta?: string; thumb?: string | null }
|
||||
interface Collection {
|
||||
id: string
|
||||
name: string
|
||||
dir: string
|
||||
indexPath: string
|
||||
}
|
||||
|
||||
const ORIGINAL_MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
interface Node {
|
||||
path: string
|
||||
name: string
|
||||
ext: string
|
||||
family: string
|
||||
mode: 'extracted' | 'meeting' | 'link'
|
||||
bytes: number
|
||||
modified: string
|
||||
url: string | null
|
||||
out?: string
|
||||
title?: string
|
||||
warnings?: string[]
|
||||
}
|
||||
|
||||
interface Index {
|
||||
root: string
|
||||
files: Node[]
|
||||
counts?: Record<string, number>
|
||||
}
|
||||
|
||||
const TEXT_EXTS = new Set(['md', 'markdown', 'txt', 'json', 'yaml', 'yml', 'csv', 'html', 'htm'])
|
||||
|
||||
const MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf', '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||
'.html': 'text/html', '.htm': 'text/html', '.md': 'text/markdown', '.txt': 'text/plain',
|
||||
'.csv': 'text/csv', '.json': 'application/json', '.yaml': 'text/yaml', '.yml': 'text/yaml',
|
||||
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'.csv': 'text/csv',
|
||||
'.html': 'text/html',
|
||||
'.htm': 'text/html',
|
||||
'.json': 'application/json',
|
||||
'.md': 'text/markdown',
|
||||
'.txt': 'text/plain',
|
||||
'.yaml': 'text/yaml',
|
||||
'.yml': 'text/yaml',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.mp4': 'video/mp4',
|
||||
}
|
||||
|
||||
function buildCollections(dirs: string[]): Collection[] {
|
||||
const seen = new Map<string, number>()
|
||||
return dirs.map((d) => {
|
||||
const dir = path.resolve(d)
|
||||
let id = path.basename(dir) || 'output'
|
||||
const n = seen.get(id) ?? 0
|
||||
seen.set(id, n + 1)
|
||||
if (n > 0) id = `${id}-${n + 1}`
|
||||
return { id, name: id, dir, indexPath: path.join(dir, 'index.json') }
|
||||
})
|
||||
}
|
||||
|
||||
export function doocusApi(opts: Options): Plugin {
|
||||
const outputDir = path.resolve(opts.outputDir)
|
||||
const toPosix = (p: string) => p.split(path.sep).join('/')
|
||||
|
||||
/** Resolve a doc id (possibly nested) to its directory, rejecting traversal. */
|
||||
function docDir(id: string): string | null {
|
||||
const dir = path.resolve(outputDir, id)
|
||||
if (dir !== outputDir && !dir.startsWith(outputDir + path.sep)) return null
|
||||
try {
|
||||
if (fs.statSync(dir).isDirectory()) return dir
|
||||
} catch { /* missing */ }
|
||||
return null
|
||||
}
|
||||
|
||||
/** Find every doc run directory (one containing meta.json) under `base`. */
|
||||
async function findDocDirs(base: string, maxDepth = 8): Promise<string[]> {
|
||||
const found: string[] = []
|
||||
const walk = async (dir: string, depth: number): Promise<void> => {
|
||||
if (depth > maxDepth) return
|
||||
let ents
|
||||
try {
|
||||
ents = await fsp.readdir(dir, { withFileTypes: true })
|
||||
} catch { return }
|
||||
if (ents.some((e) => e.isFile() && e.name === 'meta.json')) {
|
||||
found.push(dir)
|
||||
return // a doc dir; don't descend into assets/
|
||||
}
|
||||
for (const e of ents) {
|
||||
if (!e.isDirectory()) continue
|
||||
if (e.name === 'assets' || e.name === 'node_modules' || e.name.startsWith('.')) continue
|
||||
await walk(path.join(dir, e.name), depth + 1)
|
||||
/** Collections discovered across the managed roots (each is <root>/<source>/). */
|
||||
function discoverInRoots(): Collection[] {
|
||||
const found: Collection[] = []
|
||||
for (const root of opts.outputsRoots) {
|
||||
let subs: fs.Dirent[] = []
|
||||
try { subs = fs.readdirSync(root, { withFileTypes: true }) } catch { continue }
|
||||
for (const d of subs) {
|
||||
if (d.isDirectory() && fs.existsSync(path.join(root, d.name, 'index.json'))) {
|
||||
found.push({
|
||||
id: d.name, name: d.name,
|
||||
dir: path.join(root, d.name),
|
||||
indexPath: path.join(root, d.name, 'index.json'),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk(base, 0)
|
||||
return found
|
||||
}
|
||||
|
||||
function readJson<T>(file: string): T | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf-8')) as T
|
||||
} catch {
|
||||
return null
|
||||
/** Current collections = explicit env dirs + managed-root subdirs (deduped,
|
||||
* recomputed each request so UI scans appear without a restart). */
|
||||
function currentCollections(): Collection[] {
|
||||
const out: Collection[] = []
|
||||
const seen = new Set<string>()
|
||||
for (const c of [...buildCollections(opts.outputDirs), ...discoverInRoots()]) {
|
||||
if (out.some((o) => o.dir === c.dir)) continue
|
||||
let id = c.id, n = 1
|
||||
while (seen.has(id)) { n++; id = `${c.id}-${n}` }
|
||||
seen.add(id)
|
||||
out.push({ ...c, id })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function collectionById(id: string): Collection | undefined {
|
||||
return currentCollections().find((c) => c.id === id)
|
||||
}
|
||||
|
||||
function readIndexFor(c: Collection): Index | null {
|
||||
try { return JSON.parse(fs.readFileSync(c.indexPath, 'utf-8')) } catch { return null }
|
||||
}
|
||||
|
||||
/** Split a composite "<collectionId>/<rel>" into its collection + rel path. */
|
||||
function splitPath(p: string): { col: Collection; rel: string } | null {
|
||||
const i = p.indexOf('/')
|
||||
const id = i < 0 ? p : p.slice(0, i)
|
||||
const rel = i < 0 ? '' : p.slice(i + 1)
|
||||
const col = collectionById(id)
|
||||
return col ? { col, rel } : null
|
||||
}
|
||||
|
||||
/** Resolve a file within its collection's source root, rejecting traversal. */
|
||||
function originalAbs(index: Index, rel: string): string | null {
|
||||
const root = path.resolve(index.root)
|
||||
const abs = path.resolve(root, rel)
|
||||
if (abs !== root && !abs.startsWith(root + path.sep)) return null
|
||||
return abs
|
||||
}
|
||||
|
||||
function meetusDir(col: Collection, rel: string): { dir: string; stem: string } {
|
||||
return { dir: path.join(col.dir, rel + '.meetus'), stem: path.parse(rel).name }
|
||||
}
|
||||
|
||||
// Per-collection search index over cached text — rebuilt on index.json change.
|
||||
const searchCache = new Map<string, { mtime: number; entries: Array<{ path: string; hay: string; bytes: number }> }>()
|
||||
|
||||
async function readCap(p: string | null, cap = 20_000_000): Promise<string> {
|
||||
if (!p) return ''
|
||||
try { return (await fsp.readFile(p, 'utf-8')).slice(0, cap) } catch { return '' }
|
||||
}
|
||||
|
||||
async function buildSearchIndex(c: Collection): Promise<void> {
|
||||
const index = readIndexFor(c)
|
||||
let mtime = 0
|
||||
try { mtime = fs.statSync(c.indexPath).mtimeMs } catch { /* */ }
|
||||
const entries: Array<{ path: string; hay: string; bytes: number }> = []
|
||||
for (const f of index?.files ?? []) {
|
||||
let text = `${f.path} ${f.title ?? ''}`
|
||||
if (f.mode === 'extracted' && f.out) {
|
||||
text += ' ' + await readCap(path.join(c.dir, f.out, 'content.md'))
|
||||
} else if (f.mode === 'meeting') {
|
||||
const { dir, stem } = meetusDir(c, f.path)
|
||||
text += ' ' + await readCap(path.join(dir, `${stem}_enhanced.txt`))
|
||||
} else if (TEXT_EXTS.has(f.ext) && index) {
|
||||
text += ' ' + await readCap(originalAbs(index, f.path))
|
||||
}
|
||||
entries.push({ path: `${c.id}/${f.path}`, hay: text.toLowerCase(), bytes: f.bytes })
|
||||
}
|
||||
searchCache.set(c.id, { mtime, entries })
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'doocus-api',
|
||||
configureServer(server) {
|
||||
server.config.logger.info(`[doocus-api] serving docs from: ${outputDir}`)
|
||||
const cols = currentCollections()
|
||||
server.config.logger.info(
|
||||
`[doocus-api] ${cols.length} collection(s); managed roots: ${opts.outputsRoots.join(', ')}`)
|
||||
server.middlewares.use('/api', (req, res, next) => {
|
||||
const url = new URL(req.url ?? '/', 'http://localhost')
|
||||
const parts = url.pathname.split('/').filter(Boolean)
|
||||
handle(req, res, parts).catch((err) => {
|
||||
handle(req, res, parts, url.searchParams).catch((err) => {
|
||||
server.config.logger.error(`[doocus-api] ${String(err)}`)
|
||||
sendJson(res, 500, { error: String(err) })
|
||||
}).then((handled) => {
|
||||
if (!handled) next()
|
||||
})
|
||||
}).then((handled) => { if (!handled) next() })
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[]): Promise<boolean> {
|
||||
if (parts[0] !== 'docs') return false
|
||||
|
||||
// GET /api/docs
|
||||
if (parts.length === 1 && req.method === 'GET') {
|
||||
await listDocs(res)
|
||||
return true
|
||||
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[], q: URLSearchParams): Promise<boolean> {
|
||||
if (parts[0] === 'collections') { listCollections(res); return true }
|
||||
if (parts[0] === 'scan' && req.method === 'POST') { await runScan(req, res, false); return true }
|
||||
if (parts[0] === 'rescan' && req.method === 'POST') { await runScan(req, res, true); return true }
|
||||
if (parts[0] === 'tree') { getTree(res, q.get('collections')); return true }
|
||||
if (parts[0] === 'detail') { await getDetail(res, q.get('path') ?? ''); return true }
|
||||
if (parts[0] === 'original') { await serveOriginal(res, q.get('path') ?? '', req.headers.range); return true }
|
||||
if (parts[0] === 'meeting' && parts[1] === 'frame') { serveMeetingFrame(res, q.get('path') ?? '', q.get('file') ?? ''); return true }
|
||||
if (parts[0] === 'meeting') { await getMeeting(res, q.get('path') ?? ''); return true }
|
||||
if (parts[0] === 'search') { await search(res, q.get('q') ?? '', q.get('collections')); return true }
|
||||
if (parts[0] === 'runs' && parts.length >= 2) {
|
||||
const id = decodeURIComponent(parts[1])
|
||||
if (parts.length === 2 && req.method === 'GET') { await getRunShaped(res, id); return true }
|
||||
if (parts.length === 3 && parts[2] === 'review' && req.method === 'PUT') { await saveRunReview(req, res, id); return true }
|
||||
}
|
||||
|
||||
const id = parts[1] ? decodeURIComponent(parts[1]) : ''
|
||||
const dir = id ? docDir(id) : null
|
||||
if (!dir) { sendJson(res, 404, { error: 'doc not found' }); return true }
|
||||
|
||||
// GET /api/docs/:id
|
||||
if (parts.length === 2 && req.method === 'GET') {
|
||||
await getDoc(res, id, dir)
|
||||
return true
|
||||
}
|
||||
// GET /api/docs/:id/thumb
|
||||
if (parts.length === 3 && parts[2] === 'thumb' && req.method === 'GET') {
|
||||
serveThumb(res, dir)
|
||||
return true
|
||||
}
|
||||
// GET /api/docs/:id/original
|
||||
if (parts.length === 3 && parts[2] === 'original' && req.method === 'GET') {
|
||||
serveOriginal(res, dir)
|
||||
return true
|
||||
}
|
||||
|
||||
sendJson(res, 404, { error: 'not found' })
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
async function listDocs(res: ServerResponse): Promise<void> {
|
||||
const dirs = await findDocDirs(outputDir)
|
||||
const docs = []
|
||||
for (const dir of dirs) {
|
||||
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||
if (!meta) continue
|
||||
const id = toPosix(path.relative(outputDir, dir))
|
||||
const source = (meta.source ?? {}) as { name?: string; bytes?: number }
|
||||
docs.push({
|
||||
id,
|
||||
name: source.name ?? id,
|
||||
type: meta.type ?? null,
|
||||
family: meta.family ?? null,
|
||||
title: meta.title ?? null,
|
||||
extractedAt: meta.extracted_at ?? null,
|
||||
wordCount: meta.word_count ?? 0,
|
||||
bytes: source.bytes ?? 0,
|
||||
hasThumb: fs.existsSync(path.join(dir, 'thumb.jpg')),
|
||||
warnings: Array.isArray(meta.warnings) ? meta.warnings.length : 0,
|
||||
})
|
||||
}
|
||||
docs.sort((a, b) =>
|
||||
String(b.extractedAt ?? '').localeCompare(String(a.extractedAt ?? '')) || a.id.localeCompare(b.id))
|
||||
sendJson(res, 200, { docs, outputDir })
|
||||
function listCollections(res: ServerResponse): void {
|
||||
const out = currentCollections().map((c) => {
|
||||
const idx = readIndexFor(c)
|
||||
const meetings = idx?.files.filter((f) => f.mode === 'meeting').length ?? 0
|
||||
return {
|
||||
id: c.id, name: c.name, available: !!idx,
|
||||
count: idx?.files.length ?? 0,
|
||||
root: idx?.root ?? null, // shared root → same source
|
||||
only: (idx as any)?.only ?? 'all',
|
||||
meetings,
|
||||
}
|
||||
})
|
||||
sendJson(res, 200, { collections: out })
|
||||
}
|
||||
|
||||
async function getDoc(res: ServerResponse, id: string, dir: string): Promise<void> {
|
||||
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||
if (!meta) { sendJson(res, 404, { error: 'meta missing' }); return }
|
||||
const manifest = readJson<Manifest>(path.join(dir, 'manifest.json'))
|
||||
/** Run process_tree.py over a source folder (new scan) or an existing
|
||||
* collection's recorded source (rescan), writing to the managed root. */
|
||||
async function runScan(req: IncomingMessage, res: ServerResponse, rerun: boolean): Promise<void> {
|
||||
let body: { source?: string; id?: string }
|
||||
try { body = JSON.parse((await readBody(req)) || '{}') } catch { sendJson(res, 400, { error: 'invalid JSON' }); return }
|
||||
|
||||
let source: string
|
||||
let dir: string
|
||||
let only = 'docs' // UI scans create document collections (meetings come from the batch)
|
||||
if (rerun) {
|
||||
const col = body.id ? collectionById(body.id) : undefined
|
||||
const idx = col ? readIndexFor(col) : null
|
||||
if (!col || !idx?.root) { sendJson(res, 404, { error: 'collection not found' }); return }
|
||||
source = idx.root
|
||||
dir = col.dir
|
||||
only = (idx as any).only ?? 'all' // preserve the collection's kind on rescan
|
||||
} else {
|
||||
source = (body.source ?? '').trim()
|
||||
if (!source || !isDir(source)) { sendJson(res, 400, { error: `not a directory: ${source}` }); return }
|
||||
const slug = path.basename(path.resolve(source)).replace(/[^\w.-]+/g, '_') || 'source'
|
||||
dir = path.join(opts.outputsRoots[0], slug) // docs land in the first root (doocus-data)
|
||||
fs.mkdirSync(opts.outputsRoots[0], { recursive: true })
|
||||
}
|
||||
|
||||
let content = ''
|
||||
try {
|
||||
content = await fsp.readFile(path.join(dir, 'content.md'), 'utf-8')
|
||||
} catch { /* none */ }
|
||||
await runProcessTree(source, dir, only)
|
||||
sendJson(res, 200, { ok: true, id: path.basename(dir) })
|
||||
} catch (e) {
|
||||
sendJson(res, 500, { error: String(e) })
|
||||
}
|
||||
}
|
||||
|
||||
const hasThumb = fs.existsSync(path.join(dir, 'thumb.jpg'))
|
||||
const originalPath = manifest?.source?.path
|
||||
const hasOriginal = !!originalPath && isFile(originalPath)
|
||||
|
||||
sendJson(res, 200, {
|
||||
id,
|
||||
meta,
|
||||
content,
|
||||
hasThumb,
|
||||
thumbUrl: hasThumb ? `/api/docs/${encodeURIComponent(id)}/thumb` : null,
|
||||
hasOriginal,
|
||||
originalUrl: hasOriginal ? `/api/docs/${encodeURIComponent(id)}/original` : null,
|
||||
/** Spawn `uv run python process_tree.py <source> --output <dir> --only <only>`. */
|
||||
function runProcessTree(source: string, dir: string, only: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('uv',
|
||||
['run', 'python', 'process_tree.py', source, '--output', dir, '--only', only],
|
||||
{ cwd: opts.repoRoot })
|
||||
let err = ''
|
||||
child.stderr.on('data', (d) => { err += d })
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => code === 0 ? resolve() : reject(new Error(err || `exit ${code}`)))
|
||||
})
|
||||
}
|
||||
|
||||
function serveThumb(res: ServerResponse, dir: string): void {
|
||||
const full = path.join(dir, 'thumb.jpg')
|
||||
if (!fs.existsSync(full)) { sendJson(res, 404, { error: 'no thumb' }); return }
|
||||
/** Higher = "richer" version of a file — the one whose sidecar resolves wins,
|
||||
* so a meetus-data meeting (with .meetus) overrides a doocus copy without it. */
|
||||
function scoreNode(c: Collection, f: Node): number {
|
||||
if (f.mode === 'meeting') return fs.existsSync(meetusDir(c, f.path).dir) ? 4 : 2
|
||||
if (f.mode === 'extracted') {
|
||||
return f.out && fs.existsSync(path.join(c.dir, f.out, 'content.md')) ? 3 : 1.5
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
function getTree(res: ServerResponse, activeCsv: string | null): void {
|
||||
const active = activeCsv ? new Set(activeCsv.split(',').filter(Boolean)) : null
|
||||
const cols = currentCollections().filter((c) => !active || active.has(c.id))
|
||||
|
||||
// Dedup across collections by shared root + rel; the resolving copy wins.
|
||||
const best = new Map<string, { file: Node & { collection: string; root: string; rel: string }; score: number }>()
|
||||
for (const c of cols) {
|
||||
const idx = readIndexFor(c)
|
||||
if (!idx) continue
|
||||
for (const f of idx.files) {
|
||||
const key = `${idx.root} | ||||