doocus first ver
This commit is contained in:
6
.gitignore
vendored
6
.gitignore
vendored
@@ -6,9 +6,15 @@ samples/*
|
|||||||
output/*
|
output/*
|
||||||
!output/.gitkeep
|
!output/.gitkeep
|
||||||
|
|
||||||
|
# doocus document extraction output (textified docs + metadata; local-only)
|
||||||
|
docs-output/
|
||||||
|
|
||||||
# Python cache
|
# Python cache
|
||||||
__pycache__
|
__pycache__
|
||||||
*.pyc
|
*.pyc
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
|
||||||
|
# uv virtual environment (uv.lock IS committed)
|
||||||
|
.venv/
|
||||||
|
|
||||||
local-run.sh
|
local-run.sh
|
||||||
|
|||||||
43
INDEX.md
43
INDEX.md
@@ -31,6 +31,28 @@ output/<run>/ run folder: YYYYMMDD-NNN-<stem>/
|
|||||||
`make batch` (→ `ctrl/batch.sh`) runs the pipeline over a whole directory tree,
|
`make batch` (→ `ctrl/batch.sh`) runs the pipeline over a whole directory tree,
|
||||||
mirroring the input folder structure into the output.
|
mirroring the input folder structure into the output.
|
||||||
|
|
||||||
|
## doocus — document extraction (alongside meetus)
|
||||||
|
|
||||||
|
The same idea for documents. `process_doc.py` turns local files (Drive exports,
|
||||||
|
PDFs, images, ...) into a textified product + metadata, deterministically and
|
||||||
|
offline. Its output is what feeds permitted services (Gemini web, NotebookLM)
|
||||||
|
and the local search index; the raw source never leaves the machine.
|
||||||
|
|
||||||
|
```
|
||||||
|
document (docx/pdf/xlsx/img/...)
|
||||||
|
│ process_doc.py (deterministic, offline, no cloud AI)
|
||||||
|
▼
|
||||||
|
docs-output/<run>/ run folder: YYYYMMDD-NNN-<stem>/
|
||||||
|
├── content.md textified document ◀── the product
|
||||||
|
├── meta.json source/sha/dates/author/pages/... metadata
|
||||||
|
├── thumb.jpg optional single preview
|
||||||
|
└── manifest.json
|
||||||
|
```
|
||||||
|
|
||||||
|
`make docs` (→ `ctrl/batch_docs.sh`) runs it over a whole tree. Full doocus docs:
|
||||||
|
[`doocus/README.md`](doocus/README.md). Cross-search (meetus transcripts + doocus
|
||||||
|
docs, textified-only) is a later phase. 🚧
|
||||||
|
|
||||||
## Status legend
|
## Status legend
|
||||||
|
|
||||||
✅ active · 🚧 WIP, on hold · 🗄️ deprecated (kept for reference) · 🧰 one-off / niche · 📄 docs
|
✅ active · 🚧 WIP, on hold · 🗄️ deprecated (kept for reference) · 🧰 one-off / niche · 📄 docs
|
||||||
@@ -39,7 +61,8 @@ mirroring the input folder structure into the output.
|
|||||||
|
|
||||||
```
|
```
|
||||||
process_meeting.py ✅ the CLI tool — entry point (stays at root)
|
process_meeting.py ✅ the CLI tool — entry point (stays at root)
|
||||||
Makefile ✅ batch convenience wrapper
|
process_doc.py ✅ doocus CLI — document extraction entry point
|
||||||
|
Makefile ✅ batch convenience wrapper (make batch / make docs)
|
||||||
meetus/ ✅ core package
|
meetus/ ✅ core package
|
||||||
├─ workflow.py orchestrator (whisper → frames → merge)
|
├─ workflow.py orchestrator (whisper → frames → merge)
|
||||||
├─ frame_extractor.py FFmpeg scene-detection / interval frames
|
├─ frame_extractor.py FFmpeg scene-detection / interval frames
|
||||||
@@ -51,8 +74,15 @@ meetus/ ✅ core package
|
|||||||
├─ vision_processor.py (was --use-vision)
|
├─ vision_processor.py (was --use-vision)
|
||||||
├─ hybrid_processor.py (was --use-hybrid)
|
├─ hybrid_processor.py (was --use-hybrid)
|
||||||
└─ prompts/ vision context prompts
|
└─ prompts/ vision context prompts
|
||||||
|
doocus/ ✅ document-extraction package (see doocus/README.md)
|
||||||
|
├─ workflow.py DocConfig + DocWorkflow (dispatch → extract → write → manifest)
|
||||||
|
├─ registry.py extension → extractor family dispatch (lazy, graceful)
|
||||||
|
├─ output_manager.py run-folder + content.md/meta.json/thumb writer
|
||||||
|
├─ naming.py YYYYMMDD-NNN naming + sha256 (reuses meetus convention)
|
||||||
|
└─ extractors/ text, tabular(csv), office(docx/pptx/xlsx), pdf, web(html), image, media(mp4)
|
||||||
ctrl/ control plane / operational scripts
|
ctrl/ control plane / operational scripts
|
||||||
├─ batch.sh ✅ recursive batch runner (mirrors tree, continues past failures)
|
├─ batch.sh ✅ recursive batch runner (mirrors tree, continues past failures)
|
||||||
|
├─ batch_docs.sh ✅ doocus twin of batch.sh (make docs)
|
||||||
├─ transcribe_oneoff.sh 🧰 high-quality re-transcription over an existing run
|
├─ transcribe_oneoff.sh 🧰 high-quality re-transcription over an existing run
|
||||||
├─ summarize/ 🚧 last step — local-LLM summarization (WIP, on hold)
|
├─ summarize/ 🚧 last step — local-LLM summarization (WIP, on hold)
|
||||||
│ ├─ summarize_simple.py minimal map-and-append; reads every referenced frame
|
│ ├─ summarize_simple.py minimal map-and-append; reads every referenced frame
|
||||||
@@ -60,6 +90,10 @@ ctrl/ control plane / operational scripts
|
|||||||
│ └─ summarize_meeting.py map→extract(validated facts)→reduce
|
│ └─ summarize_meeting.py map→extract(validated facts)→reduce
|
||||||
└─ cht/ 🧰 bridge to the separate realtime `cht` project
|
└─ cht/ 🧰 bridge to the separate realtime `cht` project
|
||||||
└─ interleave_cht_frames.py whisperx JSON + cht frames/index.json → enhanced.txt
|
└─ interleave_cht_frames.py whisperx JSON + cht frames/index.json → enhanced.txt
|
||||||
|
ui/ local browser UIs (Vue 3 + Vite), shared framework
|
||||||
|
├─ framework/ ✅ shared component/renderer library + design tokens
|
||||||
|
├─ meetus-app/ ✅ review meeting runs (was ui/app/)
|
||||||
|
└─ doocus-app/ ✅ browse extracted docs + build packages (Gemini/NotebookLM)
|
||||||
def/ 📄 design/decision notes, in order (the feature history)
|
def/ 📄 design/decision notes, in order (the feature history)
|
||||||
README.md 📄 manual for process_meeting.py
|
README.md 📄 manual for process_meeting.py
|
||||||
INDEX.md 📄 this file
|
INDEX.md 📄 this file
|
||||||
@@ -74,7 +108,12 @@ local-run.sh 📄 personal scratch invocations (gitignored)
|
|||||||
`--transcript-formats srt` and writes outputs next to the sources. See `README.md`.
|
`--transcript-formats srt` and writes outputs next to the sources. See `README.md`.
|
||||||
- **`meetus/deprecated/`** is no longer imported or reachable from the CLI (its flags
|
- **`meetus/deprecated/`** is no longer imported or reachable from the CLI (its flags
|
||||||
were removed and `workflow.py` no longer imports it). Kept for reference only; the
|
were removed and `workflow.py` no longer imports it). Kept for reference only; the
|
||||||
realtime continuation of the idea is the separate `cht` project.
|
realtime continuation of the idea is the separate `cht` project. Its only unique
|
||||||
|
dependency (`ollama`) is isolated to the `deprecated` uv group in `pyproject.toml`,
|
||||||
|
so a normal `uv sync --group meetus`/`--group doocus` never installs it.
|
||||||
|
- **Dependencies** are uv feature groups in `pyproject.toml` (no `requirements.txt`):
|
||||||
|
`meetus`, `doocus`, `ocr`, `pdf-render`, `deprecated`. whisper/whisperx are
|
||||||
|
external CLIs (like ffmpeg), installed separately — not uv-managed.
|
||||||
- **`ctrl/summarize/`** scripts are standalone; run them under an env with the `openai`
|
- **`ctrl/summarize/`** scripts are standalone; run them under an env with the `openai`
|
||||||
client (e.g. `~/wdir/llm/.venv`) against a local OpenAI-compatible server.
|
client (e.g. `~/wdir/llm/.venv`) against a local OpenAI-compatible server.
|
||||||
|
|
||||||
|
|||||||
22
Makefile
22
Makefile
@@ -24,7 +24,8 @@ FLAGS ?= --embed-images --scene-detection --scene-threshold 10 --diarize --trans
|
|||||||
# Per-run knobs (appended after FLAGS): language, plus any one-off extra flags
|
# Per-run knobs (appended after FLAGS): language, plus any one-off extra flags
|
||||||
# (e.g. --skip-cache-whisper, --transcript-formats srt).
|
# (e.g. --skip-cache-whisper, --transcript-formats srt).
|
||||||
# NB: named AUDIO_LANG, not LANG — LANG is the shell locale env var.
|
# NB: named AUDIO_LANG, not LANG — LANG is the shell locale env var.
|
||||||
AUDIO_LANG ?=
|
# Defaults to English; override (e.g. AUDIO_LANG=es) for other languages.
|
||||||
|
AUDIO_LANG ?= en
|
||||||
EXTRA ?=
|
EXTRA ?=
|
||||||
|
|
||||||
# Optional: restrict scanned extensions / pick the python interpreter.
|
# Optional: restrict scanned extensions / pick the python interpreter.
|
||||||
@@ -32,7 +33,13 @@ EXT ?=
|
|||||||
PYTHON ?=
|
PYTHON ?=
|
||||||
export PYTHON
|
export PYTHON
|
||||||
|
|
||||||
.PHONY: batch dry help
|
# doocus (document extraction) knobs. DOCS_OUT defaults to docs-output; the input
|
||||||
|
# tree is mirrored under it. DOC_EXTRA forwards one-off flags to process_doc.py
|
||||||
|
# (e.g. --render for pdf page thumbnails, --ocr for images/scanned pdfs).
|
||||||
|
DOCS_OUT ?= docs-output
|
||||||
|
DOC_EXTRA ?=
|
||||||
|
|
||||||
|
.PHONY: batch dry docs docs-dry help
|
||||||
|
|
||||||
batch:
|
batch:
|
||||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -- \
|
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -- \
|
||||||
@@ -41,5 +48,16 @@ batch:
|
|||||||
dry:
|
dry:
|
||||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -n
|
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -n
|
||||||
|
|
||||||
|
# Document extraction (doocus), mirrors the input tree into DOCS_OUT.
|
||||||
|
# make docs IN="/mnt/win/drive" # extract all docs
|
||||||
|
# make docs IN="..." DOC_EXTRA="--render --ocr" # thumbnails + OCR
|
||||||
|
# make docs-dry IN="..." # list what would run
|
||||||
|
docs:
|
||||||
|
@ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -- \
|
||||||
|
$(DOC_EXTRA)
|
||||||
|
|
||||||
|
docs-dry:
|
||||||
|
@ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -n
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@sed -n '3,14p' Makefile
|
@sed -n '3,14p' Makefile
|
||||||
|
|||||||
24
README.md
24
README.md
@@ -24,21 +24,31 @@ sudo apt-get install ffmpeg
|
|||||||
brew install ffmpeg
|
brew install ffmpeg
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Python Dependencies
|
### 2. Python Dependencies (uv)
|
||||||
|
|
||||||
|
Dependencies live in `pyproject.toml` as [uv](https://docs.astral.sh/uv/)
|
||||||
|
feature groups — install only what you need:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install -r requirements.txt
|
uv sync --group meetus # meeting pipeline (frame extraction)
|
||||||
|
uv sync --group doocus # document extraction (see doocus/README.md)
|
||||||
|
uv sync --group doocus --group ocr # + OCR for images / scanned pdfs
|
||||||
|
uv run process_meeting.py samples/meeting.mkv --embed-images --scene-detection --diarize
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Groups: `meetus`, `doocus`, `ocr`, `pdf-render`, `deprecated` (the last is the
|
||||||
|
unwired OCR/vision path — the only user of `ollama`, kept out of every default
|
||||||
|
install).
|
||||||
|
|
||||||
### 3. Whisper or WhisperX (for audio transcription)
|
### 3. Whisper or WhisperX (for audio transcription)
|
||||||
|
|
||||||
**Standard Whisper:**
|
meetus calls these as **external CLI tools** (like ffmpeg), so they are *not*
|
||||||
```bash
|
uv-managed — install them however suits your machine (often a separate GPU env):
|
||||||
pip install openai-whisper
|
|
||||||
```
|
|
||||||
|
|
||||||
**WhisperX** (recommended - includes speaker diarization):
|
|
||||||
```bash
|
```bash
|
||||||
|
# standard whisper
|
||||||
|
pip install openai-whisper
|
||||||
|
# or WhisperX (recommended - adds speaker diarization)
|
||||||
pip install whisperx
|
pip install whisperx
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
119
ctrl/batch_docs.sh
Executable file
119
ctrl/batch_docs.sh
Executable file
@@ -0,0 +1,119 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Batch-extract every document under a directory through process_doc.py,
|
||||||
|
# mirroring the input folder structure into the output base. The doocus twin of
|
||||||
|
# ctrl/batch.sh (which does the same for meeting videos via process_meeting.py).
|
||||||
|
#
|
||||||
|
# Recursive and space-safe (paths come off a Windows mount, so they often have
|
||||||
|
# spaces). Each document's run folder is auto-created by process_doc.py inside
|
||||||
|
# the mirrored output subfolder.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ctrl/batch_docs.sh -i <input-dir> -o <output-dir> [-e "pdf docx ..."] [-n] \
|
||||||
|
# [-- <process_doc.py flags>]
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
# # Everything under a downloaded Drive folder, mirrored into docs-output
|
||||||
|
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output
|
||||||
|
#
|
||||||
|
# # Dry run: just show which docs map to which output folders
|
||||||
|
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -n
|
||||||
|
#
|
||||||
|
# # Only pdfs/docx, render page thumbnails
|
||||||
|
# ctrl/batch_docs.sh -i "./download" -o docs-output -e "pdf docx" -- --render
|
||||||
|
#
|
||||||
|
# A doc at <input>/2026/team a/notes.docx produces
|
||||||
|
# <output>/2026/team a/<YYYYMMDD-NNN-notes>/...
|
||||||
|
# i.e. the run folder lands inside the mirrored subtree.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$PROJECT_DIR"
|
||||||
|
|
||||||
|
# python: honor $PYTHON, else prefer python3
|
||||||
|
PYTHON="${PYTHON:-}"
|
||||||
|
if [ -z "$PYTHON" ]; then
|
||||||
|
if command -v python3 >/dev/null 2>&1; then PYTHON=python3; else PYTHON=python; fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
usage() { sed -n '2,26p' "$0"; }
|
||||||
|
|
||||||
|
INPUT=""
|
||||||
|
OUTPUT=""
|
||||||
|
EXTS="csv docx html htm jpg jpeg json md txt pdf png pptx xlsx yaml yml mp4"
|
||||||
|
DRY=false
|
||||||
|
FORWARD=()
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
-i|--input) INPUT="$2"; shift 2 ;;
|
||||||
|
-o|--output) OUTPUT="$2"; shift 2 ;;
|
||||||
|
-e|--ext) EXTS="$2"; shift 2 ;;
|
||||||
|
-n|--dry-run) DRY=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
--) shift; FORWARD=("$@"); break ;;
|
||||||
|
*) echo "Unknown arg: $1" >&2; usage; exit 1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; }
|
||||||
|
[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; }
|
||||||
|
[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; }
|
||||||
|
|
||||||
|
# Absolute paths so relative-path math is stable regardless of where we cd'd.
|
||||||
|
INPUT="$(realpath "$INPUT")"
|
||||||
|
mkdir -p "$OUTPUT"
|
||||||
|
OUTPUT="$(realpath "$OUTPUT")"
|
||||||
|
|
||||||
|
# Build the find extension filter: \( -iname '*.pdf' -o -iname '*.docx' ... \)
|
||||||
|
find_expr=()
|
||||||
|
for ext in $EXTS; do
|
||||||
|
find_expr+=( -iname "*.${ext}" -o )
|
||||||
|
done
|
||||||
|
unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o
|
||||||
|
|
||||||
|
echo "Input : $INPUT"
|
||||||
|
echo "Output: $OUTPUT"
|
||||||
|
echo "Exts : $EXTS"
|
||||||
|
[ "$DRY" = true ] && echo "(dry run — nothing will be processed)"
|
||||||
|
echo
|
||||||
|
|
||||||
|
total=0 ok=0 fail=0
|
||||||
|
# Process substitution (not a pipe) so counters survive into the summary.
|
||||||
|
while IFS= read -r -d '' doc; do
|
||||||
|
total=$((total + 1))
|
||||||
|
|
||||||
|
rel="${doc#"$INPUT"/}" # path relative to the input root
|
||||||
|
reldir="$(dirname "$rel")"
|
||||||
|
if [ "$reldir" = "." ]; then
|
||||||
|
outdir="$OUTPUT" # doc sat directly in the input root
|
||||||
|
else
|
||||||
|
outdir="$OUTPUT/$reldir" # mirror the subfolder structure
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[$total] $rel"
|
||||||
|
echo " -> $outdir"
|
||||||
|
|
||||||
|
if [ "$DRY" = true ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$outdir"
|
||||||
|
# stdin from /dev/null so any child process can't eat the file list and stall
|
||||||
|
# the batch (same guard as ctrl/batch.sh).
|
||||||
|
if "$PYTHON" "$PROJECT_DIR/process_doc.py" "$doc" \
|
||||||
|
--output-dir "$outdir" "${FORWARD[@]+"${FORWARD[@]}"}" </dev/null; then
|
||||||
|
ok=$((ok + 1))
|
||||||
|
else
|
||||||
|
echo " !! FAILED (continuing)" >&2
|
||||||
|
fail=$((fail + 1))
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z)
|
||||||
|
|
||||||
|
echo "----------------------------------------"
|
||||||
|
if [ "$DRY" = true ]; then
|
||||||
|
echo "Found $total document(s)."
|
||||||
|
else
|
||||||
|
echo "Done. $total document(s): $ok ok, $fail failed."
|
||||||
|
fi
|
||||||
|
[ "$fail" -eq 0 ]
|
||||||
70
ctrl/sync.sh
Executable file
70
ctrl/sync.sh
Executable file
@@ -0,0 +1,70 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Sync (copy) this project to another location, excluding git-ignored files,
|
||||||
|
# the .git directory, and local-only cruft.
|
||||||
|
#
|
||||||
|
# Respects every .gitignore in the tree (root and nested, e.g. ui/.gitignore)
|
||||||
|
# via rsync's per-directory merge filter, so node_modules/, build output,
|
||||||
|
# output/ runs, samples/, local-run.sh, etc. are never copied. Tracked files
|
||||||
|
# (including the .gitignore files themselves) are copied so the destination is
|
||||||
|
# a clean, working copy of the project.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ctrl/sync.sh [-n] [-d] <destination>
|
||||||
|
#
|
||||||
|
# <destination> local path or rsync target
|
||||||
|
# (e.g. ../meetus-copy, /mnt/backup/meetus, host:~/meetus)
|
||||||
|
# -n dry run — list what would change, copy nothing
|
||||||
|
# -d mirror mode — also delete files in <destination> that are
|
||||||
|
# not in the source (destructive; off by default)
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
# ctrl/sync.sh ../meetus-copy
|
||||||
|
# ctrl/sync.sh -n /mnt/backup/meetus # preview
|
||||||
|
# ctrl/sync.sh -d server:~/projects/meetus # exact mirror
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
# Print the header comment block (skip shebang, stop at first code line).
|
||||||
|
awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"
|
||||||
|
}
|
||||||
|
|
||||||
|
DRY_RUN=""
|
||||||
|
DELETE=""
|
||||||
|
DEST=""
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
-n) DRY_RUN="--dry-run" ;;
|
||||||
|
-d) DELETE="--delete" ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
--) shift; break ;;
|
||||||
|
-*) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
|
||||||
|
*) if [ -z "$DEST" ]; then DEST="$1"; else echo "Unexpected argument: $1" >&2; exit 1; fi ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
# Any trailing args after -- are the destination too (support one).
|
||||||
|
if [ -z "$DEST" ] && [ $# -gt 0 ]; then DEST="$1"; fi
|
||||||
|
|
||||||
|
if [ -z "$DEST" ]; then
|
||||||
|
echo "Error: destination required." >&2
|
||||||
|
echo "Usage: ctrl/sync.sh [-n] [-d] <destination>" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Syncing project -> $DEST"
|
||||||
|
[ -n "$DRY_RUN" ] && echo " (dry run: no changes will be made)"
|
||||||
|
[ -n "$DELETE" ] && echo " (mirror mode: extraneous files in destination will be DELETED)"
|
||||||
|
|
||||||
|
# --filter ':- .gitignore' reads a .gitignore in every directory rsync visits
|
||||||
|
# and applies its ignore patterns to that subtree.
|
||||||
|
rsync -ah --info=stats1 $DRY_RUN $DELETE \
|
||||||
|
--exclude='.git/' \
|
||||||
|
--exclude='.DS_Store' \
|
||||||
|
--filter=':- .gitignore' \
|
||||||
|
"$PROJECT_DIR"/ "$DEST"/
|
||||||
|
|
||||||
|
echo "Sync complete."
|
||||||
10
def/doocuscmds
Normal file
10
def/doocuscmds
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
cd ~/wdir/own/mts
|
||||||
|
|
||||||
|
# whole drive tree → docs-output/ (uv run so it uses the .venv you created)
|
||||||
|
uv run make docs IN="/path/to/local/drive"
|
||||||
|
|
||||||
|
# choose a different output folder:
|
||||||
|
uv run make docs IN="/path/to/local/drive" DOCS_OUT="$PWD/docs-output"
|
||||||
|
|
||||||
|
# see what it would process, no extraction:
|
||||||
|
uv run make docs-dry IN="/path/to/local/drive"
|
||||||
76
doocus/README.md
Normal file
76
doocus/README.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# doocus — local document extraction
|
||||||
|
|
||||||
|
The document twin of meetus. Turns locally-downloaded files (Drive exports,
|
||||||
|
PDFs, images, ...) into a shareable **textified** product (`content.md`) plus
|
||||||
|
structured `meta.json`, mirroring the meetus `output/<run>/` contract.
|
||||||
|
|
||||||
|
Extraction is **deterministic and offline** — no cloud AI touches the raw source.
|
||||||
|
The textified output is what feeds permitted services (Gemini web, NotebookLM)
|
||||||
|
and the local search index; the raw source stays on disk.
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# one document
|
||||||
|
python process_doc.py notes.docx # → docs-output/<YYYYMMDD-NNN-notes>/
|
||||||
|
python process_doc.py report.pdf --render # also render page 1 → thumb.jpg
|
||||||
|
python process_doc.py scan.png --ocr # OCR the image into content.md
|
||||||
|
|
||||||
|
# a whole downloaded folder (mirrors the input tree)
|
||||||
|
make docs IN="/mnt/win/drive" # → docs-output/...
|
||||||
|
make docs IN="..." DOC_EXTRA="--render --ocr"
|
||||||
|
make docs-dry IN="..." # list what would run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output contract
|
||||||
|
|
||||||
|
```
|
||||||
|
docs-output/<YYYYMMDD-NNN-stem>/
|
||||||
|
content.md textified document — the shareable product
|
||||||
|
meta.json source info, sha256, dates, author, title, pages/sheets/slides, word_count, warnings
|
||||||
|
thumb.jpg optional single preview (image / rendered page / video frame)
|
||||||
|
manifest.json run config + outputs inventory + source pointer (path, not a copy)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Supported types
|
||||||
|
|
||||||
|
| Family | Extensions | Notes |
|
||||||
|
|---------|-------------------------|-------|
|
||||||
|
| text | md, txt, yaml/yml, json | passthrough; json/yaml add a structure summary |
|
||||||
|
| tabular | csv | markdown table + row/col metadata (stdlib) |
|
||||||
|
| office | docx, pptx, xlsx | text + core properties (author, dates, title) |
|
||||||
|
| pdf | pdf | text per page + info dict; `--render` for page-1 thumb |
|
||||||
|
| web | html/htm | visible text + title/meta tags |
|
||||||
|
| image | jpg/jpeg, png | EXIF (incl. capture date) + thumbnail; `--ocr` for text |
|
||||||
|
| media | mp4 | ffprobe metadata + thumbnail; **delegates transcription to meetus** |
|
||||||
|
|
||||||
|
Each extractor is isolated: a missing optional dependency or a corrupt file is
|
||||||
|
recorded as a warning in `meta.json`, never aborting a batch (same resilience as
|
||||||
|
`ctrl/batch.sh`).
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
Core text/tabular use only the stdlib. Everything else is a uv group in the
|
||||||
|
repo's `pyproject.toml`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv sync --group doocus # Pillow, PyYAML, python-docx, python-pptx,
|
||||||
|
# openpyxl, pypdf, beautifulsoup4, lxml
|
||||||
|
uv sync --group doocus --group ocr # + pytesseract (needs system `tesseract`)
|
||||||
|
```
|
||||||
|
|
||||||
|
`mp4` uses system `ffmpeg`/`ffprobe` (shared with meetus). PDF page rendering
|
||||||
|
(`--render`) additionally needs the `pdf-render` group (`pdf2image`) + system
|
||||||
|
`poppler`.
|
||||||
|
|
||||||
|
## Browser UI
|
||||||
|
|
||||||
|
`ui/doocus-app/` (sibling of `ui/meetus-app/`, shares `ui/framework/`) browses the
|
||||||
|
extracted docs with per-type viewers, a metadata panel, and a package builder
|
||||||
|
that zips the **original file + content.md + meta.json** per doc for a chosen
|
||||||
|
target (Gemini web / NotebookLM):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ui/doocus-app && npm install
|
||||||
|
DOOCUS_OUTPUT=/path/to/docs-output npm run dev
|
||||||
|
```
|
||||||
10
doocus/__init__.py
Normal file
10
doocus/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
"""
|
||||||
|
doocus — local, offline document extraction, alongside meetus.
|
||||||
|
|
||||||
|
Turns locally-downloaded documents (Drive exports, PDFs, images, ...) into a
|
||||||
|
shareable *textified* product (`content.md`) plus structured `meta.json`,
|
||||||
|
mirroring the meetus `output/<run>/` contract. The extraction step is
|
||||||
|
deterministic and offline — no cloud AI ever touches the raw source. The
|
||||||
|
textified output is what feeds permitted services (Gemini web, NotebookLM) and
|
||||||
|
the local search index; the raw source stays on disk.
|
||||||
|
"""
|
||||||
1
doocus/extractors/__init__.py
Normal file
1
doocus/extractors/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""Per-family document extractors. See base.py for the contract."""
|
||||||
28
doocus/extractors/base.py
Normal file
28
doocus/extractors/base.py
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
"""
|
||||||
|
Shared extractor contract.
|
||||||
|
|
||||||
|
Every extractor exposes `extract(source: Path, options: dict) -> ExtractionResult`
|
||||||
|
and is isolated: a failure on one file is caught by the workflow and recorded,
|
||||||
|
never aborting a batch (same resilience as ctrl/batch.sh).
|
||||||
|
"""
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Optional, List, Dict, Any
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ExtractionResult:
|
||||||
|
"""What an extractor returns.
|
||||||
|
|
||||||
|
content textified document as markdown/plain text — the shareable product.
|
||||||
|
metadata type-specific fields (title, author, created/modified, pages,
|
||||||
|
sheet/slide names, structure summary, ...). Merged into meta.json.
|
||||||
|
thumb optional JPEG bytes for thumb.jpg (page 1 / slide 1 / the image).
|
||||||
|
warnings non-fatal issues to surface in meta.json.
|
||||||
|
extractor id string, e.g. "office/python-docx@1".
|
||||||
|
"""
|
||||||
|
|
||||||
|
content: str = ""
|
||||||
|
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||||
|
thumb: Optional[bytes] = None
|
||||||
|
warnings: List[str] = field(default_factory=list)
|
||||||
|
extractor: str = ""
|
||||||
79
doocus/extractors/image.py
Normal file
79
doocus/extractors/image.py
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
"""
|
||||||
|
Image-family extractor: jpg/jpeg, png.
|
||||||
|
|
||||||
|
Reads EXIF (including capture date) into metadata, writes a downscaled JPEG
|
||||||
|
thumbnail, and — when options['ocr'] is set and pytesseract is available — OCRs
|
||||||
|
the image into content. Without OCR, content is a short placeholder note.
|
||||||
|
"""
|
||||||
|
import io
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "image/pillow@1"
|
||||||
|
THUMB_MAX = 640 # longest edge, px
|
||||||
|
|
||||||
|
|
||||||
|
def _exif(img) -> dict:
|
||||||
|
try:
|
||||||
|
from PIL.ExifTags import TAGS
|
||||||
|
except ImportError:
|
||||||
|
return {}
|
||||||
|
raw = getattr(img, "_getexif", lambda: None)()
|
||||||
|
if not raw:
|
||||||
|
return {}
|
||||||
|
out = {}
|
||||||
|
for tag_id, value in raw.items():
|
||||||
|
name = TAGS.get(tag_id, str(tag_id))
|
||||||
|
if name in ("DateTime", "DateTimeOriginal", "Make", "Model", "Orientation"):
|
||||||
|
out[name] = str(value)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _thumbnail(img) -> bytes:
|
||||||
|
from PIL import Image # noqa: F401 (ensure Pillow present)
|
||||||
|
im = img.copy()
|
||||||
|
im.thumbnail((THUMB_MAX, THUMB_MAX))
|
||||||
|
if im.mode not in ("RGB", "L"):
|
||||||
|
im = im.convert("RGB")
|
||||||
|
buf = io.BytesIO()
|
||||||
|
im.save(buf, format="JPEG", quality=80)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
from PIL import Image # lazy
|
||||||
|
|
||||||
|
warnings = []
|
||||||
|
metadata = {}
|
||||||
|
content = ""
|
||||||
|
thumb = None
|
||||||
|
|
||||||
|
with Image.open(source) as img:
|
||||||
|
metadata["width"], metadata["height"] = img.size
|
||||||
|
metadata["mode"] = img.mode
|
||||||
|
exif = _exif(img)
|
||||||
|
if exif:
|
||||||
|
metadata["exif"] = exif
|
||||||
|
if "DateTimeOriginal" in exif or "DateTime" in exif:
|
||||||
|
metadata["captured"] = exif.get("DateTimeOriginal", exif.get("DateTime"))
|
||||||
|
try:
|
||||||
|
thumb = _thumbnail(img)
|
||||||
|
except Exception as e:
|
||||||
|
warnings.append(f"thumbnail failed: {e}")
|
||||||
|
|
||||||
|
if options.get("ocr"):
|
||||||
|
try:
|
||||||
|
import pytesseract
|
||||||
|
content = pytesseract.image_to_string(img).strip()
|
||||||
|
metadata["ocr"] = True
|
||||||
|
except ImportError:
|
||||||
|
warnings.append("--ocr requested but pytesseract not installed")
|
||||||
|
except Exception as e:
|
||||||
|
warnings.append(f"ocr failed: {e}")
|
||||||
|
|
||||||
|
if not content:
|
||||||
|
content = f"\n\n_Image {metadata.get('width')}×{metadata.get('height')}. Run with --ocr to extract any text._\n"
|
||||||
|
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
|
||||||
|
warnings=warnings, extractor=EXTRACTOR_ID)
|
||||||
79
doocus/extractors/media.py
Normal file
79
doocus/extractors/media.py
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
"""
|
||||||
|
Media-family extractor: mp4.
|
||||||
|
|
||||||
|
doocus does not transcribe here. It probes container metadata via ffprobe, grabs
|
||||||
|
a thumbnail via ffmpeg, and records a pointer delegating full transcription to
|
||||||
|
the meetus pipeline (process_meeting.py). Meeting videos are thus discoverable
|
||||||
|
and previewable in doocus while their transcript lives in meetus.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "media/ffprobe@1"
|
||||||
|
|
||||||
|
|
||||||
|
def _ffprobe(source: Path) -> dict:
|
||||||
|
if not shutil.which("ffprobe"):
|
||||||
|
return {}
|
||||||
|
cmd = ["ffprobe", "-v", "quiet", "-print_format", "json",
|
||||||
|
"-show_format", "-show_streams", str(source)]
|
||||||
|
try:
|
||||||
|
out = subprocess.run(cmd, capture_output=True, text=True, check=True).stdout
|
||||||
|
return json.loads(out)
|
||||||
|
except (subprocess.CalledProcessError, json.JSONDecodeError):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _thumbnail(source: Path, at_seconds: float = 1.0) -> bytes:
|
||||||
|
if not shutil.which("ffmpeg"):
|
||||||
|
return b""
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
|
||||||
|
tmp_path = Path(tmp.name)
|
||||||
|
try:
|
||||||
|
cmd = ["ffmpeg", "-v", "quiet", "-y", "-ss", str(at_seconds),
|
||||||
|
"-i", str(source), "-frames:v", "1", "-vf", "scale=640:-1",
|
||||||
|
str(tmp_path)]
|
||||||
|
subprocess.run(cmd, check=True)
|
||||||
|
return tmp_path.read_bytes() if tmp_path.exists() else b""
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
return b""
|
||||||
|
finally:
|
||||||
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
warnings = []
|
||||||
|
probe = _ffprobe(source)
|
||||||
|
metadata = {"delegate": "meetus", "delegate_hint": "run process_meeting.py for transcript"}
|
||||||
|
|
||||||
|
fmt = probe.get("format", {})
|
||||||
|
if fmt:
|
||||||
|
if "duration" in fmt:
|
||||||
|
metadata["duration_seconds"] = float(fmt["duration"])
|
||||||
|
tags = fmt.get("tags", {})
|
||||||
|
if "creation_time" in tags:
|
||||||
|
metadata["created"] = tags["creation_time"]
|
||||||
|
video_streams = [s for s in probe.get("streams", []) if s.get("codec_type") == "video"]
|
||||||
|
if video_streams:
|
||||||
|
v = video_streams[0]
|
||||||
|
metadata["width"], metadata["height"] = v.get("width"), v.get("height")
|
||||||
|
if not probe:
|
||||||
|
warnings.append("ffprobe unavailable or failed; metadata limited")
|
||||||
|
|
||||||
|
thumb = _thumbnail(source) or None
|
||||||
|
if thumb is None:
|
||||||
|
warnings.append("thumbnail not generated (ffmpeg unavailable or failed)")
|
||||||
|
|
||||||
|
dur = metadata.get("duration_seconds")
|
||||||
|
content = (f"_Video {source.name}"
|
||||||
|
+ (f", {dur:.0f}s" if dur else "")
|
||||||
|
+ ". Transcript is produced by the meetus pipeline "
|
||||||
|
"(`process_meeting.py`), not doocus._\n")
|
||||||
|
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
|
||||||
|
warnings=warnings, extractor=EXTRACTOR_ID)
|
||||||
114
doocus/extractors/office.py
Normal file
114
doocus/extractors/office.py
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
"""
|
||||||
|
Office-family extractor: docx, pptx, xlsx.
|
||||||
|
|
||||||
|
Each format is handled with its own lazy import so a missing dependency degrades
|
||||||
|
only that format. These formats carry core properties (author, created/modified
|
||||||
|
dates, title) which go straight into metadata.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
MAX_SHEET_ROWS = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _core_props(props) -> dict:
|
||||||
|
"""Common OOXML core properties → metadata (only non-empty ones).
|
||||||
|
|
||||||
|
python-docx/pptx expose `.author`/`.last_modified_by`; openpyxl uses
|
||||||
|
`.creator`/`.lastModifiedBy`. Each target checks both spellings.
|
||||||
|
"""
|
||||||
|
# dst -> candidate source attribute names (first non-empty wins)
|
||||||
|
fields = {
|
||||||
|
"title": ("title",),
|
||||||
|
"author": ("author", "creator"),
|
||||||
|
"created": ("created",),
|
||||||
|
"modified_doc": ("modified",),
|
||||||
|
"last_modified_by": ("last_modified_by", "lastModifiedBy"),
|
||||||
|
}
|
||||||
|
out = {}
|
||||||
|
for dst, sources in fields.items():
|
||||||
|
for src in sources:
|
||||||
|
val = getattr(props, src, None)
|
||||||
|
if val:
|
||||||
|
out[dst] = str(val)
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_docx(source: Path) -> ExtractionResult:
|
||||||
|
import docx # python-docx
|
||||||
|
doc = docx.Document(str(source))
|
||||||
|
paras = [p.text for p in doc.paragraphs]
|
||||||
|
content = "\n\n".join(p for p in paras if p.strip()) + "\n"
|
||||||
|
metadata = _core_props(doc.core_properties)
|
||||||
|
metadata["paragraphs"] = len([p for p in paras if p.strip()])
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, extractor="office/python-docx@1")
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_pptx(source: Path) -> ExtractionResult:
|
||||||
|
from pptx import Presentation
|
||||||
|
prs = Presentation(str(source))
|
||||||
|
blocks = []
|
||||||
|
for i, slide in enumerate(prs.slides, 1):
|
||||||
|
texts = []
|
||||||
|
for shape in slide.shapes:
|
||||||
|
if shape.has_text_frame:
|
||||||
|
for para in shape.text_frame.paragraphs:
|
||||||
|
line = "".join(run.text for run in para.runs)
|
||||||
|
if line.strip():
|
||||||
|
texts.append(line)
|
||||||
|
blocks.append(f"## Slide {i}\n\n" + "\n".join(texts))
|
||||||
|
content = "\n\n".join(blocks) + "\n"
|
||||||
|
metadata = _core_props(prs.core_properties)
|
||||||
|
metadata["slides"] = len(prs.slides)
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, extractor="office/python-pptx@1")
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_xlsx(source: Path) -> ExtractionResult:
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
wb = load_workbook(str(source), read_only=True, data_only=True)
|
||||||
|
warnings = []
|
||||||
|
blocks = []
|
||||||
|
for ws in wb.worksheets:
|
||||||
|
rows = []
|
||||||
|
for r, row in enumerate(ws.iter_rows(values_only=True)):
|
||||||
|
if r > MAX_SHEET_ROWS:
|
||||||
|
warnings.append(f"sheet '{ws.title}' truncated to {MAX_SHEET_ROWS} rows")
|
||||||
|
break
|
||||||
|
rows.append(["" if c is None else str(c) for c in row])
|
||||||
|
blocks.append(f"## {ws.title}\n\n" + _rows_to_md(rows))
|
||||||
|
wb.close()
|
||||||
|
content = "\n\n".join(blocks) + "\n"
|
||||||
|
metadata = _core_props(wb.properties)
|
||||||
|
metadata["sheets"] = [ws.title for ws in wb.worksheets]
|
||||||
|
return ExtractionResult(content=content, metadata=metadata,
|
||||||
|
warnings=warnings, extractor="office/openpyxl@1")
|
||||||
|
|
||||||
|
|
||||||
|
def _rows_to_md(rows: list) -> str:
|
||||||
|
if not rows:
|
||||||
|
return "_(empty)_\n"
|
||||||
|
width = max(len(r) for r in rows)
|
||||||
|
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||||
|
|
||||||
|
def esc(c):
|
||||||
|
return str(c).replace("|", "\\|").replace("\n", " ")
|
||||||
|
|
||||||
|
lines = ["| " + " | ".join(esc(c) for c in rows[0]) + " |",
|
||||||
|
"| " + " | ".join(["---"] * width) + " |"]
|
||||||
|
for r in rows[1:]:
|
||||||
|
lines.append("| " + " | ".join(esc(c) for c in r) + " |")
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
ext = source.suffix.lower().lstrip(".")
|
||||||
|
if ext == "docx":
|
||||||
|
return _extract_docx(source)
|
||||||
|
if ext == "pptx":
|
||||||
|
return _extract_pptx(source)
|
||||||
|
if ext == "xlsx":
|
||||||
|
return _extract_xlsx(source)
|
||||||
|
return ExtractionResult(extractor="office/unhandled",
|
||||||
|
warnings=[f"office extractor got unexpected '.{ext}'"])
|
||||||
73
doocus/extractors/pdf.py
Normal file
73
doocus/extractors/pdf.py
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
"""
|
||||||
|
PDF-family extractor.
|
||||||
|
|
||||||
|
Text per page via pypdf, plus the document info dict (title/author/dates). When
|
||||||
|
options['render'] is set and pdf2image (+ system poppler) is available, page 1 is
|
||||||
|
rendered to thumb.jpg.
|
||||||
|
"""
|
||||||
|
import io
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "pdf/pypdf@1"
|
||||||
|
|
||||||
|
|
||||||
|
def _info_metadata(reader) -> dict:
|
||||||
|
out = {}
|
||||||
|
info = getattr(reader, "metadata", None)
|
||||||
|
if not info:
|
||||||
|
return out
|
||||||
|
for src, dst in (("title", "title"), ("author", "author"),
|
||||||
|
("creation_date", "created"), ("modification_date", "modified_doc")):
|
||||||
|
try:
|
||||||
|
val = getattr(info, src, None)
|
||||||
|
except Exception:
|
||||||
|
val = None
|
||||||
|
if val:
|
||||||
|
out[dst] = str(val)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _render_first_page(source: Path) -> bytes:
|
||||||
|
from pdf2image import convert_from_path # needs system poppler
|
||||||
|
images = convert_from_path(str(source), first_page=1, last_page=1, dpi=100)
|
||||||
|
if not images:
|
||||||
|
return b""
|
||||||
|
im = images[0]
|
||||||
|
im.thumbnail((640, 640))
|
||||||
|
buf = io.BytesIO()
|
||||||
|
im.convert("RGB").save(buf, format="JPEG", quality=80)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
from pypdf import PdfReader # lazy
|
||||||
|
|
||||||
|
warnings = []
|
||||||
|
reader = PdfReader(str(source))
|
||||||
|
pages = reader.pages
|
||||||
|
blocks = []
|
||||||
|
for i, page in enumerate(pages, 1):
|
||||||
|
try:
|
||||||
|
text = page.extract_text() or ""
|
||||||
|
except Exception as e:
|
||||||
|
text = ""
|
||||||
|
warnings.append(f"page {i} text extraction failed: {e}")
|
||||||
|
blocks.append(f"## Page {i}\n\n{text.strip()}")
|
||||||
|
content = "\n\n".join(blocks) + "\n"
|
||||||
|
|
||||||
|
metadata = _info_metadata(reader)
|
||||||
|
metadata["pages"] = len(pages)
|
||||||
|
|
||||||
|
thumb = None
|
||||||
|
if options.get("render"):
|
||||||
|
try:
|
||||||
|
thumb = _render_first_page(source) or None
|
||||||
|
except ImportError:
|
||||||
|
warnings.append("--render requested but pdf2image/poppler not available")
|
||||||
|
except Exception as e:
|
||||||
|
warnings.append(f"page render failed: {e}")
|
||||||
|
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
|
||||||
|
warnings=warnings, extractor=EXTRACTOR_ID)
|
||||||
57
doocus/extractors/tabular.py
Normal file
57
doocus/extractors/tabular.py
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
"""
|
||||||
|
Tabular-family extractor: csv.
|
||||||
|
|
||||||
|
Renders to a markdown table (capped preview) with row/column metadata. Uses only
|
||||||
|
the stdlib so it always works.
|
||||||
|
"""
|
||||||
|
import csv
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "tabular/csv@1"
|
||||||
|
MAX_PREVIEW_ROWS = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _md_table(rows: list) -> str:
|
||||||
|
if not rows:
|
||||||
|
return ""
|
||||||
|
width = max(len(r) for r in rows)
|
||||||
|
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||||
|
header, body = rows[0], rows[1:]
|
||||||
|
|
||||||
|
def esc(c: str) -> str:
|
||||||
|
return str(c).replace("|", "\\|").replace("\n", " ")
|
||||||
|
|
||||||
|
lines = ["| " + " | ".join(esc(c) for c in header) + " |",
|
||||||
|
"| " + " | ".join(["---"] * width) + " |"]
|
||||||
|
for r in body:
|
||||||
|
lines.append("| " + " | ".join(esc(c) for c in r) + " |")
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
warnings = []
|
||||||
|
with open(source, newline="", encoding="utf-8", errors="replace") as f:
|
||||||
|
sample = f.read(8192)
|
||||||
|
f.seek(0)
|
||||||
|
try:
|
||||||
|
dialect = csv.Sniffer().sniff(sample) if sample.strip() else csv.excel
|
||||||
|
except csv.Error:
|
||||||
|
dialect = csv.excel
|
||||||
|
rows = list(csv.reader(f, dialect))
|
||||||
|
|
||||||
|
total = len(rows)
|
||||||
|
cols = max((len(r) for r in rows), default=0)
|
||||||
|
preview = rows[: MAX_PREVIEW_ROWS + 1] # +1 for header
|
||||||
|
if total > MAX_PREVIEW_ROWS + 1:
|
||||||
|
warnings.append(f"table has {total} rows; previewed first {MAX_PREVIEW_ROWS}")
|
||||||
|
|
||||||
|
content = _md_table(preview)
|
||||||
|
metadata = {
|
||||||
|
"rows": max(total - 1, 0),
|
||||||
|
"columns": cols,
|
||||||
|
"header": rows[0] if rows else [],
|
||||||
|
}
|
||||||
|
return ExtractionResult(content=content, metadata=metadata,
|
||||||
|
warnings=warnings, extractor=EXTRACTOR_ID)
|
||||||
92
doocus/extractors/text.py
Normal file
92
doocus/extractors/text.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""
|
||||||
|
Text-family extractor: md, txt, yaml/yml, json.
|
||||||
|
|
||||||
|
Plain text (md/txt) passes through as-is ("the texts as they are"). Structured
|
||||||
|
text (json/yaml) is kept verbatim inside a fenced block so content.md renders,
|
||||||
|
with a shallow structure summary added to metadata for search/browse.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "text@1"
|
||||||
|
|
||||||
|
|
||||||
|
def _structure_summary(data) -> dict:
|
||||||
|
"""Shallow shape of parsed json/yaml — top-level keys or item count."""
|
||||||
|
if isinstance(data, dict):
|
||||||
|
keys = list(data.keys())
|
||||||
|
return {"root": "object", "keys": keys[:50], "key_count": len(keys)}
|
||||||
|
if isinstance(data, list):
|
||||||
|
return {"root": "array", "length": len(data)}
|
||||||
|
return {"root": type(data).__name__}
|
||||||
|
|
||||||
|
|
||||||
|
def _md_metadata(raw: str) -> dict:
|
||||||
|
"""Headings + trivial front-matter for markdown."""
|
||||||
|
meta = {}
|
||||||
|
lines = raw.splitlines()
|
||||||
|
|
||||||
|
# YAML-ish front matter delimited by --- ... ---
|
||||||
|
if lines and lines[0].strip() == "---":
|
||||||
|
for i in range(1, len(lines)):
|
||||||
|
if lines[i].strip() == "---":
|
||||||
|
front = lines[1:i]
|
||||||
|
fm = {}
|
||||||
|
for ln in front:
|
||||||
|
if ":" in ln:
|
||||||
|
k, v = ln.split(":", 1)
|
||||||
|
fm[k.strip()] = v.strip()
|
||||||
|
if fm:
|
||||||
|
meta["frontmatter"] = fm
|
||||||
|
break
|
||||||
|
|
||||||
|
headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")]
|
||||||
|
if headings:
|
||||||
|
meta["headings"] = headings[:50]
|
||||||
|
if not meta.get("title"):
|
||||||
|
first = headings[0].lstrip("#").strip()
|
||||||
|
if first:
|
||||||
|
meta["title"] = first
|
||||||
|
return meta
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
ext = source.suffix.lower().lstrip(".")
|
||||||
|
raw = source.read_text(encoding="utf-8", errors="replace")
|
||||||
|
metadata: dict = {}
|
||||||
|
warnings: list = []
|
||||||
|
content = raw
|
||||||
|
|
||||||
|
if ext == "json":
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
metadata["structure"] = _structure_summary(data)
|
||||||
|
pretty = json.dumps(data, indent=2, ensure_ascii=False)
|
||||||
|
content = f"```json\n{pretty}\n```\n"
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
warnings.append(f"invalid json, kept raw: {e}")
|
||||||
|
content = f"```\n{raw}\n```\n"
|
||||||
|
elif ext in ("yaml", "yml"):
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
data = yaml.safe_load(raw)
|
||||||
|
metadata["structure"] = _structure_summary(data)
|
||||||
|
except ImportError:
|
||||||
|
warnings.append("PyYAML not installed; kept raw without structure summary")
|
||||||
|
except Exception as e: # malformed yaml
|
||||||
|
warnings.append(f"invalid yaml, kept raw: {e}")
|
||||||
|
content = f"```yaml\n{raw}\n```\n"
|
||||||
|
elif ext in ("md", "markdown"):
|
||||||
|
metadata.update(_md_metadata(raw))
|
||||||
|
content = raw
|
||||||
|
else: # txt and anything else routed here
|
||||||
|
content = raw
|
||||||
|
|
||||||
|
return ExtractionResult(
|
||||||
|
content=content,
|
||||||
|
metadata=metadata,
|
||||||
|
warnings=warnings,
|
||||||
|
extractor=EXTRACTOR_ID,
|
||||||
|
)
|
||||||
40
doocus/extractors/web.py
Normal file
40
doocus/extractors/web.py
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
"""
|
||||||
|
Web-family extractor: html/htm.
|
||||||
|
|
||||||
|
Strips scripts/styles, keeps the visible text, and pulls the <title> plus common
|
||||||
|
<meta> tags into metadata. Uses BeautifulSoup (bs4 + lxml).
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .base import ExtractionResult
|
||||||
|
|
||||||
|
EXTRACTOR_ID = "web/bs4@1"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
from bs4 import BeautifulSoup # lazy: degrade to "unavailable" if missing
|
||||||
|
|
||||||
|
html = source.read_text(encoding="utf-8", errors="replace")
|
||||||
|
soup = BeautifulSoup(html, "lxml")
|
||||||
|
|
||||||
|
metadata = {}
|
||||||
|
if soup.title and soup.title.string:
|
||||||
|
metadata["title"] = soup.title.string.strip()
|
||||||
|
|
||||||
|
metas = {}
|
||||||
|
for tag in soup.find_all("meta"):
|
||||||
|
key = tag.get("name") or tag.get("property")
|
||||||
|
val = tag.get("content")
|
||||||
|
if key and val:
|
||||||
|
metas[key.strip()] = val.strip()
|
||||||
|
if metas:
|
||||||
|
metadata["meta_tags"] = {k: metas[k] for k in list(metas)[:30]}
|
||||||
|
|
||||||
|
for bad in soup(["script", "style", "noscript"]):
|
||||||
|
bad.decompose()
|
||||||
|
|
||||||
|
text = soup.get_text(separator="\n")
|
||||||
|
lines = [ln.strip() for ln in text.splitlines()]
|
||||||
|
content = "\n".join(ln for ln in lines if ln) + "\n"
|
||||||
|
|
||||||
|
return ExtractionResult(content=content, metadata=metadata, extractor=EXTRACTOR_ID)
|
||||||
52
doocus/naming.py
Normal file
52
doocus/naming.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
"""
|
||||||
|
Run-folder naming + content hashing for doocus.
|
||||||
|
|
||||||
|
Reuses meetus's `YYYYMMDD-NNN-<stem>` run-folder convention (see
|
||||||
|
meetus/output_manager.py) so both trees look the same. Kept here to keep doocus
|
||||||
|
self-contained; meetus could adopt this helper later.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
|
||||||
|
"""Streaming sha256 of a file — never loads the whole file into memory."""
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
for block in iter(lambda: f.read(chunk), b""):
|
||||||
|
h.update(block)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_run_dir(base_output_dir: Path, stem: str, use_cache: bool = True) -> Path:
|
||||||
|
"""Return (creating if needed) the run folder for `stem` under `base_output_dir`.
|
||||||
|
|
||||||
|
Mirrors meetus/output_manager.py:37-84 — reuse the most recent
|
||||||
|
`*-<stem>` folder when caching, else create `YYYYMMDD-NNN-<stem>` with the
|
||||||
|
next run number for today.
|
||||||
|
"""
|
||||||
|
base = Path(base_output_dir)
|
||||||
|
|
||||||
|
if use_cache and base.exists():
|
||||||
|
existing = sorted(
|
||||||
|
[d for d in base.iterdir() if d.is_dir() and d.name.endswith(f"-{stem}")],
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
return existing[0]
|
||||||
|
|
||||||
|
date_str = datetime.now().strftime("%Y%m%d")
|
||||||
|
next_run = 1
|
||||||
|
if base.exists():
|
||||||
|
nums = []
|
||||||
|
for d in base.iterdir():
|
||||||
|
if d.is_dir() and d.name.startswith(date_str) and d.name.endswith(f"-{stem}"):
|
||||||
|
parts = d.name.split("-")
|
||||||
|
if len(parts) >= 2 and parts[1].isdigit():
|
||||||
|
nums.append(int(parts[1]))
|
||||||
|
next_run = max(nums) + 1 if nums else 1
|
||||||
|
|
||||||
|
run_dir = base / f"{date_str}-{next_run:03d}-{stem}"
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
return run_dir
|
||||||
80
doocus/output_manager.py
Normal file
80
doocus/output_manager.py
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
"""
|
||||||
|
Manage per-document output folders and their sidecar files.
|
||||||
|
|
||||||
|
Parallels meetus/output_manager.py but for the document contract:
|
||||||
|
|
||||||
|
docs-output/<YYYYMMDD-NNN-stem>/
|
||||||
|
content.md textified document — the product
|
||||||
|
meta.json structured metadata
|
||||||
|
thumb.jpg optional single preview image
|
||||||
|
assets/ optional embedded images / per-page text (later)
|
||||||
|
manifest.json run config + outputs inventory + source pointer
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Dict, Any, Optional
|
||||||
|
|
||||||
|
from .naming import resolve_run_dir
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class DocOutputManager:
|
||||||
|
"""Create and populate a run folder for one source document."""
|
||||||
|
|
||||||
|
def __init__(self, source_path: Path, base_output_dir: str = "docs-output", use_cache: bool = True):
|
||||||
|
self.source_path = Path(source_path)
|
||||||
|
self.base_output_dir = Path(base_output_dir)
|
||||||
|
self.use_cache = use_cache
|
||||||
|
self.output_dir = resolve_run_dir(self.base_output_dir, self.source_path.stem, use_cache)
|
||||||
|
logger.info("Output directory: %s", self.output_dir)
|
||||||
|
|
||||||
|
def get_path(self, filename: str) -> Path:
|
||||||
|
return self.output_dir / filename
|
||||||
|
|
||||||
|
def write_content(self, markdown: str) -> Path:
|
||||||
|
path = self.output_dir / "content.md"
|
||||||
|
path.write_text(markdown, encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
def write_meta(self, meta: Dict[str, Any]) -> Path:
|
||||||
|
path = self.output_dir / "meta.json"
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(meta, f, indent=2, ensure_ascii=False)
|
||||||
|
return path
|
||||||
|
|
||||||
|
def write_thumb(self, jpeg_bytes: bytes) -> Path:
|
||||||
|
path = self.output_dir / "thumb.jpg"
|
||||||
|
path.write_bytes(jpeg_bytes)
|
||||||
|
return path
|
||||||
|
|
||||||
|
def assets_dir(self) -> Path:
|
||||||
|
d = self.output_dir / "assets"
|
||||||
|
d.mkdir(exist_ok=True)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def save_manifest(self, config: Dict[str, Any], outputs: Dict[str, Any], source_sha256: str) -> Path:
|
||||||
|
manifest_path = self.output_dir / "manifest.json"
|
||||||
|
manifest = {
|
||||||
|
"source": {
|
||||||
|
"name": self.source_path.name,
|
||||||
|
"path": str(self.source_path.absolute()),
|
||||||
|
"sha256": source_sha256,
|
||||||
|
},
|
||||||
|
"processed_at": datetime.now().isoformat(),
|
||||||
|
"configuration": config,
|
||||||
|
"outputs": outputs,
|
||||||
|
}
|
||||||
|
with open(manifest_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(manifest, f, indent=2, ensure_ascii=False)
|
||||||
|
logger.info("Saved manifest: %s", manifest_path)
|
||||||
|
return manifest_path
|
||||||
|
|
||||||
|
def load_manifest(self) -> Optional[Dict[str, Any]]:
|
||||||
|
manifest_path = self.output_dir / "manifest.json"
|
||||||
|
if manifest_path.exists():
|
||||||
|
with open(manifest_path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
return None
|
||||||
66
doocus/registry.py
Normal file
66
doocus/registry.py
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
"""
|
||||||
|
Extension → extractor dispatch.
|
||||||
|
|
||||||
|
Maps a file extension to an extractor "family" module under doocus/extractors/.
|
||||||
|
Families are imported lazily so an app that only handles text files doesn't need
|
||||||
|
the office/pdf/image dependencies installed. Unknown extensions (and families
|
||||||
|
whose optional deps are missing) are reported as "unhandled" — never fatal.
|
||||||
|
"""
|
||||||
|
import importlib
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Optional
|
||||||
|
|
||||||
|
from .extractors.base import ExtractionResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# extension (no dot, lowercase) → family module name under doocus.extractors
|
||||||
|
EXT_FAMILY: Dict[str, str] = {
|
||||||
|
"md": "text", "markdown": "text", "txt": "text",
|
||||||
|
"yaml": "text", "yml": "text", "json": "text",
|
||||||
|
"csv": "tabular",
|
||||||
|
"docx": "office", "pptx": "office", "xlsx": "office",
|
||||||
|
"pdf": "pdf",
|
||||||
|
"html": "web", "htm": "web",
|
||||||
|
"jpg": "image", "jpeg": "image", "png": "image",
|
||||||
|
"mp4": "media",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def ext_of(source: Path) -> str:
|
||||||
|
return source.suffix.lower().lstrip(".")
|
||||||
|
|
||||||
|
|
||||||
|
def family_for(source: Path) -> Optional[str]:
|
||||||
|
return EXT_FAMILY.get(ext_of(source))
|
||||||
|
|
||||||
|
|
||||||
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
||||||
|
"""Dispatch to the right family extractor.
|
||||||
|
|
||||||
|
Returns an ExtractionResult in all cases — unhandled types and import/extract
|
||||||
|
failures come back as an empty result carrying a warning, so a batch keeps
|
||||||
|
going.
|
||||||
|
"""
|
||||||
|
family = family_for(source)
|
||||||
|
if family is None:
|
||||||
|
return ExtractionResult(
|
||||||
|
extractor="unhandled",
|
||||||
|
warnings=[f"no extractor for extension '.{ext_of(source)}'"],
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
module = importlib.import_module(f".extractors.{family}", package=__package__)
|
||||||
|
except ImportError as e:
|
||||||
|
return ExtractionResult(
|
||||||
|
extractor=f"{family}/unavailable",
|
||||||
|
warnings=[f"extractor '{family}' unavailable (missing dependency): {e}"],
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return module.extract(source, options)
|
||||||
|
except Exception as e: # isolate: one bad file must not abort the batch
|
||||||
|
logger.warning("extractor '%s' failed on %s: %s", family, source.name, e)
|
||||||
|
return ExtractionResult(
|
||||||
|
extractor=f"{family}/error",
|
||||||
|
warnings=[f"extraction failed: {e}"],
|
||||||
|
)
|
||||||
111
doocus/workflow.py
Normal file
111
doocus/workflow.py
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
"""
|
||||||
|
Orchestrate document extraction: dispatch → extract → write sidecars → manifest.
|
||||||
|
|
||||||
|
Follows the meetus WorkflowConfig / ProcessingWorkflow shape
|
||||||
|
(meetus/workflow.py) so the two pipelines feel the same.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
import logging
|
||||||
|
from typing import Dict, Any
|
||||||
|
|
||||||
|
from . import registry
|
||||||
|
from .naming import sha256_file
|
||||||
|
from .output_manager import DocOutputManager
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class DocConfig:
|
||||||
|
"""Configuration for a single document extraction."""
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
self.source_path = Path(kwargs["source"])
|
||||||
|
self.output_dir = kwargs.get("output_dir", "docs-output")
|
||||||
|
self.no_cache = kwargs.get("no_cache", False)
|
||||||
|
self.render = kwargs.get("render", False) # optional page/slide images
|
||||||
|
self.ocr = kwargs.get("ocr", False) # OCR images / scanned pdf
|
||||||
|
|
||||||
|
def options(self) -> Dict[str, Any]:
|
||||||
|
return {"render": self.render, "ocr": self.ocr}
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return {"render": self.render, "ocr": self.ocr, "no_cache": self.no_cache}
|
||||||
|
|
||||||
|
|
||||||
|
class DocWorkflow:
|
||||||
|
"""Extract one document into a run folder."""
|
||||||
|
|
||||||
|
def __init__(self, config: DocConfig):
|
||||||
|
self.config = config
|
||||||
|
self.out = DocOutputManager(
|
||||||
|
config.source_path,
|
||||||
|
config.output_dir,
|
||||||
|
use_cache=not config.no_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
def run(self) -> Dict[str, Any]:
|
||||||
|
src = self.config.source_path
|
||||||
|
if not src.exists():
|
||||||
|
raise FileNotFoundError(f"source not found: {src}")
|
||||||
|
|
||||||
|
ext = registry.ext_of(src)
|
||||||
|
family = registry.family_for(src) or "unhandled"
|
||||||
|
logger.info("Extracting %s (family: %s)", src.name, family)
|
||||||
|
|
||||||
|
result = registry.extract(src, self.config.options())
|
||||||
|
|
||||||
|
content_path = self.out.write_content(result.content)
|
||||||
|
sha = sha256_file(src)
|
||||||
|
meta = self._build_meta(src, ext, family, sha, result)
|
||||||
|
self.out.write_meta(meta)
|
||||||
|
|
||||||
|
thumb_written = False
|
||||||
|
if result.thumb:
|
||||||
|
self.out.write_thumb(result.thumb)
|
||||||
|
thumb_written = True
|
||||||
|
|
||||||
|
outputs = {
|
||||||
|
"content": content_path.name,
|
||||||
|
"meta": "meta.json",
|
||||||
|
"thumb": "thumb.jpg" if thumb_written else None,
|
||||||
|
}
|
||||||
|
self.out.save_manifest(self.config.to_dict(), outputs, sha)
|
||||||
|
|
||||||
|
if result.warnings:
|
||||||
|
for w in result.warnings:
|
||||||
|
logger.warning(" %s", w)
|
||||||
|
|
||||||
|
logger.info("✓ %s → %s", src.name, self.out.output_dir)
|
||||||
|
return {
|
||||||
|
"output_dir": str(self.out.output_dir),
|
||||||
|
"content": str(content_path),
|
||||||
|
"meta": str(self.out.get_path("meta.json")),
|
||||||
|
"family": family,
|
||||||
|
"extractor": result.extractor,
|
||||||
|
"warnings": result.warnings,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _build_meta(self, src: Path, ext: str, family: str, sha: str, result) -> Dict[str, Any]:
|
||||||
|
stat = src.stat()
|
||||||
|
meta = {
|
||||||
|
"source": {
|
||||||
|
"name": src.name,
|
||||||
|
"path": str(src.absolute()),
|
||||||
|
"sha256": sha,
|
||||||
|
"bytes": stat.st_size,
|
||||||
|
"modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
||||||
|
},
|
||||||
|
"type": ext,
|
||||||
|
"family": family,
|
||||||
|
"extractor": result.extractor,
|
||||||
|
"extracted_at": datetime.now().isoformat(),
|
||||||
|
"word_count": len(result.content.split()),
|
||||||
|
"warnings": result.warnings,
|
||||||
|
}
|
||||||
|
# Type-specific fields (title, author, dates, pages, structure, ...) sit
|
||||||
|
# at the top level alongside the core fields.
|
||||||
|
for k, v in result.metadata.items():
|
||||||
|
if k not in meta:
|
||||||
|
meta[k] = v
|
||||||
|
return meta
|
||||||
59
process_doc.py
Normal file
59
process_doc.py
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
doocus CLI — extract one document into a textified run folder.
|
||||||
|
|
||||||
|
Parallels process_meeting.py. Deterministic and offline: no cloud AI touches the
|
||||||
|
source. Produces docs-output/<YYYYMMDD-NNN-stem>/{content.md, meta.json,
|
||||||
|
thumb.jpg?, manifest.json}.
|
||||||
|
|
||||||
|
python process_doc.py notes.docx
|
||||||
|
python process_doc.py report.pdf --output-dir docs-output/run1
|
||||||
|
python process_doc.py scan.png --ocr
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from doocus.workflow import DocConfig, DocWorkflow
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Extract a document into a textified run folder.")
|
||||||
|
parser.add_argument("source", help="Path to the document to extract")
|
||||||
|
parser.add_argument("--output-dir", default="docs-output",
|
||||||
|
help="Base output directory (default: docs-output)")
|
||||||
|
parser.add_argument("--no-cache", action="store_true",
|
||||||
|
help="Force a fresh run folder instead of reusing the latest")
|
||||||
|
parser.add_argument("--render", action="store_true",
|
||||||
|
help="Render page/slide images where supported (needs extra deps)")
|
||||||
|
parser.add_argument("--ocr", action="store_true",
|
||||||
|
help="OCR images / scanned PDFs (needs pytesseract + tesseract)")
|
||||||
|
parser.add_argument("--verbose", action="store_true", help="Verbose logging")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||||
|
format="%(message)s",
|
||||||
|
)
|
||||||
|
|
||||||
|
config = DocConfig(
|
||||||
|
source=args.source,
|
||||||
|
output_dir=args.output_dir,
|
||||||
|
no_cache=args.no_cache,
|
||||||
|
render=args.render,
|
||||||
|
ocr=args.ocr,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = DocWorkflow(config).run()
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(f"\n✓ {result['family']} → {result['output_dir']}")
|
||||||
|
if result["warnings"]:
|
||||||
|
print(" warnings: " + "; ".join(result["warnings"]))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
51
pyproject.toml
Normal file
51
pyproject.toml
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
[project]
|
||||||
|
name = "meetus"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Local, offline toolkit: meetus (meeting recordings → enhanced transcripts) + doocus (documents → textified content + metadata)."
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
# Base install pulls nothing — pick the feature groups you need (see below).
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
# Feature groups (PEP 735), installed with uv. Examples:
|
||||||
|
# uv sync --group meetus meeting pipeline
|
||||||
|
# uv sync --group doocus document extraction
|
||||||
|
# uv sync --group doocus --group ocr + OCR for images/scanned pdfs
|
||||||
|
# uv run process_doc.py file.pdf run inside the synced venv
|
||||||
|
[dependency-groups]
|
||||||
|
# meetus: frame extraction from recordings. whisper/whisperx are installed
|
||||||
|
# separately (heavy; see `transcribe` group / README) and called as CLIs.
|
||||||
|
meetus = [
|
||||||
|
"opencv-python>=4.8.0",
|
||||||
|
"ffmpeg-python>=0.2.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
# doocus: local, offline document extraction (office/pdf/web/text/image).
|
||||||
|
doocus = [
|
||||||
|
"Pillow>=10.0.0",
|
||||||
|
"PyYAML>=6.0",
|
||||||
|
"python-docx>=1.1.0",
|
||||||
|
"python-pptx>=0.6.23",
|
||||||
|
"openpyxl>=3.1.0",
|
||||||
|
"pypdf>=4.0.0",
|
||||||
|
"beautifulsoup4>=4.12.0",
|
||||||
|
"lxml>=5.0.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
# OCR for doocus images / scanned pdfs (--ocr). Needs system `tesseract`.
|
||||||
|
ocr = ["pytesseract>=0.3.10"]
|
||||||
|
|
||||||
|
# doocus pdf page-image thumbnails (--render). Needs system `poppler`.
|
||||||
|
pdf-render = ["pdf2image>=1.17.0"]
|
||||||
|
|
||||||
|
# NB: whisper / whisperx are NOT here. meetus invokes them as external CLIs
|
||||||
|
# (via subprocess), like ffmpeg — install them your own way (e.g. a separate
|
||||||
|
# GPU env). They're heavy and don't belong in the managed deps.
|
||||||
|
|
||||||
|
# Old OCR/vision path in meetus/deprecated/ — unwired, reference only. Kept out
|
||||||
|
# of every default install; this is the ONLY place `ollama` is used.
|
||||||
|
deprecated = ["ollama>=0.1.0", "pytesseract>=0.3.10"]
|
||||||
|
|
||||||
|
[tool.uv]
|
||||||
|
# This repo is a collection of scripts (process_meeting.py, process_doc.py) plus
|
||||||
|
# the meetus/ and doocus/ packages, not a distributable — don't build it.
|
||||||
|
package = false
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
# Core dependencies
|
|
||||||
opencv-python>=4.8.0
|
|
||||||
Pillow>=10.0.0
|
|
||||||
ffmpeg-python>=0.2.0
|
|
||||||
|
|
||||||
# Vision analysis (recommended for better results)
|
|
||||||
# Requires Ollama to be installed: https://ollama.ai/download
|
|
||||||
ollama>=0.1.0
|
|
||||||
|
|
||||||
# OCR engines (alternative to vision analysis)
|
|
||||||
# Tesseract (lightweight, basic text extraction)
|
|
||||||
pytesseract>=0.3.10
|
|
||||||
|
|
||||||
# Alternative OCR engines (optional)
|
|
||||||
# easyocr>=1.7.0
|
|
||||||
# paddleocr>=2.7.0
|
|
||||||
|
|
||||||
# For Whisper transcription (recommended)
|
|
||||||
# openai-whisper>=20230918
|
|
||||||
140
tests/test_doocus.py
Normal file
140
tests/test_doocus.py
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
"""Tests for the doocus document-extraction pipeline.
|
||||||
|
|
||||||
|
Covers registry dispatch and one assertion per extractor. Formats whose optional
|
||||||
|
dependency is absent are skipped (never failed), matching the graceful-degrade
|
||||||
|
design. Everything runs on synthetic fixtures — never real content.
|
||||||
|
"""
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from tests import REPO_ROOT # noqa: F401 (puts repo root on sys.path)
|
||||||
|
from doocus import registry
|
||||||
|
from doocus.workflow import DocConfig, DocWorkflow
|
||||||
|
|
||||||
|
|
||||||
|
def has(mod: str) -> bool:
|
||||||
|
try:
|
||||||
|
importlib.import_module(mod)
|
||||||
|
return True
|
||||||
|
except ImportError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def run(source: Path, out: Path, **opts) -> dict:
|
||||||
|
cfg = DocConfig(source=str(source), output_dir=str(out), **opts)
|
||||||
|
return DocWorkflow(cfg).run()
|
||||||
|
|
||||||
|
|
||||||
|
def read_meta(result: dict) -> dict:
|
||||||
|
return json.loads(Path(result["meta"]).read_text())
|
||||||
|
|
||||||
|
|
||||||
|
class TestRegistry(unittest.TestCase):
|
||||||
|
def test_family_dispatch(self):
|
||||||
|
self.assertEqual(registry.family_for(Path("a.md")), "text")
|
||||||
|
self.assertEqual(registry.family_for(Path("a.CSV")), "tabular")
|
||||||
|
self.assertEqual(registry.family_for(Path("a.docx")), "office")
|
||||||
|
self.assertIsNone(registry.family_for(Path("a.xyz")))
|
||||||
|
|
||||||
|
def test_unhandled_is_not_fatal(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
src = Path(d) / "thing.xyz"
|
||||||
|
src.write_text("hi")
|
||||||
|
result = run(src, Path(d) / "out")
|
||||||
|
self.assertEqual(result["family"], "unhandled")
|
||||||
|
self.assertTrue(result["warnings"])
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractors(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.d = Path(self._tmp.name)
|
||||||
|
self.out = self.d / "out"
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self._tmp.cleanup()
|
||||||
|
|
||||||
|
def _write(self, name, text):
|
||||||
|
p = self.d / name
|
||||||
|
p.write_text(text, encoding="utf-8")
|
||||||
|
return p
|
||||||
|
|
||||||
|
def test_markdown(self):
|
||||||
|
src = self._write("notes.md", "# Title\n\nbody text\n")
|
||||||
|
meta = read_meta(run(src, self.out))
|
||||||
|
self.assertEqual(meta["title"], "Title")
|
||||||
|
self.assertEqual(meta["family"], "text")
|
||||||
|
|
||||||
|
def test_json_structure(self):
|
||||||
|
src = self._write("data.json", '{"a":1,"b":2}')
|
||||||
|
meta = read_meta(run(src, self.out))
|
||||||
|
self.assertEqual(meta["structure"]["key_count"], 2)
|
||||||
|
|
||||||
|
def test_csv_table(self):
|
||||||
|
src = self._write("t.csv", "a,b\n1,2\n3,4\n")
|
||||||
|
result = run(src, self.out)
|
||||||
|
meta = read_meta(result)
|
||||||
|
self.assertEqual(meta["rows"], 2)
|
||||||
|
self.assertEqual(meta["columns"], 2)
|
||||||
|
self.assertIn("| a | b |", Path(result["content"]).read_text())
|
||||||
|
|
||||||
|
@unittest.skipUnless(has("bs4"), "beautifulsoup4 not installed")
|
||||||
|
def test_html(self):
|
||||||
|
src = self._write("p.html", "<title>T</title><body><p>hello</p></body>")
|
||||||
|
result = run(src, self.out)
|
||||||
|
meta = read_meta(result)
|
||||||
|
self.assertEqual(meta["title"], "T")
|
||||||
|
self.assertIn("hello", Path(result["content"]).read_text())
|
||||||
|
|
||||||
|
@unittest.skipUnless(has("PIL"), "Pillow not installed")
|
||||||
|
def test_image_thumb(self):
|
||||||
|
from PIL import Image
|
||||||
|
src = self.d / "pic.png"
|
||||||
|
Image.new("RGB", (120, 80), "navy").save(src)
|
||||||
|
result = run(src, self.out)
|
||||||
|
meta = read_meta(result)
|
||||||
|
self.assertEqual((meta["width"], meta["height"]), (120, 80))
|
||||||
|
self.assertTrue((Path(result["output_dir"]) / "thumb.jpg").exists())
|
||||||
|
|
||||||
|
@unittest.skipUnless(has("docx"), "python-docx not installed")
|
||||||
|
def test_docx(self):
|
||||||
|
import docx
|
||||||
|
src = self.d / "r.docx"
|
||||||
|
doc = docx.Document()
|
||||||
|
doc.core_properties.author = "Ada"
|
||||||
|
doc.add_paragraph("hello world")
|
||||||
|
doc.save(src)
|
||||||
|
meta = read_meta(run(src, self.out))
|
||||||
|
self.assertEqual(meta["author"], "Ada")
|
||||||
|
self.assertEqual(meta["family"], "office")
|
||||||
|
|
||||||
|
@unittest.skipUnless(has("openpyxl"), "openpyxl not installed")
|
||||||
|
def test_xlsx(self):
|
||||||
|
from openpyxl import Workbook
|
||||||
|
src = self.d / "s.xlsx"
|
||||||
|
wb = Workbook()
|
||||||
|
wb.active.title = "Sales"
|
||||||
|
wb.active.append(["q", "rev"])
|
||||||
|
wb.save(src)
|
||||||
|
meta = read_meta(run(src, self.out))
|
||||||
|
self.assertEqual(meta["sheets"], ["Sales"])
|
||||||
|
|
||||||
|
@unittest.skipUnless(has("pypdf"), "pypdf not installed")
|
||||||
|
def test_pdf(self):
|
||||||
|
from pypdf import PdfWriter
|
||||||
|
src = self.d / "d.pdf"
|
||||||
|
w = PdfWriter()
|
||||||
|
w.add_blank_page(width=200, height=200)
|
||||||
|
w.add_metadata({"/Title": "PDF Doc"})
|
||||||
|
with open(src, "wb") as f:
|
||||||
|
w.write(f)
|
||||||
|
meta = read_meta(run(src, self.out))
|
||||||
|
self.assertEqual(meta["pages"], 1)
|
||||||
|
self.assertEqual(meta["title"], "PDF Doc")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -39,7 +39,7 @@ class TestMakefile(unittest.TestCase):
|
|||||||
self.assertIn("--scene-threshold 10", line)
|
self.assertIn("--scene-threshold 10", line)
|
||||||
self.assertIn("--transcript-formats srt", line)
|
self.assertIn("--transcript-formats srt", line)
|
||||||
self.assertIn("--embed-images", line)
|
self.assertIn("--embed-images", line)
|
||||||
self.assertNotIn("--language", line) # no AUDIO_LANG -> no --language
|
self.assertIn("--language en", line) # AUDIO_LANG defaults to en
|
||||||
|
|
||||||
def test_audio_lang_appends_language(self):
|
def test_audio_lang_appends_language(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
|||||||
12
ui/doocus-app/index.html
Normal file
12
ui/doocus-app/index.html
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Doocus — Browse</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="app"></div>
|
||||||
|
<script type="module" src="/src/main.ts"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
1618
ui/doocus-app/package-lock.json
generated
Normal file
1618
ui/doocus-app/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
23
ui/doocus-app/package.json
Normal file
23
ui/doocus-app/package.json
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"name": "doocus-browser-ui",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "vue-tsc --noEmit && vite build",
|
||||||
|
"preview": "vite preview",
|
||||||
|
"typecheck": "vue-tsc --noEmit"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"jszip": "^3.10",
|
||||||
|
"vue": "^3.5"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^22",
|
||||||
|
"@vitejs/plugin-vue": "^5",
|
||||||
|
"typescript": "^5.6",
|
||||||
|
"vite": "^6",
|
||||||
|
"vue-tsc": "^2"
|
||||||
|
}
|
||||||
|
}
|
||||||
228
ui/doocus-app/server/doocusApi.ts
Normal file
228
ui/doocus-app/server/doocusApi.ts
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
/**
|
||||||
|
* Vite dev middleware exposing the local doocus docs-output tree to the app.
|
||||||
|
*
|
||||||
|
* Runs in the dev server process (Node), so it can reach the original source
|
||||||
|
* file by the absolute path stored in each doc's manifest.json — needed for the
|
||||||
|
* package builder, which bundles the original alongside content.md + meta.json.
|
||||||
|
* All reads are local; nothing leaves the machine.
|
||||||
|
*
|
||||||
|
* Routes:
|
||||||
|
* GET /api/docs list extracted docs
|
||||||
|
* GET /api/docs/:id meta.json + content.md + thumb/original flags
|
||||||
|
* GET /api/docs/:id/thumb stream thumb.jpg
|
||||||
|
* GET /api/docs/:id/original stream the original source file
|
||||||
|
*/
|
||||||
|
import type { Plugin } from 'vite'
|
||||||
|
import type { IncomingMessage, ServerResponse } from 'node:http'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
import fsp from 'node:fs/promises'
|
||||||
|
import path from 'node:path'
|
||||||
|
|
||||||
|
interface Options {
|
||||||
|
outputDir: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Manifest {
|
||||||
|
source?: { name?: string; path?: string; sha256?: string }
|
||||||
|
processed_at?: string
|
||||||
|
outputs?: { content?: string; meta?: string; thumb?: string | null }
|
||||||
|
}
|
||||||
|
|
||||||
|
const ORIGINAL_MIME: Record<string, string> = {
|
||||||
|
'.pdf': 'application/pdf',
|
||||||
|
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||||
|
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||||
|
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
'.csv': 'text/csv',
|
||||||
|
'.html': 'text/html',
|
||||||
|
'.htm': 'text/html',
|
||||||
|
'.json': 'application/json',
|
||||||
|
'.md': 'text/markdown',
|
||||||
|
'.txt': 'text/plain',
|
||||||
|
'.yaml': 'text/yaml',
|
||||||
|
'.yml': 'text/yaml',
|
||||||
|
'.jpg': 'image/jpeg',
|
||||||
|
'.jpeg': 'image/jpeg',
|
||||||
|
'.png': 'image/png',
|
||||||
|
'.mp4': 'video/mp4',
|
||||||
|
}
|
||||||
|
|
||||||
|
export function doocusApi(opts: Options): Plugin {
|
||||||
|
const outputDir = path.resolve(opts.outputDir)
|
||||||
|
const toPosix = (p: string) => p.split(path.sep).join('/')
|
||||||
|
|
||||||
|
/** Resolve a doc id (possibly nested) to its directory, rejecting traversal. */
|
||||||
|
function docDir(id: string): string | null {
|
||||||
|
const dir = path.resolve(outputDir, id)
|
||||||
|
if (dir !== outputDir && !dir.startsWith(outputDir + path.sep)) return null
|
||||||
|
try {
|
||||||
|
if (fs.statSync(dir).isDirectory()) return dir
|
||||||
|
} catch { /* missing */ }
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Find every doc run directory (one containing meta.json) under `base`. */
|
||||||
|
async function findDocDirs(base: string, maxDepth = 8): Promise<string[]> {
|
||||||
|
const found: string[] = []
|
||||||
|
const walk = async (dir: string, depth: number): Promise<void> => {
|
||||||
|
if (depth > maxDepth) return
|
||||||
|
let ents
|
||||||
|
try {
|
||||||
|
ents = await fsp.readdir(dir, { withFileTypes: true })
|
||||||
|
} catch { return }
|
||||||
|
if (ents.some((e) => e.isFile() && e.name === 'meta.json')) {
|
||||||
|
found.push(dir)
|
||||||
|
return // a doc dir; don't descend into assets/
|
||||||
|
}
|
||||||
|
for (const e of ents) {
|
||||||
|
if (!e.isDirectory()) continue
|
||||||
|
if (e.name === 'assets' || e.name === 'node_modules' || e.name.startsWith('.')) continue
|
||||||
|
await walk(path.join(dir, e.name), depth + 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await walk(base, 0)
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
function readJson<T>(file: string): T | null {
|
||||||
|
try {
|
||||||
|
return JSON.parse(fs.readFileSync(file, 'utf-8')) as T
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: 'doocus-api',
|
||||||
|
configureServer(server) {
|
||||||
|
server.config.logger.info(`[doocus-api] serving docs from: ${outputDir}`)
|
||||||
|
server.middlewares.use('/api', (req, res, next) => {
|
||||||
|
const url = new URL(req.url ?? '/', 'http://localhost')
|
||||||
|
const parts = url.pathname.split('/').filter(Boolean)
|
||||||
|
handle(req, res, parts).catch((err) => {
|
||||||
|
server.config.logger.error(`[doocus-api] ${String(err)}`)
|
||||||
|
sendJson(res, 500, { error: String(err) })
|
||||||
|
}).then((handled) => {
|
||||||
|
if (!handled) next()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handle(req: IncomingMessage, res: ServerResponse, parts: string[]): Promise<boolean> {
|
||||||
|
if (parts[0] !== 'docs') return false
|
||||||
|
|
||||||
|
// GET /api/docs
|
||||||
|
if (parts.length === 1 && req.method === 'GET') {
|
||||||
|
await listDocs(res)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
const id = parts[1] ? decodeURIComponent(parts[1]) : ''
|
||||||
|
const dir = id ? docDir(id) : null
|
||||||
|
if (!dir) { sendJson(res, 404, { error: 'doc not found' }); return true }
|
||||||
|
|
||||||
|
// GET /api/docs/:id
|
||||||
|
if (parts.length === 2 && req.method === 'GET') {
|
||||||
|
await getDoc(res, id, dir)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// GET /api/docs/:id/thumb
|
||||||
|
if (parts.length === 3 && parts[2] === 'thumb' && req.method === 'GET') {
|
||||||
|
serveThumb(res, dir)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// GET /api/docs/:id/original
|
||||||
|
if (parts.length === 3 && parts[2] === 'original' && req.method === 'GET') {
|
||||||
|
serveOriginal(res, dir)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
sendJson(res, 404, { error: 'not found' })
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listDocs(res: ServerResponse): Promise<void> {
|
||||||
|
const dirs = await findDocDirs(outputDir)
|
||||||
|
const docs = []
|
||||||
|
for (const dir of dirs) {
|
||||||
|
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||||
|
if (!meta) continue
|
||||||
|
const id = toPosix(path.relative(outputDir, dir))
|
||||||
|
const source = (meta.source ?? {}) as { name?: string; bytes?: number }
|
||||||
|
docs.push({
|
||||||
|
id,
|
||||||
|
name: source.name ?? id,
|
||||||
|
type: meta.type ?? null,
|
||||||
|
family: meta.family ?? null,
|
||||||
|
title: meta.title ?? null,
|
||||||
|
extractedAt: meta.extracted_at ?? null,
|
||||||
|
wordCount: meta.word_count ?? 0,
|
||||||
|
bytes: source.bytes ?? 0,
|
||||||
|
hasThumb: fs.existsSync(path.join(dir, 'thumb.jpg')),
|
||||||
|
warnings: Array.isArray(meta.warnings) ? meta.warnings.length : 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
docs.sort((a, b) =>
|
||||||
|
String(b.extractedAt ?? '').localeCompare(String(a.extractedAt ?? '')) || a.id.localeCompare(b.id))
|
||||||
|
sendJson(res, 200, { docs, outputDir })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getDoc(res: ServerResponse, id: string, dir: string): Promise<void> {
|
||||||
|
const meta = readJson<Record<string, unknown>>(path.join(dir, 'meta.json'))
|
||||||
|
if (!meta) { sendJson(res, 404, { error: 'meta missing' }); return }
|
||||||
|
const manifest = readJson<Manifest>(path.join(dir, 'manifest.json'))
|
||||||
|
|
||||||
|
let content = ''
|
||||||
|
try {
|
||||||
|
content = await fsp.readFile(path.join(dir, 'content.md'), 'utf-8')
|
||||||
|
} catch { /* none */ }
|
||||||
|
|
||||||
|
const hasThumb = fs.existsSync(path.join(dir, 'thumb.jpg'))
|
||||||
|
const originalPath = manifest?.source?.path
|
||||||
|
const hasOriginal = !!originalPath && isFile(originalPath)
|
||||||
|
|
||||||
|
sendJson(res, 200, {
|
||||||
|
id,
|
||||||
|
meta,
|
||||||
|
content,
|
||||||
|
hasThumb,
|
||||||
|
thumbUrl: hasThumb ? `/api/docs/${encodeURIComponent(id)}/thumb` : null,
|
||||||
|
hasOriginal,
|
||||||
|
originalUrl: hasOriginal ? `/api/docs/${encodeURIComponent(id)}/original` : null,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function serveThumb(res: ServerResponse, dir: string): void {
|
||||||
|
const full = path.join(dir, 'thumb.jpg')
|
||||||
|
if (!fs.existsSync(full)) { sendJson(res, 404, { error: 'no thumb' }); return }
|
||||||
|
res.setHeader('Content-Type', 'image/jpeg')
|
||||||
|
res.setHeader('Cache-Control', 'no-cache')
|
||||||
|
fs.createReadStream(full).pipe(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
function serveOriginal(res: ServerResponse, dir: string): void {
|
||||||
|
const manifest = readJson<Manifest>(path.join(dir, 'manifest.json'))
|
||||||
|
const original = manifest?.source?.path
|
||||||
|
if (!original || !isFile(original)) {
|
||||||
|
sendJson(res, 404, { error: 'original source not reachable on this machine' })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const mime = ORIGINAL_MIME[path.extname(original).toLowerCase()] ?? 'application/octet-stream'
|
||||||
|
res.setHeader('Content-Type', mime)
|
||||||
|
res.setHeader('Content-Disposition', `attachment; filename="${path.basename(original)}"`)
|
||||||
|
fs.createReadStream(original).pipe(res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendJson(res: ServerResponse, status: number, obj: unknown): void {
|
||||||
|
const body = JSON.stringify(obj)
|
||||||
|
res.statusCode = status
|
||||||
|
res.setHeader('Content-Type', 'application/json')
|
||||||
|
res.setHeader('Content-Length', Buffer.byteLength(body))
|
||||||
|
res.end(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
function isFile(p: string): boolean {
|
||||||
|
try { return fs.statSync(p).isFile() } catch { return false }
|
||||||
|
}
|
||||||
71
ui/doocus-app/src/App.vue
Normal file
71
ui/doocus-app/src/App.vue
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import Panel from '@framework/components/Panel.vue'
|
||||||
|
import SplitPane from '@framework/components/SplitPane.vue'
|
||||||
|
import DocPicker from '@/components/DocPicker.vue'
|
||||||
|
import DocViewer from '@/components/DocViewer.vue'
|
||||||
|
import MetadataPanel from '@/components/MetadataPanel.vue'
|
||||||
|
import ExportBar from '@/components/ExportBar.vue'
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="app">
|
||||||
|
<header class="topbar">
|
||||||
|
<span class="brand">doocus</span>
|
||||||
|
<span class="spacer" />
|
||||||
|
<ExportBar />
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<main class="body">
|
||||||
|
<SplitPane :initial-size="0.9" :min="0.5" :max="2">
|
||||||
|
<template #first>
|
||||||
|
<Panel title="Documents">
|
||||||
|
<DocPicker />
|
||||||
|
</Panel>
|
||||||
|
</template>
|
||||||
|
<template #second>
|
||||||
|
<SplitPane :initial-size="2.2" :min="1" :max="5">
|
||||||
|
<template #first>
|
||||||
|
<Panel title="Content">
|
||||||
|
<DocViewer />
|
||||||
|
</Panel>
|
||||||
|
</template>
|
||||||
|
<template #second>
|
||||||
|
<Panel title="Metadata">
|
||||||
|
<MetadataPanel />
|
||||||
|
</Panel>
|
||||||
|
</template>
|
||||||
|
</SplitPane>
|
||||||
|
</template>
|
||||||
|
</SplitPane>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.app {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
.topbar {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-3);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
background: var(--surface-1);
|
||||||
|
border-bottom: var(--panel-border);
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.brand {
|
||||||
|
font-weight: 700;
|
||||||
|
letter-spacing: 0.02em;
|
||||||
|
}
|
||||||
|
.spacer {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.body {
|
||||||
|
flex: 1;
|
||||||
|
min-height: 0;
|
||||||
|
padding: var(--space-2);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
85
ui/doocus-app/src/components/DocPicker.vue
Normal file
85
ui/doocus-app/src/components/DocPicker.vue
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { onMounted } from 'vue'
|
||||||
|
import { store, loadDocs, openDoc, toggleSelect, isSelected } from '@/store'
|
||||||
|
|
||||||
|
onMounted(loadDocs)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="doc-picker">
|
||||||
|
<div class="picker-head">
|
||||||
|
<strong>{{ store.docs.length }}</strong> document{{ store.docs.length === 1 ? '' : 's' }}
|
||||||
|
</div>
|
||||||
|
<ul class="doc-list">
|
||||||
|
<li
|
||||||
|
v-for="d in store.docs"
|
||||||
|
:key="d.id"
|
||||||
|
:class="{ active: d.id === store.currentId }"
|
||||||
|
@click="openDoc(d.id)"
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
:checked="isSelected(d.id)"
|
||||||
|
title="Add to package"
|
||||||
|
@click.stop="toggleSelect(d.id)"
|
||||||
|
/>
|
||||||
|
<span class="type-badge" :data-type="d.type ?? '?'">{{ d.type ?? '?' }}</span>
|
||||||
|
<span class="doc-name">{{ d.title || d.name }}</span>
|
||||||
|
<span v-if="d.warnings" class="warn" :title="`${d.warnings} warning(s)`">⚠</span>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.doc-picker {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.picker-head {
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
color: var(--text-secondary);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
border-bottom: var(--panel-border);
|
||||||
|
}
|
||||||
|
.doc-list {
|
||||||
|
list-style: none;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.doc-list li {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
padding: var(--space-2) var(--space-3);
|
||||||
|
cursor: pointer;
|
||||||
|
border-bottom: 1px solid var(--surface-2);
|
||||||
|
}
|
||||||
|
.doc-list li:hover {
|
||||||
|
background: var(--surface-2);
|
||||||
|
}
|
||||||
|
.doc-list li.active {
|
||||||
|
background: var(--surface-3);
|
||||||
|
}
|
||||||
|
.type-badge {
|
||||||
|
font-size: 10px;
|
||||||
|
text-transform: uppercase;
|
||||||
|
padding: 1px 5px;
|
||||||
|
border-radius: 3px;
|
||||||
|
background: var(--surface-3);
|
||||||
|
color: var(--text-secondary);
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.doc-name {
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.warn {
|
||||||
|
margin-left: auto;
|
||||||
|
color: var(--status-error, #e0a030);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
76
ui/doocus-app/src/components/DocViewer.vue
Normal file
76
ui/doocus-app/src/components/DocViewer.vue
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import VideoPlayer from '@framework/components/VideoPlayer.vue'
|
||||||
|
import { store } from '@/store'
|
||||||
|
|
||||||
|
// Choose a viewer by file type. Images and video render the original directly;
|
||||||
|
// everything else renders the textified content.md (monospace for structured
|
||||||
|
// text, prose for documents).
|
||||||
|
const kind = computed<'image' | 'video' | 'mono' | 'prose'>(() => {
|
||||||
|
const t = (store.current?.meta as any)?.type as string | undefined
|
||||||
|
if (t === 'jpg' || t === 'jpeg' || t === 'png') return 'image'
|
||||||
|
if (t === 'mp4') return 'video'
|
||||||
|
if (t === 'json' || t === 'yaml' || t === 'yml' || t === 'csv') return 'mono'
|
||||||
|
return 'prose'
|
||||||
|
})
|
||||||
|
|
||||||
|
const content = computed(() => store.current?.content ?? '')
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="viewer">
|
||||||
|
<div v-if="!store.current" class="empty">Select a document to view its extracted content.</div>
|
||||||
|
|
||||||
|
<div v-else-if="kind === 'image'" class="image-wrap">
|
||||||
|
<img :src="store.current.originalUrl ?? store.current.thumbUrl ?? ''" alt="document" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-else-if="kind === 'video'" class="video-wrap">
|
||||||
|
<VideoPlayer v-if="store.current.originalUrl" :src="store.current.originalUrl" />
|
||||||
|
<p class="note">
|
||||||
|
Video preview. The transcript is produced by the meetus pipeline, not doocus.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<pre v-else-if="kind === 'mono'" class="mono">{{ content }}</pre>
|
||||||
|
|
||||||
|
<div v-else class="prose">{{ content }}</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.viewer {
|
||||||
|
height: 100%;
|
||||||
|
overflow: auto;
|
||||||
|
padding: var(--space-4);
|
||||||
|
}
|
||||||
|
.empty {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
.image-wrap img {
|
||||||
|
max-width: 100%;
|
||||||
|
border-radius: var(--panel-radius);
|
||||||
|
border: var(--panel-border);
|
||||||
|
}
|
||||||
|
.video-wrap {
|
||||||
|
max-width: 900px;
|
||||||
|
}
|
||||||
|
.note {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
}
|
||||||
|
.mono {
|
||||||
|
margin: 0;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
font-family: var(--font-mono, monospace);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
.prose {
|
||||||
|
white-space: pre-wrap;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
max-width: 820px;
|
||||||
|
line-height: 1.6;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
110
ui/doocus-app/src/components/ExportBar.vue
Normal file
110
ui/doocus-app/src/components/ExportBar.vue
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import JSZip from 'jszip'
|
||||||
|
import {
|
||||||
|
store, TARGETS, target, packageEstimate, humanBytes,
|
||||||
|
} from '@/store'
|
||||||
|
|
||||||
|
const building = ref(false)
|
||||||
|
const error = ref('')
|
||||||
|
|
||||||
|
// Build a zip: per selected doc, a folder holding the original source file plus
|
||||||
|
// the textified content.md and meta.json. Originals are included because the
|
||||||
|
// permitted services (Gemini web / NotebookLM) receive them directly; content.md
|
||||||
|
// keeps each doc greppable and small.
|
||||||
|
async function buildPackage(): Promise<void> {
|
||||||
|
if (!store.selected.size) return
|
||||||
|
building.value = true
|
||||||
|
error.value = ''
|
||||||
|
try {
|
||||||
|
const zip = new JSZip()
|
||||||
|
for (const id of store.selected) {
|
||||||
|
const summary = store.docs.find((d) => d.id === id)
|
||||||
|
const res = await fetch(`/api/docs/${encodeURIComponent(id)}`)
|
||||||
|
if (!res.ok) throw new Error(`fetch ${id} failed`)
|
||||||
|
const detail = await res.json()
|
||||||
|
const folder = zip.folder(id.replace(/[\\/]+/g, '__')) ?? zip
|
||||||
|
folder.file('content.md', detail.content ?? '')
|
||||||
|
folder.file('meta.json', JSON.stringify(detail.meta ?? {}, null, 2))
|
||||||
|
if (detail.originalUrl) {
|
||||||
|
const bin = await fetch(detail.originalUrl)
|
||||||
|
if (bin.ok) {
|
||||||
|
const name = (summary?.name) || 'original'
|
||||||
|
folder.file(name, await bin.blob())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const blob = await zip.generateAsync({ type: 'blob' })
|
||||||
|
const url = URL.createObjectURL(blob)
|
||||||
|
const a = document.createElement('a')
|
||||||
|
a.href = url
|
||||||
|
a.download = `doocus-package-${target.value.key}.zip`
|
||||||
|
a.click()
|
||||||
|
URL.revokeObjectURL(url)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = String(e)
|
||||||
|
} finally {
|
||||||
|
building.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearSelection(): void {
|
||||||
|
store.selected.clear()
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="export-bar">
|
||||||
|
<label class="target">
|
||||||
|
Target
|
||||||
|
<select v-model="store.targetKey">
|
||||||
|
<option v-for="t in TARGETS" :key="t.key" :value="t.key">{{ t.label }}</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<div class="estimate" :class="{ over: packageEstimate.overFiles || packageEstimate.overBytes }">
|
||||||
|
{{ packageEstimate.docCount }} docs ·
|
||||||
|
<span :class="{ bad: packageEstimate.overFiles }">
|
||||||
|
{{ packageEstimate.files }}/{{ target.maxFiles }} files
|
||||||
|
</span>
|
||||||
|
·
|
||||||
|
<span :class="{ bad: packageEstimate.overBytes }">
|
||||||
|
{{ humanBytes(packageEstimate.bytes) }} / {{ humanBytes(target.maxBytes) }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button :disabled="!store.selected.size" @click="clearSelection">Clear</button>
|
||||||
|
<button :disabled="!store.selected.size || building" @click="buildPackage">
|
||||||
|
{{ building ? 'Building…' : 'Download package' }}
|
||||||
|
</button>
|
||||||
|
<span v-if="error" class="err">{{ error }}</span>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.export-bar {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-3);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
}
|
||||||
|
.target {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-1);
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
.estimate {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
.estimate.over {
|
||||||
|
color: var(--status-error, #e0a030);
|
||||||
|
}
|
||||||
|
.estimate .bad {
|
||||||
|
color: var(--status-error, #e0a030);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.err {
|
||||||
|
color: var(--status-error, #e0a030);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
123
ui/doocus-app/src/components/MetadataPanel.vue
Normal file
123
ui/doocus-app/src/components/MetadataPanel.vue
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { store, humanBytes } from '@/store'
|
||||||
|
|
||||||
|
// Flatten meta.json into label/value rows, promoting the common fields to the
|
||||||
|
// top and rendering nested objects/arrays compactly.
|
||||||
|
const rows = computed(() => {
|
||||||
|
const m = store.current?.meta as Record<string, any> | undefined
|
||||||
|
if (!m) return []
|
||||||
|
const out: Array<{ label: string; value: string }> = []
|
||||||
|
const push = (label: string, value: unknown) => {
|
||||||
|
if (value === null || value === undefined || value === '') return
|
||||||
|
let s: string
|
||||||
|
if (Array.isArray(value)) s = value.join(', ')
|
||||||
|
else if (typeof value === 'object') s = JSON.stringify(value)
|
||||||
|
else s = String(value)
|
||||||
|
out.push({ label, value: s })
|
||||||
|
}
|
||||||
|
|
||||||
|
const src = (m.source ?? {}) as Record<string, unknown>
|
||||||
|
push('File', src.name)
|
||||||
|
push('Type', m.type)
|
||||||
|
push('Family', m.family)
|
||||||
|
push('Size', typeof src.bytes === 'number' ? humanBytes(src.bytes) : undefined)
|
||||||
|
push('Modified', src.modified)
|
||||||
|
push('Title', m.title)
|
||||||
|
push('Author', m.author)
|
||||||
|
push('Created', m.created)
|
||||||
|
push('Doc modified', m.modified_doc)
|
||||||
|
push('Pages', m.pages)
|
||||||
|
push('Slides', m.slides)
|
||||||
|
push('Sheets', m.sheets)
|
||||||
|
push('Rows', m.rows)
|
||||||
|
push('Columns', m.columns)
|
||||||
|
push('Words', m.word_count)
|
||||||
|
push('Captured', m.captured)
|
||||||
|
push('Duration (s)', m.duration_seconds)
|
||||||
|
push('Delegate', m.delegate)
|
||||||
|
push('Extractor', m.extractor)
|
||||||
|
push('sha256', src.sha256)
|
||||||
|
|
||||||
|
// Any remaining structured extras not already shown.
|
||||||
|
if (m.structure) push('Structure', m.structure)
|
||||||
|
if (m.headings) push('Headings', m.headings)
|
||||||
|
return out
|
||||||
|
})
|
||||||
|
|
||||||
|
const warnings = computed(() => {
|
||||||
|
const w = (store.current?.meta as any)?.warnings
|
||||||
|
return Array.isArray(w) ? w : []
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="meta-panel">
|
||||||
|
<div v-if="!store.current" class="empty">No document selected.</div>
|
||||||
|
<template v-else>
|
||||||
|
<img
|
||||||
|
v-if="store.current.thumbUrl"
|
||||||
|
class="thumb"
|
||||||
|
:src="store.current.thumbUrl"
|
||||||
|
alt="preview"
|
||||||
|
/>
|
||||||
|
<dl class="rows">
|
||||||
|
<template v-for="r in rows" :key="r.label">
|
||||||
|
<dt>{{ r.label }}</dt>
|
||||||
|
<dd>{{ r.value }}</dd>
|
||||||
|
</template>
|
||||||
|
</dl>
|
||||||
|
<div v-if="warnings.length" class="warnings">
|
||||||
|
<div class="warn-title">Warnings</div>
|
||||||
|
<ul>
|
||||||
|
<li v-for="(w, i) in warnings" :key="i">{{ w }}</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.meta-panel {
|
||||||
|
height: 100%;
|
||||||
|
overflow-y: auto;
|
||||||
|
padding: var(--space-3);
|
||||||
|
}
|
||||||
|
.empty {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
}
|
||||||
|
.thumb {
|
||||||
|
width: 100%;
|
||||||
|
border-radius: var(--panel-radius);
|
||||||
|
border: var(--panel-border);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
.rows {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: max-content 1fr;
|
||||||
|
gap: var(--space-1) var(--space-3);
|
||||||
|
margin: 0;
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
}
|
||||||
|
.rows dt {
|
||||||
|
color: var(--text-secondary);
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.rows dd {
|
||||||
|
margin: 0;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
.warnings {
|
||||||
|
margin-top: var(--space-4);
|
||||||
|
color: var(--status-error, #e0a030);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
}
|
||||||
|
.warn-title {
|
||||||
|
font-weight: 600;
|
||||||
|
margin-bottom: var(--space-1);
|
||||||
|
}
|
||||||
|
.warnings ul {
|
||||||
|
margin: 0;
|
||||||
|
padding-left: var(--space-4);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
120
ui/doocus-app/src/store.ts
Normal file
120
ui/doocus-app/src/store.ts
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
/**
|
||||||
|
* Reactive app store for the doocus browser.
|
||||||
|
*
|
||||||
|
* Reads the local docs-output tree via /api/docs (see server/doocusApi.ts).
|
||||||
|
* Holds the doc list, the currently open doc (meta + textified content), and the
|
||||||
|
* selection used by the package builder. Read-only over the pipeline output —
|
||||||
|
* the UI never mutates extracted artifacts.
|
||||||
|
*/
|
||||||
|
import { reactive, computed } from 'vue'
|
||||||
|
|
||||||
|
export interface DocSummary {
|
||||||
|
id: string
|
||||||
|
name: string
|
||||||
|
type: string | null
|
||||||
|
family: string | null
|
||||||
|
title: string | null
|
||||||
|
extractedAt: string | null
|
||||||
|
wordCount: number
|
||||||
|
bytes: number
|
||||||
|
hasThumb: boolean
|
||||||
|
warnings: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DocDetail {
|
||||||
|
id: string
|
||||||
|
meta: Record<string, unknown>
|
||||||
|
content: string
|
||||||
|
hasThumb: boolean
|
||||||
|
thumbUrl: string | null
|
||||||
|
hasOriginal: boolean
|
||||||
|
originalUrl: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Package targets and their (soft) limits. Surfaced, not hard-enforced. */
|
||||||
|
export interface Target {
|
||||||
|
key: string
|
||||||
|
label: string
|
||||||
|
maxFiles: number
|
||||||
|
maxBytes: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export const TARGETS: Target[] = [
|
||||||
|
{ key: 'gemini', label: 'Gemini web', maxFiles: 10, maxBytes: 100 * 1024 * 1024 },
|
||||||
|
{ key: 'notebooklm', label: 'NotebookLM', maxFiles: 50, maxBytes: 200 * 1024 * 1024 },
|
||||||
|
]
|
||||||
|
|
||||||
|
interface State {
|
||||||
|
docs: DocSummary[]
|
||||||
|
outputDir: string
|
||||||
|
currentId: string | null
|
||||||
|
current: DocDetail | null
|
||||||
|
loading: boolean
|
||||||
|
selected: Set<string>
|
||||||
|
targetKey: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export const store = reactive<State>({
|
||||||
|
docs: [],
|
||||||
|
outputDir: '',
|
||||||
|
currentId: null,
|
||||||
|
current: null,
|
||||||
|
loading: false,
|
||||||
|
selected: new Set<string>(),
|
||||||
|
targetKey: TARGETS[0].key,
|
||||||
|
})
|
||||||
|
|
||||||
|
export async function loadDocs(): Promise<void> {
|
||||||
|
const res = await fetch('/api/docs')
|
||||||
|
const data = await res.json()
|
||||||
|
store.docs = data.docs ?? []
|
||||||
|
store.outputDir = data.outputDir ?? ''
|
||||||
|
if (!store.currentId && store.docs.length) {
|
||||||
|
await openDoc(store.docs[0].id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function openDoc(id: string): Promise<void> {
|
||||||
|
store.loading = true
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/docs/${encodeURIComponent(id)}`)
|
||||||
|
if (!res.ok) throw new Error(`open failed: ${res.status}`)
|
||||||
|
store.current = await res.json()
|
||||||
|
store.currentId = id
|
||||||
|
} finally {
|
||||||
|
store.loading = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toggleSelect(id: string): void {
|
||||||
|
if (store.selected.has(id)) store.selected.delete(id)
|
||||||
|
else store.selected.add(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isSelected(id: string): boolean {
|
||||||
|
return store.selected.has(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
export const target = computed(() => TARGETS.find((t) => t.key === store.targetKey) ?? TARGETS[0])
|
||||||
|
|
||||||
|
/** Rough package estimate: each doc contributes original + content.md + meta.json. */
|
||||||
|
export const packageEstimate = computed(() => {
|
||||||
|
const chosen = store.docs.filter((d) => store.selected.has(d.id))
|
||||||
|
const files = chosen.length * 3 // original + content.md + meta.json
|
||||||
|
// originals dominate size; content/meta are small — approximate with wordCount.
|
||||||
|
const bytes = chosen.reduce((sum, d) => sum + d.bytes + d.wordCount * 6 + 512, 0)
|
||||||
|
const t = target.value
|
||||||
|
return {
|
||||||
|
docCount: chosen.length,
|
||||||
|
files,
|
||||||
|
bytes,
|
||||||
|
overFiles: files > t.maxFiles,
|
||||||
|
overBytes: bytes > t.maxBytes,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
export function humanBytes(n: number): string {
|
||||||
|
if (n < 1024) return `${n} B`
|
||||||
|
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
|
||||||
|
return `${(n / 1024 / 1024).toFixed(1)} MB`
|
||||||
|
}
|
||||||
29
ui/doocus-app/vite.config.ts
Normal file
29
ui/doocus-app/vite.config.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { defineConfig } from 'vite'
|
||||||
|
import vue from '@vitejs/plugin-vue'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
import { doocusApi } from './server/doocusApi'
|
||||||
|
|
||||||
|
// Output directory produced by the doocus pipeline (process_doc.py /
|
||||||
|
// ctrl/batch_docs.sh). Override with DOOCUS_OUTPUT.
|
||||||
|
const outputDir = process.env.DOOCUS_OUTPUT
|
||||||
|
?? fileURLToPath(new URL('../../docs-output', import.meta.url))
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
plugins: [vue(), doocusApi({ outputDir })],
|
||||||
|
resolve: {
|
||||||
|
alias: {
|
||||||
|
'@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),
|
||||||
|
'@': fileURLToPath(new URL('./src', import.meta.url)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
server: {
|
||||||
|
fs: {
|
||||||
|
// Allow the dev server to read the framework source and the docs-output tree.
|
||||||
|
allow: [
|
||||||
|
fileURLToPath(new URL('.', import.meta.url)),
|
||||||
|
fileURLToPath(new URL('../framework', import.meta.url)),
|
||||||
|
outputDir,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
@@ -20,6 +20,9 @@ import path from 'node:path'
|
|||||||
|
|
||||||
interface Options {
|
interface Options {
|
||||||
outputDir: string
|
outputDir: string
|
||||||
|
/** Optional directory to look for source videos by name when the absolute
|
||||||
|
* path recorded in the manifest is not reachable on this machine. */
|
||||||
|
videoDir?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
interface Manifest {
|
interface Manifest {
|
||||||
@@ -43,19 +46,66 @@ const VIDEO_MIME: Record<string, string> = {
|
|||||||
|
|
||||||
export function meetusApi(opts: Options): Plugin {
|
export function meetusApi(opts: Options): Plugin {
|
||||||
const outputDir = path.resolve(opts.outputDir)
|
const outputDir = path.resolve(opts.outputDir)
|
||||||
|
const videoDir = opts.videoDir ? path.resolve(opts.videoDir) : null
|
||||||
|
|
||||||
/** Resolve a run id to its directory, rejecting traversal. */
|
/** Locate a run's source video. The manifest records an absolute path from
|
||||||
|
* the machine that processed it, which may not exist here — so fall back to
|
||||||
|
* finding the file by name in MEETUS_VIDEO_DIR, the run dir, its parent, and
|
||||||
|
* the output root (videos are sometimes stored alongside the output). */
|
||||||
|
function resolveVideoPath(dir: string, manifest: Manifest): string | null {
|
||||||
|
const recorded = manifest.video?.path
|
||||||
|
if (recorded && isFile(recorded)) return recorded
|
||||||
|
const name = manifest.video?.name
|
||||||
|
if (name) {
|
||||||
|
const candidates: string[] = []
|
||||||
|
if (videoDir) candidates.push(path.join(videoDir, name))
|
||||||
|
candidates.push(
|
||||||
|
path.join(dir, name), // inside the run dir
|
||||||
|
path.join(path.dirname(dir), name), // sibling of the run dir
|
||||||
|
path.join(outputDir, name), // output root
|
||||||
|
)
|
||||||
|
for (const c of candidates) if (isFile(c)) return c
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolve a run id (possibly nested, e.g. "batch/2026/team a/<run>") to its
|
||||||
|
* directory, rejecting traversal outside the output root. */
|
||||||
function runDir(id: string): string | null {
|
function runDir(id: string): string | null {
|
||||||
const safe = path.basename(id)
|
const dir = path.resolve(outputDir, id)
|
||||||
if (safe !== id || !safe || safe.startsWith('.')) return null
|
if (dir !== outputDir && !dir.startsWith(outputDir + path.sep)) return null
|
||||||
const dir = path.join(outputDir, safe)
|
|
||||||
if (!dir.startsWith(outputDir + path.sep)) return null
|
|
||||||
try {
|
try {
|
||||||
if (fs.statSync(dir).isDirectory()) return dir
|
if (fs.statSync(dir).isDirectory()) return dir
|
||||||
} catch { /* missing */ }
|
} catch { /* missing */ }
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const toPosix = (p: string) => p.split(path.sep).join('/')
|
||||||
|
|
||||||
|
/** Find every run directory (one containing manifest.json) under `base`,
|
||||||
|
* at any depth — batch.sh mirrors the input tree, so runs are nested. */
|
||||||
|
async function findRunDirs(base: string, maxDepth = 8): Promise<string[]> {
|
||||||
|
const found: string[] = []
|
||||||
|
const walk = async (dir: string, depth: number): Promise<void> => {
|
||||||
|
if (depth > maxDepth) return
|
||||||
|
let ents
|
||||||
|
try {
|
||||||
|
ents = await fsp.readdir(dir, { withFileTypes: true })
|
||||||
|
} catch { return }
|
||||||
|
if (ents.some((e) => e.isFile() && e.name === 'manifest.json')) {
|
||||||
|
found.push(dir)
|
||||||
|
return // a run dir; don't descend into it (frames/ etc.)
|
||||||
|
}
|
||||||
|
for (const e of ents) {
|
||||||
|
if (!e.isDirectory()) continue
|
||||||
|
if (e.name === 'frames' || e.name === 'node_modules' || e.name.startsWith('.')) continue
|
||||||
|
await walk(path.join(dir, e.name), depth + 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await walk(base, 0)
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
function readManifest(dir: string): Manifest | null {
|
function readManifest(dir: string): Manifest | null {
|
||||||
try {
|
try {
|
||||||
return JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf-8'))
|
return JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf-8'))
|
||||||
@@ -67,6 +117,8 @@ export function meetusApi(opts: Options): Plugin {
|
|||||||
return {
|
return {
|
||||||
name: 'meetus-api',
|
name: 'meetus-api',
|
||||||
configureServer(server) {
|
configureServer(server) {
|
||||||
|
server.config.logger.info(`[meetus-api] serving meeting runs from: ${outputDir}`)
|
||||||
|
if (videoDir) server.config.logger.info(`[meetus-api] video fallback dir: ${videoDir}`)
|
||||||
server.middlewares.use('/api', (req, res, next) => {
|
server.middlewares.use('/api', (req, res, next) => {
|
||||||
const url = new URL(req.url ?? '/', 'http://localhost')
|
const url = new URL(req.url ?? '/', 'http://localhost')
|
||||||
const parts = url.pathname.split('/').filter(Boolean) // after /api strip
|
const parts = url.pathname.split('/').filter(Boolean) // after /api strip
|
||||||
@@ -128,20 +180,12 @@ export function meetusApi(opts: Options): Plugin {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function listRuns(res: ServerResponse): Promise<void> {
|
async function listRuns(res: ServerResponse): Promise<void> {
|
||||||
let entries: string[] = []
|
const dirs = await findRunDirs(outputDir)
|
||||||
try {
|
|
||||||
entries = (await fsp.readdir(outputDir, { withFileTypes: true }))
|
|
||||||
.filter((e) => e.isDirectory())
|
|
||||||
.map((e) => e.name)
|
|
||||||
} catch {
|
|
||||||
sendJson(res, 200, { runs: [], outputDir })
|
|
||||||
return
|
|
||||||
}
|
|
||||||
const runs = []
|
const runs = []
|
||||||
for (const name of entries.sort().reverse()) {
|
for (const dir of dirs) {
|
||||||
const dir = path.join(outputDir, name)
|
|
||||||
const manifest = readManifest(dir)
|
const manifest = readManifest(dir)
|
||||||
if (!manifest) continue
|
if (!manifest) continue
|
||||||
|
const id = toPosix(path.relative(outputDir, dir))
|
||||||
const framesRel = manifest.outputs?.frames ?? 'frames'
|
const framesRel = manifest.outputs?.frames ?? 'frames'
|
||||||
let frameCount = 0
|
let frameCount = 0
|
||||||
try {
|
try {
|
||||||
@@ -149,13 +193,16 @@ export function meetusApi(opts: Options): Plugin {
|
|||||||
.filter((f) => f.toLowerCase().endsWith('.jpg')).length
|
.filter((f) => f.toLowerCase().endsWith('.jpg')).length
|
||||||
} catch { /* no frames */ }
|
} catch { /* no frames */ }
|
||||||
runs.push({
|
runs.push({
|
||||||
id: name,
|
id,
|
||||||
name: manifest.video?.name ?? name,
|
name: manifest.video?.name ?? id,
|
||||||
processedAt: manifest.processed_at ?? null,
|
processedAt: manifest.processed_at ?? null,
|
||||||
frameCount,
|
frameCount,
|
||||||
hasVideo: !!manifest.video?.path && fs.existsSync(manifest.video.path),
|
hasVideo: !!resolveVideoPath(dir, manifest),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
// Most recent first (by processed time, then id).
|
||||||
|
runs.sort((a, b) =>
|
||||||
|
(b.processedAt ?? '').localeCompare(a.processedAt ?? '') || b.id.localeCompare(a.id))
|
||||||
sendJson(res, 200, { runs, outputDir })
|
sendJson(res, 200, { runs, outputDir })
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,7 +289,7 @@ export function meetusApi(opts: Options): Plugin {
|
|||||||
segments,
|
segments,
|
||||||
frames,
|
frames,
|
||||||
enhancedAvailable,
|
enhancedAvailable,
|
||||||
hasVideo: !!manifest.video?.path && fs.existsSync(manifest.video.path),
|
hasVideo: !!resolveVideoPath(dir, manifest),
|
||||||
videoUrl: `/api/runs/${encodeURIComponent(id)}/video`,
|
videoUrl: `/api/runs/${encodeURIComponent(id)}/video`,
|
||||||
review,
|
review,
|
||||||
})
|
})
|
||||||
@@ -266,10 +313,10 @@ export function meetusApi(opts: Options): Plugin {
|
|||||||
logger: { warn: (m: string) => void },
|
logger: { warn: (m: string) => void },
|
||||||
): void {
|
): void {
|
||||||
const manifest = readManifest(dir)
|
const manifest = readManifest(dir)
|
||||||
const videoPath = manifest?.video?.path
|
const videoPath = manifest ? resolveVideoPath(dir, manifest) : null
|
||||||
if (!videoPath || !fs.existsSync(videoPath)) {
|
if (!videoPath) {
|
||||||
logger.warn(`[meetus-api] source video missing for run: ${dir}`)
|
logger.warn(`[meetus-api] source video not found for run: ${dir} (recorded: ${manifest?.video?.path ?? '?'})`)
|
||||||
sendJson(res, 404, { error: 'source video not available at manifest path' })
|
sendJson(res, 404, { error: 'source video not found (recorded path unreachable and no match by name)' })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const stat = fs.statSync(videoPath)
|
const stat = fs.statSync(videoPath)
|
||||||
@@ -330,6 +377,10 @@ function sendJson(res: ServerResponse, status: number, obj: unknown): void {
|
|||||||
res.end(body)
|
res.end(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isFile(p: string): boolean {
|
||||||
|
try { return fs.statSync(p).isFile() } catch { return false }
|
||||||
|
}
|
||||||
|
|
||||||
function readBody(req: IncomingMessage): Promise<string> {
|
function readBody(req: IncomingMessage): Promise<string> {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
let data = ''
|
let data = ''
|
||||||
@@ -23,6 +23,9 @@ function onChange(e: Event) {
|
|||||||
</select>
|
</select>
|
||||||
<span v-if="state.loading" class="hint">loading…</span>
|
<span v-if="state.loading" class="hint">loading…</span>
|
||||||
<span v-else-if="state.error" class="hint err">{{ state.error }}</span>
|
<span v-else-if="state.error" class="hint err">{{ state.error }}</span>
|
||||||
|
<span v-else-if="!state.runs.length" class="hint" :title="state.outputDir ?? ''">
|
||||||
|
no runs found in {{ state.outputDir ?? '(unknown)' }}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
7
ui/meetus-app/src/env.d.ts
vendored
Normal file
7
ui/meetus-app/src/env.d.ts
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
/// <reference types="vite/client" />
|
||||||
|
|
||||||
|
declare module '*.vue' {
|
||||||
|
import type { DefineComponent } from 'vue'
|
||||||
|
const component: DefineComponent<Record<string, unknown>, Record<string, unknown>, unknown>
|
||||||
|
export default component
|
||||||
|
}
|
||||||
6
ui/meetus-app/src/main.ts
Normal file
6
ui/meetus-app/src/main.ts
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
import { createApp } from 'vue'
|
||||||
|
import '@framework/tokens.css'
|
||||||
|
import './styles.css'
|
||||||
|
import App from './App.vue'
|
||||||
|
|
||||||
|
createApp(App).mount('#app')
|
||||||
61
ui/meetus-app/src/styles.css
Normal file
61
ui/meetus-app/src/styles.css
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
* {
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
html,
|
||||||
|
body,
|
||||||
|
#app {
|
||||||
|
margin: 0;
|
||||||
|
height: 100%;
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
background: var(--surface-0);
|
||||||
|
color: var(--text-primary);
|
||||||
|
font-family: var(--font-ui);
|
||||||
|
font-size: var(--font-size-base);
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
font-family: var(--font-ui);
|
||||||
|
font-size: var(--font-size-base);
|
||||||
|
color: var(--text-primary);
|
||||||
|
background: var(--surface-2);
|
||||||
|
border: var(--panel-border);
|
||||||
|
border-radius: var(--panel-radius);
|
||||||
|
padding: var(--space-1) var(--space-3);
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:hover:not(:disabled) {
|
||||||
|
background: var(--surface-3);
|
||||||
|
}
|
||||||
|
|
||||||
|
button:disabled {
|
||||||
|
opacity: 0.5;
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
|
input,
|
||||||
|
select,
|
||||||
|
textarea {
|
||||||
|
font-family: var(--font-ui);
|
||||||
|
font-size: var(--font-size-base);
|
||||||
|
color: var(--text-primary);
|
||||||
|
background: var(--surface-0);
|
||||||
|
border: var(--panel-border);
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
}
|
||||||
|
::-webkit-scrollbar-thumb {
|
||||||
|
background: var(--surface-3);
|
||||||
|
border-radius: 5px;
|
||||||
|
}
|
||||||
|
::-webkit-scrollbar-track {
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
22
ui/meetus-app/tsconfig.json
Normal file
22
ui/meetus-app/tsconfig.json
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"strict": true,
|
||||||
|
"jsx": "preserve",
|
||||||
|
"noEmit": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||||
|
"types": ["vite/client", "node"],
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": {
|
||||||
|
"@framework/*": ["../framework/src/*"],
|
||||||
|
"@/*": ["src/*"],
|
||||||
|
"vue": ["node_modules/vue"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts", "src/**/*.vue", "server/**/*.ts", "vite.config.ts"]
|
||||||
|
}
|
||||||
@@ -7,8 +7,12 @@ import { meetusApi } from './server/meetusApi'
|
|||||||
const outputDir = process.env.MEETUS_OUTPUT
|
const outputDir = process.env.MEETUS_OUTPUT
|
||||||
?? fileURLToPath(new URL('../../output', import.meta.url))
|
?? fileURLToPath(new URL('../../output', import.meta.url))
|
||||||
|
|
||||||
|
// Optional: where source videos live, if the absolute paths recorded in the
|
||||||
|
// manifests are not reachable on this machine. Videos are matched by name.
|
||||||
|
const videoDir = process.env.MEETUS_VIDEO_DIR || undefined
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
plugins: [vue(), meetusApi({ outputDir })],
|
plugins: [vue(), meetusApi({ outputDir, videoDir })],
|
||||||
resolve: {
|
resolve: {
|
||||||
alias: {
|
alias: {
|
||||||
'@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),
|
'@framework': fileURLToPath(new URL('../framework/src', import.meta.url)),
|
||||||
@@ -22,6 +26,7 @@ export default defineConfig({
|
|||||||
fileURLToPath(new URL('.', import.meta.url)),
|
fileURLToPath(new URL('.', import.meta.url)),
|
||||||
fileURLToPath(new URL('../framework', import.meta.url)),
|
fileURLToPath(new URL('../framework', import.meta.url)),
|
||||||
outputDir,
|
outputDir,
|
||||||
|
...(videoDir ? [videoDir] : []),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Reference in New Issue
Block a user