batch docs update
This commit is contained in:
Mariano Gabriel
32
INDEX.md
32
INDEX.md
@@ -39,19 +39,22 @@ offline. Its output is what feeds permitted services (Gemini web, NotebookLM)
|
||||
and the local search index; the raw source never leaves the machine.
|
||||
|
||||
```
|
||||
document (docx/pdf/xlsx/img/...)
|
||||
│ process_doc.py (deterministic, offline, no cloud AI)
|
||||
local drive tree (downloaded)
|
||||
│ process_tree.py (deterministic, offline, no cloud AI)
|
||||
▼
|
||||
docs-output/<run>/ run folder: YYYYMMDD-NNN-<stem>/
|
||||
├── content.md textified document ◀── the product
|
||||
├── meta.json source/sha/dates/author/pages/... metadata
|
||||
├── thumb.jpg optional single preview
|
||||
└── manifest.json
|
||||
docs-output/ whole tree replicated, no flattening
|
||||
├── index.json every file as a node (path, type, mode, url:null)
|
||||
└── <mirrored path>/<file>.doocus/ sidecar, only for docx/pdf/pptx/xlsx:
|
||||
├── content.md extracted text — SEARCH INDEX ONLY
|
||||
└── meta.json (never replaces the original)
|
||||
```
|
||||
|
||||
`make docs` (→ `ctrl/batch_docs.sh`) runs it over a whole tree. Full doocus docs:
|
||||
[`doocus/README.md`](doocus/README.md). Cross-search (meetus transcripts + doocus
|
||||
docs, textified-only) is a later phase. 🚧
|
||||
Node modes: **extracted** (docx/pdf/pptx/xlsx → text sidecar), **meeting**
|
||||
(mp4/... → belongs to meetus), **link** (everything else → original is the
|
||||
artifact, not duplicated). Each node has a `url` slot for the eventual Drive link.
|
||||
`make docs` (→ `process_tree.py`) indexes a whole tree; `process_doc.py` still does
|
||||
one-off single-file extraction. Full doocus docs: [`doocus/README.md`](doocus/README.md).
|
||||
Cross-search (docs frame + meetings frame, over cached text) is a later phase. 🚧
|
||||
|
||||
## Status legend
|
||||
|
||||
@@ -61,7 +64,8 @@ docs, textified-only) is a later phase. 🚧
|
||||
|
||||
```
|
||||
process_meeting.py ✅ the CLI tool — entry point (stays at root)
|
||||
process_doc.py ✅ doocus CLI — document extraction entry point
|
||||
process_tree.py ✅ doocus — index a whole local drive tree → index.json
|
||||
process_doc.py ✅ doocus — one-off single-file extraction
|
||||
Makefile ✅ batch convenience wrapper (make batch / make docs)
|
||||
meetus/ ✅ core package
|
||||
├─ workflow.py orchestrator (whisper → frames → merge)
|
||||
@@ -75,14 +79,14 @@ meetus/ ✅ core package
|
||||
├─ hybrid_processor.py (was --use-hybrid)
|
||||
└─ prompts/ vision context prompts
|
||||
doocus/ ✅ document-extraction package (see doocus/README.md)
|
||||
├─ workflow.py DocConfig + DocWorkflow (dispatch → extract → write → manifest)
|
||||
├─ tree.py build_index — walk drive tree → index.json (+ extract sidecars)
|
||||
├─ registry.py extension → extractor family dispatch (lazy, graceful)
|
||||
├─ output_manager.py run-folder + content.md/meta.json/thumb writer
|
||||
├─ workflow.py single-file DocWorkflow (process_doc.py)
|
||||
├─ output_manager.py single-file run-folder + content.md/meta.json writer
|
||||
├─ naming.py YYYYMMDD-NNN naming + sha256 (reuses meetus convention)
|
||||
└─ extractors/ text, tabular(csv), office(docx/pptx/xlsx), pdf, web(html), image, media(mp4)
|
||||
ctrl/ control plane / operational scripts
|
||||
├─ batch.sh ✅ recursive batch runner (mirrors tree, continues past failures)
|
||||
├─ batch_docs.sh ✅ doocus twin of batch.sh (make docs)
|
||||
├─ transcribe_oneoff.sh 🧰 high-quality re-transcription over an existing run
|
||||
├─ summarize/ 🚧 last step — local-LLM summarization (WIP, on hold)
|
||||
│ ├─ summarize_simple.py minimal map-and-append; reads every referenced frame
|
||||
|
||||
23
Makefile
23
Makefile
@@ -33,12 +33,15 @@ EXT ?=
|
||||
PYTHON ?=
|
||||
export PYTHON
|
||||
|
||||
# doocus (document extraction) knobs. DOCS_OUT defaults to docs-output; the input
|
||||
# tree is mirrored under it. DOC_EXTRA forwards one-off flags to process_doc.py
|
||||
# (e.g. --render for pdf page thumbnails, --ocr for images/scanned pdfs).
|
||||
# doocus (tree indexing) knobs. DOCS_OUT defaults to docs-output; the whole input
|
||||
# tree is replicated into index.json under it. DOC_EXTRA forwards one-off flags to
|
||||
# process_tree.py (e.g. --render for pdf page thumbnails, --ocr for scanned pdfs).
|
||||
DOCS_OUT ?= docs-output
|
||||
DOC_EXTRA ?=
|
||||
|
||||
# Python interpreter for doocus (override PYTHON, else python3 — use `uv run make`).
|
||||
PY := $(if $(PYTHON),$(PYTHON),python3)
|
||||
|
||||
.PHONY: batch dry docs docs-dry help
|
||||
|
||||
batch:
|
||||
@@ -48,16 +51,16 @@ batch:
|
||||
dry:
|
||||
@ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -n
|
||||
|
||||
# Document extraction (doocus), mirrors the input tree into DOCS_OUT.
|
||||
# make docs IN="/mnt/win/drive" # extract all docs
|
||||
# make docs IN="..." DOC_EXTRA="--render --ocr" # thumbnails + OCR
|
||||
# make docs-dry IN="..." # list what would run
|
||||
# Document indexing (doocus): replicate the whole IN tree into DOCS_OUT/index.json,
|
||||
# extracting text sidecars only for docx/pdf/pptx/xlsx.
|
||||
# uv run make docs IN="/mnt/win/drive" # index the tree
|
||||
# uv run make docs IN="..." DOC_EXTRA="--ocr" # + OCR scanned pdfs
|
||||
# make docs-dry IN="..." # counts only, no writes
|
||||
docs:
|
||||
@ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -- \
|
||||
$(DOC_EXTRA)
|
||||
@$(PY) process_tree.py "$(IN)" --output "$(DOCS_OUT)" $(DOC_EXTRA)
|
||||
|
||||
docs-dry:
|
||||
@ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -n
|
||||
@$(PY) process_tree.py "$(IN)" --output "$(DOCS_OUT)" --dry-run
|
||||
|
||||
help:
|
||||
@sed -n '3,14p' Makefile
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Batch-extract every document under a directory through process_doc.py,
|
||||
# mirroring the input folder structure into the output base. The doocus twin of
|
||||
# ctrl/batch.sh (which does the same for meeting videos via process_meeting.py).
|
||||
#
|
||||
# Recursive and space-safe (paths come off a Windows mount, so they often have
|
||||
# spaces). Each document's run folder is auto-created by process_doc.py inside
|
||||
# the mirrored output subfolder.
|
||||
#
|
||||
# Usage:
|
||||
# ctrl/batch_docs.sh -i <input-dir> -o <output-dir> [-e "pdf docx ..."] [-n] \
|
||||
# [-- <process_doc.py flags>]
|
||||
#
|
||||
# Examples:
|
||||
# # Everything under a downloaded Drive folder, mirrored into docs-output
|
||||
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output
|
||||
#
|
||||
# # Dry run: just show which docs map to which output folders
|
||||
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -n
|
||||
#
|
||||
# # Only pdfs/docx, render page thumbnails
|
||||
# ctrl/batch_docs.sh -i "./download" -o docs-output -e "pdf docx" -- --render
|
||||
#
|
||||
# A doc at <input>/2026/team a/notes.docx produces
|
||||
# <output>/2026/team a/<YYYYMMDD-NNN-notes>/...
|
||||
# i.e. the run folder lands inside the mirrored subtree.
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# python: honor $PYTHON, else prefer python3
|
||||
PYTHON="${PYTHON:-}"
|
||||
if [ -z "$PYTHON" ]; then
|
||||
if command -v python3 >/dev/null 2>&1; then PYTHON=python3; else PYTHON=python; fi
|
||||
fi
|
||||
|
||||
usage() { sed -n '2,26p' "$0"; }
|
||||
|
||||
INPUT=""
|
||||
OUTPUT=""
|
||||
EXTS="csv docx html htm jpg jpeg json md txt pdf png pptx xlsx yaml yml mp4"
|
||||
DRY=false
|
||||
FORWARD=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-i|--input) INPUT="$2"; shift 2 ;;
|
||||
-o|--output) OUTPUT="$2"; shift 2 ;;
|
||||
-e|--ext) EXTS="$2"; shift 2 ;;
|
||||
-n|--dry-run) DRY=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--) shift; FORWARD=("$@"); break ;;
|
||||
*) echo "Unknown arg: $1" >&2; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; }
|
||||
[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; }
|
||||
[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; }
|
||||
|
||||
# Absolute paths so relative-path math is stable regardless of where we cd'd.
|
||||
INPUT="$(realpath "$INPUT")"
|
||||
mkdir -p "$OUTPUT"
|
||||
OUTPUT="$(realpath "$OUTPUT")"
|
||||
|
||||
# Build the find extension filter: \( -iname '*.pdf' -o -iname '*.docx' ... \)
|
||||
find_expr=()
|
||||
for ext in $EXTS; do
|
||||
find_expr+=( -iname "*.${ext}" -o )
|
||||
done
|
||||
unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o
|
||||
|
||||
echo "Input : $INPUT"
|
||||
echo "Output: $OUTPUT"
|
||||
echo "Exts : $EXTS"
|
||||
[ "$DRY" = true ] && echo "(dry run — nothing will be processed)"
|
||||
echo
|
||||
|
||||
total=0 ok=0 fail=0
|
||||
# Process substitution (not a pipe) so counters survive into the summary.
|
||||
while IFS= read -r -d '' doc; do
|
||||
total=$((total + 1))
|
||||
|
||||
rel="${doc#"$INPUT"/}" # path relative to the input root
|
||||
reldir="$(dirname "$rel")"
|
||||
if [ "$reldir" = "." ]; then
|
||||
outdir="$OUTPUT" # doc sat directly in the input root
|
||||
else
|
||||
outdir="$OUTPUT/$reldir" # mirror the subfolder structure
|
||||
fi
|
||||
|
||||
echo "[$total] $rel"
|
||||
echo " -> $outdir"
|
||||
|
||||
if [ "$DRY" = true ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir -p "$outdir"
|
||||
# stdin from /dev/null so any child process can't eat the file list and stall
|
||||
# the batch (same guard as ctrl/batch.sh).
|
||||
if "$PYTHON" "$PROJECT_DIR/process_doc.py" "$doc" \
|
||||
--output-dir "$outdir" "${FORWARD[@]+"${FORWARD[@]}"}" </dev/null; then
|
||||
ok=$((ok + 1))
|
||||
else
|
||||
echo " !! FAILED (continuing)" >&2
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
echo
|
||||
done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z)
|
||||
|
||||
echo "----------------------------------------"
|
||||
if [ "$DRY" = true ]; then
|
||||
echo "Found $total document(s)."
|
||||
else
|
||||
echo "Done. $total document(s): $ok ok, $fail failed."
|
||||
fi
|
||||
[ "$fail" -eq 0 ]
|
||||
130
doocus/tree.py
Normal file
130
doocus/tree.py
Normal file
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Build a tree index over a local drive-download root.
|
||||
|
||||
The whole source tree is replicated as `index.json` — every file is a node, so
|
||||
the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes
|
||||
are classified into three modes:
|
||||
|
||||
extracted docx/pdf/pptx/xlsx → local text extraction written to a mirrored
|
||||
`<file>.doocus/` sidecar (content.md + meta.json). Search-index
|
||||
only — never a replacement for the original.
|
||||
meeting video (mp4/...) → belongs to meetus, not a doc resource. Flagged,
|
||||
not extracted; the meetus pipeline produces its transcript.
|
||||
link everything else (md/txt/json/yaml/csv/html/images/other) → the
|
||||
original is the artifact; nothing is duplicated.
|
||||
|
||||
Each node carries a `url: null` slot for the eventual Drive link (filled later
|
||||
from a URL source — Drive API / rclone metadata / export listing).
|
||||
"""
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
|
||||
from . import registry
|
||||
from .naming import sha256_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Formats we locally extract text from (the "not media, not plain-text" set).
|
||||
EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"}
|
||||
# Videos are meetings → meetus territory.
|
||||
MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"}
|
||||
|
||||
|
||||
def classify(ext: str) -> str:
|
||||
if ext in EXTRACT_EXTS:
|
||||
return "extracted"
|
||||
if ext in MEETING_EXTS:
|
||||
return "meeting"
|
||||
return "link"
|
||||
|
||||
|
||||
def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
||||
options = options or {}
|
||||
root = Path(source_root).resolve()
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(f"source root not found: {root}")
|
||||
out = Path(output_dir)
|
||||
if not dry_run:
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
files = []
|
||||
counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0}
|
||||
|
||||
for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()):
|
||||
if p.is_dir() or p.name.startswith("."):
|
||||
continue
|
||||
rel = p.relative_to(root).as_posix()
|
||||
ext = p.suffix.lower().lstrip(".")
|
||||
mode = classify(ext)
|
||||
counts[mode] += 1
|
||||
stat = p.stat()
|
||||
node = {
|
||||
"path": rel,
|
||||
"name": p.name,
|
||||
"ext": ext,
|
||||
"family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"),
|
||||
"mode": mode,
|
||||
"bytes": stat.st_size,
|
||||
"modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
||||
"url": None, # Drive link — filled later from a URL source
|
||||
}
|
||||
|
||||
if mode == "extracted" and not dry_run:
|
||||
_extract_sidecar(p, rel, out, options, node, counts)
|
||||
|
||||
files.append(node)
|
||||
|
||||
index = {
|
||||
"root": str(root),
|
||||
"generatedAt": datetime.now().isoformat(),
|
||||
"counts": {**counts, "total": len(files)},
|
||||
"files": files,
|
||||
}
|
||||
if not dry_run:
|
||||
(out / "index.json").write_text(
|
||||
json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
logger.info("Wrote index: %s (%d files)", out / "index.json", len(files))
|
||||
return index
|
||||
|
||||
|
||||
def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None:
|
||||
"""Extract text for a heavy format into a mirrored `<file>.doocus/` sidecar."""
|
||||
sidecar = out / (rel + ".doocus")
|
||||
sidecar.mkdir(parents=True, exist_ok=True)
|
||||
result = registry.extract(src, options)
|
||||
(sidecar / "content.md").write_text(result.content, encoding="utf-8")
|
||||
|
||||
stat = src.stat()
|
||||
meta = {
|
||||
"source": {
|
||||
"name": src.name,
|
||||
"path": str(src.absolute()),
|
||||
"sha256": sha256_file(src),
|
||||
"bytes": stat.st_size,
|
||||
"modified": node["modified"],
|
||||
},
|
||||
"type": node["ext"],
|
||||
"family": node["family"],
|
||||
"extractor": result.extractor,
|
||||
"extracted_at": datetime.now().isoformat(),
|
||||
"word_count": len(result.content.split()),
|
||||
"warnings": result.warnings,
|
||||
}
|
||||
for k, v in result.metadata.items():
|
||||
meta.setdefault(k, v)
|
||||
(sidecar / "meta.json").write_text(
|
||||
json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
node["out"] = sidecar.relative_to(out).as_posix()
|
||||
node["extractor"] = result.extractor
|
||||
node["words"] = meta["word_count"]
|
||||
if result.warnings:
|
||||
node["warnings"] = result.warnings
|
||||
counts["warnings"] += 1
|
||||
for k in ("title", "author", "pages", "slides", "sheets"):
|
||||
if k in result.metadata:
|
||||
node[k] = result.metadata[k]
|
||||
58
process_tree.py
Normal file
58
process_tree.py
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
doocus tree indexer — replicate a local drive tree into docs-output/index.json.
|
||||
|
||||
Walks a downloaded-drive root, preserving the full folder hierarchy. Locally
|
||||
extracts text for docx/pdf/pptx/xlsx (search-index sidecars, never a replacement
|
||||
for the original), flags videos as meetus meetings, and links everything else to
|
||||
the original without duplicating it. Deterministic and offline.
|
||||
|
||||
uv run process_tree.py /mnt/drive # → docs-output/index.json
|
||||
uv run process_tree.py /mnt/drive --output out/docs
|
||||
uv run process_tree.py /mnt/drive --dry-run # counts only, no writes
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from doocus.tree import build_index
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Index a local drive tree for doocus.")
|
||||
parser.add_argument("source", help="Local drive-download root to index")
|
||||
parser.add_argument("--output", default="docs-output",
|
||||
help="Output directory for index.json + sidecars (default: docs-output)")
|
||||
parser.add_argument("--render", action="store_true",
|
||||
help="Render page/slide images where supported (needs extra deps)")
|
||||
parser.add_argument("--ocr", action="store_true",
|
||||
help="OCR scanned PDFs where supported")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Walk and classify only; write nothing")
|
||||
parser.add_argument("--verbose", action="store_true", help="Verbose logging")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(message)s")
|
||||
|
||||
try:
|
||||
index = build_index(
|
||||
args.source,
|
||||
args.output,
|
||||
options={"render": args.render, "ocr": args.ocr},
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
except NotADirectoryError as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
c = index["counts"]
|
||||
print(f"\n{c['total']} files: {c['extracted']} extracted, "
|
||||
f"{c['meeting']} meeting, {c['link']} linked"
|
||||
+ (f", {c['warnings']} with warnings" if c.get("warnings") else "")
|
||||
+ (" (dry run — nothing written)" if args.dry_run else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user