From fa3a0e3d1a070eb4a7f920ddc4a7c9916a049dd4 Mon Sep 17 00:00:00 2001 From: Mariano Gabriel Date: Sun, 5 Jul 2026 10:38:29 -0300 Subject: [PATCH] batch docs update --- INDEX.md | 32 ++++++----- Makefile | 23 ++++---- ctrl/batch_docs.sh | 119 ----------------------------------------- doocus/tree.py | 130 +++++++++++++++++++++++++++++++++++++++++++++ process_tree.py | 58 ++++++++++++++++++++ 5 files changed, 219 insertions(+), 143 deletions(-) delete mode 100755 ctrl/batch_docs.sh create mode 100644 doocus/tree.py create mode 100644 process_tree.py diff --git a/INDEX.md b/INDEX.md index 2d192a2..9c2b539 100644 --- a/INDEX.md +++ b/INDEX.md @@ -39,19 +39,22 @@ offline. Its output is what feeds permitted services (Gemini web, NotebookLM) and the local search index; the raw source never leaves the machine. ``` -document (docx/pdf/xlsx/img/...) - │ process_doc.py (deterministic, offline, no cloud AI) +local drive tree (downloaded) + │ process_tree.py (deterministic, offline, no cloud AI) ▼ -docs-output// run folder: YYYYMMDD-NNN-/ - ├── content.md textified document ◀── the product - ├── meta.json source/sha/dates/author/pages/... metadata - ├── thumb.jpg optional single preview - └── manifest.json +docs-output/ whole tree replicated, no flattening + ├── index.json every file as a node (path, type, mode, url:null) + └── /.doocus/ sidecar, only for docx/pdf/pptx/xlsx: + ├── content.md extracted text — SEARCH INDEX ONLY + └── meta.json (never replaces the original) ``` -`make docs` (→ `ctrl/batch_docs.sh`) runs it over a whole tree. Full doocus docs: -[`doocus/README.md`](doocus/README.md). Cross-search (meetus transcripts + doocus -docs, textified-only) is a later phase. 🚧 +Node modes: **extracted** (docx/pdf/pptx/xlsx → text sidecar), **meeting** +(mp4/... → belongs to meetus), **link** (everything else → original is the +artifact, not duplicated). Each node has a `url` slot for the eventual Drive link. +`make docs` (→ `process_tree.py`) indexes a whole tree; `process_doc.py` still does +one-off single-file extraction. Full doocus docs: [`doocus/README.md`](doocus/README.md). +Cross-search (docs frame + meetings frame, over cached text) is a later phase. 🚧 ## Status legend @@ -61,7 +64,8 @@ docs, textified-only) is a later phase. 🚧 ``` process_meeting.py ✅ the CLI tool — entry point (stays at root) -process_doc.py ✅ doocus CLI — document extraction entry point +process_tree.py ✅ doocus — index a whole local drive tree → index.json +process_doc.py ✅ doocus — one-off single-file extraction Makefile ✅ batch convenience wrapper (make batch / make docs) meetus/ ✅ core package ├─ workflow.py orchestrator (whisper → frames → merge) @@ -75,14 +79,14 @@ meetus/ ✅ core package ├─ hybrid_processor.py (was --use-hybrid) └─ prompts/ vision context prompts doocus/ ✅ document-extraction package (see doocus/README.md) - ├─ workflow.py DocConfig + DocWorkflow (dispatch → extract → write → manifest) + ├─ tree.py build_index — walk drive tree → index.json (+ extract sidecars) ├─ registry.py extension → extractor family dispatch (lazy, graceful) - ├─ output_manager.py run-folder + content.md/meta.json/thumb writer + ├─ workflow.py single-file DocWorkflow (process_doc.py) + ├─ output_manager.py single-file run-folder + content.md/meta.json writer ├─ naming.py YYYYMMDD-NNN naming + sha256 (reuses meetus convention) └─ extractors/ text, tabular(csv), office(docx/pptx/xlsx), pdf, web(html), image, media(mp4) ctrl/ control plane / operational scripts ├─ batch.sh ✅ recursive batch runner (mirrors tree, continues past failures) - ├─ batch_docs.sh ✅ doocus twin of batch.sh (make docs) ├─ transcribe_oneoff.sh 🧰 high-quality re-transcription over an existing run ├─ summarize/ 🚧 last step — local-LLM summarization (WIP, on hold) │ ├─ summarize_simple.py minimal map-and-append; reads every referenced frame diff --git a/Makefile b/Makefile index e25dabb..b6847df 100644 --- a/Makefile +++ b/Makefile @@ -33,12 +33,15 @@ EXT ?= PYTHON ?= export PYTHON -# doocus (document extraction) knobs. DOCS_OUT defaults to docs-output; the input -# tree is mirrored under it. DOC_EXTRA forwards one-off flags to process_doc.py -# (e.g. --render for pdf page thumbnails, --ocr for images/scanned pdfs). +# doocus (tree indexing) knobs. DOCS_OUT defaults to docs-output; the whole input +# tree is replicated into index.json under it. DOC_EXTRA forwards one-off flags to +# process_tree.py (e.g. --render for pdf page thumbnails, --ocr for scanned pdfs). DOCS_OUT ?= docs-output DOC_EXTRA ?= +# Python interpreter for doocus (override PYTHON, else python3 — use `uv run make`). +PY := $(if $(PYTHON),$(PYTHON),python3) + .PHONY: batch dry docs docs-dry help batch: @@ -48,16 +51,16 @@ batch: dry: @ctrl/batch.sh -i "$(IN)" -o "$(OUT)" $(if $(EXT),-e "$(EXT)") -n -# Document extraction (doocus), mirrors the input tree into DOCS_OUT. -# make docs IN="/mnt/win/drive" # extract all docs -# make docs IN="..." DOC_EXTRA="--render --ocr" # thumbnails + OCR -# make docs-dry IN="..." # list what would run +# Document indexing (doocus): replicate the whole IN tree into DOCS_OUT/index.json, +# extracting text sidecars only for docx/pdf/pptx/xlsx. +# uv run make docs IN="/mnt/win/drive" # index the tree +# uv run make docs IN="..." DOC_EXTRA="--ocr" # + OCR scanned pdfs +# make docs-dry IN="..." # counts only, no writes docs: - @ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -- \ - $(DOC_EXTRA) + @$(PY) process_tree.py "$(IN)" --output "$(DOCS_OUT)" $(DOC_EXTRA) docs-dry: - @ctrl/batch_docs.sh -i "$(IN)" -o "$(DOCS_OUT)" $(if $(EXT),-e "$(EXT)") -n + @$(PY) process_tree.py "$(IN)" --output "$(DOCS_OUT)" --dry-run help: @sed -n '3,14p' Makefile diff --git a/ctrl/batch_docs.sh b/ctrl/batch_docs.sh deleted file mode 100755 index 1dc9583..0000000 --- a/ctrl/batch_docs.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -# Batch-extract every document under a directory through process_doc.py, -# mirroring the input folder structure into the output base. The doocus twin of -# ctrl/batch.sh (which does the same for meeting videos via process_meeting.py). -# -# Recursive and space-safe (paths come off a Windows mount, so they often have -# spaces). Each document's run folder is auto-created by process_doc.py inside -# the mirrored output subfolder. -# -# Usage: -# ctrl/batch_docs.sh -i -o [-e "pdf docx ..."] [-n] \ -# [-- ] -# -# Examples: -# # Everything under a downloaded Drive folder, mirrored into docs-output -# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -# -# # Dry run: just show which docs map to which output folders -# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -n -# -# # Only pdfs/docx, render page thumbnails -# ctrl/batch_docs.sh -i "./download" -o docs-output -e "pdf docx" -- --render -# -# A doc at /2026/team a/notes.docx produces -# /2026/team a//... -# i.e. the run folder lands inside the mirrored subtree. -set -euo pipefail - -PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -cd "$PROJECT_DIR" - -# python: honor $PYTHON, else prefer python3 -PYTHON="${PYTHON:-}" -if [ -z "$PYTHON" ]; then - if command -v python3 >/dev/null 2>&1; then PYTHON=python3; else PYTHON=python; fi -fi - -usage() { sed -n '2,26p' "$0"; } - -INPUT="" -OUTPUT="" -EXTS="csv docx html htm jpg jpeg json md txt pdf png pptx xlsx yaml yml mp4" -DRY=false -FORWARD=() - -while [[ $# -gt 0 ]]; do - case "$1" in - -i|--input) INPUT="$2"; shift 2 ;; - -o|--output) OUTPUT="$2"; shift 2 ;; - -e|--ext) EXTS="$2"; shift 2 ;; - -n|--dry-run) DRY=true; shift ;; - -h|--help) usage; exit 0 ;; - --) shift; FORWARD=("$@"); break ;; - *) echo "Unknown arg: $1" >&2; usage; exit 1 ;; - esac -done - -[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; } -[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; } -[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; } - -# Absolute paths so relative-path math is stable regardless of where we cd'd. -INPUT="$(realpath "$INPUT")" -mkdir -p "$OUTPUT" -OUTPUT="$(realpath "$OUTPUT")" - -# Build the find extension filter: \( -iname '*.pdf' -o -iname '*.docx' ... \) -find_expr=() -for ext in $EXTS; do - find_expr+=( -iname "*.${ext}" -o ) -done -unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o - -echo "Input : $INPUT" -echo "Output: $OUTPUT" -echo "Exts : $EXTS" -[ "$DRY" = true ] && echo "(dry run — nothing will be processed)" -echo - -total=0 ok=0 fail=0 -# Process substitution (not a pipe) so counters survive into the summary. -while IFS= read -r -d '' doc; do - total=$((total + 1)) - - rel="${doc#"$INPUT"/}" # path relative to the input root - reldir="$(dirname "$rel")" - if [ "$reldir" = "." ]; then - outdir="$OUTPUT" # doc sat directly in the input root - else - outdir="$OUTPUT/$reldir" # mirror the subfolder structure - fi - - echo "[$total] $rel" - echo " -> $outdir" - - if [ "$DRY" = true ]; then - continue - fi - - mkdir -p "$outdir" - # stdin from /dev/null so any child process can't eat the file list and stall - # the batch (same guard as ctrl/batch.sh). - if "$PYTHON" "$PROJECT_DIR/process_doc.py" "$doc" \ - --output-dir "$outdir" "${FORWARD[@]+"${FORWARD[@]}"}" &2 - fail=$((fail + 1)) - fi - echo -done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z) - -echo "----------------------------------------" -if [ "$DRY" = true ]; then - echo "Found $total document(s)." -else - echo "Done. $total document(s): $ok ok, $fail failed." -fi -[ "$fail" -eq 0 ] diff --git a/doocus/tree.py b/doocus/tree.py new file mode 100644 index 0000000..a910895 --- /dev/null +++ b/doocus/tree.py @@ -0,0 +1,130 @@ +""" +Build a tree index over a local drive-download root. + +The whole source tree is replicated as `index.json` — every file is a node, so +the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes +are classified into three modes: + + extracted docx/pdf/pptx/xlsx → local text extraction written to a mirrored + `.doocus/` sidecar (content.md + meta.json). Search-index + only — never a replacement for the original. + meeting video (mp4/...) → belongs to meetus, not a doc resource. Flagged, + not extracted; the meetus pipeline produces its transcript. + link everything else (md/txt/json/yaml/csv/html/images/other) → the + original is the artifact; nothing is duplicated. + +Each node carries a `url: null` slot for the eventual Drive link (filled later +from a URL source — Drive API / rclone metadata / export listing). +""" +from pathlib import Path +from datetime import datetime +import json +import logging + +from . import registry +from .naming import sha256_file + +logger = logging.getLogger(__name__) + +# Formats we locally extract text from (the "not media, not plain-text" set). +EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"} +# Videos are meetings → meetus territory. +MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"} + + +def classify(ext: str) -> str: + if ext in EXTRACT_EXTS: + return "extracted" + if ext in MEETING_EXTS: + return "meeting" + return "link" + + +def build_index(source_root, output_dir, options=None, dry_run=False) -> dict: + options = options or {} + root = Path(source_root).resolve() + if not root.is_dir(): + raise NotADirectoryError(f"source root not found: {root}") + out = Path(output_dir) + if not dry_run: + out.mkdir(parents=True, exist_ok=True) + + files = [] + counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0} + + for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()): + if p.is_dir() or p.name.startswith("."): + continue + rel = p.relative_to(root).as_posix() + ext = p.suffix.lower().lstrip(".") + mode = classify(ext) + counts[mode] += 1 + stat = p.stat() + node = { + "path": rel, + "name": p.name, + "ext": ext, + "family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"), + "mode": mode, + "bytes": stat.st_size, + "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(), + "url": None, # Drive link — filled later from a URL source + } + + if mode == "extracted" and not dry_run: + _extract_sidecar(p, rel, out, options, node, counts) + + files.append(node) + + index = { + "root": str(root), + "generatedAt": datetime.now().isoformat(), + "counts": {**counts, "total": len(files)}, + "files": files, + } + if not dry_run: + (out / "index.json").write_text( + json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8" + ) + logger.info("Wrote index: %s (%d files)", out / "index.json", len(files)) + return index + + +def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None: + """Extract text for a heavy format into a mirrored `.doocus/` sidecar.""" + sidecar = out / (rel + ".doocus") + sidecar.mkdir(parents=True, exist_ok=True) + result = registry.extract(src, options) + (sidecar / "content.md").write_text(result.content, encoding="utf-8") + + stat = src.stat() + meta = { + "source": { + "name": src.name, + "path": str(src.absolute()), + "sha256": sha256_file(src), + "bytes": stat.st_size, + "modified": node["modified"], + }, + "type": node["ext"], + "family": node["family"], + "extractor": result.extractor, + "extracted_at": datetime.now().isoformat(), + "word_count": len(result.content.split()), + "warnings": result.warnings, + } + for k, v in result.metadata.items(): + meta.setdefault(k, v) + (sidecar / "meta.json").write_text( + json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8" + ) + + node["out"] = sidecar.relative_to(out).as_posix() + node["extractor"] = result.extractor + node["words"] = meta["word_count"] + if result.warnings: + node["warnings"] = result.warnings + counts["warnings"] += 1 + for k in ("title", "author", "pages", "slides", "sheets"): + if k in result.metadata: + node[k] = result.metadata[k] diff --git a/process_tree.py b/process_tree.py new file mode 100644 index 0000000..52a8832 --- /dev/null +++ b/process_tree.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +doocus tree indexer — replicate a local drive tree into docs-output/index.json. + +Walks a downloaded-drive root, preserving the full folder hierarchy. Locally +extracts text for docx/pdf/pptx/xlsx (search-index sidecars, never a replacement +for the original), flags videos as meetus meetings, and links everything else to +the original without duplicating it. Deterministic and offline. + + uv run process_tree.py /mnt/drive # → docs-output/index.json + uv run process_tree.py /mnt/drive --output out/docs + uv run process_tree.py /mnt/drive --dry-run # counts only, no writes +""" +import argparse +import logging +import sys + +from doocus.tree import build_index + + +def main() -> int: + parser = argparse.ArgumentParser(description="Index a local drive tree for doocus.") + parser.add_argument("source", help="Local drive-download root to index") + parser.add_argument("--output", default="docs-output", + help="Output directory for index.json + sidecars (default: docs-output)") + parser.add_argument("--render", action="store_true", + help="Render page/slide images where supported (needs extra deps)") + parser.add_argument("--ocr", action="store_true", + help="OCR scanned PDFs where supported") + parser.add_argument("--dry-run", action="store_true", + help="Walk and classify only; write nothing") + parser.add_argument("--verbose", action="store_true", help="Verbose logging") + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, + format="%(message)s") + + try: + index = build_index( + args.source, + args.output, + options={"render": args.render, "ocr": args.ocr}, + dry_run=args.dry_run, + ) + except NotADirectoryError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + + c = index["counts"] + print(f"\n{c['total']} files: {c['extracted']} extracted, " + f"{c['meeting']} meeting, {c['link']} linked" + + (f", {c['warnings']} with warnings" if c.get("warnings") else "") + + (" (dry run — nothing written)" if args.dry_run else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main())