batch docs update

doocus/tree.py

@@ -0,0 +1,130 @@
+"""
+Build a tree index over a local drive-download root.
+
+The whole source tree is replicated as `index.json` — every file is a node, so
+the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes
+are classified into three modes:
+
+  extracted  docx/pdf/pptx/xlsx → local text extraction written to a mirrored
+             `<file>.doocus/` sidecar (content.md + meta.json). Search-index
+             only — never a replacement for the original.
+  meeting    video (mp4/...) → belongs to meetus, not a doc resource. Flagged,
+             not extracted; the meetus pipeline produces its transcript.
+  link       everything else (md/txt/json/yaml/csv/html/images/other) → the
+             original is the artifact; nothing is duplicated.
+
+Each node carries a `url: null` slot for the eventual Drive link (filled later
+from a URL source — Drive API / rclone metadata / export listing).
+"""
+from pathlib import Path
+from datetime import datetime
+import json
+import logging
+
+from . import registry
+from .naming import sha256_file
+
+logger = logging.getLogger(__name__)
+
+# Formats we locally extract text from (the "not media, not plain-text" set).
+EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"}
+# Videos are meetings → meetus territory.
+MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"}
+
+
+def classify(ext: str) -> str:
+    if ext in EXTRACT_EXTS:
+        return "extracted"
+    if ext in MEETING_EXTS:
+        return "meeting"
+    return "link"
+
+
+def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
+    options = options or {}
+    root = Path(source_root).resolve()
+    if not root.is_dir():
+        raise NotADirectoryError(f"source root not found: {root}")
+    out = Path(output_dir)
+    if not dry_run:
+        out.mkdir(parents=True, exist_ok=True)
+
+    files = []
+    counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0}
+
+    for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()):
+        if p.is_dir() or p.name.startswith("."):
+            continue
+        rel = p.relative_to(root).as_posix()
+        ext = p.suffix.lower().lstrip(".")
+        mode = classify(ext)
+        counts[mode] += 1
+        stat = p.stat()
+        node = {
+            "path": rel,
+            "name": p.name,
+            "ext": ext,
+            "family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"),
+            "mode": mode,
+            "bytes": stat.st_size,
+            "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
+            "url": None,  # Drive link — filled later from a URL source
+        }
+
+        if mode == "extracted" and not dry_run:
+            _extract_sidecar(p, rel, out, options, node, counts)
+
+        files.append(node)
+
+    index = {
+        "root": str(root),
+        "generatedAt": datetime.now().isoformat(),
+        "counts": {**counts, "total": len(files)},
+        "files": files,
+    }
+    if not dry_run:
+        (out / "index.json").write_text(
+            json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+        logger.info("Wrote index: %s (%d files)", out / "index.json", len(files))
+    return index
+
+
+def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None:
+    """Extract text for a heavy format into a mirrored `<file>.doocus/` sidecar."""
+    sidecar = out / (rel + ".doocus")
+    sidecar.mkdir(parents=True, exist_ok=True)
+    result = registry.extract(src, options)
+    (sidecar / "content.md").write_text(result.content, encoding="utf-8")
+
+    stat = src.stat()
+    meta = {
+        "source": {
+            "name": src.name,
+            "path": str(src.absolute()),
+            "sha256": sha256_file(src),
+            "bytes": stat.st_size,
+            "modified": node["modified"],
+        },
+        "type": node["ext"],
+        "family": node["family"],
+        "extractor": result.extractor,
+        "extracted_at": datetime.now().isoformat(),
+        "word_count": len(result.content.split()),
+        "warnings": result.warnings,
+    }
+    for k, v in result.metadata.items():
+        meta.setdefault(k, v)
+    (sidecar / "meta.json").write_text(
+        json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+
+    node["out"] = sidecar.relative_to(out).as_posix()
+    node["extractor"] = result.extractor
+    node["words"] = meta["word_count"]
+    if result.warnings:
+        node["warnings"] = result.warnings
+        counts["warnings"] += 1
+    for k in ("title", "author", "pages", "slides", "sheets"):
+        if k in result.metadata:
+            node[k] = result.metadata[k]