meetus/doocus/tree.py

"""
Build a tree index over a local drive-download root.

The whole source tree is replicated as `index.json` — every file is a node, so
the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes
are classified into three modes:

  extracted  docx/pdf/pptx/xlsx → local text extraction written to a mirrored
             `<file>.doocus/` sidecar (content.md + meta.json). Search-index
             only — never a replacement for the original.
  meeting    video (mp4/...) → belongs to meetus, not a doc resource. Flagged,
             not extracted; the meetus pipeline produces its transcript.
  link       everything else (md/txt/json/yaml/csv/html/images/other) → the
             original is the artifact; nothing is duplicated.

Each node carries a `url: null` slot for the eventual Drive link (filled later
from a URL source — Drive API / rclone metadata / export listing).
"""
from pathlib import Path
from datetime import datetime
import json
import logging

from . import registry
from .naming import sha256_file

logger = logging.getLogger(__name__)

# Formats we locally extract text from (the "not media, not plain-text" set).
EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"}
# Videos are meetings → meetus territory.
MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"}

# Sidecar suffix meetus writes a meeting's output into. Must match the batch
# invocation `make batch ... OUT_FORMAT="{name}.meetus"` so the index pointer set
# here (deterministically, at index time) resolves once meetus has run.
MEETING_SIDECAR_SUFFIX = ".meetus"


def classify(ext: str) -> str:
    if ext in EXTRACT_EXTS:
        return "extracted"
    if ext in MEETING_EXTS:
        return "meeting"
    return "link"


def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
    options = options or {}
    root = Path(source_root).resolve()
    if not root.is_dir():
        raise NotADirectoryError(f"source root not found: {root}")
    out = Path(output_dir)
    if not dry_run:
        out.mkdir(parents=True, exist_ok=True)

    files = []
    counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0}

    for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()):
        if p.is_dir() or p.name.startswith("."):
            continue
        rel = p.relative_to(root).as_posix()
        ext = p.suffix.lower().lstrip(".")
        mode = classify(ext)
        counts[mode] += 1
        stat = p.stat()
        birth = getattr(stat, "st_birthtime", None)  # not on every filesystem
        node = {
            "path": rel,
            "name": p.name,
            "ext": ext,
            "family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"),
            "mode": mode,
            "bytes": stat.st_size,
            "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
            "created": datetime.fromtimestamp(birth or stat.st_mtime).isoformat(),
            "url": None,  # Drive link — filled later from a URL source
        }

        if mode == "extracted" and not dry_run:
            _extract_sidecar(p, rel, out, options, node, counts)
        elif mode == "meeting":
            # Deterministic pointer to meetus's output sidecar (produced later by
            # `make batch ... OUT_FORMAT="{name}.meetus"`). No relink needed.
            node["out"] = rel + MEETING_SIDECAR_SUFFIX
            node["transcript"] = f"{rel}{MEETING_SIDECAR_SUFFIX}/{p.stem}_enhanced.txt"

        files.append(node)

    index = {
        "root": str(root),
        "generatedAt": datetime.now().isoformat(),
        "counts": {**counts, "total": len(files)},
        "files": files,
    }
    if not dry_run:
        (out / "index.json").write_text(
            json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8"
        )
        logger.info("Wrote index: %s (%d files)", out / "index.json", len(files))
        # Export ONLY the meeting paths (relative), so a meetus run elsewhere reads
        # just what it needs: mount the drive at a new root and feed this list to
        # `ctrl/batch.sh -l`. Relative paths re-root cleanly under the new mount.
        meetings = [f["path"] for f in files if f["mode"] == "meeting"]
        (out / "meetings.txt").write_text(
            "".join(m + "\n" for m in meetings), encoding="utf-8"
        )
        logger.info("Wrote meetings list: %s (%d meetings)", out / "meetings.txt", len(meetings))
    return index


def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None:
    """Extract text for a heavy format into a mirrored `<file>.doocus/` sidecar."""
    sidecar = out / (rel + ".doocus")
    sidecar.mkdir(parents=True, exist_ok=True)
    result = registry.extract(src, options)
    (sidecar / "content.md").write_text(result.content, encoding="utf-8")

    stat = src.stat()
    meta = {
        "source": {
            "name": src.name,
            "path": str(src.absolute()),
            "sha256": sha256_file(src),
            "bytes": stat.st_size,
            "modified": node["modified"],
        },
        "type": node["ext"],
        "family": node["family"],
        "extractor": result.extractor,
        "extracted_at": datetime.now().isoformat(),
        "word_count": len(result.content.split()),
        "warnings": result.warnings,
    }
    for k, v in result.metadata.items():
        meta.setdefault(k, v)
    (sidecar / "meta.json").write_text(
        json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8"
    )

    node["out"] = sidecar.relative_to(out).as_posix()
    node["extractor"] = result.extractor
    node["words"] = meta["word_count"]
    if result.warnings:
        node["warnings"] = result.warnings
        counts["warnings"] += 1
    for k in ("title", "author", "pages", "slides", "sheets"):
        if k in result.metadata:
            node[k] = result.metadata[k]