""" Build a tree index over a local drive-download root. The whole source tree is replicated as `index.json` — every file is a node, so the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes are classified into three modes: extracted docx/pdf/pptx/xlsx → local text extraction written to a mirrored `.doocus/` sidecar (content.md + meta.json). Search-index only — never a replacement for the original. meeting video (mp4/...) → belongs to meetus, not a doc resource. Flagged, not extracted; the meetus pipeline produces its transcript. link everything else (md/txt/json/yaml/csv/html/images/other) → the original is the artifact; nothing is duplicated. Each node carries a `url: null` slot for the eventual Drive link (filled later from a URL source — Drive API / rclone metadata / export listing). """ from pathlib import Path from datetime import datetime import json import logging from . import registry from .naming import sha256_file logger = logging.getLogger(__name__) # Formats we locally extract text from (the "not media, not plain-text" set). EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"} # Videos are meetings → meetus territory. MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"} def classify(ext: str) -> str: if ext in EXTRACT_EXTS: return "extracted" if ext in MEETING_EXTS: return "meeting" return "link" def build_index(source_root, output_dir, options=None, dry_run=False) -> dict: options = options or {} root = Path(source_root).resolve() if not root.is_dir(): raise NotADirectoryError(f"source root not found: {root}") out = Path(output_dir) if not dry_run: out.mkdir(parents=True, exist_ok=True) files = [] counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0} for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()): if p.is_dir() or p.name.startswith("."): continue rel = p.relative_to(root).as_posix() ext = p.suffix.lower().lstrip(".") mode = classify(ext) counts[mode] += 1 stat = p.stat() node = { "path": rel, "name": p.name, "ext": ext, "family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"), "mode": mode, "bytes": stat.st_size, "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(), "url": None, # Drive link — filled later from a URL source } if mode == "extracted" and not dry_run: _extract_sidecar(p, rel, out, options, node, counts) files.append(node) index = { "root": str(root), "generatedAt": datetime.now().isoformat(), "counts": {**counts, "total": len(files)}, "files": files, } if not dry_run: (out / "index.json").write_text( json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8" ) logger.info("Wrote index: %s (%d files)", out / "index.json", len(files)) return index def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None: """Extract text for a heavy format into a mirrored `.doocus/` sidecar.""" sidecar = out / (rel + ".doocus") sidecar.mkdir(parents=True, exist_ok=True) result = registry.extract(src, options) (sidecar / "content.md").write_text(result.content, encoding="utf-8") stat = src.stat() meta = { "source": { "name": src.name, "path": str(src.absolute()), "sha256": sha256_file(src), "bytes": stat.st_size, "modified": node["modified"], }, "type": node["ext"], "family": node["family"], "extractor": result.extractor, "extracted_at": datetime.now().isoformat(), "word_count": len(result.content.split()), "warnings": result.warnings, } for k, v in result.metadata.items(): meta.setdefault(k, v) (sidecar / "meta.json").write_text( json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8" ) node["out"] = sidecar.relative_to(out).as_posix() node["extractor"] = result.extractor node["words"] = meta["word_count"] if result.warnings: node["warnings"] = result.warnings counts["warnings"] += 1 for k in ("title", "author", "pages", "slides", "sheets"): if k in result.metadata: node[k] = result.metadata[k]