131 lines
4.5 KiB
Python
131 lines
4.5 KiB
Python
"""
|
|
Build a tree index over a local drive-download root.
|
|
|
|
The whole source tree is replicated as `index.json` — every file is a node, so
|
|
the folder hierarchy is preserved (no flattening) and nothing is dropped. Nodes
|
|
are classified into three modes:
|
|
|
|
extracted docx/pdf/pptx/xlsx → local text extraction written to a mirrored
|
|
`<file>.doocus/` sidecar (content.md + meta.json). Search-index
|
|
only — never a replacement for the original.
|
|
meeting video (mp4/...) → belongs to meetus, not a doc resource. Flagged,
|
|
not extracted; the meetus pipeline produces its transcript.
|
|
link everything else (md/txt/json/yaml/csv/html/images/other) → the
|
|
original is the artifact; nothing is duplicated.
|
|
|
|
Each node carries a `url: null` slot for the eventual Drive link (filled later
|
|
from a URL source — Drive API / rclone metadata / export listing).
|
|
"""
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
import json
|
|
import logging
|
|
|
|
from . import registry
|
|
from .naming import sha256_file
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Formats we locally extract text from (the "not media, not plain-text" set).
|
|
EXTRACT_EXTS = {"docx", "pdf", "pptx", "xlsx"}
|
|
# Videos are meetings → meetus territory.
|
|
MEETING_EXTS = {"mp4", "mkv", "mov", "m4v", "webm", "avi", "wmv"}
|
|
|
|
|
|
def classify(ext: str) -> str:
|
|
if ext in EXTRACT_EXTS:
|
|
return "extracted"
|
|
if ext in MEETING_EXTS:
|
|
return "meeting"
|
|
return "link"
|
|
|
|
|
|
def build_index(source_root, output_dir, options=None, dry_run=False) -> dict:
|
|
options = options or {}
|
|
root = Path(source_root).resolve()
|
|
if not root.is_dir():
|
|
raise NotADirectoryError(f"source root not found: {root}")
|
|
out = Path(output_dir)
|
|
if not dry_run:
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
|
|
files = []
|
|
counts = {"extracted": 0, "meeting": 0, "link": 0, "warnings": 0}
|
|
|
|
for p in sorted(root.rglob("*"), key=lambda x: str(x).lower()):
|
|
if p.is_dir() or p.name.startswith("."):
|
|
continue
|
|
rel = p.relative_to(root).as_posix()
|
|
ext = p.suffix.lower().lstrip(".")
|
|
mode = classify(ext)
|
|
counts[mode] += 1
|
|
stat = p.stat()
|
|
node = {
|
|
"path": rel,
|
|
"name": p.name,
|
|
"ext": ext,
|
|
"family": registry.family_for(p) or ("meeting" if mode == "meeting" else "other"),
|
|
"mode": mode,
|
|
"bytes": stat.st_size,
|
|
"modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
|
|
"url": None, # Drive link — filled later from a URL source
|
|
}
|
|
|
|
if mode == "extracted" and not dry_run:
|
|
_extract_sidecar(p, rel, out, options, node, counts)
|
|
|
|
files.append(node)
|
|
|
|
index = {
|
|
"root": str(root),
|
|
"generatedAt": datetime.now().isoformat(),
|
|
"counts": {**counts, "total": len(files)},
|
|
"files": files,
|
|
}
|
|
if not dry_run:
|
|
(out / "index.json").write_text(
|
|
json.dumps(index, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
logger.info("Wrote index: %s (%d files)", out / "index.json", len(files))
|
|
return index
|
|
|
|
|
|
def _extract_sidecar(src: Path, rel: str, out: Path, options: dict, node: dict, counts: dict) -> None:
|
|
"""Extract text for a heavy format into a mirrored `<file>.doocus/` sidecar."""
|
|
sidecar = out / (rel + ".doocus")
|
|
sidecar.mkdir(parents=True, exist_ok=True)
|
|
result = registry.extract(src, options)
|
|
(sidecar / "content.md").write_text(result.content, encoding="utf-8")
|
|
|
|
stat = src.stat()
|
|
meta = {
|
|
"source": {
|
|
"name": src.name,
|
|
"path": str(src.absolute()),
|
|
"sha256": sha256_file(src),
|
|
"bytes": stat.st_size,
|
|
"modified": node["modified"],
|
|
},
|
|
"type": node["ext"],
|
|
"family": node["family"],
|
|
"extractor": result.extractor,
|
|
"extracted_at": datetime.now().isoformat(),
|
|
"word_count": len(result.content.split()),
|
|
"warnings": result.warnings,
|
|
}
|
|
for k, v in result.metadata.items():
|
|
meta.setdefault(k, v)
|
|
(sidecar / "meta.json").write_text(
|
|
json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
|
|
node["out"] = sidecar.relative_to(out).as_posix()
|
|
node["extractor"] = result.extractor
|
|
node["words"] = meta["word_count"]
|
|
if result.warnings:
|
|
node["warnings"] = result.warnings
|
|
counts["warnings"] += 1
|
|
for k in ("title", "author", "pages", "slides", "sheets"):
|
|
if k in result.metadata:
|
|
node[k] = result.metadata[k]
|