doocus first ver

doocus/README.md

@@ -0,0 +1,76 @@
+# doocus — local document extraction
+
+The document twin of meetus. Turns locally-downloaded files (Drive exports,
+PDFs, images, ...) into a shareable **textified** product (`content.md`) plus
+structured `meta.json`, mirroring the meetus `output/<run>/` contract.
+
+Extraction is **deterministic and offline** — no cloud AI touches the raw source.
+The textified output is what feeds permitted services (Gemini web, NotebookLM)
+and the local search index; the raw source stays on disk.
+
+## Quick start
+
+```bash
+# one document
+python process_doc.py notes.docx                 # → docs-output/<YYYYMMDD-NNN-notes>/
+python process_doc.py report.pdf --render        # also render page 1 → thumb.jpg
+python process_doc.py scan.png --ocr             # OCR the image into content.md
+
+# a whole downloaded folder (mirrors the input tree)
+make docs IN="/mnt/win/drive"                    # → docs-output/...
+make docs IN="..." DOC_EXTRA="--render --ocr"
+make docs-dry IN="..."                           # list what would run
+```
+
+## Output contract
+
+```
+docs-output/<YYYYMMDD-NNN-stem>/
+  content.md      textified document — the shareable product
+  meta.json       source info, sha256, dates, author, title, pages/sheets/slides, word_count, warnings
+  thumb.jpg       optional single preview (image / rendered page / video frame)
+  manifest.json   run config + outputs inventory + source pointer (path, not a copy)
+```
+
+## Supported types
+
+| Family  | Extensions              | Notes |
+|---------|-------------------------|-------|
+| text    | md, txt, yaml/yml, json | passthrough; json/yaml add a structure summary |
+| tabular | csv                     | markdown table + row/col metadata (stdlib) |
+| office  | docx, pptx, xlsx        | text + core properties (author, dates, title) |
+| pdf     | pdf                     | text per page + info dict; `--render` for page-1 thumb |
+| web     | html/htm                | visible text + title/meta tags |
+| image   | jpg/jpeg, png           | EXIF (incl. capture date) + thumbnail; `--ocr` for text |
+| media   | mp4                     | ffprobe metadata + thumbnail; **delegates transcription to meetus** |
+
+Each extractor is isolated: a missing optional dependency or a corrupt file is
+recorded as a warning in `meta.json`, never aborting a batch (same resilience as
+`ctrl/batch.sh`).
+
+## Dependencies
+
+Core text/tabular use only the stdlib. Everything else is a uv group in the
+repo's `pyproject.toml`:
+
+```bash
+uv sync --group doocus                 # Pillow, PyYAML, python-docx, python-pptx,
+                                       # openpyxl, pypdf, beautifulsoup4, lxml
+uv sync --group doocus --group ocr     # + pytesseract (needs system `tesseract`)
+```
+
+`mp4` uses system `ffmpeg`/`ffprobe` (shared with meetus). PDF page rendering
+(`--render`) additionally needs the `pdf-render` group (`pdf2image`) + system
+`poppler`.
+
+## Browser UI
+
+`ui/doocus-app/` (sibling of `ui/meetus-app/`, shares `ui/framework/`) browses the
+extracted docs with per-type viewers, a metadata panel, and a package builder
+that zips the **original file + content.md + meta.json** per doc for a chosen
+target (Gemini web / NotebookLM):
+
+```bash
+cd ui/doocus-app && npm install
+DOOCUS_OUTPUT=/path/to/docs-output npm run dev
+```

doocus/__init__.py

@@ -0,0 +1,10 @@
+"""
+doocus — local, offline document extraction, alongside meetus.
+
+Turns locally-downloaded documents (Drive exports, PDFs, images, ...) into a
+shareable *textified* product (`content.md`) plus structured `meta.json`,
+mirroring the meetus `output/<run>/` contract. The extraction step is
+deterministic and offline — no cloud AI ever touches the raw source. The
+textified output is what feeds permitted services (Gemini web, NotebookLM) and
+the local search index; the raw source stays on disk.
+"""

doocus/extractors/__init__.py

@@ -0,0 +1 @@
+"""Per-family document extractors. See base.py for the contract."""

doocus/extractors/base.py

@@ -0,0 +1,28 @@
+"""
+Shared extractor contract.
+
+Every extractor exposes `extract(source: Path, options: dict) -> ExtractionResult`
+and is isolated: a failure on one file is caught by the workflow and recorded,
+never aborting a batch (same resilience as ctrl/batch.sh).
+"""
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+
+
+@dataclass
+class ExtractionResult:
+    """What an extractor returns.
+
+    content   textified document as markdown/plain text — the shareable product.
+    metadata  type-specific fields (title, author, created/modified, pages,
+              sheet/slide names, structure summary, ...). Merged into meta.json.
+    thumb     optional JPEG bytes for thumb.jpg (page 1 / slide 1 / the image).
+    warnings  non-fatal issues to surface in meta.json.
+    extractor id string, e.g. "office/python-docx@1".
+    """
+
+    content: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    thumb: Optional[bytes] = None
+    warnings: List[str] = field(default_factory=list)
+    extractor: str = ""

doocus/extractors/image.py

@@ -0,0 +1,79 @@
+"""
+Image-family extractor: jpg/jpeg, png.
+
+Reads EXIF (including capture date) into metadata, writes a downscaled JPEG
+thumbnail, and — when options['ocr'] is set and pytesseract is available — OCRs
+the image into content. Without OCR, content is a short placeholder note.
+"""
+import io
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "image/pillow@1"
+THUMB_MAX = 640  # longest edge, px
+
+
+def _exif(img) -> dict:
+    try:
+        from PIL.ExifTags import TAGS
+    except ImportError:
+        return {}
+    raw = getattr(img, "_getexif", lambda: None)()
+    if not raw:
+        return {}
+    out = {}
+    for tag_id, value in raw.items():
+        name = TAGS.get(tag_id, str(tag_id))
+        if name in ("DateTime", "DateTimeOriginal", "Make", "Model", "Orientation"):
+            out[name] = str(value)
+    return out
+
+
+def _thumbnail(img) -> bytes:
+    from PIL import Image  # noqa: F401 (ensure Pillow present)
+    im = img.copy()
+    im.thumbnail((THUMB_MAX, THUMB_MAX))
+    if im.mode not in ("RGB", "L"):
+        im = im.convert("RGB")
+    buf = io.BytesIO()
+    im.save(buf, format="JPEG", quality=80)
+    return buf.getvalue()
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from PIL import Image  # lazy
+
+    warnings = []
+    metadata = {}
+    content = ""
+    thumb = None
+
+    with Image.open(source) as img:
+        metadata["width"], metadata["height"] = img.size
+        metadata["mode"] = img.mode
+        exif = _exif(img)
+        if exif:
+            metadata["exif"] = exif
+            if "DateTimeOriginal" in exif or "DateTime" in exif:
+                metadata["captured"] = exif.get("DateTimeOriginal", exif.get("DateTime"))
+        try:
+            thumb = _thumbnail(img)
+        except Exception as e:
+            warnings.append(f"thumbnail failed: {e}")
+
+        if options.get("ocr"):
+            try:
+                import pytesseract
+                content = pytesseract.image_to_string(img).strip()
+                metadata["ocr"] = True
+            except ImportError:
+                warnings.append("--ocr requested but pytesseract not installed")
+            except Exception as e:
+                warnings.append(f"ocr failed: {e}")
+
+    if not content:
+        content = f"![{source.name}](thumb.jpg)\n\n_Image {metadata.get('width')}×{metadata.get('height')}. Run with --ocr to extract any text._\n"
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/media.py

@@ -0,0 +1,79 @@
+"""
+Media-family extractor: mp4.
+
+doocus does not transcribe here. It probes container metadata via ffprobe, grabs
+a thumbnail via ffmpeg, and records a pointer delegating full transcription to
+the meetus pipeline (process_meeting.py). Meeting videos are thus discoverable
+and previewable in doocus while their transcript lives in meetus.
+"""
+import json
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "media/ffprobe@1"
+
+
+def _ffprobe(source: Path) -> dict:
+    if not shutil.which("ffprobe"):
+        return {}
+    cmd = ["ffprobe", "-v", "quiet", "-print_format", "json",
+           "-show_format", "-show_streams", str(source)]
+    try:
+        out = subprocess.run(cmd, capture_output=True, text=True, check=True).stdout
+        return json.loads(out)
+    except (subprocess.CalledProcessError, json.JSONDecodeError):
+        return {}
+
+
+def _thumbnail(source: Path, at_seconds: float = 1.0) -> bytes:
+    if not shutil.which("ffmpeg"):
+        return b""
+    with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
+        tmp_path = Path(tmp.name)
+    try:
+        cmd = ["ffmpeg", "-v", "quiet", "-y", "-ss", str(at_seconds),
+               "-i", str(source), "-frames:v", "1", "-vf", "scale=640:-1",
+               str(tmp_path)]
+        subprocess.run(cmd, check=True)
+        return tmp_path.read_bytes() if tmp_path.exists() else b""
+    except subprocess.CalledProcessError:
+        return b""
+    finally:
+        tmp_path.unlink(missing_ok=True)
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    warnings = []
+    probe = _ffprobe(source)
+    metadata = {"delegate": "meetus", "delegate_hint": "run process_meeting.py for transcript"}
+
+    fmt = probe.get("format", {})
+    if fmt:
+        if "duration" in fmt:
+            metadata["duration_seconds"] = float(fmt["duration"])
+        tags = fmt.get("tags", {})
+        if "creation_time" in tags:
+            metadata["created"] = tags["creation_time"]
+    video_streams = [s for s in probe.get("streams", []) if s.get("codec_type") == "video"]
+    if video_streams:
+        v = video_streams[0]
+        metadata["width"], metadata["height"] = v.get("width"), v.get("height")
+    if not probe:
+        warnings.append("ffprobe unavailable or failed; metadata limited")
+
+    thumb = _thumbnail(source) or None
+    if thumb is None:
+        warnings.append("thumbnail not generated (ffmpeg unavailable or failed)")
+
+    dur = metadata.get("duration_seconds")
+    content = (f"_Video {source.name}"
+               + (f", {dur:.0f}s" if dur else "")
+               + ". Transcript is produced by the meetus pipeline "
+                 "(`process_meeting.py`), not doocus._\n")
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/office.py

@@ -0,0 +1,114 @@
+"""
+Office-family extractor: docx, pptx, xlsx.
+
+Each format is handled with its own lazy import so a missing dependency degrades
+only that format. These formats carry core properties (author, created/modified
+dates, title) which go straight into metadata.
+"""
+from pathlib import Path
+
+from .base import ExtractionResult
+
+MAX_SHEET_ROWS = 200
+
+
+def _core_props(props) -> dict:
+    """Common OOXML core properties → metadata (only non-empty ones).
+
+    python-docx/pptx expose `.author`/`.last_modified_by`; openpyxl uses
+    `.creator`/`.lastModifiedBy`. Each target checks both spellings.
+    """
+    # dst -> candidate source attribute names (first non-empty wins)
+    fields = {
+        "title": ("title",),
+        "author": ("author", "creator"),
+        "created": ("created",),
+        "modified_doc": ("modified",),
+        "last_modified_by": ("last_modified_by", "lastModifiedBy"),
+    }
+    out = {}
+    for dst, sources in fields.items():
+        for src in sources:
+            val = getattr(props, src, None)
+            if val:
+                out[dst] = str(val)
+                break
+    return out
+
+
+def _extract_docx(source: Path) -> ExtractionResult:
+    import docx  # python-docx
+    doc = docx.Document(str(source))
+    paras = [p.text for p in doc.paragraphs]
+    content = "\n\n".join(p for p in paras if p.strip()) + "\n"
+    metadata = _core_props(doc.core_properties)
+    metadata["paragraphs"] = len([p for p in paras if p.strip()])
+    return ExtractionResult(content=content, metadata=metadata, extractor="office/python-docx@1")
+
+
+def _extract_pptx(source: Path) -> ExtractionResult:
+    from pptx import Presentation
+    prs = Presentation(str(source))
+    blocks = []
+    for i, slide in enumerate(prs.slides, 1):
+        texts = []
+        for shape in slide.shapes:
+            if shape.has_text_frame:
+                for para in shape.text_frame.paragraphs:
+                    line = "".join(run.text for run in para.runs)
+                    if line.strip():
+                        texts.append(line)
+        blocks.append(f"## Slide {i}\n\n" + "\n".join(texts))
+    content = "\n\n".join(blocks) + "\n"
+    metadata = _core_props(prs.core_properties)
+    metadata["slides"] = len(prs.slides)
+    return ExtractionResult(content=content, metadata=metadata, extractor="office/python-pptx@1")
+
+
+def _extract_xlsx(source: Path) -> ExtractionResult:
+    from openpyxl import load_workbook
+    wb = load_workbook(str(source), read_only=True, data_only=True)
+    warnings = []
+    blocks = []
+    for ws in wb.worksheets:
+        rows = []
+        for r, row in enumerate(ws.iter_rows(values_only=True)):
+            if r > MAX_SHEET_ROWS:
+                warnings.append(f"sheet '{ws.title}' truncated to {MAX_SHEET_ROWS} rows")
+                break
+            rows.append(["" if c is None else str(c) for c in row])
+        blocks.append(f"## {ws.title}\n\n" + _rows_to_md(rows))
+    wb.close()
+    content = "\n\n".join(blocks) + "\n"
+    metadata = _core_props(wb.properties)
+    metadata["sheets"] = [ws.title for ws in wb.worksheets]
+    return ExtractionResult(content=content, metadata=metadata,
+                            warnings=warnings, extractor="office/openpyxl@1")
+
+
+def _rows_to_md(rows: list) -> str:
+    if not rows:
+        return "_(empty)_\n"
+    width = max(len(r) for r in rows)
+    rows = [r + [""] * (width - len(r)) for r in rows]
+
+    def esc(c):
+        return str(c).replace("|", "\\|").replace("\n", " ")
+
+    lines = ["| " + " | ".join(esc(c) for c in rows[0]) + " |",
+             "| " + " | ".join(["---"] * width) + " |"]
+    for r in rows[1:]:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    ext = source.suffix.lower().lstrip(".")
+    if ext == "docx":
+        return _extract_docx(source)
+    if ext == "pptx":
+        return _extract_pptx(source)
+    if ext == "xlsx":
+        return _extract_xlsx(source)
+    return ExtractionResult(extractor="office/unhandled",
+                            warnings=[f"office extractor got unexpected '.{ext}'"])

doocus/extractors/pdf.py

@@ -0,0 +1,73 @@
+"""
+PDF-family extractor.
+
+Text per page via pypdf, plus the document info dict (title/author/dates). When
+options['render'] is set and pdf2image (+ system poppler) is available, page 1 is
+rendered to thumb.jpg.
+"""
+import io
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "pdf/pypdf@1"
+
+
+def _info_metadata(reader) -> dict:
+    out = {}
+    info = getattr(reader, "metadata", None)
+    if not info:
+        return out
+    for src, dst in (("title", "title"), ("author", "author"),
+                     ("creation_date", "created"), ("modification_date", "modified_doc")):
+        try:
+            val = getattr(info, src, None)
+        except Exception:
+            val = None
+        if val:
+            out[dst] = str(val)
+    return out
+
+
+def _render_first_page(source: Path) -> bytes:
+    from pdf2image import convert_from_path  # needs system poppler
+    images = convert_from_path(str(source), first_page=1, last_page=1, dpi=100)
+    if not images:
+        return b""
+    im = images[0]
+    im.thumbnail((640, 640))
+    buf = io.BytesIO()
+    im.convert("RGB").save(buf, format="JPEG", quality=80)
+    return buf.getvalue()
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from pypdf import PdfReader  # lazy
+
+    warnings = []
+    reader = PdfReader(str(source))
+    pages = reader.pages
+    blocks = []
+    for i, page in enumerate(pages, 1):
+        try:
+            text = page.extract_text() or ""
+        except Exception as e:
+            text = ""
+            warnings.append(f"page {i} text extraction failed: {e}")
+        blocks.append(f"## Page {i}\n\n{text.strip()}")
+    content = "\n\n".join(blocks) + "\n"
+
+    metadata = _info_metadata(reader)
+    metadata["pages"] = len(pages)
+
+    thumb = None
+    if options.get("render"):
+        try:
+            thumb = _render_first_page(source) or None
+        except ImportError:
+            warnings.append("--render requested but pdf2image/poppler not available")
+        except Exception as e:
+            warnings.append(f"page render failed: {e}")
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/tabular.py

@@ -0,0 +1,57 @@
+"""
+Tabular-family extractor: csv.
+
+Renders to a markdown table (capped preview) with row/column metadata. Uses only
+the stdlib so it always works.
+"""
+import csv
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "tabular/csv@1"
+MAX_PREVIEW_ROWS = 200
+
+
+def _md_table(rows: list) -> str:
+    if not rows:
+        return ""
+    width = max(len(r) for r in rows)
+    rows = [r + [""] * (width - len(r)) for r in rows]
+    header, body = rows[0], rows[1:]
+
+    def esc(c: str) -> str:
+        return str(c).replace("|", "\\|").replace("\n", " ")
+
+    lines = ["| " + " | ".join(esc(c) for c in header) + " |",
+             "| " + " | ".join(["---"] * width) + " |"]
+    for r in body:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    warnings = []
+    with open(source, newline="", encoding="utf-8", errors="replace") as f:
+        sample = f.read(8192)
+        f.seek(0)
+        try:
+            dialect = csv.Sniffer().sniff(sample) if sample.strip() else csv.excel
+        except csv.Error:
+            dialect = csv.excel
+        rows = list(csv.reader(f, dialect))
+
+    total = len(rows)
+    cols = max((len(r) for r in rows), default=0)
+    preview = rows[: MAX_PREVIEW_ROWS + 1]  # +1 for header
+    if total > MAX_PREVIEW_ROWS + 1:
+        warnings.append(f"table has {total} rows; previewed first {MAX_PREVIEW_ROWS}")
+
+    content = _md_table(preview)
+    metadata = {
+        "rows": max(total - 1, 0),
+        "columns": cols,
+        "header": rows[0] if rows else [],
+    }
+    return ExtractionResult(content=content, metadata=metadata,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/text.py

@@ -0,0 +1,92 @@
+"""
+Text-family extractor: md, txt, yaml/yml, json.
+
+Plain text (md/txt) passes through as-is ("the texts as they are"). Structured
+text (json/yaml) is kept verbatim inside a fenced block so content.md renders,
+with a shallow structure summary added to metadata for search/browse.
+"""
+import json
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "text@1"
+
+
+def _structure_summary(data) -> dict:
+    """Shallow shape of parsed json/yaml — top-level keys or item count."""
+    if isinstance(data, dict):
+        keys = list(data.keys())
+        return {"root": "object", "keys": keys[:50], "key_count": len(keys)}
+    if isinstance(data, list):
+        return {"root": "array", "length": len(data)}
+    return {"root": type(data).__name__}
+
+
+def _md_metadata(raw: str) -> dict:
+    """Headings + trivial front-matter for markdown."""
+    meta = {}
+    lines = raw.splitlines()
+
+    # YAML-ish front matter delimited by --- ... ---
+    if lines and lines[0].strip() == "---":
+        for i in range(1, len(lines)):
+            if lines[i].strip() == "---":
+                front = lines[1:i]
+                fm = {}
+                for ln in front:
+                    if ":" in ln:
+                        k, v = ln.split(":", 1)
+                        fm[k.strip()] = v.strip()
+                if fm:
+                    meta["frontmatter"] = fm
+                break
+
+    headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")]
+    if headings:
+        meta["headings"] = headings[:50]
+        if not meta.get("title"):
+            first = headings[0].lstrip("#").strip()
+            if first:
+                meta["title"] = first
+    return meta
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    ext = source.suffix.lower().lstrip(".")
+    raw = source.read_text(encoding="utf-8", errors="replace")
+    metadata: dict = {}
+    warnings: list = []
+    content = raw
+
+    if ext == "json":
+        try:
+            data = json.loads(raw)
+            metadata["structure"] = _structure_summary(data)
+            pretty = json.dumps(data, indent=2, ensure_ascii=False)
+            content = f"```json\n{pretty}\n```\n"
+        except json.JSONDecodeError as e:
+            warnings.append(f"invalid json, kept raw: {e}")
+            content = f"```\n{raw}\n```\n"
+    elif ext in ("yaml", "yml"):
+        try:
+            import yaml  # PyYAML
+            data = yaml.safe_load(raw)
+            metadata["structure"] = _structure_summary(data)
+        except ImportError:
+            warnings.append("PyYAML not installed; kept raw without structure summary")
+        except Exception as e:  # malformed yaml
+            warnings.append(f"invalid yaml, kept raw: {e}")
+        content = f"```yaml\n{raw}\n```\n"
+    elif ext in ("md", "markdown"):
+        metadata.update(_md_metadata(raw))
+        content = raw
+    else:  # txt and anything else routed here
+        content = raw
+
+    return ExtractionResult(
+        content=content,
+        metadata=metadata,
+        warnings=warnings,
+        extractor=EXTRACTOR_ID,
+    )

doocus/extractors/web.py

@@ -0,0 +1,40 @@
+"""
+Web-family extractor: html/htm.
+
+Strips scripts/styles, keeps the visible text, and pulls the <title> plus common
+<meta> tags into metadata. Uses BeautifulSoup (bs4 + lxml).
+"""
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "web/bs4@1"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from bs4 import BeautifulSoup  # lazy: degrade to "unavailable" if missing
+
+    html = source.read_text(encoding="utf-8", errors="replace")
+    soup = BeautifulSoup(html, "lxml")
+
+    metadata = {}
+    if soup.title and soup.title.string:
+        metadata["title"] = soup.title.string.strip()
+
+    metas = {}
+    for tag in soup.find_all("meta"):
+        key = tag.get("name") or tag.get("property")
+        val = tag.get("content")
+        if key and val:
+            metas[key.strip()] = val.strip()
+    if metas:
+        metadata["meta_tags"] = {k: metas[k] for k in list(metas)[:30]}
+
+    for bad in soup(["script", "style", "noscript"]):
+        bad.decompose()
+
+    text = soup.get_text(separator="\n")
+    lines = [ln.strip() for ln in text.splitlines()]
+    content = "\n".join(ln for ln in lines if ln) + "\n"
+
+    return ExtractionResult(content=content, metadata=metadata, extractor=EXTRACTOR_ID)

doocus/naming.py

@@ -0,0 +1,52 @@
+"""
+Run-folder naming + content hashing for doocus.
+
+Reuses meetus's `YYYYMMDD-NNN-<stem>` run-folder convention (see
+meetus/output_manager.py) so both trees look the same. Kept here to keep doocus
+self-contained; meetus could adopt this helper later.
+"""
+from pathlib import Path
+from datetime import datetime
+import hashlib
+
+
+def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
+    """Streaming sha256 of a file — never loads the whole file into memory."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(chunk), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def resolve_run_dir(base_output_dir: Path, stem: str, use_cache: bool = True) -> Path:
+    """Return (creating if needed) the run folder for `stem` under `base_output_dir`.
+
+    Mirrors meetus/output_manager.py:37-84 — reuse the most recent
+    `*-<stem>` folder when caching, else create `YYYYMMDD-NNN-<stem>` with the
+    next run number for today.
+    """
+    base = Path(base_output_dir)
+
+    if use_cache and base.exists():
+        existing = sorted(
+            [d for d in base.iterdir() if d.is_dir() and d.name.endswith(f"-{stem}")],
+            reverse=True,
+        )
+        if existing:
+            return existing[0]
+
+    date_str = datetime.now().strftime("%Y%m%d")
+    next_run = 1
+    if base.exists():
+        nums = []
+        for d in base.iterdir():
+            if d.is_dir() and d.name.startswith(date_str) and d.name.endswith(f"-{stem}"):
+                parts = d.name.split("-")
+                if len(parts) >= 2 and parts[1].isdigit():
+                    nums.append(int(parts[1]))
+        next_run = max(nums) + 1 if nums else 1
+
+    run_dir = base / f"{date_str}-{next_run:03d}-{stem}"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    return run_dir

doocus/output_manager.py

@@ -0,0 +1,80 @@
+"""
+Manage per-document output folders and their sidecar files.
+
+Parallels meetus/output_manager.py but for the document contract:
+
+    docs-output/<YYYYMMDD-NNN-stem>/
+        content.md      textified document — the product
+        meta.json       structured metadata
+        thumb.jpg       optional single preview image
+        assets/         optional embedded images / per-page text (later)
+        manifest.json   run config + outputs inventory + source pointer
+"""
+from pathlib import Path
+from datetime import datetime
+import json
+import logging
+from typing import Dict, Any, Optional
+
+from .naming import resolve_run_dir
+
+logger = logging.getLogger(__name__)
+
+
+class DocOutputManager:
+    """Create and populate a run folder for one source document."""
+
+    def __init__(self, source_path: Path, base_output_dir: str = "docs-output", use_cache: bool = True):
+        self.source_path = Path(source_path)
+        self.base_output_dir = Path(base_output_dir)
+        self.use_cache = use_cache
+        self.output_dir = resolve_run_dir(self.base_output_dir, self.source_path.stem, use_cache)
+        logger.info("Output directory: %s", self.output_dir)
+
+    def get_path(self, filename: str) -> Path:
+        return self.output_dir / filename
+
+    def write_content(self, markdown: str) -> Path:
+        path = self.output_dir / "content.md"
+        path.write_text(markdown, encoding="utf-8")
+        return path
+
+    def write_meta(self, meta: Dict[str, Any]) -> Path:
+        path = self.output_dir / "meta.json"
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(meta, f, indent=2, ensure_ascii=False)
+        return path
+
+    def write_thumb(self, jpeg_bytes: bytes) -> Path:
+        path = self.output_dir / "thumb.jpg"
+        path.write_bytes(jpeg_bytes)
+        return path
+
+    def assets_dir(self) -> Path:
+        d = self.output_dir / "assets"
+        d.mkdir(exist_ok=True)
+        return d
+
+    def save_manifest(self, config: Dict[str, Any], outputs: Dict[str, Any], source_sha256: str) -> Path:
+        manifest_path = self.output_dir / "manifest.json"
+        manifest = {
+            "source": {
+                "name": self.source_path.name,
+                "path": str(self.source_path.absolute()),
+                "sha256": source_sha256,
+            },
+            "processed_at": datetime.now().isoformat(),
+            "configuration": config,
+            "outputs": outputs,
+        }
+        with open(manifest_path, "w", encoding="utf-8") as f:
+            json.dump(manifest, f, indent=2, ensure_ascii=False)
+        logger.info("Saved manifest: %s", manifest_path)
+        return manifest_path
+
+    def load_manifest(self) -> Optional[Dict[str, Any]]:
+        manifest_path = self.output_dir / "manifest.json"
+        if manifest_path.exists():
+            with open(manifest_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        return None

doocus/registry.py

@@ -0,0 +1,66 @@
+"""
+Extension → extractor dispatch.
+
+Maps a file extension to an extractor "family" module under doocus/extractors/.
+Families are imported lazily so an app that only handles text files doesn't need
+the office/pdf/image dependencies installed. Unknown extensions (and families
+whose optional deps are missing) are reported as "unhandled" — never fatal.
+"""
+import importlib
+import logging
+from pathlib import Path
+from typing import Dict, Optional
+
+from .extractors.base import ExtractionResult
+
+logger = logging.getLogger(__name__)
+
+# extension (no dot, lowercase) → family module name under doocus.extractors
+EXT_FAMILY: Dict[str, str] = {
+    "md": "text", "markdown": "text", "txt": "text",
+    "yaml": "text", "yml": "text", "json": "text",
+    "csv": "tabular",
+    "docx": "office", "pptx": "office", "xlsx": "office",
+    "pdf": "pdf",
+    "html": "web", "htm": "web",
+    "jpg": "image", "jpeg": "image", "png": "image",
+    "mp4": "media",
+}
+
+
+def ext_of(source: Path) -> str:
+    return source.suffix.lower().lstrip(".")
+
+
+def family_for(source: Path) -> Optional[str]:
+    return EXT_FAMILY.get(ext_of(source))
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    """Dispatch to the right family extractor.
+
+    Returns an ExtractionResult in all cases — unhandled types and import/extract
+    failures come back as an empty result carrying a warning, so a batch keeps
+    going.
+    """
+    family = family_for(source)
+    if family is None:
+        return ExtractionResult(
+            extractor="unhandled",
+            warnings=[f"no extractor for extension '.{ext_of(source)}'"],
+        )
+    try:
+        module = importlib.import_module(f".extractors.{family}", package=__package__)
+    except ImportError as e:
+        return ExtractionResult(
+            extractor=f"{family}/unavailable",
+            warnings=[f"extractor '{family}' unavailable (missing dependency): {e}"],
+        )
+    try:
+        return module.extract(source, options)
+    except Exception as e:  # isolate: one bad file must not abort the batch
+        logger.warning("extractor '%s' failed on %s: %s", family, source.name, e)
+        return ExtractionResult(
+            extractor=f"{family}/error",
+            warnings=[f"extraction failed: {e}"],
+        )

doocus/workflow.py

@@ -0,0 +1,111 @@
+"""
+Orchestrate document extraction: dispatch → extract → write sidecars → manifest.
+
+Follows the meetus WorkflowConfig / ProcessingWorkflow shape
+(meetus/workflow.py) so the two pipelines feel the same.
+"""
+from pathlib import Path
+from datetime import datetime
+import logging
+from typing import Dict, Any
+
+from . import registry
+from .naming import sha256_file
+from .output_manager import DocOutputManager
+
+logger = logging.getLogger(__name__)
+
+
+class DocConfig:
+    """Configuration for a single document extraction."""
+
+    def __init__(self, **kwargs):
+        self.source_path = Path(kwargs["source"])
+        self.output_dir = kwargs.get("output_dir", "docs-output")
+        self.no_cache = kwargs.get("no_cache", False)
+        self.render = kwargs.get("render", False)  # optional page/slide images
+        self.ocr = kwargs.get("ocr", False)        # OCR images / scanned pdf
+
+    def options(self) -> Dict[str, Any]:
+        return {"render": self.render, "ocr": self.ocr}
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"render": self.render, "ocr": self.ocr, "no_cache": self.no_cache}
+
+
+class DocWorkflow:
+    """Extract one document into a run folder."""
+
+    def __init__(self, config: DocConfig):
+        self.config = config
+        self.out = DocOutputManager(
+            config.source_path,
+            config.output_dir,
+            use_cache=not config.no_cache,
+        )
+
+    def run(self) -> Dict[str, Any]:
+        src = self.config.source_path
+        if not src.exists():
+            raise FileNotFoundError(f"source not found: {src}")
+
+        ext = registry.ext_of(src)
+        family = registry.family_for(src) or "unhandled"
+        logger.info("Extracting %s (family: %s)", src.name, family)
+
+        result = registry.extract(src, self.config.options())
+
+        content_path = self.out.write_content(result.content)
+        sha = sha256_file(src)
+        meta = self._build_meta(src, ext, family, sha, result)
+        self.out.write_meta(meta)
+
+        thumb_written = False
+        if result.thumb:
+            self.out.write_thumb(result.thumb)
+            thumb_written = True
+
+        outputs = {
+            "content": content_path.name,
+            "meta": "meta.json",
+            "thumb": "thumb.jpg" if thumb_written else None,
+        }
+        self.out.save_manifest(self.config.to_dict(), outputs, sha)
+
+        if result.warnings:
+            for w in result.warnings:
+                logger.warning("  %s", w)
+
+        logger.info("✓ %s → %s", src.name, self.out.output_dir)
+        return {
+            "output_dir": str(self.out.output_dir),
+            "content": str(content_path),
+            "meta": str(self.out.get_path("meta.json")),
+            "family": family,
+            "extractor": result.extractor,
+            "warnings": result.warnings,
+        }
+
+    def _build_meta(self, src: Path, ext: str, family: str, sha: str, result) -> Dict[str, Any]:
+        stat = src.stat()
+        meta = {
+            "source": {
+                "name": src.name,
+                "path": str(src.absolute()),
+                "sha256": sha,
+                "bytes": stat.st_size,
+                "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
+            },
+            "type": ext,
+            "family": family,
+            "extractor": result.extractor,
+            "extracted_at": datetime.now().isoformat(),
+            "word_count": len(result.content.split()),
+            "warnings": result.warnings,
+        }
+        # Type-specific fields (title, author, dates, pages, structure, ...) sit
+        # at the top level alongside the core fields.
+        for k, v in result.metadata.items():
+            if k not in meta:
+                meta[k] = v
+        return meta