doocus first ver

doocus/extractors/__init__.py

@@ -0,0 +1 @@
+"""Per-family document extractors. See base.py for the contract."""

doocus/extractors/base.py

@@ -0,0 +1,28 @@
+"""
+Shared extractor contract.
+
+Every extractor exposes `extract(source: Path, options: dict) -> ExtractionResult`
+and is isolated: a failure on one file is caught by the workflow and recorded,
+never aborting a batch (same resilience as ctrl/batch.sh).
+"""
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+
+
+@dataclass
+class ExtractionResult:
+    """What an extractor returns.
+
+    content   textified document as markdown/plain text — the shareable product.
+    metadata  type-specific fields (title, author, created/modified, pages,
+              sheet/slide names, structure summary, ...). Merged into meta.json.
+    thumb     optional JPEG bytes for thumb.jpg (page 1 / slide 1 / the image).
+    warnings  non-fatal issues to surface in meta.json.
+    extractor id string, e.g. "office/python-docx@1".
+    """
+
+    content: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    thumb: Optional[bytes] = None
+    warnings: List[str] = field(default_factory=list)
+    extractor: str = ""

doocus/extractors/image.py

@@ -0,0 +1,79 @@
+"""
+Image-family extractor: jpg/jpeg, png.
+
+Reads EXIF (including capture date) into metadata, writes a downscaled JPEG
+thumbnail, and — when options['ocr'] is set and pytesseract is available — OCRs
+the image into content. Without OCR, content is a short placeholder note.
+"""
+import io
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "image/pillow@1"
+THUMB_MAX = 640  # longest edge, px
+
+
+def _exif(img) -> dict:
+    try:
+        from PIL.ExifTags import TAGS
+    except ImportError:
+        return {}
+    raw = getattr(img, "_getexif", lambda: None)()
+    if not raw:
+        return {}
+    out = {}
+    for tag_id, value in raw.items():
+        name = TAGS.get(tag_id, str(tag_id))
+        if name in ("DateTime", "DateTimeOriginal", "Make", "Model", "Orientation"):
+            out[name] = str(value)
+    return out
+
+
+def _thumbnail(img) -> bytes:
+    from PIL import Image  # noqa: F401 (ensure Pillow present)
+    im = img.copy()
+    im.thumbnail((THUMB_MAX, THUMB_MAX))
+    if im.mode not in ("RGB", "L"):
+        im = im.convert("RGB")
+    buf = io.BytesIO()
+    im.save(buf, format="JPEG", quality=80)
+    return buf.getvalue()
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from PIL import Image  # lazy
+
+    warnings = []
+    metadata = {}
+    content = ""
+    thumb = None
+
+    with Image.open(source) as img:
+        metadata["width"], metadata["height"] = img.size
+        metadata["mode"] = img.mode
+        exif = _exif(img)
+        if exif:
+            metadata["exif"] = exif
+            if "DateTimeOriginal" in exif or "DateTime" in exif:
+                metadata["captured"] = exif.get("DateTimeOriginal", exif.get("DateTime"))
+        try:
+            thumb = _thumbnail(img)
+        except Exception as e:
+            warnings.append(f"thumbnail failed: {e}")
+
+        if options.get("ocr"):
+            try:
+                import pytesseract
+                content = pytesseract.image_to_string(img).strip()
+                metadata["ocr"] = True
+            except ImportError:
+                warnings.append("--ocr requested but pytesseract not installed")
+            except Exception as e:
+                warnings.append(f"ocr failed: {e}")
+
+    if not content:
+        content = f"![{source.name}](thumb.jpg)\n\n_Image {metadata.get('width')}×{metadata.get('height')}. Run with --ocr to extract any text._\n"
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/media.py

@@ -0,0 +1,79 @@
+"""
+Media-family extractor: mp4.
+
+doocus does not transcribe here. It probes container metadata via ffprobe, grabs
+a thumbnail via ffmpeg, and records a pointer delegating full transcription to
+the meetus pipeline (process_meeting.py). Meeting videos are thus discoverable
+and previewable in doocus while their transcript lives in meetus.
+"""
+import json
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "media/ffprobe@1"
+
+
+def _ffprobe(source: Path) -> dict:
+    if not shutil.which("ffprobe"):
+        return {}
+    cmd = ["ffprobe", "-v", "quiet", "-print_format", "json",
+           "-show_format", "-show_streams", str(source)]
+    try:
+        out = subprocess.run(cmd, capture_output=True, text=True, check=True).stdout
+        return json.loads(out)
+    except (subprocess.CalledProcessError, json.JSONDecodeError):
+        return {}
+
+
+def _thumbnail(source: Path, at_seconds: float = 1.0) -> bytes:
+    if not shutil.which("ffmpeg"):
+        return b""
+    with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
+        tmp_path = Path(tmp.name)
+    try:
+        cmd = ["ffmpeg", "-v", "quiet", "-y", "-ss", str(at_seconds),
+               "-i", str(source), "-frames:v", "1", "-vf", "scale=640:-1",
+               str(tmp_path)]
+        subprocess.run(cmd, check=True)
+        return tmp_path.read_bytes() if tmp_path.exists() else b""
+    except subprocess.CalledProcessError:
+        return b""
+    finally:
+        tmp_path.unlink(missing_ok=True)
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    warnings = []
+    probe = _ffprobe(source)
+    metadata = {"delegate": "meetus", "delegate_hint": "run process_meeting.py for transcript"}
+
+    fmt = probe.get("format", {})
+    if fmt:
+        if "duration" in fmt:
+            metadata["duration_seconds"] = float(fmt["duration"])
+        tags = fmt.get("tags", {})
+        if "creation_time" in tags:
+            metadata["created"] = tags["creation_time"]
+    video_streams = [s for s in probe.get("streams", []) if s.get("codec_type") == "video"]
+    if video_streams:
+        v = video_streams[0]
+        metadata["width"], metadata["height"] = v.get("width"), v.get("height")
+    if not probe:
+        warnings.append("ffprobe unavailable or failed; metadata limited")
+
+    thumb = _thumbnail(source) or None
+    if thumb is None:
+        warnings.append("thumbnail not generated (ffmpeg unavailable or failed)")
+
+    dur = metadata.get("duration_seconds")
+    content = (f"_Video {source.name}"
+               + (f", {dur:.0f}s" if dur else "")
+               + ". Transcript is produced by the meetus pipeline "
+                 "(`process_meeting.py`), not doocus._\n")
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/office.py

@@ -0,0 +1,114 @@
+"""
+Office-family extractor: docx, pptx, xlsx.
+
+Each format is handled with its own lazy import so a missing dependency degrades
+only that format. These formats carry core properties (author, created/modified
+dates, title) which go straight into metadata.
+"""
+from pathlib import Path
+
+from .base import ExtractionResult
+
+MAX_SHEET_ROWS = 200
+
+
+def _core_props(props) -> dict:
+    """Common OOXML core properties → metadata (only non-empty ones).
+
+    python-docx/pptx expose `.author`/`.last_modified_by`; openpyxl uses
+    `.creator`/`.lastModifiedBy`. Each target checks both spellings.
+    """
+    # dst -> candidate source attribute names (first non-empty wins)
+    fields = {
+        "title": ("title",),
+        "author": ("author", "creator"),
+        "created": ("created",),
+        "modified_doc": ("modified",),
+        "last_modified_by": ("last_modified_by", "lastModifiedBy"),
+    }
+    out = {}
+    for dst, sources in fields.items():
+        for src in sources:
+            val = getattr(props, src, None)
+            if val:
+                out[dst] = str(val)
+                break
+    return out
+
+
+def _extract_docx(source: Path) -> ExtractionResult:
+    import docx  # python-docx
+    doc = docx.Document(str(source))
+    paras = [p.text for p in doc.paragraphs]
+    content = "\n\n".join(p for p in paras if p.strip()) + "\n"
+    metadata = _core_props(doc.core_properties)
+    metadata["paragraphs"] = len([p for p in paras if p.strip()])
+    return ExtractionResult(content=content, metadata=metadata, extractor="office/python-docx@1")
+
+
+def _extract_pptx(source: Path) -> ExtractionResult:
+    from pptx import Presentation
+    prs = Presentation(str(source))
+    blocks = []
+    for i, slide in enumerate(prs.slides, 1):
+        texts = []
+        for shape in slide.shapes:
+            if shape.has_text_frame:
+                for para in shape.text_frame.paragraphs:
+                    line = "".join(run.text for run in para.runs)
+                    if line.strip():
+                        texts.append(line)
+        blocks.append(f"## Slide {i}\n\n" + "\n".join(texts))
+    content = "\n\n".join(blocks) + "\n"
+    metadata = _core_props(prs.core_properties)
+    metadata["slides"] = len(prs.slides)
+    return ExtractionResult(content=content, metadata=metadata, extractor="office/python-pptx@1")
+
+
+def _extract_xlsx(source: Path) -> ExtractionResult:
+    from openpyxl import load_workbook
+    wb = load_workbook(str(source), read_only=True, data_only=True)
+    warnings = []
+    blocks = []
+    for ws in wb.worksheets:
+        rows = []
+        for r, row in enumerate(ws.iter_rows(values_only=True)):
+            if r > MAX_SHEET_ROWS:
+                warnings.append(f"sheet '{ws.title}' truncated to {MAX_SHEET_ROWS} rows")
+                break
+            rows.append(["" if c is None else str(c) for c in row])
+        blocks.append(f"## {ws.title}\n\n" + _rows_to_md(rows))
+    wb.close()
+    content = "\n\n".join(blocks) + "\n"
+    metadata = _core_props(wb.properties)
+    metadata["sheets"] = [ws.title for ws in wb.worksheets]
+    return ExtractionResult(content=content, metadata=metadata,
+                            warnings=warnings, extractor="office/openpyxl@1")
+
+
+def _rows_to_md(rows: list) -> str:
+    if not rows:
+        return "_(empty)_\n"
+    width = max(len(r) for r in rows)
+    rows = [r + [""] * (width - len(r)) for r in rows]
+
+    def esc(c):
+        return str(c).replace("|", "\\|").replace("\n", " ")
+
+    lines = ["| " + " | ".join(esc(c) for c in rows[0]) + " |",
+             "| " + " | ".join(["---"] * width) + " |"]
+    for r in rows[1:]:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    ext = source.suffix.lower().lstrip(".")
+    if ext == "docx":
+        return _extract_docx(source)
+    if ext == "pptx":
+        return _extract_pptx(source)
+    if ext == "xlsx":
+        return _extract_xlsx(source)
+    return ExtractionResult(extractor="office/unhandled",
+                            warnings=[f"office extractor got unexpected '.{ext}'"])

doocus/extractors/pdf.py

@@ -0,0 +1,73 @@
+"""
+PDF-family extractor.
+
+Text per page via pypdf, plus the document info dict (title/author/dates). When
+options['render'] is set and pdf2image (+ system poppler) is available, page 1 is
+rendered to thumb.jpg.
+"""
+import io
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "pdf/pypdf@1"
+
+
+def _info_metadata(reader) -> dict:
+    out = {}
+    info = getattr(reader, "metadata", None)
+    if not info:
+        return out
+    for src, dst in (("title", "title"), ("author", "author"),
+                     ("creation_date", "created"), ("modification_date", "modified_doc")):
+        try:
+            val = getattr(info, src, None)
+        except Exception:
+            val = None
+        if val:
+            out[dst] = str(val)
+    return out
+
+
+def _render_first_page(source: Path) -> bytes:
+    from pdf2image import convert_from_path  # needs system poppler
+    images = convert_from_path(str(source), first_page=1, last_page=1, dpi=100)
+    if not images:
+        return b""
+    im = images[0]
+    im.thumbnail((640, 640))
+    buf = io.BytesIO()
+    im.convert("RGB").save(buf, format="JPEG", quality=80)
+    return buf.getvalue()
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from pypdf import PdfReader  # lazy
+
+    warnings = []
+    reader = PdfReader(str(source))
+    pages = reader.pages
+    blocks = []
+    for i, page in enumerate(pages, 1):
+        try:
+            text = page.extract_text() or ""
+        except Exception as e:
+            text = ""
+            warnings.append(f"page {i} text extraction failed: {e}")
+        blocks.append(f"## Page {i}\n\n{text.strip()}")
+    content = "\n\n".join(blocks) + "\n"
+
+    metadata = _info_metadata(reader)
+    metadata["pages"] = len(pages)
+
+    thumb = None
+    if options.get("render"):
+        try:
+            thumb = _render_first_page(source) or None
+        except ImportError:
+            warnings.append("--render requested but pdf2image/poppler not available")
+        except Exception as e:
+            warnings.append(f"page render failed: {e}")
+
+    return ExtractionResult(content=content, metadata=metadata, thumb=thumb,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/tabular.py

@@ -0,0 +1,57 @@
+"""
+Tabular-family extractor: csv.
+
+Renders to a markdown table (capped preview) with row/column metadata. Uses only
+the stdlib so it always works.
+"""
+import csv
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "tabular/csv@1"
+MAX_PREVIEW_ROWS = 200
+
+
+def _md_table(rows: list) -> str:
+    if not rows:
+        return ""
+    width = max(len(r) for r in rows)
+    rows = [r + [""] * (width - len(r)) for r in rows]
+    header, body = rows[0], rows[1:]
+
+    def esc(c: str) -> str:
+        return str(c).replace("|", "\\|").replace("\n", " ")
+
+    lines = ["| " + " | ".join(esc(c) for c in header) + " |",
+             "| " + " | ".join(["---"] * width) + " |"]
+    for r in body:
+        lines.append("| " + " | ".join(esc(c) for c in r) + " |")
+    return "\n".join(lines) + "\n"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    warnings = []
+    with open(source, newline="", encoding="utf-8", errors="replace") as f:
+        sample = f.read(8192)
+        f.seek(0)
+        try:
+            dialect = csv.Sniffer().sniff(sample) if sample.strip() else csv.excel
+        except csv.Error:
+            dialect = csv.excel
+        rows = list(csv.reader(f, dialect))
+
+    total = len(rows)
+    cols = max((len(r) for r in rows), default=0)
+    preview = rows[: MAX_PREVIEW_ROWS + 1]  # +1 for header
+    if total > MAX_PREVIEW_ROWS + 1:
+        warnings.append(f"table has {total} rows; previewed first {MAX_PREVIEW_ROWS}")
+
+    content = _md_table(preview)
+    metadata = {
+        "rows": max(total - 1, 0),
+        "columns": cols,
+        "header": rows[0] if rows else [],
+    }
+    return ExtractionResult(content=content, metadata=metadata,
+                            warnings=warnings, extractor=EXTRACTOR_ID)

doocus/extractors/text.py

@@ -0,0 +1,92 @@
+"""
+Text-family extractor: md, txt, yaml/yml, json.
+
+Plain text (md/txt) passes through as-is ("the texts as they are"). Structured
+text (json/yaml) is kept verbatim inside a fenced block so content.md renders,
+with a shallow structure summary added to metadata for search/browse.
+"""
+import json
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "text@1"
+
+
+def _structure_summary(data) -> dict:
+    """Shallow shape of parsed json/yaml — top-level keys or item count."""
+    if isinstance(data, dict):
+        keys = list(data.keys())
+        return {"root": "object", "keys": keys[:50], "key_count": len(keys)}
+    if isinstance(data, list):
+        return {"root": "array", "length": len(data)}
+    return {"root": type(data).__name__}
+
+
+def _md_metadata(raw: str) -> dict:
+    """Headings + trivial front-matter for markdown."""
+    meta = {}
+    lines = raw.splitlines()
+
+    # YAML-ish front matter delimited by --- ... ---
+    if lines and lines[0].strip() == "---":
+        for i in range(1, len(lines)):
+            if lines[i].strip() == "---":
+                front = lines[1:i]
+                fm = {}
+                for ln in front:
+                    if ":" in ln:
+                        k, v = ln.split(":", 1)
+                        fm[k.strip()] = v.strip()
+                if fm:
+                    meta["frontmatter"] = fm
+                break
+
+    headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")]
+    if headings:
+        meta["headings"] = headings[:50]
+        if not meta.get("title"):
+            first = headings[0].lstrip("#").strip()
+            if first:
+                meta["title"] = first
+    return meta
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    ext = source.suffix.lower().lstrip(".")
+    raw = source.read_text(encoding="utf-8", errors="replace")
+    metadata: dict = {}
+    warnings: list = []
+    content = raw
+
+    if ext == "json":
+        try:
+            data = json.loads(raw)
+            metadata["structure"] = _structure_summary(data)
+            pretty = json.dumps(data, indent=2, ensure_ascii=False)
+            content = f"```json\n{pretty}\n```\n"
+        except json.JSONDecodeError as e:
+            warnings.append(f"invalid json, kept raw: {e}")
+            content = f"```\n{raw}\n```\n"
+    elif ext in ("yaml", "yml"):
+        try:
+            import yaml  # PyYAML
+            data = yaml.safe_load(raw)
+            metadata["structure"] = _structure_summary(data)
+        except ImportError:
+            warnings.append("PyYAML not installed; kept raw without structure summary")
+        except Exception as e:  # malformed yaml
+            warnings.append(f"invalid yaml, kept raw: {e}")
+        content = f"```yaml\n{raw}\n```\n"
+    elif ext in ("md", "markdown"):
+        metadata.update(_md_metadata(raw))
+        content = raw
+    else:  # txt and anything else routed here
+        content = raw
+
+    return ExtractionResult(
+        content=content,
+        metadata=metadata,
+        warnings=warnings,
+        extractor=EXTRACTOR_ID,
+    )

doocus/extractors/web.py

@@ -0,0 +1,40 @@
+"""
+Web-family extractor: html/htm.
+
+Strips scripts/styles, keeps the visible text, and pulls the <title> plus common
+<meta> tags into metadata. Uses BeautifulSoup (bs4 + lxml).
+"""
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "web/bs4@1"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from bs4 import BeautifulSoup  # lazy: degrade to "unavailable" if missing
+
+    html = source.read_text(encoding="utf-8", errors="replace")
+    soup = BeautifulSoup(html, "lxml")
+
+    metadata = {}
+    if soup.title and soup.title.string:
+        metadata["title"] = soup.title.string.strip()
+
+    metas = {}
+    for tag in soup.find_all("meta"):
+        key = tag.get("name") or tag.get("property")
+        val = tag.get("content")
+        if key and val:
+            metas[key.strip()] = val.strip()
+    if metas:
+        metadata["meta_tags"] = {k: metas[k] for k in list(metas)[:30]}
+
+    for bad in soup(["script", "style", "noscript"]):
+        bad.decompose()
+
+    text = soup.get_text(separator="\n")
+    lines = [ln.strip() for ln in text.splitlines()]
+    content = "\n".join(ln for ln in lines if ln) + "\n"
+
+    return ExtractionResult(content=content, metadata=metadata, extractor=EXTRACTOR_ID)