doocus first ver

doocus/extractors/text.py

@@ -0,0 +1,92 @@
+"""
+Text-family extractor: md, txt, yaml/yml, json.
+
+Plain text (md/txt) passes through as-is ("the texts as they are"). Structured
+text (json/yaml) is kept verbatim inside a fenced block so content.md renders,
+with a shallow structure summary added to metadata for search/browse.
+"""
+import json
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "text@1"
+
+
+def _structure_summary(data) -> dict:
+    """Shallow shape of parsed json/yaml — top-level keys or item count."""
+    if isinstance(data, dict):
+        keys = list(data.keys())
+        return {"root": "object", "keys": keys[:50], "key_count": len(keys)}
+    if isinstance(data, list):
+        return {"root": "array", "length": len(data)}
+    return {"root": type(data).__name__}
+
+
+def _md_metadata(raw: str) -> dict:
+    """Headings + trivial front-matter for markdown."""
+    meta = {}
+    lines = raw.splitlines()
+
+    # YAML-ish front matter delimited by --- ... ---
+    if lines and lines[0].strip() == "---":
+        for i in range(1, len(lines)):
+            if lines[i].strip() == "---":
+                front = lines[1:i]
+                fm = {}
+                for ln in front:
+                    if ":" in ln:
+                        k, v = ln.split(":", 1)
+                        fm[k.strip()] = v.strip()
+                if fm:
+                    meta["frontmatter"] = fm
+                break
+
+    headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")]
+    if headings:
+        meta["headings"] = headings[:50]
+        if not meta.get("title"):
+            first = headings[0].lstrip("#").strip()
+            if first:
+                meta["title"] = first
+    return meta
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    ext = source.suffix.lower().lstrip(".")
+    raw = source.read_text(encoding="utf-8", errors="replace")
+    metadata: dict = {}
+    warnings: list = []
+    content = raw
+
+    if ext == "json":
+        try:
+            data = json.loads(raw)
+            metadata["structure"] = _structure_summary(data)
+            pretty = json.dumps(data, indent=2, ensure_ascii=False)
+            content = f"```json\n{pretty}\n```\n"
+        except json.JSONDecodeError as e:
+            warnings.append(f"invalid json, kept raw: {e}")
+            content = f"```\n{raw}\n```\n"
+    elif ext in ("yaml", "yml"):
+        try:
+            import yaml  # PyYAML
+            data = yaml.safe_load(raw)
+            metadata["structure"] = _structure_summary(data)
+        except ImportError:
+            warnings.append("PyYAML not installed; kept raw without structure summary")
+        except Exception as e:  # malformed yaml
+            warnings.append(f"invalid yaml, kept raw: {e}")
+        content = f"```yaml\n{raw}\n```\n"
+    elif ext in ("md", "markdown"):
+        metadata.update(_md_metadata(raw))
+        content = raw
+    else:  # txt and anything else routed here
+        content = raw
+
+    return ExtractionResult(
+        content=content,
+        metadata=metadata,
+        warnings=warnings,
+        extractor=EXTRACTOR_ID,
+    )