""" Text-family extractor: md, txt, yaml/yml, json. Plain text (md/txt) passes through as-is ("the texts as they are"). Structured text (json/yaml) is kept verbatim inside a fenced block so content.md renders, with a shallow structure summary added to metadata for search/browse. """ import json from pathlib import Path from .base import ExtractionResult EXTRACTOR_ID = "text@1" def _structure_summary(data) -> dict: """Shallow shape of parsed json/yaml — top-level keys or item count.""" if isinstance(data, dict): keys = list(data.keys()) return {"root": "object", "keys": keys[:50], "key_count": len(keys)} if isinstance(data, list): return {"root": "array", "length": len(data)} return {"root": type(data).__name__} def _md_metadata(raw: str) -> dict: """Headings + trivial front-matter for markdown.""" meta = {} lines = raw.splitlines() # YAML-ish front matter delimited by --- ... --- if lines and lines[0].strip() == "---": for i in range(1, len(lines)): if lines[i].strip() == "---": front = lines[1:i] fm = {} for ln in front: if ":" in ln: k, v = ln.split(":", 1) fm[k.strip()] = v.strip() if fm: meta["frontmatter"] = fm break headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")] if headings: meta["headings"] = headings[:50] if not meta.get("title"): first = headings[0].lstrip("#").strip() if first: meta["title"] = first return meta def extract(source: Path, options: dict) -> ExtractionResult: ext = source.suffix.lower().lstrip(".") raw = source.read_text(encoding="utf-8", errors="replace") metadata: dict = {} warnings: list = [] content = raw if ext == "json": try: data = json.loads(raw) metadata["structure"] = _structure_summary(data) pretty = json.dumps(data, indent=2, ensure_ascii=False) content = f"```json\n{pretty}\n```\n" except json.JSONDecodeError as e: warnings.append(f"invalid json, kept raw: {e}") content = f"```\n{raw}\n```\n" elif ext in ("yaml", "yml"): try: import yaml # PyYAML data = yaml.safe_load(raw) metadata["structure"] = _structure_summary(data) except ImportError: warnings.append("PyYAML not installed; kept raw without structure summary") except Exception as e: # malformed yaml warnings.append(f"invalid yaml, kept raw: {e}") content = f"```yaml\n{raw}\n```\n" elif ext in ("md", "markdown"): metadata.update(_md_metadata(raw)) content = raw else: # txt and anything else routed here content = raw return ExtractionResult( content=content, metadata=metadata, warnings=warnings, extractor=EXTRACTOR_ID, )