93 lines
3.0 KiB
Python
93 lines
3.0 KiB
Python
"""
|
|
Text-family extractor: md, txt, yaml/yml, json.
|
|
|
|
Plain text (md/txt) passes through as-is ("the texts as they are"). Structured
|
|
text (json/yaml) is kept verbatim inside a fenced block so content.md renders,
|
|
with a shallow structure summary added to metadata for search/browse.
|
|
"""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from .base import ExtractionResult
|
|
|
|
EXTRACTOR_ID = "text@1"
|
|
|
|
|
|
def _structure_summary(data) -> dict:
|
|
"""Shallow shape of parsed json/yaml — top-level keys or item count."""
|
|
if isinstance(data, dict):
|
|
keys = list(data.keys())
|
|
return {"root": "object", "keys": keys[:50], "key_count": len(keys)}
|
|
if isinstance(data, list):
|
|
return {"root": "array", "length": len(data)}
|
|
return {"root": type(data).__name__}
|
|
|
|
|
|
def _md_metadata(raw: str) -> dict:
|
|
"""Headings + trivial front-matter for markdown."""
|
|
meta = {}
|
|
lines = raw.splitlines()
|
|
|
|
# YAML-ish front matter delimited by --- ... ---
|
|
if lines and lines[0].strip() == "---":
|
|
for i in range(1, len(lines)):
|
|
if lines[i].strip() == "---":
|
|
front = lines[1:i]
|
|
fm = {}
|
|
for ln in front:
|
|
if ":" in ln:
|
|
k, v = ln.split(":", 1)
|
|
fm[k.strip()] = v.strip()
|
|
if fm:
|
|
meta["frontmatter"] = fm
|
|
break
|
|
|
|
headings = [ln.strip() for ln in lines if ln.lstrip().startswith("#")]
|
|
if headings:
|
|
meta["headings"] = headings[:50]
|
|
if not meta.get("title"):
|
|
first = headings[0].lstrip("#").strip()
|
|
if first:
|
|
meta["title"] = first
|
|
return meta
|
|
|
|
|
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
|
ext = source.suffix.lower().lstrip(".")
|
|
raw = source.read_text(encoding="utf-8", errors="replace")
|
|
metadata: dict = {}
|
|
warnings: list = []
|
|
content = raw
|
|
|
|
if ext == "json":
|
|
try:
|
|
data = json.loads(raw)
|
|
metadata["structure"] = _structure_summary(data)
|
|
pretty = json.dumps(data, indent=2, ensure_ascii=False)
|
|
content = f"```json\n{pretty}\n```\n"
|
|
except json.JSONDecodeError as e:
|
|
warnings.append(f"invalid json, kept raw: {e}")
|
|
content = f"```\n{raw}\n```\n"
|
|
elif ext in ("yaml", "yml"):
|
|
try:
|
|
import yaml # PyYAML
|
|
data = yaml.safe_load(raw)
|
|
metadata["structure"] = _structure_summary(data)
|
|
except ImportError:
|
|
warnings.append("PyYAML not installed; kept raw without structure summary")
|
|
except Exception as e: # malformed yaml
|
|
warnings.append(f"invalid yaml, kept raw: {e}")
|
|
content = f"```yaml\n{raw}\n```\n"
|
|
elif ext in ("md", "markdown"):
|
|
metadata.update(_md_metadata(raw))
|
|
content = raw
|
|
else: # txt and anything else routed here
|
|
content = raw
|
|
|
|
return ExtractionResult(
|
|
content=content,
|
|
metadata=metadata,
|
|
warnings=warnings,
|
|
extractor=EXTRACTOR_ID,
|
|
)
|