doocus first ver

doocus/extractors/web.py

@@ -0,0 +1,40 @@
+"""
+Web-family extractor: html/htm.
+
+Strips scripts/styles, keeps the visible text, and pulls the <title> plus common
+<meta> tags into metadata. Uses BeautifulSoup (bs4 + lxml).
+"""
+from pathlib import Path
+
+from .base import ExtractionResult
+
+EXTRACTOR_ID = "web/bs4@1"
+
+
+def extract(source: Path, options: dict) -> ExtractionResult:
+    from bs4 import BeautifulSoup  # lazy: degrade to "unavailable" if missing
+
+    html = source.read_text(encoding="utf-8", errors="replace")
+    soup = BeautifulSoup(html, "lxml")
+
+    metadata = {}
+    if soup.title and soup.title.string:
+        metadata["title"] = soup.title.string.strip()
+
+    metas = {}
+    for tag in soup.find_all("meta"):
+        key = tag.get("name") or tag.get("property")
+        val = tag.get("content")
+        if key and val:
+            metas[key.strip()] = val.strip()
+    if metas:
+        metadata["meta_tags"] = {k: metas[k] for k in list(metas)[:30]}
+
+    for bad in soup(["script", "style", "noscript"]):
+        bad.decompose()
+
+    text = soup.get_text(separator="\n")
+    lines = [ln.strip() for ln in text.splitlines()]
+    content = "\n".join(ln for ln in lines if ln) + "\n"
+
+    return ExtractionResult(content=content, metadata=metadata, extractor=EXTRACTOR_ID)