41 lines
1.2 KiB
Python
41 lines
1.2 KiB
Python
"""
|
|
Web-family extractor: html/htm.
|
|
|
|
Strips scripts/styles, keeps the visible text, and pulls the <title> plus common
|
|
<meta> tags into metadata. Uses BeautifulSoup (bs4 + lxml).
|
|
"""
|
|
from pathlib import Path
|
|
|
|
from .base import ExtractionResult
|
|
|
|
EXTRACTOR_ID = "web/bs4@1"
|
|
|
|
|
|
def extract(source: Path, options: dict) -> ExtractionResult:
|
|
from bs4 import BeautifulSoup # lazy: degrade to "unavailable" if missing
|
|
|
|
html = source.read_text(encoding="utf-8", errors="replace")
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
metadata = {}
|
|
if soup.title and soup.title.string:
|
|
metadata["title"] = soup.title.string.strip()
|
|
|
|
metas = {}
|
|
for tag in soup.find_all("meta"):
|
|
key = tag.get("name") or tag.get("property")
|
|
val = tag.get("content")
|
|
if key and val:
|
|
metas[key.strip()] = val.strip()
|
|
if metas:
|
|
metadata["meta_tags"] = {k: metas[k] for k in list(metas)[:30]}
|
|
|
|
for bad in soup(["script", "style", "noscript"]):
|
|
bad.decompose()
|
|
|
|
text = soup.get_text(separator="\n")
|
|
lines = [ln.strip() for ln in text.splitlines()]
|
|
content = "\n".join(ln for ln in lines if ln) + "\n"
|
|
|
|
return ExtractionResult(content=content, metadata=metadata, extractor=EXTRACTOR_ID)
|