add compile meeting, summarize. both for local llm run
This commit is contained in:
Mariano Gabriel
282
compile_meeting.py
Executable file
282
compile_meeting.py
Executable file
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compile a long meeting/training enhanced-transcript into a detailed technical
|
||||
reference, using a LOCAL multimodal LLM that reads frames ON DEMAND.
|
||||
|
||||
This is NOT summarization — it RETAINS workflow/architecture detail and
|
||||
reorganizes it out of conversation order. It uses the REFINE pattern: walk the
|
||||
transcript top-to-bottom in windows, carrying a running compiled document as the
|
||||
only large context. The running doc IS the memory; the raw transcript is never
|
||||
held whole (which is why a 4-hour recording fits a small model).
|
||||
|
||||
Frames are consulted the way a human note-taker does: while reading each window,
|
||||
the model decides which referenced frames it actually needs to see, and only
|
||||
those images are attached (on demand) — webcam/transition frames cost nothing.
|
||||
|
||||
Standalone on purpose: not wired into process_meeting.py. Talks to a local
|
||||
OpenAI-compatible server (vLLM or llama.cpp — see ~/wdir/llm/serve.sh); --base-url
|
||||
swaps it.
|
||||
|
||||
Usage (start `~/wdir/llm/serve.sh qwen-vl` first, then):
|
||||
~/wdir/llm/.venv/bin/python compile_meeting.py \\
|
||||
output/<run>/<stem>_enhanced.txt \\
|
||||
"compile every deployment/data-flow workflow and the system architecture \\
|
||||
as a technical reference; note the [mm:ss] each was shown on screen" \\
|
||||
-o output/<run>/reference.md
|
||||
|
||||
Frame modes:
|
||||
--frames ondemand (default) two-step: model lists which frames it needs,
|
||||
then only those are attached. Cheapest on vision tokens.
|
||||
--frames window attach every frame in the current window; model uses the
|
||||
relevant ones. Simpler, more tokens.
|
||||
--frames none ignore frames entirely (text-only; for non-VL models / A-B).
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_BASE_URL = "http://localhost:11000/v1"
|
||||
DEFAULT_MODEL = "Qwen/Qwen2.5-VL-7B-Instruct-AWQ"
|
||||
CHARS_PER_TOKEN = 4.0
|
||||
|
||||
GROUNDING = """\
|
||||
Rules:
|
||||
- Be faithful. Never invent names, components, commands, numbers, or steps.
|
||||
- Preserve proper nouns and identifiers exactly as written.
|
||||
- This is a COMPILATION, not a summary: keep technical detail (workflows step by
|
||||
step, architecture components and how they connect, configs, commands, gotchas).
|
||||
- Reorganize by TOPIC, not by conversation order. Merge new info into the right
|
||||
existing section rather than appending chronologically.
|
||||
- Anchor concrete items to the [mm:ss] where they were said/shown.
|
||||
- If something is unclear or only partially stated, mark it (e.g. "(unclear)")
|
||||
rather than guessing."""
|
||||
|
||||
REFINE_SYS = """\
|
||||
You maintain a growing TECHNICAL REFERENCE compiled from a training recording.
|
||||
|
||||
The user's compilation instruction is authoritative:
|
||||
<instruction>
|
||||
{instruction}
|
||||
</instruction>
|
||||
|
||||
You are given the CURRENT REFERENCE so far and the NEXT WINDOW of transcript
|
||||
(and possibly some screen frames). Integrate any new workflow/architecture detail
|
||||
from this window into the reference, slotting it into the correct topical section
|
||||
(create sections as needed). Return the COMPLETE updated reference in Markdown —
|
||||
not a diff, not just the new part. Do not drop earlier content.
|
||||
|
||||
{rules}"""
|
||||
|
||||
TRIAGE_SYS = """\
|
||||
You are reading one window of a training transcript while compiling technical
|
||||
notes per this instruction:
|
||||
<instruction>
|
||||
{instruction}
|
||||
</instruction>
|
||||
|
||||
The window references the screen frames listed below (id + [mm:ss]). Decide which
|
||||
frames you would need to SEE to capture workflow/architecture/config detail the
|
||||
text alone doesn't convey (diagrams, slides, terminal output, code). Ignore
|
||||
webcam/transition frames. Reply with STRICT JSON only:
|
||||
{{"need": ["<frame-id>", ...]}}
|
||||
Empty list if none are needed."""
|
||||
|
||||
FRAME_RE = re.compile(r"Frame:\s+(\S+\.(?:jpg|jpeg|png))", re.IGNORECASE)
|
||||
TS_RE = re.compile(r"\[(\d+):(\d+)\]")
|
||||
|
||||
|
||||
def estimate_tokens(text):
|
||||
return int(len(text) / CHARS_PER_TOKEN)
|
||||
|
||||
|
||||
def default_output(transcript, kind):
|
||||
"""Write next to the transcript, in the same run folder, following the
|
||||
pipeline's <stem>_<kind> naming (e.g. training_reference.md)."""
|
||||
stem = transcript.stem
|
||||
if stem.endswith("_enhanced"):
|
||||
stem = stem[: -len("_enhanced")]
|
||||
return transcript.parent / f"{stem}_{kind}.md"
|
||||
|
||||
|
||||
def parse_windows(path, window_tokens):
|
||||
"""Split the enhanced transcript into windows of ~window_tokens, packing
|
||||
blank-line-separated blocks whole. Each window keeps the frame refs that
|
||||
fall inside it: {text, frames:[{id, ts, path}]}."""
|
||||
raw = path.read_text()
|
||||
blocks = re.split(r"\n\s*\n", raw)
|
||||
windows, cur_text, cur_frames, cur_tok = [], [], [], 0
|
||||
|
||||
def flush():
|
||||
if cur_text:
|
||||
windows.append({"text": "\n\n".join(cur_text),
|
||||
"frames": list(cur_frames)})
|
||||
|
||||
last_ts = "00:00"
|
||||
for block in blocks:
|
||||
ts_m = TS_RE.search(block)
|
||||
if ts_m:
|
||||
last_ts = f"{ts_m.group(1)}:{ts_m.group(2)}"
|
||||
fm = FRAME_RE.search(block)
|
||||
bt = estimate_tokens(block)
|
||||
if cur_text and cur_tok + bt > window_tokens:
|
||||
flush()
|
||||
cur_text, cur_frames, cur_tok = [], [], 0
|
||||
if fm:
|
||||
p = fm.group(1)
|
||||
cur_frames.append({"id": Path(p).stem, "ts": last_ts, "path": p})
|
||||
# keep a compact ref line in the text instead of the bare path
|
||||
cur_text.append(f"[{last_ts}] (frame {Path(p).stem})")
|
||||
else:
|
||||
cur_text.append(block)
|
||||
cur_tok += bt
|
||||
flush()
|
||||
return windows
|
||||
|
||||
|
||||
def resolve_path(ref_path, transcript_path):
|
||||
p = Path(ref_path)
|
||||
if p.is_absolute() and p.exists():
|
||||
return p
|
||||
# paths in the transcript are usually relative to the run dir
|
||||
cand = transcript_path.parent / ref_path
|
||||
return cand if cand.exists() else p
|
||||
|
||||
|
||||
def encode_image(path, max_side):
|
||||
data = path.read_bytes()
|
||||
mime = "image/png" if path.suffix.lower() == ".png" else "image/jpeg"
|
||||
if max_side:
|
||||
try:
|
||||
from PIL import Image
|
||||
import io
|
||||
img = Image.open(io.BytesIO(data))
|
||||
if max(img.size) > max_side:
|
||||
img.thumbnail((max_side, max_side))
|
||||
buf = io.BytesIO()
|
||||
img.convert("RGB").save(buf, format="JPEG", quality=85)
|
||||
data, mime = buf.getvalue(), "image/jpeg"
|
||||
except ImportError:
|
||||
pass # PIL absent: send original (more tokens, still works)
|
||||
b64 = base64.b64encode(data).decode()
|
||||
return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def make_client(base_url, api_key):
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
sys.exit("ERROR: `openai` not installed here. Run under ~/wdir/llm/.venv")
|
||||
return OpenAI(base_url=base_url, api_key=api_key)
|
||||
|
||||
|
||||
def call(client, model, system, content, temperature, max_tokens):
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "system", "content": system},
|
||||
{"role": "user", "content": content}],
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
)
|
||||
return resp.choices[0].message.content.strip()
|
||||
|
||||
|
||||
def parse_need(raw, valid_ids):
|
||||
raw = re.sub(r"^```(?:json)?|```$", "", raw.strip(), flags=re.MULTILINE)
|
||||
m = re.search(r"\{.*\}", raw, re.DOTALL)
|
||||
if not m:
|
||||
return []
|
||||
try:
|
||||
ids = json.loads(m.group(0)).get("need", [])
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [i for i in ids if i in valid_ids]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Compile a long meeting transcript into a technical reference (refine + on-demand frames).",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
||||
p.add_argument("transcript", type=Path)
|
||||
p.add_argument("instruction", nargs="?",
|
||||
default="Compile a detailed technical reference of the workflows and architecture covered.")
|
||||
p.add_argument("-o", "--output", type=Path, help="write here (default: <run>/<stem>_reference.md next to the transcript)")
|
||||
p.add_argument("--stdout", action="store_true", help="print to stdout instead of writing a file")
|
||||
p.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
p.add_argument("--model", default=DEFAULT_MODEL)
|
||||
p.add_argument("--api-key", default="local")
|
||||
p.add_argument("--frames", choices=["ondemand", "window", "none"], default="ondemand")
|
||||
p.add_argument("--window-tokens", type=int, default=3500, help="transcript tokens per refine step (default 3500)")
|
||||
p.add_argument("--max-tokens", type=int, default=8192, help="generation cap; must fit the growing doc (default 8192)")
|
||||
p.add_argument("--max-image-side", type=int, default=1280, help="downscale frames to this max side (0=off)")
|
||||
p.add_argument("--temperature", type=float, default=0.2)
|
||||
p.add_argument("--checkpoint", type=Path, help="write the running doc here after each window (resumable progress)")
|
||||
p.add_argument("-q", "--quiet", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
if not args.transcript.is_file():
|
||||
sys.exit(f"ERROR: transcript not found: {args.transcript}")
|
||||
|
||||
def log(m):
|
||||
if not args.quiet:
|
||||
print(f"[compile] {m}", file=sys.stderr)
|
||||
|
||||
windows = parse_windows(args.transcript, args.window_tokens)
|
||||
nframes = sum(len(w["frames"]) for w in windows)
|
||||
log(f"{len(windows)} windows, {nframes} frame refs, mode={args.frames}")
|
||||
|
||||
client = make_client(args.base_url, args.api_key)
|
||||
doc = "# (compilation in progress)\n"
|
||||
|
||||
for wi, w in enumerate(windows, 1):
|
||||
wanted = []
|
||||
if args.frames != "none" and w["frames"]:
|
||||
if args.frames == "window":
|
||||
wanted = w["frames"]
|
||||
else: # ondemand: ask the model which frames it needs
|
||||
listing = "\n".join(f"- {f['id']} [{f['ts']}]" for f in w["frames"])
|
||||
raw = call(client, args.model,
|
||||
TRIAGE_SYS.format(instruction=args.instruction),
|
||||
f"Window transcript:\n{w['text']}\n\nReferenced frames:\n{listing}",
|
||||
0.0, 256)
|
||||
valid = {f["id"] for f in w["frames"]}
|
||||
keep = set(parse_need(raw, valid))
|
||||
wanted = [f for f in w["frames"] if f["id"] in keep]
|
||||
|
||||
# build the refine turn (multimodal if any frames wanted)
|
||||
text = (f"CURRENT REFERENCE:\n{doc}\n\n"
|
||||
f"NEXT TRANSCRIPT WINDOW:\n{w['text']}")
|
||||
if wanted:
|
||||
text += "\n\nAttached frames: " + ", ".join(f"{f['id']} [{f['ts']}]" for f in wanted)
|
||||
content = [{"type": "text", "text": text}]
|
||||
for f in wanted:
|
||||
ip = resolve_path(f["path"], args.transcript)
|
||||
if ip.exists():
|
||||
content.append({"type": "image_url",
|
||||
"image_url": {"url": encode_image(ip, args.max_image_side)}})
|
||||
log(f" window {wi}/{len(windows)}: {len(wanted)} frame(s) attached")
|
||||
else:
|
||||
content = text
|
||||
log(f" window {wi}/{len(windows)}: text-only")
|
||||
|
||||
doc = call(client, args.model,
|
||||
REFINE_SYS.format(instruction=args.instruction, rules=GROUNDING),
|
||||
content, args.temperature, args.max_tokens)
|
||||
|
||||
if args.checkpoint:
|
||||
args.checkpoint.write_text(doc + "\n")
|
||||
if estimate_tokens(doc) > args.window_tokens * 4 and not args.quiet:
|
||||
log(f" note: running doc ~{estimate_tokens(doc)} tok and growing — "
|
||||
f"if it nears the context window, switch to a 32k-context profile (qwen14b-gguf)")
|
||||
|
||||
if args.stdout:
|
||||
print(doc)
|
||||
else:
|
||||
out = args.output or default_output(args.transcript, "reference")
|
||||
out.write_text(doc + "\n")
|
||||
log(f"wrote {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user