#!/usr/bin/env python3 """ Compile a long meeting/training enhanced-transcript into a detailed technical reference, using a LOCAL multimodal LLM that reads frames ON DEMAND. This is NOT summarization — it RETAINS workflow/architecture detail and reorganizes it out of conversation order. It uses the REFINE pattern: walk the transcript top-to-bottom in windows, carrying a running compiled document as the only large context. The running doc IS the memory; the raw transcript is never held whole (which is why a 4-hour recording fits a small model). Frames are consulted the way a human note-taker does: while reading each window, the model decides which referenced frames it actually needs to see, and only those images are attached (on demand) — webcam/transition frames cost nothing. Standalone on purpose: not wired into process_meeting.py. Talks to a local OpenAI-compatible server (vLLM or llama.cpp — see ~/wdir/llm/serve.sh); --base-url swaps it. Usage (start `~/wdir/llm/serve.sh qwen-vl` first, then): ~/wdir/llm/.venv/bin/python compile_meeting.py \\ output//_enhanced.txt \\ "compile every deployment/data-flow workflow and the system architecture \\ as a technical reference; note the [mm:ss] each was shown on screen" \\ -o output//reference.md Frame modes: --frames ondemand (default) two-step: model lists which frames it needs, then only those are attached. Cheapest on vision tokens. --frames window attach every frame in the current window; model uses the relevant ones. Simpler, more tokens. --frames none ignore frames entirely (text-only; for non-VL models / A-B). """ import argparse import base64 import json import re import sys from pathlib import Path DEFAULT_BASE_URL = "http://localhost:11000/v1" DEFAULT_MODEL = "Qwen/Qwen2.5-VL-7B-Instruct-AWQ" CHARS_PER_TOKEN = 4.0 GROUNDING = """\ Rules: - Be faithful. Never invent names, components, commands, numbers, or steps. - Preserve proper nouns and identifiers exactly as written. - This is a COMPILATION, not a summary: keep technical detail (workflows step by step, architecture components and how they connect, configs, commands, gotchas). - Reorganize by TOPIC, not by conversation order. Merge new info into the right existing section rather than appending chronologically. - Anchor concrete items to the [mm:ss] where they were said/shown. - ASR vigilance on TERMS, especially acronyms and product/tool names: this transcript is machine-transcribed, so a term that reads oddly or makes no sense in context is likely a mis-hearing (a slightly-off acronym, a homophone, a split or merged word). Flag it like "(heard: X — likely Y?)", using context to infer the intended term; never silently propagate a nonsensical token, and never silently "correct" a term you are unsure about. - If something is unclear or only partially stated, mark it (e.g. "(unclear)") rather than guessing.""" REFINE_SYS = """\ You maintain a growing TECHNICAL REFERENCE compiled from a training recording. The user's compilation instruction is authoritative: {instruction} You are given the CURRENT REFERENCE so far and the NEXT WINDOW of transcript (and possibly some screen frames). Integrate any new workflow/architecture detail from this window into the reference, slotting it into the correct topical section (create sections as needed). Return the COMPLETE updated reference in Markdown — not a diff, not just the new part. Do not drop earlier content. {rules}""" TRIAGE_SYS = """\ You are reading one window of a training transcript while compiling technical notes per this instruction: {instruction} The window references the screen frames listed below (id + [mm:ss]). Decide which frames you would need to SEE to capture workflow/architecture/config detail the text alone doesn't convey (diagrams, slides, terminal output, code). Ignore webcam/transition frames. Reply with STRICT JSON only: {{"need": ["", ...]}} Empty list if none are needed.""" FRAME_RE = re.compile(r"Frame:\s+(\S+\.(?:jpg|jpeg|png))", re.IGNORECASE) TS_RE = re.compile(r"\[(\d+):(\d+)\]") def estimate_tokens(text): return int(len(text) / CHARS_PER_TOKEN) def default_output(transcript, kind): """Write next to the transcript, in the same run folder, following the pipeline's _ naming (e.g. training_reference.md).""" stem = transcript.stem if stem.endswith("_enhanced"): stem = stem[: -len("_enhanced")] return transcript.parent / f"{stem}_{kind}.md" def content_tokens(content): """Estimate input tokens of a chat content (str or multimodal list).""" if isinstance(content, str): return estimate_tokens(content) total = 0 for part in content: if part.get("type") == "text": total += estimate_tokens(part.get("text", "")) else: # image_url etc. — rough per-image vision-token allowance total += 800 return total def parse_windows(path, window_tokens): """Split the enhanced transcript into windows of ~window_tokens, packing blank-line-separated blocks whole. Each window keeps the frame refs that fall inside it: {text, frames:[{id, ts, path}]}.""" raw = path.read_text() blocks = re.split(r"\n\s*\n", raw) windows, cur_text, cur_frames, cur_tok = [], [], [], 0 def flush(): if cur_text: windows.append({"text": "\n\n".join(cur_text), "frames": list(cur_frames)}) last_ts = "00:00" for block in blocks: ts_m = TS_RE.search(block) if ts_m: last_ts = f"{ts_m.group(1)}:{ts_m.group(2)}" fm = FRAME_RE.search(block) bt = estimate_tokens(block) if cur_text and cur_tok + bt > window_tokens: flush() cur_text, cur_frames, cur_tok = [], [], 0 if fm: p = fm.group(1) cur_frames.append({"id": Path(p).stem, "ts": last_ts, "path": p}) # keep a compact ref line in the text instead of the bare path cur_text.append(f"[{last_ts}] (frame {Path(p).stem})") else: cur_text.append(block) cur_tok += bt flush() return windows def resolve_path(ref_path, transcript_path): p = Path(ref_path) if p.is_absolute() and p.exists(): return p # paths in the transcript are usually relative to the run dir cand = transcript_path.parent / ref_path return cand if cand.exists() else p def encode_image(path, max_side): data = path.read_bytes() mime = "image/png" if path.suffix.lower() == ".png" else "image/jpeg" if max_side: try: from PIL import Image import io img = Image.open(io.BytesIO(data)) if max(img.size) > max_side: img.thumbnail((max_side, max_side)) buf = io.BytesIO() img.convert("RGB").save(buf, format="JPEG", quality=85) data, mime = buf.getvalue(), "image/jpeg" except ImportError: pass # PIL absent: send original (more tokens, still works) b64 = base64.b64encode(data).decode() return f"data:{mime};base64,{b64}" def make_client(base_url, api_key): try: from openai import OpenAI except ImportError: sys.exit("ERROR: `openai` not installed here. Run under ~/wdir/llm/.venv") return OpenAI(base_url=base_url, api_key=api_key) def call(client, model, system, content, temperature, max_tokens): resp = client.chat.completions.create( model=model, messages=[{"role": "system", "content": system}, {"role": "user", "content": content}], temperature=temperature, max_tokens=max_tokens, ) return resp.choices[0].message.content.strip() def parse_need(raw, valid_ids): raw = re.sub(r"^```(?:json)?|```$", "", raw.strip(), flags=re.MULTILINE) m = re.search(r"\{.*\}", raw, re.DOTALL) if not m: return [] try: ids = json.loads(m.group(0)).get("need", []) except json.JSONDecodeError: return [] return [i for i in ids if i in valid_ids] def main(): p = argparse.ArgumentParser( description="Compile a long meeting transcript into a technical reference (refine + on-demand frames).", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__) p.add_argument("transcript", type=Path) p.add_argument("instruction", nargs="?", default="Compile a detailed technical reference of the workflows and architecture covered.") p.add_argument("-o", "--output", type=Path, help="write here (default: /_reference.md next to the transcript)") p.add_argument("--stdout", action="store_true", help="print to stdout instead of writing a file") p.add_argument("--base-url", default=DEFAULT_BASE_URL) p.add_argument("--model", default=DEFAULT_MODEL) p.add_argument("--api-key", default="local") p.add_argument("--frames", choices=["ondemand", "window", "none"], default="ondemand") p.add_argument("--window-tokens", type=int, default=3500, help="transcript tokens per refine step (default 3500)") p.add_argument("--max-tokens", type=int, default=8192, help="upper bound on generation; auto-capped to fit --ctx (default 8192)") p.add_argument("--ctx", type=int, default=16384, help="model context window; output is auto-capped so input+output fit (default 16384)") p.add_argument("--max-image-side", type=int, default=1280, help="downscale frames to this max side (0=off)") p.add_argument("--temperature", type=float, default=0.2) p.add_argument("--checkpoint", type=Path, help="write the running doc here after each window (resumable progress)") p.add_argument("-q", "--quiet", action="store_true") args = p.parse_args() if not args.transcript.is_file(): sys.exit(f"ERROR: transcript not found: {args.transcript}") def log(m): if not args.quiet: print(f"[compile] {m}", file=sys.stderr) windows = parse_windows(args.transcript, args.window_tokens) nframes = sum(len(w["frames"]) for w in windows) log(f"{len(windows)} windows, {nframes} frame refs, mode={args.frames}") client = make_client(args.base_url, args.api_key) doc = "# (compilation in progress)\n" for wi, w in enumerate(windows, 1): wanted = [] if args.frames != "none" and w["frames"]: if args.frames == "window": wanted = w["frames"] else: # ondemand: ask the model which frames it needs listing = "\n".join(f"- {f['id']} [{f['ts']}]" for f in w["frames"]) raw = call(client, args.model, TRIAGE_SYS.format(instruction=args.instruction), f"Window transcript:\n{w['text']}\n\nReferenced frames:\n{listing}", 0.0, 256) valid = {f["id"] for f in w["frames"]} keep = set(parse_need(raw, valid)) wanted = [f for f in w["frames"] if f["id"] in keep] # build the refine turn (multimodal if any frames wanted) text = (f"CURRENT REFERENCE:\n{doc}\n\n" f"NEXT TRANSCRIPT WINDOW:\n{w['text']}") if wanted: text += "\n\nAttached frames: " + ", ".join(f"{f['id']} [{f['ts']}]" for f in wanted) content = [{"type": "text", "text": text}] for f in wanted: ip = resolve_path(f["path"], args.transcript) if ip.exists(): content.append({"type": "image_url", "image_url": {"url": encode_image(ip, args.max_image_side)}}) log(f" window {wi}/{len(windows)}: {len(wanted)} frame(s) attached") else: content = text log(f" window {wi}/{len(windows)}: text-only") sys_txt = REFINE_SYS.format(instruction=args.instruction, rules=GROUNDING) in_tok = content_tokens(content) + estimate_tokens(sys_txt) out_budget = max(512, min(args.max_tokens, args.ctx - in_tok - 256)) if in_tok > args.ctx - 512: log(f" WARNING: doc+window ~{in_tok} tok ≥ ctx {args.ctx}; output will truncate — " f"use a 32k profile (qwen14b-gguf) or lower --window-tokens for the full training") doc = call(client, args.model, sys_txt, content, args.temperature, out_budget) if args.checkpoint: args.checkpoint.write_text(doc + "\n") if estimate_tokens(doc) > args.window_tokens * 4 and not args.quiet: log(f" note: running doc ~{estimate_tokens(doc)} tok and growing — " f"if it nears the context window, switch to a 32k-context profile (qwen14b-gguf)") if args.stdout: print(doc) else: out = args.output or default_output(args.transcript, "reference") out.write_text(doc + "\n") log(f"wrote {out}") if __name__ == "__main__": main()