From 7a2ee6abd2609fa741fbe3c2c5b04dd22dd0b692 Mon Sep 17 00:00:00 2001 From: Mariano Gabriel Date: Tue, 9 Jun 2026 11:10:10 -0300 Subject: [PATCH] add compile meeting, summarize. both for local llm run --- compile_meeting.py | 282 +++++++++++++++++++++++++++++++++++++++ summarize_meeting.py | 309 +++++++++++++++++++++++++++++++++++++++++++ transcribe_oneoff.sh | 121 +++++++++++++++++ 3 files changed, 712 insertions(+) create mode 100755 compile_meeting.py create mode 100755 summarize_meeting.py create mode 100755 transcribe_oneoff.sh diff --git a/compile_meeting.py b/compile_meeting.py new file mode 100755 index 0000000..1ce7bb8 --- /dev/null +++ b/compile_meeting.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +Compile a long meeting/training enhanced-transcript into a detailed technical +reference, using a LOCAL multimodal LLM that reads frames ON DEMAND. + +This is NOT summarization — it RETAINS workflow/architecture detail and +reorganizes it out of conversation order. It uses the REFINE pattern: walk the +transcript top-to-bottom in windows, carrying a running compiled document as the +only large context. The running doc IS the memory; the raw transcript is never +held whole (which is why a 4-hour recording fits a small model). + +Frames are consulted the way a human note-taker does: while reading each window, +the model decides which referenced frames it actually needs to see, and only +those images are attached (on demand) — webcam/transition frames cost nothing. + +Standalone on purpose: not wired into process_meeting.py. Talks to a local +OpenAI-compatible server (vLLM or llama.cpp — see ~/wdir/llm/serve.sh); --base-url +swaps it. + +Usage (start `~/wdir/llm/serve.sh qwen-vl` first, then): + ~/wdir/llm/.venv/bin/python compile_meeting.py \\ + output//_enhanced.txt \\ + "compile every deployment/data-flow workflow and the system architecture \\ + as a technical reference; note the [mm:ss] each was shown on screen" \\ + -o output//reference.md + +Frame modes: + --frames ondemand (default) two-step: model lists which frames it needs, + then only those are attached. Cheapest on vision tokens. + --frames window attach every frame in the current window; model uses the + relevant ones. Simpler, more tokens. + --frames none ignore frames entirely (text-only; for non-VL models / A-B). +""" +import argparse +import base64 +import json +import re +import sys +from pathlib import Path + +DEFAULT_BASE_URL = "http://localhost:11000/v1" +DEFAULT_MODEL = "Qwen/Qwen2.5-VL-7B-Instruct-AWQ" +CHARS_PER_TOKEN = 4.0 + +GROUNDING = """\ +Rules: +- Be faithful. Never invent names, components, commands, numbers, or steps. +- Preserve proper nouns and identifiers exactly as written. +- This is a COMPILATION, not a summary: keep technical detail (workflows step by + step, architecture components and how they connect, configs, commands, gotchas). +- Reorganize by TOPIC, not by conversation order. Merge new info into the right + existing section rather than appending chronologically. +- Anchor concrete items to the [mm:ss] where they were said/shown. +- If something is unclear or only partially stated, mark it (e.g. "(unclear)") + rather than guessing.""" + +REFINE_SYS = """\ +You maintain a growing TECHNICAL REFERENCE compiled from a training recording. + +The user's compilation instruction is authoritative: + +{instruction} + + +You are given the CURRENT REFERENCE so far and the NEXT WINDOW of transcript +(and possibly some screen frames). Integrate any new workflow/architecture detail +from this window into the reference, slotting it into the correct topical section +(create sections as needed). Return the COMPLETE updated reference in Markdown — +not a diff, not just the new part. Do not drop earlier content. + +{rules}""" + +TRIAGE_SYS = """\ +You are reading one window of a training transcript while compiling technical +notes per this instruction: + +{instruction} + + +The window references the screen frames listed below (id + [mm:ss]). Decide which +frames you would need to SEE to capture workflow/architecture/config detail the +text alone doesn't convey (diagrams, slides, terminal output, code). Ignore +webcam/transition frames. Reply with STRICT JSON only: + {{"need": ["", ...]}} +Empty list if none are needed.""" + +FRAME_RE = re.compile(r"Frame:\s+(\S+\.(?:jpg|jpeg|png))", re.IGNORECASE) +TS_RE = re.compile(r"\[(\d+):(\d+)\]") + + +def estimate_tokens(text): + return int(len(text) / CHARS_PER_TOKEN) + + +def default_output(transcript, kind): + """Write next to the transcript, in the same run folder, following the + pipeline's _ naming (e.g. training_reference.md).""" + stem = transcript.stem + if stem.endswith("_enhanced"): + stem = stem[: -len("_enhanced")] + return transcript.parent / f"{stem}_{kind}.md" + + +def parse_windows(path, window_tokens): + """Split the enhanced transcript into windows of ~window_tokens, packing + blank-line-separated blocks whole. Each window keeps the frame refs that + fall inside it: {text, frames:[{id, ts, path}]}.""" + raw = path.read_text() + blocks = re.split(r"\n\s*\n", raw) + windows, cur_text, cur_frames, cur_tok = [], [], [], 0 + + def flush(): + if cur_text: + windows.append({"text": "\n\n".join(cur_text), + "frames": list(cur_frames)}) + + last_ts = "00:00" + for block in blocks: + ts_m = TS_RE.search(block) + if ts_m: + last_ts = f"{ts_m.group(1)}:{ts_m.group(2)}" + fm = FRAME_RE.search(block) + bt = estimate_tokens(block) + if cur_text and cur_tok + bt > window_tokens: + flush() + cur_text, cur_frames, cur_tok = [], [], 0 + if fm: + p = fm.group(1) + cur_frames.append({"id": Path(p).stem, "ts": last_ts, "path": p}) + # keep a compact ref line in the text instead of the bare path + cur_text.append(f"[{last_ts}] (frame {Path(p).stem})") + else: + cur_text.append(block) + cur_tok += bt + flush() + return windows + + +def resolve_path(ref_path, transcript_path): + p = Path(ref_path) + if p.is_absolute() and p.exists(): + return p + # paths in the transcript are usually relative to the run dir + cand = transcript_path.parent / ref_path + return cand if cand.exists() else p + + +def encode_image(path, max_side): + data = path.read_bytes() + mime = "image/png" if path.suffix.lower() == ".png" else "image/jpeg" + if max_side: + try: + from PIL import Image + import io + img = Image.open(io.BytesIO(data)) + if max(img.size) > max_side: + img.thumbnail((max_side, max_side)) + buf = io.BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=85) + data, mime = buf.getvalue(), "image/jpeg" + except ImportError: + pass # PIL absent: send original (more tokens, still works) + b64 = base64.b64encode(data).decode() + return f"data:{mime};base64,{b64}" + + +def make_client(base_url, api_key): + try: + from openai import OpenAI + except ImportError: + sys.exit("ERROR: `openai` not installed here. Run under ~/wdir/llm/.venv") + return OpenAI(base_url=base_url, api_key=api_key) + + +def call(client, model, system, content, temperature, max_tokens): + resp = client.chat.completions.create( + model=model, + messages=[{"role": "system", "content": system}, + {"role": "user", "content": content}], + temperature=temperature, max_tokens=max_tokens, + ) + return resp.choices[0].message.content.strip() + + +def parse_need(raw, valid_ids): + raw = re.sub(r"^```(?:json)?|```$", "", raw.strip(), flags=re.MULTILINE) + m = re.search(r"\{.*\}", raw, re.DOTALL) + if not m: + return [] + try: + ids = json.loads(m.group(0)).get("need", []) + except json.JSONDecodeError: + return [] + return [i for i in ids if i in valid_ids] + + +def main(): + p = argparse.ArgumentParser( + description="Compile a long meeting transcript into a technical reference (refine + on-demand frames).", + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__) + p.add_argument("transcript", type=Path) + p.add_argument("instruction", nargs="?", + default="Compile a detailed technical reference of the workflows and architecture covered.") + p.add_argument("-o", "--output", type=Path, help="write here (default: /_reference.md next to the transcript)") + p.add_argument("--stdout", action="store_true", help="print to stdout instead of writing a file") + p.add_argument("--base-url", default=DEFAULT_BASE_URL) + p.add_argument("--model", default=DEFAULT_MODEL) + p.add_argument("--api-key", default="local") + p.add_argument("--frames", choices=["ondemand", "window", "none"], default="ondemand") + p.add_argument("--window-tokens", type=int, default=3500, help="transcript tokens per refine step (default 3500)") + p.add_argument("--max-tokens", type=int, default=8192, help="generation cap; must fit the growing doc (default 8192)") + p.add_argument("--max-image-side", type=int, default=1280, help="downscale frames to this max side (0=off)") + p.add_argument("--temperature", type=float, default=0.2) + p.add_argument("--checkpoint", type=Path, help="write the running doc here after each window (resumable progress)") + p.add_argument("-q", "--quiet", action="store_true") + args = p.parse_args() + + if not args.transcript.is_file(): + sys.exit(f"ERROR: transcript not found: {args.transcript}") + + def log(m): + if not args.quiet: + print(f"[compile] {m}", file=sys.stderr) + + windows = parse_windows(args.transcript, args.window_tokens) + nframes = sum(len(w["frames"]) for w in windows) + log(f"{len(windows)} windows, {nframes} frame refs, mode={args.frames}") + + client = make_client(args.base_url, args.api_key) + doc = "# (compilation in progress)\n" + + for wi, w in enumerate(windows, 1): + wanted = [] + if args.frames != "none" and w["frames"]: + if args.frames == "window": + wanted = w["frames"] + else: # ondemand: ask the model which frames it needs + listing = "\n".join(f"- {f['id']} [{f['ts']}]" for f in w["frames"]) + raw = call(client, args.model, + TRIAGE_SYS.format(instruction=args.instruction), + f"Window transcript:\n{w['text']}\n\nReferenced frames:\n{listing}", + 0.0, 256) + valid = {f["id"] for f in w["frames"]} + keep = set(parse_need(raw, valid)) + wanted = [f for f in w["frames"] if f["id"] in keep] + + # build the refine turn (multimodal if any frames wanted) + text = (f"CURRENT REFERENCE:\n{doc}\n\n" + f"NEXT TRANSCRIPT WINDOW:\n{w['text']}") + if wanted: + text += "\n\nAttached frames: " + ", ".join(f"{f['id']} [{f['ts']}]" for f in wanted) + content = [{"type": "text", "text": text}] + for f in wanted: + ip = resolve_path(f["path"], args.transcript) + if ip.exists(): + content.append({"type": "image_url", + "image_url": {"url": encode_image(ip, args.max_image_side)}}) + log(f" window {wi}/{len(windows)}: {len(wanted)} frame(s) attached") + else: + content = text + log(f" window {wi}/{len(windows)}: text-only") + + doc = call(client, args.model, + REFINE_SYS.format(instruction=args.instruction, rules=GROUNDING), + content, args.temperature, args.max_tokens) + + if args.checkpoint: + args.checkpoint.write_text(doc + "\n") + if estimate_tokens(doc) > args.window_tokens * 4 and not args.quiet: + log(f" note: running doc ~{estimate_tokens(doc)} tok and growing — " + f"if it nears the context window, switch to a 32k-context profile (qwen14b-gguf)") + + if args.stdout: + print(doc) + else: + out = args.output or default_output(args.transcript, "reference") + out.write_text(doc + "\n") + log(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/summarize_meeting.py b/summarize_meeting.py new file mode 100755 index 0000000..c292b4f --- /dev/null +++ b/summarize_meeting.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +Summarize / reformat a meeting's enhanced transcript with a LOCAL LLM. + +Standalone on purpose: this is NOT wired into process_meeting.py. The pipeline +(transcribe + frames + OCR) stays fully deterministic and offline; this is the +one non-deterministic, network-*capable* step, so it gets its own entry point. +By default it talks to a local vLLM OpenAI-compatible server (no cloud), but the +--base-url swap lets you point it at a company-sanctioned endpoint instead. + +The steering instruction is a first-class argument — pass any nuance you want +("focus on names and their roles", "read the closing signals", "reformat as a +decisions+action-items table, English output"). The instruction is threaded +into every stage (map, extract, reduce), not just the final synthesis, so the +per-chunk pass never discards the detail you asked to keep. + +Architecture (ports the "let the architecture carry correctness" rule to the +summarization failure mode — hallucinated names/facts + long-input drift): + 1. map — chunk the transcript, summarize each chunk under the instruction + 2. extract — emit a validated JSON of facts (participants/roles/decisions/...) + 3. reduce — write the final output from the validated facts + chunk notes + +Usage: + # start the local server first (`~/wdir/llm/serve.sh qwen7b`), then: + python summarize_meeting.py output//_enhanced.txt \\ + "focus on the names mentioned and their roles, output in English" \\ + -o output//summary_en.md + +Run it under the venv that has the `openai` client (e.g. ~/wdir/llm/.venv). +""" +import argparse +import json +import re +import sys +from pathlib import Path + + +DEFAULT_BASE_URL = "http://localhost:11000/v1" +DEFAULT_MODEL = "Qwen/Qwen2.5-7B-Instruct-AWQ" + +# Rough char->token heuristic so we don't need a tokenizer dependency. Mixed +# ES/EN prose lands around ~3.6 chars/token; 4.0 keeps us conservative (we +# under-fill rather than overflow the context window). +CHARS_PER_TOKEN = 4.0 + +# Shared rules injected into every stage. This is the hallucination guard: the +# model condenses what is present, it never invents — especially names/roles. +GROUNDING_RULES = """\ +Rules you must follow: +- Be faithful to the transcript. Never invent names, roles, numbers, or facts. +- If something is unclear or not stated, say so — do not guess. +- Preserve proper nouns exactly as written (people, companies, tools). +- When you state a concrete claim, anchor it to its [mm:ss] timestamp. +- The transcript is machine-generated and may contain ASR errors; prefer the + most consistent reading across the whole transcript over any single garbled + line, and flag a name/term you are unsure about rather than normalizing it + silently.""" + +MAP_SYSTEM = """\ +You are condensing ONE chunk of a longer meeting transcript. + +The user's instruction for the final output is: + +{instruction} + + +Produce dense, factual notes for THIS chunk that preserve everything relevant to +that instruction. Keep every name, role, decision, date, number, and notable +quote with its [mm:ss]. Do not write a polished summary yet — these notes are +raw material for a later synthesis pass, so keep detail over readability. + +{rules}""" + +EXTRACT_SYSTEM = """\ +Extract structured facts from the meeting notes/transcript as STRICT JSON only — +no prose, no markdown fences. Use exactly this schema; use [] or "" when unknown: + +{{ + "participants": [{{"name": "", "role": "", "org": "", "evidence_ts": ""}}], + "people_mentioned": [{{"name": "", "role": "", "org": "", "evidence_ts": ""}}], + "orgs": [{{"name": "", "what": ""}}], + "decisions": [{{"decision": "", "ts": ""}}], + "action_items": [{{"item": "", "owner": "", "due": "", "ts": ""}}], + "dates": [{{"what": "", "when": "", "ts": ""}}], + "key_quotes": [{{"speaker": "", "ts": "", "quote": ""}}], + "open_questions": [""] +}} + +Only include entries actually supported by the text. Distinguish participants +(present on the call) from people merely mentioned. This JSON is the source of +truth for names/roles in the final output, so be precise and do not invent. + +{rules}""" + +REDUCE_SYSTEM = """\ +You are writing the FINAL output of a meeting from (a) the user's instruction, +(b) a validated JSON of facts, and (c) per-chunk notes. + +The user's instruction is authoritative — follow it for focus, structure, and +language: + +{instruction} + + +Use the validated facts JSON as the source of truth for all names, roles, and +dates (the notes may contain ASR noise; the JSON has been checked). Write only +what the instruction asks for. Output clean Markdown. + +{rules}""" + + +def estimate_tokens(text: str) -> int: + return int(len(text) / CHARS_PER_TOKEN) + + +def default_output(transcript: Path, kind: str) -> Path: + """Write next to the transcript, in the same run folder, following the + pipeline's _ naming (e.g. keneth_aponte_summary.md).""" + stem = transcript.stem + if stem.endswith("_enhanced"): + stem = stem[: -len("_enhanced")] + return transcript.parent / f"{stem}_{kind}.md" + + +def load_transcript(path: Path, keep_frames: bool) -> str: + text = path.read_text() + if keep_frames: + return text + # Drop the "Frame: .jpg" noise lines and their "SCREEN CONTENT:" + # headers — a text-only model can't use a file path, and they waste tokens. + # Real OCR text (if any) does not match these patterns and is kept. + lines = text.splitlines() + out = [] + skip_next_blank = False + for line in lines: + if re.match(r"\s*\[\d+:\d+\]\s+SCREEN CONTENT:\s*$", line): + skip_next_blank = True + continue + if re.match(r"\s*Frame:\s+.*\.(jpg|jpeg|png)\s*$", line): + continue + out.append(line) + return "\n".join(out) + + +def chunk_transcript(text: str, chunk_tokens: int) -> list: + """Split on blank-line block boundaries, packing blocks up to chunk_tokens + so we never cut a speaker turn in half.""" + blocks = re.split(r"\n\s*\n", text) + chunks, cur, cur_tok = [], [], 0 + for block in blocks: + bt = estimate_tokens(block) + if cur and cur_tok + bt > chunk_tokens: + chunks.append("\n\n".join(cur)) + cur, cur_tok = [], 0 + cur.append(block) + cur_tok += bt + if cur: + chunks.append("\n\n".join(cur)) + return chunks + + +def make_client(base_url: str, api_key: str): + try: + from openai import OpenAI + except ImportError: + sys.exit( + "ERROR: the `openai` client is not installed in this interpreter.\n" + "Run this under the venv that has it, e.g.:\n" + " ~/wdir/llm/.venv/bin/python summarize_meeting.py ..." + ) + return OpenAI(base_url=base_url, api_key=api_key) + + +def call(client, model, system, user, temperature, max_tokens): + resp = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + temperature=temperature, + max_tokens=max_tokens, + ) + return resp.choices[0].message.content.strip() + + +def parse_json_lenient(raw: str): + """vLLM models sometimes wrap JSON in ``` fences or add a stray prefix.""" + raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw.strip(), flags=re.MULTILINE) + try: + return json.loads(raw) + except json.JSONDecodeError: + m = re.search(r"\{.*\}", raw, re.DOTALL) + if m: + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + pass + return None + + +def summarize(client, args, transcript: str) -> str: + rules = GROUNDING_RULES + instr = args.instruction + budget = args.chunk_tokens + + def log(msg): + if not args.quiet: + print(f"[summarize] {msg}", file=sys.stderr) + + total_tok = estimate_tokens(transcript) + single_pass = args.no_map_reduce or total_tok <= budget + + # --- map --------------------------------------------------------------- + if single_pass: + log(f"single-pass (~{total_tok} tok <= chunk budget {budget})") + notes = transcript + else: + chunks = chunk_transcript(transcript, budget) + log(f"map: {len(chunks)} chunks (~{total_tok} tok total)") + chunk_notes = [] + for i, ch in enumerate(chunks, 1): + log(f" map chunk {i}/{len(chunks)}") + note = call( + client, args.model, + MAP_SYSTEM.format(instruction=instr, rules=rules), + ch, args.temperature, args.max_tokens, + ) + chunk_notes.append(f"### Chunk {i} notes\n{note}") + notes = "\n\n".join(chunk_notes) + + # --- extract ----------------------------------------------------------- + facts_json = "{}" + if not args.no_extract: + log("extract: pulling structured facts") + raw = call( + client, args.model, + EXTRACT_SYSTEM.format(rules=rules), + notes if single_pass else notes + "\n\n" + transcript[: budget * 4], + 0.0, args.max_tokens, + ) + facts = parse_json_lenient(raw) + if facts is None: + log(" WARNING: extraction did not return valid JSON; continuing without it") + else: + facts_json = json.dumps(facts, ensure_ascii=False, indent=2) + if args.extract_only: + return facts_json + + # --- reduce ------------------------------------------------------------ + log("reduce: writing final output") + user = ( + f"VALIDATED FACTS (source of truth for names/roles/dates):\n{facts_json}\n\n" + f"NOTES / TRANSCRIPT:\n{notes}" + ) + return call( + client, args.model, + REDUCE_SYSTEM.format(instruction=instr, rules=rules), + user, args.temperature, args.max_tokens, + ) + + +def main(): + p = argparse.ArgumentParser( + description="Summarize/reformat a meeting enhanced-transcript with a local LLM.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + p.add_argument("transcript", type=Path, help="path to *_enhanced.txt (or any text)") + p.add_argument( + "instruction", nargs="?", default="Summarize this meeting clearly.", + help='steering instruction, e.g. "focus on names and roles, English output"', + ) + p.add_argument("-o", "--output", type=Path, help="write here (default: /_summary.md next to the transcript)") + p.add_argument("--stdout", action="store_true", help="print to stdout instead of writing a file") + p.add_argument("--base-url", default=DEFAULT_BASE_URL, help=f"OpenAI-compatible endpoint (default: {DEFAULT_BASE_URL})") + p.add_argument("--model", default=DEFAULT_MODEL, help=f"model id (default: {DEFAULT_MODEL})") + p.add_argument("--api-key", default="local", help="ignored by vLLM; set for a real provider") + p.add_argument("--chunk-tokens", type=int, default=6000, help="map-reduce chunk budget (default: 6000)") + p.add_argument("--max-tokens", type=int, default=4096, help="generation cap per call (default: 4096)") + p.add_argument("--temperature", type=float, default=0.2, help="sampling temperature (default: 0.2)") + p.add_argument("--no-map-reduce", action="store_true", help="force single-pass (short transcripts)") + p.add_argument("--no-extract", action="store_true", help="skip the structured-facts grounding pass") + p.add_argument("--extract-only", action="store_true", help="print only the extracted JSON facts and exit") + p.add_argument("--keep-frames", action="store_true", help="keep 'Frame: ' lines (default: strip them)") + p.add_argument("-q", "--quiet", action="store_true", help="suppress progress on stderr") + args = p.parse_args() + + if not args.transcript.is_file(): + sys.exit(f"ERROR: transcript not found: {args.transcript}") + + transcript = load_transcript(args.transcript, args.keep_frames) + if not transcript.strip(): + sys.exit("ERROR: transcript is empty after loading.") + + client = make_client(args.base_url, args.api_key) + result = summarize(client, args, transcript) + + if args.stdout: + print(result) + else: + out = args.output or default_output(args.transcript, "summary") + out.write_text(result + "\n") + if not args.quiet: + print(f"[summarize] wrote {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/transcribe_oneoff.sh b/transcribe_oneoff.sh new file mode 100755 index 0000000..ad17eae --- /dev/null +++ b/transcribe_oneoff.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# One-off "high-quality" transcription that overrides the cached transcript +# in the most recent run directory for the given video, then re-runs the +# merger so the enhanced transcript is regenerated using the existing frames. +# +# Usage: +# ./transcribe_oneoff.sh