embeded stream opengl

This commit is contained in:
2026-04-01 15:16:09 -03:00
parent 453601c072
commit bdc5705022
8 changed files with 407 additions and 328 deletions

View File

@@ -3,6 +3,8 @@ Thin wrapper around ffmpeg-python for building and running ffmpeg pipelines.
All ffmpeg command construction goes through this module so manager.py
and other consumers never build raw CLI arg lists.
Uses ffmpeg-python's own run/run_async for subprocess management.
"""
import logging
@@ -15,28 +17,51 @@ import ffmpeg
log = logging.getLogger(__name__)
GLOBAL_ARGS = ("-hide_banner", "-loglevel", "warning")
def receive_and_segment(stream_url, segment_dir, segment_duration=60):
"""Receive mpegts stream and save as segmented .ts files.
Returns an ffmpeg-python output node (not yet running).
def receive_to_pipe(stream_url, segment_dir=None, segment_duration=60):
"""Receive mpegts stream and pipe to stdout for mpv.
If segment_dir is provided, also saves segments to disk.
Uses pipe (not fifo) so OS kernel buffers prevent blocking.
"""
stream = ffmpeg.input(stream_url, fflags="nobuffer", flags="low_delay")
return ffmpeg.output(
stream,
str(segment_dir / "segment_%04d.ts"),
c="copy",
f="segment",
segment_time=segment_duration,
reset_timestamps=1,
out_pipe = ffmpeg.output(stream, "pipe:", c="copy", f="mpegts")
if segment_dir:
out_segments = ffmpeg.output(
stream,
str(segment_dir / "segment_%04d.ts"),
c="copy",
f="segment",
segment_time=segment_duration,
reset_timestamps=1,
)
return ffmpeg.merge_outputs(out_pipe, out_segments).global_args(*GLOBAL_ARGS)
return out_pipe.global_args(*GLOBAL_ARGS)
def receive_and_segment(stream_url, segment_dir, segment_duration=60):
"""Receive mpegts stream and save as segmented .ts files."""
stream = ffmpeg.input(stream_url, fflags="nobuffer", flags="low_delay")
return (
ffmpeg.output(
stream,
str(segment_dir / "segment_%04d.ts"),
c="copy",
f="segment",
segment_time=segment_duration,
reset_timestamps=1,
)
.global_args(*GLOBAL_ARGS)
)
def receive_and_segment_with_monitor(stream_url, segment_dir, fifo_path, segment_duration=60):
"""Receive stream, save segments AND tee to a named pipe for monitoring.
Returns an ffmpeg-python merged output node.
"""
"""Receive stream, save segments AND tee to a named pipe for monitoring."""
if not fifo_path.exists():
os.mkfifo(str(fifo_path))
@@ -58,7 +83,7 @@ def receive_and_segment_with_monitor(stream_url, segment_dir, fifo_path, segment
f="mpegts",
)
return ffmpeg.merge_outputs(out_segments, out_monitor)
return ffmpeg.merge_outputs(out_segments, out_monitor).global_args(*GLOBAL_ARGS)
def extract_scene_frames(input_path, output_dir, scene_threshold=0.3,
@@ -66,78 +91,57 @@ def extract_scene_frames(input_path, output_dir, scene_threshold=0.3,
"""Extract frames from a file on scene change.
Uses ffmpeg select filter with scene detection and a max-interval fallback.
Returns (process_result, stderr) for timestamp parsing.
Returns (stdout bytes, stderr bytes) for timestamp parsing.
"""
select_expr = (
f"gt(scene\\,{scene_threshold})"
f"+gte(t-prev_selected_t\\,{max_interval})"
f"gt(scene,{scene_threshold})"
f"+gte(t-prev_selected_t,{max_interval})"
)
stream = ffmpeg.input(str(input_path))
stream = stream.filter("select", select_expr).filter("showinfo")
output = ffmpeg.output(
stream,
str(output_dir / "F%04d.jpg"),
vsync="vfr",
**{"q:v": "2"},
start_number=start_number,
output = (
ffmpeg.output(
stream,
str(output_dir / "F%04d.jpg"),
vsync="vfr",
**{"q:v": "2"},
start_number=start_number,
)
.global_args(*GLOBAL_ARGS)
)
return run_sync(output, timeout=120)
log.info("extract_scene_frames: %s", " ".join(output.compile()))
stdout, stderr = output.run(capture_stdout=True, capture_stderr=True)
return stdout.decode("utf-8", errors="replace"), stderr.decode("utf-8", errors="replace")
def extract_audio_pcm(input_path):
"""Extract audio as 16kHz mono PCM wav, returning an output node for piping.
Use run_async with pipe_stdout=True to stream PCM data.
"""
"""Extract audio as 16kHz mono PCM wav, returning an output node for piping."""
stream = ffmpeg.input(str(input_path))
return ffmpeg.output(
stream.audio,
"pipe:",
vn=None,
acodec="pcm_s16le",
ar=16000,
ac=1,
f="wav",
return (
ffmpeg.output(
stream.audio,
"pipe:",
vn=None,
acodec="pcm_s16le",
ar=16000,
ac=1,
f="wav",
)
.global_args(*GLOBAL_ARGS)
)
def run_async(output_node, pipe_stdout=False, pipe_stderr=False):
"""Start an ffmpeg pipeline asynchronously. Returns subprocess.Popen."""
cmd = compile_cmd(output_node)
log.info("run_async: %s", " ".join(str(c) for c in cmd))
return subprocess.Popen(
cmd,
stdout=subprocess.PIPE if pipe_stdout else subprocess.DEVNULL,
stderr=subprocess.PIPE if pipe_stderr else subprocess.DEVNULL,
"""Start an ffmpeg pipeline asynchronously via ffmpeg-python's run_async."""
log.info("run_async: %s", " ".join(output_node.compile()))
return output_node.run_async(
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
)
def run_sync(output_node, timeout=None):
"""Run an ffmpeg pipeline synchronously. Returns (stdout, stderr) as strings."""
cmd = compile_cmd(output_node)
log.info("run_sync: %s", " ".join(str(c) for c in cmd))
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout,
)
return result.stdout, result.stderr
def compile_cmd(output_node):
"""Compile an ffmpeg-python node to a command list, adding global flags."""
cmd = output_node.compile()
# Insert global flags after 'ffmpeg'
idx = 1
for flag in ["-hide_banner", "-loglevel", "warning"]:
cmd.insert(idx, flag)
idx += 1
return cmd
def stop_proc(proc, timeout=5):
"""Gracefully stop an ffmpeg subprocess."""
if proc and proc.poll() is None:

View File

@@ -75,6 +75,20 @@ class StreamManager:
log.info("Recorder started: pid=%s url=%s", proc.pid, self.stream_url)
self._start_stderr_reader("recorder", proc)
def start_receiver_pipe(self):
"""Receive stream, pipe stdout to mpv, save segments to disk."""
self.setup_dirs()
node = ff.receive_to_pipe(
self.stream_url,
segment_dir=self.stream_dir,
segment_duration=SEGMENT_DURATION,
)
proc = ff.run_async(node, pipe_stdout=True, pipe_stderr=True)
self._procs["receiver"] = proc
log.info("Receiver started: pid=%s url=%s (pipe + segments)", proc.pid, self.stream_url)
self._start_stderr_reader("receiver", proc)
return proc
def start_recorder_with_monitor(self):
self.setup_dirs()
fifo_path = self.session_dir / "monitor.pipe"
@@ -104,6 +118,30 @@ class StreamManager:
t = Thread(target=_read, daemon=True, name=f"{name}_stderr")
t.start()
def start_frame_extractor_on_recording(self, recording_path):
"""Extract frames periodically from a growing recording file."""
log.info("Starting frame extractor on recording: %s", recording_path)
self._recording_path = recording_path
self._start_recording_frame_watcher()
def _start_recording_frame_watcher(self):
def _watch():
last_size = 0
log.info("Recording frame watcher running, watching %s", self._recording_path)
while "stop" not in self._stop_flags:
if self._recording_path.exists():
size = self._recording_path.stat().st_size
if size > last_size and size > 100_000: # wait for some data
log.info("Recording grew: %d -> %d bytes, extracting frames", last_size, size)
last_size = size
self._extract_frames_from_file(self._recording_path)
time.sleep(10) # check every 10s
log.info("Recording frame watcher stopped")
t = Thread(target=_watch, daemon=True, name="recording_frame_watcher")
t.start()
self._threads["recording_frame_watcher"] = t
def start_frame_extractor(self):
log.info("Starting frame watcher...")
self._start_frame_watcher()

View File

@@ -1,159 +1,100 @@
"""
MonitorWidget: embeds ffplay into the GTK4 window for live stream monitoring.
MonitorWidget: mpv-based live stream monitor embedded in GTK4.
On X11, uses the --wid flag of mpv for native embedding, or reparents
ffplay's window via xdotool. GTK4 dropped GtkSocket so we use X11-level tricks.
Uses libmpv's OpenGL render API + Gtk.GLArea for proper embedding.
No X11 wid hacks — renders directly to a GL texture in the GTK layout.
Works on both X11 and Wayland.
"""
import logging
import subprocess
import signal
from threading import Timer
import gi
gi.require_version("Gtk", "4.0")
from gi.repository import Gtk, GLib, Gdk
from cht.ui.mpv import Player
log = logging.getLogger(__name__)
class MonitorWidget(Gtk.Box):
"""Widget that embeds an ffplay/mpv window for live stream monitoring."""
"""Widget that embeds mpv video via OpenGL into the GTK4 layout."""
def __init__(self, **kwargs):
super().__init__(orientation=Gtk.Orientation.VERTICAL, **kwargs)
self._proc = None
self._xid = None
self._player = None
self._drawing_area = Gtk.DrawingArea()
self._drawing_area.set_hexpand(True)
self._drawing_area.set_vexpand(True)
self._drawing_area.set_content_height(250)
self.append(self._drawing_area)
log.info("MonitorWidget initialized")
self._gl_area = Gtk.GLArea()
self._gl_area.set_hexpand(True)
self._gl_area.set_vexpand(True)
self._gl_area.set_auto_render(False)
self._gl_area.set_has_depth_buffer(False)
self._gl_area.set_has_stencil_buffer(False)
def start_ffplay(self, input_path):
log.info("Starting ffplay with input: %s", input_path)
cmd = [
"ffplay",
"-fflags", "nobuffer",
"-flags", "low_delay",
"-framedrop",
"-noborder",
"-i", str(input_path),
]
log.info("ffplay cmd: %s", " ".join(cmd))
self._gl_area.connect("realize", self._on_realize)
self._gl_area.connect("unrealize", self._on_unrealize)
self._gl_area.connect("render", self._on_render)
self._proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
log.info("ffplay started: pid=%s", self._proc.pid)
self._start_stderr_log("ffplay")
Timer(1.0, self._try_reparent).start()
self.append(self._gl_area)
log.info("MonitorWidget initialized (GLArea)")
def start_mpv(self, input_path):
log.info("Starting mpv with input: %s", input_path)
surface = self._drawing_area.get_native().get_surface()
xid = None
if hasattr(surface, "get_xid"):
xid = surface.get_xid()
log.info("Got X11 window ID: %s", xid)
else:
log.warning("No get_xid on surface (type=%s), mpv will open separate window", type(surface).__name__)
cmd = [
"mpv",
"--no-terminal",
"--no-osc",
"--no-input-default-bindings",
"--profile=low-latency",
"--demuxer-lavf-o=fflags=+nobuffer",
]
if xid:
cmd.append(f"--wid={xid}")
cmd.append(str(input_path))
log.info("mpv cmd: %s", " ".join(cmd))
self._proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
log.info("mpv started: pid=%s", self._proc.pid)
self._start_stderr_log("mpv")
def _start_stderr_log(self, name):
"""Read stderr in background thread and log it."""
import threading
def _read():
try:
for line in self._proc.stderr:
text = line.decode("utf-8", errors="replace").rstrip()
if text:
log.info("[%s:stderr] %s", name, text)
except Exception as e:
log.warning("[%s:stderr] read error: %s", name, e)
retcode = self._proc.poll() if self._proc else None
log.info("[%s] process exited: code=%s", name, retcode)
t = threading.Thread(target=_read, daemon=True, name=f"{name}_stderr")
t.start()
def _try_reparent(self):
if self._proc is None or self._proc.poll() is not None:
log.warning("ffplay not running, cannot reparent")
def _on_realize(self, gl_area):
"""GL context is ready — initialize mpv's render context."""
log.info("GLArea realized")
gl_area.make_current()
if gl_area.get_error():
log.error("GLArea error: %s", gl_area.get_error())
return
log.info("Attempting to reparent ffplay window (pid=%s)", self._proc.pid)
try:
result = subprocess.run(
["xdotool", "search", "--pid", str(self._proc.pid)],
capture_output=True,
text=True,
timeout=5,
)
windows = result.stdout.strip().split("\n")
log.info("xdotool found windows: %s", windows)
if windows and windows[0]:
ffplay_wid = windows[0]
surface = self._drawing_area.get_native().get_surface()
if hasattr(surface, "get_xid"):
parent_xid = surface.get_xid()
log.info("Reparenting ffplay %s into %s", ffplay_wid, parent_xid)
subprocess.run(
["xdotool", "windowreparent", ffplay_wid, str(parent_xid)],
timeout=5,
)
subprocess.run(
["xdotool", "windowsize", ffplay_wid, "100%", "100%"],
timeout=5,
)
log.info("Reparenting done")
else:
log.warning("No get_xid on surface, cannot reparent")
else:
log.warning("No windows found for ffplay pid=%s", self._proc.pid)
except FileNotFoundError:
log.error("xdotool not found, cannot reparent ffplay window")
except subprocess.TimeoutExpired:
log.error("xdotool timed out")
def _on_unrealize(self, gl_area):
"""Clean up mpv render context."""
log.info("GLArea unrealized")
if self._player:
self._player.terminate()
self._player = None
def _on_render(self, gl_area, gl_context):
"""Render mpv's current frame to the GLArea."""
if not self._player:
return True
# Get the default FBO that GTK4 GLArea renders to
fbo = gl_area.get_buffer() if hasattr(gl_area, 'get_buffer') else 0
width = gl_area.get_width()
height = gl_area.get_height()
# GTK4 GLArea uses its own FBO, get it from GL state
import ctypes
fbo_id = ctypes.c_int(0)
gl = ctypes.cdll.LoadLibrary("libGL.so.1")
gl.glGetIntegerv(0x8CA6, ctypes.byref(fbo_id)) # GL_DRAW_FRAMEBUFFER_BINDING
self._player.render(fbo_id.value, width, height)
return True
def _on_mpv_update(self):
"""Called by mpv when a new frame is ready. Triggers re-render."""
GLib.idle_add(self._gl_area.queue_render)
def start_stream(self, source, record_path=None):
"""Start playing from a URL and optionally record to disk.
Args:
source: TCP URL (tcp://...), file path, etc.
record_path: if set, mpv dumps the raw stream to this file
"""
self._gl_area.make_current()
self._player = Player(record_path=record_path)
self._player.init_gl(update_callback=self._on_mpv_update)
self._player.play(source)
log.info("Monitor streaming from: %s (record=%s)", source, record_path)
def stop(self):
if self._proc and self._proc.poll() is None:
log.info("Stopping monitor process pid=%s", self._proc.pid)
self._proc.send_signal(signal.SIGINT)
try:
self._proc.wait(timeout=3)
log.info("Monitor process stopped gracefully")
except subprocess.TimeoutExpired:
self._proc.kill()
log.warning("Monitor process killed (did not stop in 3s)")
self._proc = None
"""Stop playback and release mpv."""
if self._player:
log.info("Stopping monitor")
self._player.terminate()
self._player = None
else:
log.info("Monitor process already stopped or never started")
log.info("Monitor already stopped")

160
cht/ui/mpv.py Normal file
View File

@@ -0,0 +1,160 @@
"""
MPV wrapper using python-mpv (libmpv bindings) with OpenGL render API.
Renders video frames to an OpenGL context provided by GTK4's GLArea.
No subprocess calls, no X11 wid hacks.
"""
import ctypes
import logging
import mpv as libmpv
log = logging.getLogger(__name__)
def _make_get_proc_address():
"""Create a ctypes callback for OpenGL function loading."""
libgl = ctypes.cdll.LoadLibrary("libGL.so.1")
libgl.glXGetProcAddressARB.restype = ctypes.c_void_p
libgl.glXGetProcAddressARB.argtypes = [ctypes.c_char_p]
@ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p)
def get_proc_address(_, name):
return libgl.glXGetProcAddressARB(name)
return get_proc_address
_get_proc_address = _make_get_proc_address()
class Player:
"""Wraps a libmpv player with OpenGL render context for GTK4 embedding."""
def __init__(self, record_path=None):
opts = {
"input_default_bindings": False,
"input_vo_keyboard": False,
"osc": False,
"profile": "low-latency",
"cache": "no",
"untimed": True,
"demuxer_thread": "no",
"demuxer_lavf_o": "fflags=+nobuffer",
"video_sync": "display-desync",
"vo": "libmpv",
"hwdec": "auto",
}
if record_path:
opts["stream_record"] = str(record_path)
log.info("mpv will record stream to: %s", record_path)
log.info("Creating mpv player (OpenGL render mode)")
self._player = libmpv.MPV(**opts)
self._ctx = None
self._update_callback = None
log.info("mpv player created")
def init_gl(self, update_callback):
"""Initialize the OpenGL render context.
Must be called from a thread with an active GL context (e.g. GLArea realize).
Args:
update_callback: called by mpv when a new frame is ready to render.
Should trigger a GLArea queue_render().
"""
self._update_callback = update_callback
self._ctx = libmpv.MpvRenderContext(
self._player,
"opengl",
opengl_init_params={
"get_proc_address": _get_proc_address,
},
)
# Keep reference to prevent GC of the ctypes callback
self._get_proc_address_ref = _get_proc_address
self._ctx.update_cb = self._on_mpv_update
log.info("mpv OpenGL render context initialized")
def _on_mpv_update(self):
"""Called by mpv from any thread when a new frame is available."""
if self._update_callback:
self._update_callback()
def render(self, fbo, width, height):
"""Render the current frame to the given OpenGL FBO.
Call from the GLArea render signal handler.
"""
if self._ctx:
self._ctx.render(
flip_y=True,
opengl_fbo={
"fbo": fbo,
"w": width,
"h": height,
},
)
def play(self, source):
"""Play from a file path or URL."""
log.info("mpv play: %s", source)
self._player.play(str(source))
def play_fd(self, fd):
"""Play from a raw file descriptor."""
source = f"fd://{fd}"
log.info("mpv play from fd: %s", source)
self._player.play(source)
def pause(self):
self._player.pause = True
def resume(self):
self._player.pause = False
@property
def paused(self):
return self._player.pause
def seek(self, seconds):
"""Seek to absolute position in seconds."""
self._player.seek(seconds, reference="absolute")
def seek_relative(self, seconds):
"""Seek relative to current position."""
self._player.seek(seconds, reference="relative")
def stop(self):
log.info("mpv stop")
self._player.stop()
def terminate(self):
log.info("mpv terminate")
try:
if self._ctx:
self._ctx.free()
self._ctx = None
self._player.terminate()
except Exception as e:
log.warning("mpv terminate error: %s", e)
@property
def idle(self):
return self._player.core_idle
@property
def duration(self):
try:
return self._player.duration
except Exception:
return None
@property
def time_pos(self):
try:
return self._player.time_pos
except Exception:
return None

View File

@@ -70,24 +70,23 @@ class ChtWindow(Adw.ApplicationWindow):
self._stream_mgr = StreamManager()
log.info("Session: %s", self._stream_mgr.session_id)
log.info("Session dir: %s", self._stream_mgr.session_dir)
self._stream_mgr.setup_dirs()
fifo_path = self._stream_mgr.start_recorder_with_monitor()
log.info("FIFO path: %s", fifo_path)
log.info("Stream URL: %s", self._stream_mgr.stream_url)
log.info("Recorder started, waiting for sender connection...")
# mpv listens on TCP directly (lowest latency, like def scripts)
# and records the raw stream to disk for frame extraction
stream_url = self._stream_mgr.stream_url
record_path = self._stream_mgr.stream_dir / "recording.ts"
log.info("Starting mpv on %s, recording to %s", stream_url, record_path)
self._monitor.start_stream(stream_url, record_path=record_path)
log.info("Monitor started, waiting for sender...")
self._monitor.start_mpv(fifo_path)
log.info("Monitor (mpv) started")
self._stream_mgr.start_frame_extractor()
self._stream_mgr.start_frame_extractor_on_recording(record_path)
log.info("Frame extractor started")
def _stop_stream(self):
log.info("Stopping stream...")
self._monitor.stop()
log.info("Monitor stopped")
if self._stream_mgr:
self._stream_mgr.stop_all()
log.info("Stream manager stopped")