mariano/mediaproc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chunker ui redo

Browse Source
This commit is contained in:
buenosairesam
2026-03-15 16:03:53 -03:00
parent d5a3372d6b
commit b40bd68411
62 changed files with 5460 additions and 1493 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
core/api/chunker_sse.py
View File

@@ -1,8 +1,10 @@
""" """
SSE endpoint for chunker pipeline events. SSE endpoint for chunker pipeline events.
Bridges gRPC StreamProgress to browser-native EventSource. Uses Redis as the event bus between Celery workers and the SSE stream.
GET /api/chunker/stream/{job_id} → text/event-stream Celery worker pushes events via core.events, SSE endpoint polls them.
GET /chunker/stream/{job_id} → text/event-stream
""" """
import asyncio import asyncio
@@ -14,46 +16,39 @@ from typing import AsyncGenerator
from fastapi import APIRouter from fastapi import APIRouter
from starlette.responses import StreamingResponse from starlette.responses import StreamingResponse
from core.events import poll_events
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/chunker", tags=["chunker"]) router = APIRouter(prefix="/chunker", tags=["chunker"])
async def _event_generator(job_id: str) -> AsyncGenerator[str, None]: async def _event_generator(job_id: str) -> AsyncGenerator[str, None]:
""" """
Generate SSE events by polling gRPC job state. Generate SSE events by polling Redis for chunk job events.
Yields server-sent events in the format:
event: <event_type>
data: <json_payload>
""" """
from core.rpc.server import _active_jobs cursor = 0
last_state = None
timeout = time.monotonic() + 600 # 10 min max timeout = time.monotonic() + 600 # 10 min max
while time.monotonic() < timeout: while time.monotonic() < timeout:
job_state = _active_jobs.get(job_id) events, cursor = poll_events(job_id, cursor)
if job_state is None: if not events:
# Job not found yet — may not have started
yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n" yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n"
await asyncio.sleep(0.5) await asyncio.sleep(0.1)
continue continue
# Only send if state changed for data in events:
if job_state != last_state: event_type = data.pop("event", "update")
last_state = dict(job_state) payload = {**data, "job_id": job_id}
event_type = job_state.get("status", "update")
yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n" yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n"
# End stream when job is terminal if event_type in ("pipeline_complete", "pipeline_error", "cancelled"):
if event_type in ("completed", "failed", "cancelled"):
yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n" yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n"
break return
await asyncio.sleep(0.2) await asyncio.sleep(0.05)
yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n" yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n"

54
core/api/graphql.py
View File

@@ -15,7 +15,9 @@ from strawberry.schema.config import StrawberryConfig
from strawberry.types import Info from strawberry.types import Info
from core.api.schema.graphql import ( from core.api.schema.graphql import (
CancelResultType,
ChunkJobType, ChunkJobType,
ChunkOutputFileType,
CreateChunkJobInput, CreateChunkJobInput,
CreateJobInput, CreateJobInput,
DeleteResultType, DeleteResultType,
@@ -26,7 +28,7 @@ from core.api.schema.graphql import (
TranscodePresetType, TranscodePresetType,
UpdateAssetInput, UpdateAssetInput,
) )
from core.storage import BUCKET_IN, list_objects from core.storage import BUCKET_IN, list_objects, upload_file
VIDEO_EXTS = {".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v"} VIDEO_EXTS = {".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v"}
AUDIO_EXTS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"} AUDIO_EXTS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"}
@@ -90,6 +92,25 @@ class Query:
def system_status(self, info: Info) -> SystemStatusType: def system_status(self, info: Info) -> SystemStatusType:
return SystemStatusType(status="ok", version="0.1.0") return SystemStatusType(status="ok", version="0.1.0")
@strawberry.field
def chunk_output_files(self, info: Info, job_id: str) -> List[ChunkOutputFileType]:
"""List output chunk files for a completed job from media/out/."""
from pathlib import Path
media_out = os.environ.get("MEDIA_OUT_DIR", "/app/media/out")
output_dir = Path(media_out) / "chunks" / job_id
if not output_dir.is_dir():
return []
return [
ChunkOutputFileType(
key=f.name,
size=f.stat().st_size,
url=f"/media/out/chunks/{job_id}/{f.name}",
)
for f in sorted(output_dir.iterdir())
if f.is_file()
]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Mutations # Mutations
@@ -100,8 +121,26 @@ class Query:
class Mutation: class Mutation:
@strawberry.mutation @strawberry.mutation
def scan_media_folder(self, info: Info) -> ScanResultType: def scan_media_folder(self, info: Info) -> ScanResultType:
import logging
from pathlib import Path
from core.db import create_asset, get_asset_filenames from core.db import create_asset, get_asset_filenames
logger = logging.getLogger(__name__)
# Sync local media/in/ files to MinIO (handles fresh installs / pruned volumes)
local_media = Path("/app/media/in")
if local_media.is_dir():
existing_keys = {o["key"] for o in list_objects(BUCKET_IN)}
for f in local_media.iterdir():
if f.is_file() and f.suffix.lower() in MEDIA_EXTS:
if f.name not in existing_keys:
try:
upload_file(str(f), BUCKET_IN, f.name)
logger.info("Uploaded %s to MinIO", f.name)
except Exception as e:
logger.warning("Failed to upload %s: %s", f.name, e)
objects = list_objects(BUCKET_IN, extensions=MEDIA_EXTS) objects = list_objects(BUCKET_IN, extensions=MEDIA_EXTS)
existing = get_asset_filenames() existing = get_asset_filenames()
@@ -284,6 +323,8 @@ class Mutation:
"num_workers": input.num_workers, "num_workers": input.num_workers,
"max_retries": input.max_retries, "max_retries": input.max_retries,
"processor_type": input.processor_type, "processor_type": input.processor_type,
"start_time": input.start_time,
"end_time": input.end_time,
} }
executor_mode = os.environ.get("MPR_EXECUTOR", "local") executor_mode = os.environ.get("MPR_EXECUTOR", "local")
@@ -320,6 +361,17 @@ class Mutation:
celery_task_id=celery_task_id, celery_task_id=celery_task_id,
) )
@strawberry.mutation
def cancel_chunk_job(self, info: Info, celery_task_id: str) -> CancelResultType:
"""Cancel a running chunk job by revoking its Celery task."""
try:
from admin.mpr.celery import app as celery_app
celery_app.control.revoke(celery_task_id, terminate=True, signal="SIGTERM")
return CancelResultType(ok=True, message="Task revoked")
except Exception as e:
return CancelResultType(ok=False, message=str(e))
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Schema # Schema

21
core/api/schema/graphql.py
View File

@@ -37,7 +37,7 @@ class MediaAssetType:
file_path: Optional[str] = None file_path: Optional[str] = None
status: Optional[str] = None status: Optional[str] = None
error_message: Optional[str] = None error_message: Optional[str] = None
file_size: Optional[int] = None file_size: Optional[float] = None
duration: Optional[float] = None duration: Optional[float] = None
video_codec: Optional[str] = None video_codec: Optional[str] = None
audio_codec: Optional[str] = None audio_codec: Optional[str] = None
@@ -205,3 +205,22 @@ class CreateChunkJobInput:
max_retries: int = 3 max_retries: int = 3
processor_type: str = "ffmpeg" processor_type: str = "ffmpeg"
priority: int = 0 priority: int = 0
start_time: Optional[float] = None
end_time: Optional[float] = None
@strawberry.type
class CancelResultType:
"""Result of cancelling a chunk job."""
ok: bool = False
message: Optional[str] = None
@strawberry.type
class ChunkOutputFileType:
"""A chunk output file in S3/MinIO with presigned download URL."""
key: str
size: int = 0
url: str = ""

23
core/chunker/chunker.py
View File

@@ -28,7 +28,13 @@ class Chunker:
chunk_duration: Duration of each chunk in seconds (default: 10.0) chunk_duration: Duration of each chunk in seconds (default: 10.0)
""" """
def __init__(self, file_path: str, chunk_duration: float = 10.0): def __init__(
self,
file_path: str,
chunk_duration: float = 10.0,
start_time: float | None = None,
end_time: float | None = None,
):
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
raise ChunkReadError(f"File not found: {file_path}") raise ChunkReadError(f"File not found: {file_path}")
if chunk_duration <= 0: if chunk_duration <= 0:
@@ -37,7 +43,16 @@ class Chunker:
self.file_path = file_path self.file_path = file_path
self.chunk_duration = chunk_duration self.chunk_duration = chunk_duration
self.file_size = os.path.getsize(file_path) self.file_size = os.path.getsize(file_path)
self.source_duration = self._probe_duration() full_duration = self._probe_duration()
# Apply time range
self.range_start = max(start_time or 0.0, 0.0)
self.range_end = min(end_time or full_duration, full_duration)
if self.range_start >= self.range_end:
raise ValueError(
f"Invalid range: start={self.range_start} >= end={self.range_end}"
)
self.source_duration = self.range_end - self.range_start
def _probe_duration(self) -> float: def _probe_duration(self) -> float:
"""Get source file duration via FFmpeg probe.""" """Get source file duration via FFmpeg probe."""
@@ -71,9 +86,9 @@ class Chunker:
""" """
total = self.expected_chunks total = self.expected_chunks
for sequence in range(total): for sequence in range(total):
start_time = sequence * self.chunk_duration start_time = self.range_start + sequence * self.chunk_duration
end_time = min( end_time = min(
start_time + self.chunk_duration, self.source_duration start_time + self.chunk_duration, self.range_end
) )
duration = end_time - start_time duration = end_time - start_time

39
core/chunker/pipeline.py
View File

@@ -57,6 +57,8 @@ class Pipeline:
queue_size: int = 10, queue_size: int = 10,
event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None,
output_dir: Optional[str] = None, output_dir: Optional[str] = None,
start_time: Optional[float] = None,
end_time: Optional[float] = None,
): ):
self.source = source self.source = source
self.chunk_duration = chunk_duration self.chunk_duration = chunk_duration
@@ -66,6 +68,8 @@ class Pipeline:
self.queue_size = queue_size self.queue_size = queue_size
self.event_callback = event_callback self.event_callback = event_callback
self.output_dir = output_dir self.output_dir = output_dir
self.start_time = start_time
self.end_time = end_time
def _emit(self, event_type: str, data: Dict[str, Any]) -> None: def _emit(self, event_type: str, data: Dict[str, Any]) -> None:
"""Emit an event if callback is registered.""" """Emit an event if callback is registered."""
@@ -92,6 +96,19 @@ class Pipeline:
finally: finally:
chunk_queue.close() chunk_queue.close()
def _monitor_progress(
self, start_time: float, file_size: int, stop_event: threading.Event
) -> None:
"""Monitor thread: emit pipeline_progress every 500ms."""
while not stop_event.is_set():
elapsed = time.monotonic() - start_time
mb = file_size / (1024 * 1024)
self._emit("pipeline_progress", {
"elapsed": round(elapsed, 2),
"throughput_mbps": round(mb / elapsed, 2) if elapsed > 0 else 0,
})
stop_event.wait(0.5)
def _write_manifest( def _write_manifest(
self, result: PipelineResult, source_duration: float self, result: PipelineResult, source_duration: float
) -> None: ) -> None:
@@ -146,7 +163,12 @@ class Pipeline:
try: try:
# Stage 1: Set up chunker (probes file for duration) # Stage 1: Set up chunker (probes file for duration)
chunker = Chunker(self.source, self.chunk_duration) chunker = Chunker(
self.source,
self.chunk_duration,
start_time=self.start_time,
end_time=self.end_time,
)
total_chunks = chunker.expected_chunks total_chunks = chunker.expected_chunks
if total_chunks == 0: if total_chunks == 0:
@@ -170,9 +192,18 @@ class Pipeline:
output_dir=self.output_dir, output_dir=self.output_dir,
) )
# Stage 3: Start workers, then produce chunks # Stage 3: Start workers, monitor, then produce chunks
pool.start() pool.start()
monitor_stop = threading.Event()
monitor = threading.Thread(
target=self._monitor_progress,
args=(start_time, chunker.file_size, monitor_stop),
name="progress-monitor",
daemon=True,
)
monitor.start()
producer = threading.Thread( producer = threading.Thread(
target=self._produce_chunks, target=self._produce_chunks,
args=(chunker, chunk_queue), args=(chunker, chunk_queue),
@@ -185,6 +216,10 @@ class Pipeline:
all_results = pool.wait() all_results = pool.wait()
producer.join(timeout=5.0) producer.join(timeout=5.0)
# Stop monitor
monitor_stop.set()
monitor.join(timeout=2.0)
# Stage 5: Collect results in order # Stage 5: Collect results in order
collector = ResultCollector(total_chunks) collector = ResultCollector(total_chunks)
for r in all_results: for r in all_results:

2
core/chunker/worker.py
View File

@@ -124,6 +124,7 @@ class Worker:
self._emit("chunk_processing", { self._emit("chunk_processing", {
"sequence": chunk.sequence, "sequence": chunk.sequence,
"state": "processing", "state": "processing",
"queue_size": self.chunk_queue.qsize(),
}) })
result = self._process_with_retry(chunk) result = self._process_with_retry(chunk)
@@ -135,6 +136,7 @@ class Worker:
"success": result.success, "success": result.success,
"processing_time": result.processing_time, "processing_time": result.processing_time,
"retries": result.retries, "retries": result.retries,
"queue_size": self.chunk_queue.qsize(),
}) })
self._emit("worker_status", {"state": "stopped"}) self._emit("worker_status", {"state": "stopped"})

40
core/events.py Normal file
View File

@@ -0,0 +1,40 @@
"""
Redis-based event bus for pipeline job progress.
Celery workers push events, SSE endpoints poll them.
Only depends on redis — safe to import from any context.
"""
import json
import os
import redis
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
def _get_redis():
return redis.from_url(REDIS_URL, decode_responses=True)
def push_event(job_id: str, event_type: str, data: dict) -> None:
"""Push an event to the Redis list for a job."""
r = _get_redis()
key = f"chunk_events:{job_id}"
event = json.dumps({"event": event_type, **data})
r.rpush(key, event)
r.expire(key, 3600)
def poll_events(job_id: str, cursor: int = 0) -> tuple[list[dict], int]:
"""Poll new events from Redis. Returns (events, new_cursor)."""
r = _get_redis()
key = f"chunk_events:{job_id}"
raw_events = r.lrange(key, cursor, -1)
parsed = []
for raw in raw_events:
try:
parsed.append(json.loads(raw))
except (json.JSONDecodeError, TypeError):
pass
return parsed, cursor + len(raw_events)

66
core/jobs/handlers/chunk.py
View File

@@ -2,22 +2,24 @@
ChunkHandler — job handler that wraps the chunker Pipeline. ChunkHandler — job handler that wraps the chunker Pipeline.
Downloads source from S3/MinIO, runs FFmpeg chunking pipeline, Downloads source from S3/MinIO, runs FFmpeg chunking pipeline,
uploads mp4 segments + manifest back to S3/MinIO. writes mp4 segments + manifest to media/out/chunks/{job_id}/.
Pushes real-time events to Redis for SSE consumption.
""" """
import logging import logging
import os import os
import shutil
import tempfile
from typing import Any, Callable, Dict, Optional from typing import Any, Callable, Dict, Optional
from core.events import push_event as push_chunk_event
from core.chunker import Pipeline from core.chunker import Pipeline
from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file from core.storage import BUCKET_IN, download_to_temp
from .base import Handler from .base import Handler
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MEDIA_OUT_DIR = os.environ.get("MEDIA_OUT_DIR", "/app/media/out")
class ChunkHandler(Handler): class ChunkHandler(Handler):
""" """
@@ -44,14 +46,19 @@ class ChunkHandler(Handler):
logger.info(f"ChunkHandler starting job {job_id}: {source_key}") logger.info(f"ChunkHandler starting job {job_id}: {source_key}")
# Download source from S3/MinIO # Download source from S3/MinIO
push_chunk_event(job_id, "pipeline_start", {"status": "downloading", "source_key": source_key})
tmp_source = download_to_temp(BUCKET_IN, source_key) tmp_source = download_to_temp(BUCKET_IN, source_key)
# Create temp output directory for chunks # Output directory: media/out/chunks/{job_id}/
tmp_output_dir = tempfile.mkdtemp(prefix=f"chunks-{job_id}-") output_dir = os.path.join(MEDIA_OUT_DIR, "chunks", job_id)
if processor_type == "ffmpeg":
os.makedirs(output_dir, exist_ok=True)
try: try:
def event_bridge(event_type: str, data: Dict[str, Any]) -> None: def event_bridge(event_type: str, data: Dict[str, Any]) -> None:
"""Bridge pipeline events to the job progress callback.""" """Bridge pipeline events to Redis + optional progress callback."""
push_chunk_event(job_id, event_type, data)
if progress_callback and event_type == "pipeline_complete": if progress_callback and event_type == "pipeline_complete":
progress_callback(100, data) progress_callback(100, data)
elif progress_callback and event_type == "chunk_done": elif progress_callback and event_type == "chunk_done":
@@ -68,29 +75,28 @@ class ChunkHandler(Handler):
processor_type=processor_type, processor_type=processor_type,
queue_size=payload.get("queue_size", 10), queue_size=payload.get("queue_size", 10),
event_callback=event_bridge, event_callback=event_bridge,
output_dir=tmp_output_dir if processor_type == "ffmpeg" else None, output_dir=output_dir if processor_type == "ffmpeg" else None,
start_time=payload.get("start_time"),
end_time=payload.get("end_time"),
) )
result = pipeline.run() result = pipeline.run()
# Upload chunks + manifest to S3/MinIO # Files are already in media/out/chunks/{job_id}/
output_prefix = f"chunks/{job_id}" output_prefix = f"chunks/{job_id}"
uploaded_files = [] output_files = [
f"{output_prefix}/{os.path.basename(f)}"
for f in result.chunk_files
]
for chunk_file in result.chunk_files: push_chunk_event(job_id, "pipeline_complete", {
filename = os.path.basename(chunk_file) "status": "completed",
output_key = f"{output_prefix}/{filename}" "total_chunks": result.total_chunks,
upload_file(chunk_file, BUCKET_OUT, output_key) "processed": result.processed,
uploaded_files.append(output_key) "failed": result.failed,
logger.info(f"Uploaded {output_key}") "elapsed": result.elapsed_time,
"throughput_mbps": result.throughput_mbps,
# Upload manifest })
manifest_path = os.path.join(tmp_output_dir, "manifest.json")
if os.path.exists(manifest_path):
manifest_key = f"{output_prefix}/manifest.json"
upload_file(manifest_path, BUCKET_OUT, manifest_key)
uploaded_files.append(manifest_key)
logger.info(f"Uploaded {manifest_key}")
return { return {
"status": "completed" if result.failed == 0 else "completed_with_errors", "status": "completed" if result.failed == 0 else "completed_with_errors",
@@ -104,16 +110,16 @@ class ChunkHandler(Handler):
"errors": result.errors, "errors": result.errors,
"chunks_in_order": result.chunks_in_order, "chunks_in_order": result.chunks_in_order,
"output_prefix": output_prefix, "output_prefix": output_prefix,
"uploaded_files": uploaded_files, "output_files": output_files,
} }
except Exception as e:
push_chunk_event(job_id, "pipeline_error", {"status": "failed", "error": str(e)})
raise
finally: finally:
# Cleanup temp files # Cleanup temp source file only (output dir is persistent)
try: try:
os.unlink(tmp_source) os.unlink(tmp_source)
except OSError: except OSError:
pass pass
try:
shutil.rmtree(tmp_output_dir, ignore_errors=True)
except OSError:
pass

22
core/rpc/protos/worker.proto
View File

@@ -11,6 +11,7 @@ service WorkerService {
rpc StreamProgress(ProgressRequest) returns (stream ProgressUpdate); rpc StreamProgress(ProgressRequest) returns (stream ProgressUpdate);
rpc CancelJob(CancelRequest) returns (CancelResponse); rpc CancelJob(CancelRequest) returns (CancelResponse);
rpc GetWorkerStatus(Empty) returns (WorkerStatus); rpc GetWorkerStatus(Empty) returns (WorkerStatus);
rpc StreamChunkPipeline(ChunkStreamRequest) returns (stream ChunkPipelineEvent);
} }
message JobRequest { message JobRequest {
@@ -62,3 +63,24 @@ message WorkerStatus {
message Empty { message Empty {
// Empty // Empty
} }
message ChunkStreamRequest {
string job_id = 1;
}
message ChunkPipelineEvent {
string job_id = 1;
string event_type = 2;
int32 sequence = 3;
string worker_id = 4;
string state = 5;
int32 queue_size = 6;
float elapsed = 7;
float throughput_mbps = 8;
int32 total_chunks = 9;
int32 processed_chunks = 10;
int32 failed_chunks = 11;
string error = 12;
float processing_time = 13;
int32 retries = 14;
}

37
core/rpc/server.py
View File

@@ -173,6 +173,43 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer):
message="Job not found", message="Job not found",
) )
def StreamChunkPipeline(self, request, context) -> Iterator[worker_pb2.ChunkPipelineEvent]:
"""Stream chunk pipeline events for a job."""
from core.events import poll_events
job_id = request.job_id
logger.info(f"StreamChunkPipeline: {job_id}")
cursor = 0
timeout = time.monotonic() + 600 # 10 min max
while context.is_active() and time.monotonic() < timeout:
events, cursor = poll_events(job_id, cursor)
for data in events:
event_type = data.pop("event", "")
yield worker_pb2.ChunkPipelineEvent(
job_id=job_id,
event_type=event_type,
sequence=data.get("sequence", 0),
worker_id=data.get("worker_id", ""),
state=data.get("state", ""),
queue_size=data.get("queue_size", 0),
elapsed=data.get("elapsed", 0.0),
throughput_mbps=data.get("throughput_mbps", 0.0),
total_chunks=data.get("total_chunks", 0),
processed_chunks=data.get("processed_chunks", 0),
failed_chunks=data.get("failed_chunks", 0),
error=data.get("error", ""),
processing_time=data.get("processing_time", 0.0),
retries=data.get("retries", 0),
)
if event_type in ("pipeline_complete", "pipeline_error"):
return
time.sleep(0.05)
def GetWorkerStatus(self, request, context): def GetWorkerStatus(self, request, context):
"""Get worker health and capabilities.""" """Get worker health and capabilities."""
try: try:

10
core/rpc/worker_pb2.py
View File

@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty2\x9e\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatusb\x06proto3') DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty\"$\n\x12\x43hunkStreamRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\xaa\x02\n\x12\x43hunkPipelineEvent\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x12\n\nevent_type\x18\x02 \x01(\t\x12\x10\n\x08sequence\x18\x03 \x01(\x05\x12\x11\n\tworker_id\x18\x04 \x01(\t\x12\r\n\x05state\x18\x05 \x01(\t\x12\x12\n\nqueue_size\x18\x06 \x01(\x05\x12\x0f\n\x07\x65lapsed\x18\x07 \x01(\x02\x12\x17\n\x0fthroughput_mbps\x18\x08 \x01(\x02\x12\x14\n\x0ctotal_chunks\x18\t \x01(\x05\x12\x18\n\x10processed_chunks\x18\n \x01(\x05\x12\x15\n\rfailed_chunks\x18\x0b \x01(\x05\x12\r\n\x05\x65rror\x18\x0c \x01(\t\x12\x17\n\x0fprocessing_time\x18\r \x01(\x02\x12\x0f\n\x07retries\x18\x0e \x01(\x05\x32\xf7\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatus\x12W\n\x13StreamChunkPipeline\x12\x1e.mpr.worker.ChunkStreamRequest\x1a\x1e.mpr.worker.ChunkPipelineEvent0\x01\x62\x06proto3')
_globals = globals() _globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -47,6 +47,10 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals['_WORKERSTATUS']._serialized_end=664 _globals['_WORKERSTATUS']._serialized_end=664
_globals['_EMPTY']._serialized_start=666 _globals['_EMPTY']._serialized_start=666
_globals['_EMPTY']._serialized_end=673 _globals['_EMPTY']._serialized_end=673
_globals['_WORKERSERVICE']._serialized_start=676 _globals['_CHUNKSTREAMREQUEST']._serialized_start=675
_globals['_WORKERSERVICE']._serialized_end=962 _globals['_CHUNKSTREAMREQUEST']._serialized_end=711
_globals['_CHUNKPIPELINEEVENT']._serialized_start=714
_globals['_CHUNKPIPELINEEVENT']._serialized_end=1012
_globals['_WORKERSERVICE']._serialized_start=1015
_globals['_WORKERSERVICE']._serialized_end=1390
# @@protoc_insertion_point(module_scope) # @@protoc_insertion_point(module_scope)

45
core/rpc/worker_pb2_grpc.py
View File

@@ -5,7 +5,7 @@ import warnings
from . import worker_pb2 as worker__pb2 from . import worker_pb2 as worker__pb2
GRPC_GENERATED_VERSION = '1.76.0' GRPC_GENERATED_VERSION = '1.78.0'
GRPC_VERSION = grpc.__version__ GRPC_VERSION = grpc.__version__
_version_not_supported = False _version_not_supported = False
@@ -54,6 +54,11 @@ class WorkerServiceStub(object):
request_serializer=worker__pb2.Empty.SerializeToString, request_serializer=worker__pb2.Empty.SerializeToString,
response_deserializer=worker__pb2.WorkerStatus.FromString, response_deserializer=worker__pb2.WorkerStatus.FromString,
_registered_method=True) _registered_method=True)
self.StreamChunkPipeline = channel.unary_stream(
'/mpr.worker.WorkerService/StreamChunkPipeline',
request_serializer=worker__pb2.ChunkStreamRequest.SerializeToString,
response_deserializer=worker__pb2.ChunkPipelineEvent.FromString,
_registered_method=True)
class WorkerServiceServicer(object): class WorkerServiceServicer(object):
@@ -83,6 +88,12 @@ class WorkerServiceServicer(object):
context.set_details('Method not implemented!') context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!') raise NotImplementedError('Method not implemented!')
def StreamChunkPipeline(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_WorkerServiceServicer_to_server(servicer, server): def add_WorkerServiceServicer_to_server(servicer, server):
rpc_method_handlers = { rpc_method_handlers = {
@@ -106,6 +117,11 @@ def add_WorkerServiceServicer_to_server(servicer, server):
request_deserializer=worker__pb2.Empty.FromString, request_deserializer=worker__pb2.Empty.FromString,
response_serializer=worker__pb2.WorkerStatus.SerializeToString, response_serializer=worker__pb2.WorkerStatus.SerializeToString,
), ),
'StreamChunkPipeline': grpc.unary_stream_rpc_method_handler(
servicer.StreamChunkPipeline,
request_deserializer=worker__pb2.ChunkStreamRequest.FromString,
response_serializer=worker__pb2.ChunkPipelineEvent.SerializeToString,
),
} }
generic_handler = grpc.method_handlers_generic_handler( generic_handler = grpc.method_handlers_generic_handler(
'mpr.worker.WorkerService', rpc_method_handlers) 'mpr.worker.WorkerService', rpc_method_handlers)
@@ -224,3 +240,30 @@ class WorkerService(object):
timeout, timeout,
metadata, metadata,
_registered_method=True) _registered_method=True)
@staticmethod
def StreamChunkPipeline(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_stream(
request,
target,
'/mpr.worker.WorkerService/StreamChunkPipeline',
worker__pb2.ChunkStreamRequest.SerializeToString,
worker__pb2.ChunkPipelineEvent.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)

4
core/schema/modelgen.json
View File

@@ -13,8 +13,8 @@
}, },
{ {
"target": "typescript", "target": "typescript",
"output": "ui/timeline/src/types.ts", "output": "ui/common/types/generated.ts",
"include": ["dataclasses", "enums", "api"] "include": ["dataclasses", "enums", "api", "views"]
}, },
{ {
"target": "protobuf", "target": "protobuf",

16
core/schema/models/__init__.py
View File

@@ -16,6 +16,8 @@ from .grpc import (
GRPC_SERVICE, GRPC_SERVICE,
CancelRequest, CancelRequest,
CancelResponse, CancelResponse,
ChunkPipelineEvent,
ChunkStreamRequest,
Empty, Empty,
JobRequest, JobRequest,
JobResponse, JobResponse,
@@ -26,6 +28,7 @@ from .grpc import (
from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob
from .media import AssetStatus, MediaAsset from .media import AssetStatus, MediaAsset
from .presets import BUILTIN_PRESETS, TranscodePreset from .presets import BUILTIN_PRESETS, TranscodePreset
from .views import ChunkEvent, ChunkOutputFile, PipelineStats, WorkerEvent
# Core domain models - generates Django, Pydantic, TypeScript # Core domain models - generates Django, Pydantic, TypeScript
DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob] DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob]
@@ -44,6 +47,9 @@ API_MODELS = [
# Status enums - included in generated code # Status enums - included in generated code
ENUMS = [AssetStatus, JobStatus, ChunkJobStatus] ENUMS = [AssetStatus, JobStatus, ChunkJobStatus]
# View/event models - generates TypeScript for UI consumption
VIEWS = [ChunkEvent, WorkerEvent, PipelineStats, ChunkOutputFile]
# gRPC messages - generates Proto # gRPC messages - generates Proto
GRPC_MESSAGES = [ GRPC_MESSAGES = [
JobRequest, JobRequest,
@@ -54,6 +60,8 @@ GRPC_MESSAGES = [
CancelResponse, CancelResponse,
WorkerStatus, WorkerStatus,
Empty, Empty,
ChunkStreamRequest,
ChunkPipelineEvent,
] ]
__all__ = [ __all__ = [
@@ -82,10 +90,18 @@ __all__ = [
"CancelResponse", "CancelResponse",
"WorkerStatus", "WorkerStatus",
"Empty", "Empty",
"ChunkStreamRequest",
"ChunkPipelineEvent",
# Views
"ChunkEvent",
"WorkerEvent",
"PipelineStats",
"ChunkOutputFile",
# For generator # For generator
"DATACLASSES", "DATACLASSES",
"API_MODELS", "API_MODELS",
"ENUMS", "ENUMS",
"VIEWS",
"GRPC_MESSAGES", "GRPC_MESSAGES",
"BUILTIN_PRESETS", "BUILTIN_PRESETS",
] ]

33
core/schema/models/grpc.py
View File

@@ -41,6 +41,13 @@ class CancelRequest:
job_id: str job_id: str
@dataclass
class ChunkStreamRequest:
"""Request to stream chunk pipeline events."""
job_id: str
@dataclass @dataclass
class Empty: class Empty:
"""Empty message for requests with no parameters.""" """Empty message for requests with no parameters."""
@@ -94,6 +101,26 @@ class WorkerStatus:
gpu_available: bool gpu_available: bool
@dataclass
class ChunkPipelineEvent:
"""Streaming chunk pipeline event."""
job_id: str
event_type: str # pipeline_start, chunk_queued, chunk_done, etc.
sequence: int = 0
worker_id: str = ""
state: str = ""
queue_size: int = 0
elapsed: float = 0.0
throughput_mbps: float = 0.0
total_chunks: int = 0
processed_chunks: int = 0
failed_chunks: int = 0
error: str = ""
processing_time: float = 0.0
retries: int = 0
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Service Definition (for documentation, generator uses this) # Service Definition (for documentation, generator uses this)
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -126,5 +153,11 @@ GRPC_SERVICE = {
"response": WorkerStatus, "response": WorkerStatus,
"stream_response": False, "stream_response": False,
}, },
{
"name": "StreamChunkPipeline",
"request": ChunkStreamRequest,
"response": ChunkPipelineEvent,
"stream_response": True, # Server streaming
},
], ],
} }

57
core/schema/models/views.py Normal file
View File

@@ -0,0 +1,57 @@
"""
View/Event Schema Definitions
Projections of domain models for UI consumption via SSE events.
These reference existing schema types (e.g., ChunkJobStatus) to maintain
type-level dependencies — if the domain model changes, views update too.
"""
from dataclasses import dataclass
from typing import Optional
@dataclass
class ChunkEvent:
"""SSE event for a single chunk's lifecycle."""
sequence: int
status: str
size: Optional[int] = None
worker_id: Optional[str] = None
processing_time: Optional[float] = None
error: Optional[str] = None
retries: int = 0
@dataclass
class WorkerEvent:
"""SSE event for worker state changes."""
worker_id: str
state: str
current_chunk: Optional[int] = None
processed: int = 0
errors: int = 0
retries: int = 0
@dataclass
class PipelineStats:
"""Aggregate pipeline statistics, updated via SSE."""
total_chunks: int = 0
processed: int = 0
failed: int = 0
retries: int = 0
elapsed: float = 0.0
throughput_mbps: float = 0.0
queue_size: int = 0
@dataclass
class ChunkOutputFile:
"""A chunk output file in S3/MinIO with presigned download URL."""
key: str
size: int = 0
url: str = ""

17
ctrl/docker-compose.yml
View File

@@ -89,6 +89,15 @@ services:
mc anonymous set download local/mpr-media-in mc anonymous set download local/mpr-media-in
mc anonymous set download local/mpr-media-out mc anonymous set download local/mpr-media-out
envoy:
image: envoyproxy/envoy:v1.28-latest
ports:
- "8090:8090"
volumes:
- ./envoy.yaml:/etc/envoy/envoy.yaml:ro
depends_on:
- grpc
nginx: nginx:
image: nginx:alpine image: nginx:alpine
ports: ports:
@@ -96,12 +105,14 @@ services:
volumes: volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro - ./nginx.conf:/etc/nginx/nginx.conf:ro
- ./landing.html:/etc/nginx/landing.html:ro - ./landing.html:/etc/nginx/landing.html:ro
- ../media/out:/app/media/out:ro
depends_on: depends_on:
- django - django
- fastapi - fastapi
- timeline - timeline
- chunker - chunker
- minio - minio
- envoy
# ============================================================================= # =============================================================================
# Application Services # Application Services
@@ -139,7 +150,7 @@ services:
build: build:
context: .. context: ..
dockerfile: ctrl/Dockerfile.worker dockerfile: ctrl/Dockerfile.worker
command: celery -A admin.mpr worker -l info -Q transcode -c 2 command: celery -A admin.mpr worker -l info -Q celery,transcode -c 2
environment: environment:
<<: *common-env <<: *common-env
MPR_EXECUTOR: local MPR_EXECUTOR: local
@@ -163,6 +174,8 @@ services:
VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-} VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-}
volumes: volumes:
- ../ui/timeline/src:/app/src - ../ui/timeline/src:/app/src
- ../ui/timeline/vite.config.ts:/app/vite.config.ts
- ../ui/common:/common
chunker: chunker:
build: build:
@@ -174,6 +187,8 @@ services:
VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-} VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-}
volumes: volumes:
- ../ui/chunker/src:/app/src - ../ui/chunker/src:/app/src
- ../ui/chunker/vite.config.ts:/app/vite.config.ts
- ../ui/common:/common
volumes: volumes:
postgres-data: postgres-data:

64
ctrl/envoy.yaml Normal file
View File

@@ -0,0 +1,64 @@
admin:
address:
socket_address: { address: 0.0.0.0, port_value: 9901 }
static_resources:
listeners:
- name: listener_0
address:
socket_address: { address: 0.0.0.0, port_value: 8090 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
codec_type: auto
stat_prefix: ingress_http
route_config:
name: local_route
virtual_hosts:
- name: local_service
domains: ["*"]
routes:
- match: { prefix: "/" }
route:
cluster: grpc_service
timeout: 600s
max_stream_duration:
grpc_timeout_header_max: 600s
cors:
allow_origin_string_match:
- prefix: "*"
allow_methods: GET, PUT, DELETE, POST, OPTIONS
allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout
expose_headers: grpc-status,grpc-message
max_age: "1728000"
http_filters:
- name: envoy.filters.http.grpc_web
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
- name: envoy.filters.http.cors
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
clusters:
- name: grpc_service
connect_timeout: 5s
type: logical_dns
lb_policy: round_robin
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options: {}
load_assignment:
cluster_name: grpc_service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: grpc
port_value: 50051

9
ctrl/generate.sh
View File

@@ -19,4 +19,13 @@ python -m grpc_tools.protoc \
# Fix relative import in generated grpc stub # Fix relative import in generated grpc stub
sed -i 's/^import worker_pb2/from . import worker_pb2/' core/rpc/worker_pb2_grpc.py sed -i 's/^import worker_pb2/from . import worker_pb2/' core/rpc/worker_pb2_grpc.py
# Generate TypeScript gRPC-Web client from proto
echo "Generating TypeScript gRPC-Web client..."
cd ui/chunker
npx protoc \
--ts_out ../common/api/grpc \
--proto_path ../../core/rpc/protos \
worker.proto
cd ../..
echo "Done!" echo "Done!"

24
ctrl/nginx.conf
View File

@@ -29,6 +29,10 @@ http {
server minio:9000; server minio:9000;
} }
upstream envoy {
server envoy:8090;
}
server { server {
listen 80; listen 80;
server_name mpr.local.ar; server_name mpr.local.ar;
@@ -106,8 +110,24 @@ http {
} }
location /media/out/ { location /media/out/ {
proxy_pass http://minio/mpr-media-out/; alias /app/media/out/;
proxy_set_header Host $http_host; autoindex on;
}
# gRPC-Web proxy via Envoy
location /grpc-web/ {
proxy_pass http://envoy/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 600s;
# Critical for streaming: disable nginx response buffering
proxy_buffering off;
proxy_cache off;
chunked_transfer_encoding on;
} }
} }
} }

290
docs/architecture/05-chunker-pipeline.md Normal file
View File

@@ -0,0 +1,290 @@
# Chunker Pipeline — Execution Path
## Overview
The chunker pipeline splits a media file into time-based segments using FFmpeg stream-copy. Events flow from worker threads through Redis and gRPC-Web streaming to the browser UI in real time.
**7 hops from worker thread to pixel:**
```
Worker thread → Pipeline._emit() → event_bridge() → Redis RPUSH
→ [50ms poll] gRPC server LRANGE → yield protobuf
→ HTTP/2 frame → Envoy (grpc-web filter)
→ HTTP/1.1 chunk → nginx (proxy_buffering off)
→ fetch ReadableStream → protobuf-ts decode
→ setEvents([...prev, evt]) → React re-render
```
---
## Step 1: Job Creation (Browser → GraphQL → Celery)
```
User clicks "Start"
→ App.tsx: handleStart(config)
→ api.ts: createChunkJob(config)
→ POST /graphql (nginx :80 → fastapi:8702)
→ graphql.py: Mutation.create_chunk_job()
→ core.db: creates ChunkJob row in Postgres
→ Celery: run_job.delay(job_type="chunk", job_id=..., payload=...)
→ Returns { id, celery_task_id } to browser
→ App.tsx: setJobId(id) — triggers gRPC stream subscription
```
**Files:** `ui/chunker/src/api.ts`, `core/api/graphql.py`, `core/jobs/task.py`
---
## Step 2: gRPC-Web Stream (Browser → nginx → Envoy → gRPC Server)
Once `jobId` is set, `useGrpcStream(jobId)` opens a server-streaming RPC:
```
useGrpcStream(jobId) fires useEffect
→ GrpcWebFetchTransport({ baseUrl: "/grpc-web" })
→ WorkerServiceClient.streamChunkPipeline({ jobId })
→ fetch() POST to /grpc-web/worker.WorkerService/StreamChunkPipeline
→ nginx :80 /grpc-web/ (proxy_pass → envoy:8090, proxy_buffering off)
→ Envoy :8090 (grpc_web filter: HTTP/1.1 grpc-web → HTTP/2 native gRPC)
→ gRPC server :50051 WorkerServicer.StreamChunkPipeline()
→ Enters Redis polling loop (Step 5)
```
**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ctrl/nginx.conf`, `ctrl/envoy.yaml`, `core/rpc/server.py`
**Key nginx config:** `proxy_buffering off` is critical — without it, nginx collects the entire upstream response before forwarding, defeating streaming entirely.
---
## Step 3: Celery Worker → ChunkHandler
```
Celery picks up run_job task
→ task.py: run_job(job_type="chunk", job_id, payload)
→ registry.get_handler("chunk") → ChunkHandler
→ chunk.py: ChunkHandler.process(job_id, payload)
→ download_to_temp(BUCKET_IN, source_key) — pulls source from MinIO/S3
→ Creates output_dir: /app/media/out/chunks/{job_id}/
→ Constructs event_bridge callback (bridges Pipeline events → Redis)
→ pipeline = Pipeline(source, ..., event_callback=event_bridge, output_dir=...)
→ pipeline.run()
```
**Files:** `core/jobs/task.py`, `core/jobs/handlers/chunk.py`
The `event_bridge` closure wraps every `Pipeline._emit()` call, forwarding to `push_event(job_id, event_type, data)` which writes to Redis.
---
## Step 4: Pipeline Orchestration (inside Celery worker process)
`Pipeline.run()` spawns multiple threads:
```
pipeline.run():
│
├─ Chunker(source, chunk_duration)
│ → ffprobe source file → gets duration, file_size
│ → calculates total_chunks = ceil(duration / chunk_duration)
│
├─ _emit("pipeline_start", {...}) → event_bridge → Redis
├─ _emit("pipeline_info", {file_size, duration, total_chunks}) → Redis
│
├─ Creates ChunkQueue(maxsize=10)
├─ Creates WorkerPool(num_workers=N, chunk_queue, processor, event_callback)
│
├─ pool.start() — spawns N worker threads
│
├─ MONITOR THREAD starts (_monitor_progress)
│ → Every 500ms: _emit("pipeline_progress", {elapsed, throughput_mbps}) → Redis
│
├─ PRODUCER THREAD starts (_produce_chunks)
│ → Iterates chunker.chunks() → yields Chunk(sequence, start_time, end_time)
│ → For each: chunk_queue.put(chunk)
│ → _emit("chunk_queued", {sequence, start_time, end_time, queue_size}) → Redis
│ → chunk_queue.close() when done (sends N sentinel Nones)
│
├─ WORKER THREADS (N concurrent, each runs worker.py:Worker.run())
│ │ Each worker loops:
│ │
│ ├─ chunk = chunk_queue.get(timeout=1.0)
│ ├─ _emit("chunk_processing", {sequence, state:"processing", queue_size}) → Redis
│ │
│ ├─ processor.process(chunk)
│ │ ├─ ffmpeg: runs `ffmpeg -ss start -to end -c copy chunk_NNNN.mp4`
│ │ ├─ simulated_decode: sleep(random) + checksum
│ │ └─ checksum: reads bytes, computes hash
│ │
│ ├─ On success: _emit("chunk_done", {sequence, processing_time, retries, queue_size}) → Redis
│ ├─ On failure: retries with exponential backoff (0.1s, 0.2s, 0.4s...)
│ │ └─ _emit("chunk_retry", {sequence, attempt, backoff}) → Redis
│ │ └─ _emit("chunk_error", {sequence, error, retries}) → Redis (after exhaustion)
│ │
│ └─ On sentinel (None): _emit("worker_status", {state:"stopped"}) → Redis
│
├─ pool.wait() — joins all worker threads, collects results
├─ monitor_stop.set() — stops progress monitor
│
├─ ResultCollector — reassembles results in sequence order
│ └─ _emit("chunk_collected", {sequence, buffered, emitted}) → Redis
│
├─ Writes manifest.json to output_dir
│
└─ _emit("pipeline_complete", {total_chunks, processed, failed, elapsed, throughput}) → Redis
```
**Files:** `core/chunker/pipeline.py`, `core/chunker/worker.py`, `core/chunker/pool.py`, `core/chunker/chunker.py`, `core/chunker/collector.py`
---
## Step 5: Redis — the Event Bus
```
WRITE side (Celery worker, all threads):
push_event(job_id, event_type, data)
→ json.dumps({"event": event_type, ...data})
→ Redis RPUSH to key "chunk_events:{job_id}"
→ Redis EXPIRE 3600 (1 hour TTL)
READ side (gRPC server, StreamChunkPipeline):
poll_events(job_id, cursor)
→ Redis LRANGE "chunk_events:{job_id}" cursor -1
→ Returns (parsed_events, new_cursor)
→ Called every 50ms (time.sleep(0.05) in server loop)
```
Redis acts as a decoupling layer between the Celery worker process (which runs the pipeline) and the gRPC server process (which streams to browsers). Events are appended with RPUSH and read with cursor-based LRANGE polling.
**Files:** `core/events.py`
---
## Step 6: gRPC Server → Envoy → nginx → Browser
```
server.py: StreamChunkPipeline polling loop:
while context.is_active():
events, cursor = poll_events(job_id, cursor) ← Redis LRANGE
for data in events:
yield worker_pb2.ChunkPipelineEvent( ← serialized protobuf message
job_id, event_type, sequence, worker_id,
state, queue_size, elapsed, throughput_mbps,
total_chunks, processed_chunks, failed_chunks,
error, processing_time, retries
)
if event_type in ("pipeline_complete", "pipeline_error"):
return ← ends the stream
time.sleep(0.05) ← 50ms poll interval
Each yield sends:
→ gRPC HTTP/2 DATA frame to Envoy
→ Envoy grpc_web filter: HTTP/2 → base64-encoded grpc-web-text
→ nginx proxy_pass (proxy_buffering off) → chunked HTTP/1.1 to browser
→ fetch() ReadableStream in GrpcWebFetchTransport
→ @protobuf-ts decodes protobuf → ChunkPipelineEvent TypeScript object
```
**Files:** `core/rpc/server.py`, `ctrl/envoy.yaml`, `ctrl/nginx.conf`, `ui/common/api/grpc/worker.ts`, `ui/common/api/grpc/worker.client.ts`
---
## Step 7: React State Derivation and Rendering
```
useGrpcStream.ts:
for await (const msg of stream.responses):
const evt = toEvent(msg) ← maps protobuf camelCase → snake_case PipelineEvent
setEvents(prev => [...prev, evt]) ← appends to events array
if pipeline_complete/error → setDone(true), break
App.tsx useMemo(events):
Iterates ALL events on every update, derives:
├─ chunkMap: Map<sequence, ChunkInfo> — state machine per chunk
│ pending → queued → processing → done/error/retry
├─ workerMap: Map<worker_id, WorkerInfo> — state per worker
│ idle → processing → idle → ... → stopped
├─ stats: PipelineStats
│ total_chunks, processed, failed, retries, elapsed, throughput_mbps, queue_size
├─ errors: ErrorEntry[] — every event containing an error field
└─ queueSize: number — last seen queue_size value
Renders:
├─ ChunkGrid — colored cells per chunk (pending/queued/processing/done/error)
├─ QueueGauge — current queue depth / max
├─ WorkerPanel — per-worker state + current chunk assignment
├─ StatsPanel — elapsed time, throughput, processed/failed counts
├─ ErrorLog — scrollable error list
└─ OutputFiles — download links (when done)
```
**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ui/chunker/src/App.tsx`
---
## Step 8: Output File Access (after pipeline completes)
```
App.tsx useEffect([done, jobId]):
→ api.ts: getChunkOutputFiles(jobId)
→ POST /graphql → graphql.py: chunk_output_files(job_id)
→ Reads /app/media/out/chunks/{job_id}/ directory listing from disk
→ Returns [{key, size, url: "/media/out/chunks/{job_id}/chunk_0001.mp4"}]
→ Browser renders download links
→ Click link → nginx /media/out/ → alias /app/media/out/ → serves file from disk
```
Chunks are written directly to `media/out/chunks/{job_id}/` by the ffmpeg processor — no MinIO upload needed for output. Nginx serves them with `autoindex on`.
**Files:** `core/api/graphql.py`, `core/jobs/handlers/chunk.py`, `ctrl/nginx.conf`
---
## Event Types Reference
| Event | Source | Key Fields |
|-------|--------|------------|
| `pipeline_start` | Pipeline.run() | source, chunk_duration, num_workers, processor_type |
| `pipeline_info` | Pipeline.run() | file_size, source_duration, total_chunks |
| `pipeline_progress` | Monitor thread (500ms) | elapsed, throughput_mbps |
| `chunk_queued` | Producer thread | sequence, start_time, end_time, duration, queue_size |
| `chunk_processing` | Worker thread | sequence, worker_id, state, queue_size |
| `chunk_done` | Worker thread | sequence, processing_time, retries, queue_size |
| `chunk_retry` | Worker thread | sequence, attempt, backoff |
| `chunk_error` | Worker thread | sequence, error, retries |
| `chunk_collected` | ResultCollector | sequence, buffered, emitted |
| `worker_status` | Worker thread | worker_id, state (idle/processing/stopped) |
| `pipeline_complete` | Pipeline.run() | total_chunks, processed, failed, elapsed, throughput_mbps |
| `pipeline_error` | Pipeline.run() | error |
---
## Thread Model (inside Celery worker)
```
Celery worker process
└─ run_job task thread
└─ Pipeline.run()
├─ Producer thread — enqueues chunks
├─ Monitor thread — emits progress every 500ms
├─ Worker thread 0 — pulls from queue, processes
├─ Worker thread 1 — pulls from queue, processes
├─ Worker thread 2 — pulls from queue, processes
└─ Worker thread 3 — pulls from queue, processes
```
All threads share the same `event_callback` → `event_bridge` → `push_event()`, which creates a new Redis connection per call. Thread-safe via Redis atomic RPUSH.
---
## Infrastructure
| Service | Port | Role |
|---------|------|------|
| nginx | 80 | Reverse proxy, static file serving |
| fastapi | 8702 | GraphQL API (Strawberry) |
| celery | — | Task worker (runs pipeline) |
| redis | 6379 | Event bus + Celery broker |
| grpc | 50051 | gRPC server (StreamChunkPipeline) |
| envoy | 8090 | gRPC-Web ↔ native gRPC translation |
| minio | 9000 | S3-compatible source media storage |
| postgres | 5432 | Job/asset metadata |

212
docs/architecture/index.html
View File

@@ -1,212 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>MPR - Architecture</title>
<link rel="stylesheet" href="styles.css" />
</head>
<body>
<h1>MPR - Media Processor</h1>
<p>
Media transcoding platform with dual execution modes: local (Celery
+ MinIO) and cloud (AWS Step Functions + Lambda + S3).
</p>
<nav>
<a href="#overview">System Overview</a>
<a href="#data-model">Data Model</a>
<a href="#job-flow">Job Flow</a>
<a href="#media-storage">Media Storage</a>
</nav>
<h2 id="overview">System Overview</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Local Architecture (Development)</h3>
<object type="image/svg+xml" data="01a-local-architecture.svg">
<img
src="01a-local-architecture.svg"
alt="Local Architecture"
/>
</object>
<a href="01a-local-architecture.svg" target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>AWS Architecture (Production)</h3>
<object type="image/svg+xml" data="01b-aws-architecture.svg">
<img
src="01b-aws-architecture.svg"
alt="AWS Architecture"
/>
</object>
<a href="01b-aws-architecture.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Components</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4f8"></span>
Reverse Proxy (nginx)
</li>
<li>
<span class="color-box" style="background: #f0f8e8"></span>
Application Layer (Django Admin, GraphQL API, Timeline UI)
</li>
<li>
<span class="color-box" style="background: #fff8e8"></span>
Worker Layer (Celery local mode)
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
AWS (Step Functions, Lambda - cloud mode)
</li>
<li>
<span class="color-box" style="background: #f8e8f0"></span>
Data Layer (PostgreSQL, Redis)
</li>
<li>
<span class="color-box" style="background: #f0f0f0"></span>
S3 Storage (MinIO local / AWS S3 cloud)
</li>
</ul>
</div>
<h2 id="data-model">Data Model</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Entity Relationships</h3>
<object type="image/svg+xml" data="02-data-model.svg">
<img src="02-data-model.svg" alt="Data Model" />
</object>
<a href="02-data-model.svg" target="_blank">Open full size</a>
</div>
</div>
<div class="legend">
<h3>Entities</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
MediaAsset - Video/audio files (S3 keys as paths)
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
TranscodePreset - Encoding configurations
</li>
<li>
<span class="color-box" style="background: #d9534f"></span>
TranscodeJob - Processing queue (celery_task_id or
execution_arn)
</li>
</ul>
</div>
<h2 id="job-flow">Job Flow</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Job Lifecycle</h3>
<object type="image/svg+xml" data="03-job-flow.svg">
<img src="03-job-flow.svg" alt="Job Flow" />
</object>
<a href="03-job-flow.svg" target="_blank">Open full size</a>
</div>
</div>
<div class="legend">
<h3>Job States</h3>
<ul>
<li>
<span class="color-box" style="background: #ffc107"></span>
PENDING - Waiting in queue
</li>
<li>
<span class="color-box" style="background: #17a2b8"></span>
PROCESSING - Worker executing
</li>
<li>
<span class="color-box" style="background: #28a745"></span>
COMPLETED - Success
</li>
<li>
<span class="color-box" style="background: #dc3545"></span>
FAILED - Error occurred
</li>
<li>
<span class="color-box" style="background: #6c757d"></span>
CANCELLED - User cancelled
</li>
</ul>
<h3>Execution Modes</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4e8"></span>
Local: Celery + MinIO (S3 API) + FFmpeg
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
Lambda: Step Functions + Lambda + AWS S3
</li>
</ul>
</div>
<h2 id="media-storage">Media Storage</h2>
<div class="diagram-container">
<p>
MPR separates media into input and output paths for flexible
storage configuration.
</p>
<p>
<a href="04-media-storage.md" target="_blank"
>View Media Storage Documentation →</a
>
</p>
</div>
<h2>API (GraphQL)</h2>
<pre><code># GraphiQL IDE
http://mpr.local.ar/graphql
# Queries
query { assets(status: "ready") { id filename duration } }
query { jobs(status: "processing") { id status progress } }
query { presets { id name container videoCodec } }
query { systemStatus { status version } }
# Mutations
mutation { scanMediaFolder { found registered skipped } }
mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
mutation { cancelJob(id: "...") { id status } }
mutation { retryJob(id: "...") { id status } }
mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
mutation { deleteAsset(id: "...") { ok } }
# Lambda callback (REST)
POST /api/jobs/{id}/callback - Lambda completion webhook</code></pre>
<h2>Access Points</h2>
<pre><code># Local development
127.0.0.1 mpr.local.ar
http://mpr.local.ar/admin - Django Admin
http://mpr.local.ar/graphql - GraphiQL
http://mpr.local.ar/ - Timeline UI
http://localhost:9001 - MinIO Console
# AWS deployment
https://mpr.mcrn.ar/ - Production</code></pre>
<h2>Quick Reference</h2>
<pre><code># Render SVGs from DOT files
for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
# Switch executor mode
MPR_EXECUTOR=local # Celery + MinIO
MPR_EXECUTOR=lambda # Step Functions + Lambda + S3</code></pre>
</body>
</html>

96
docs/architecture/styles.css
View File

@@ -3,6 +3,8 @@
--text-color: #e8e8e8; --text-color: #e8e8e8;
--accent-color: #4a90d9; --accent-color: #4a90d9;
--border-color: #333; --border-color: #333;
--sidebar-width: 220px;
--sidebar-bg: #151528;
} }
* { * {
@@ -16,6 +18,59 @@ body {
background-color: var(--bg-color); background-color: var(--bg-color);
color: var(--text-color); color: var(--text-color);
line-height: 1.6; line-height: 1.6;
}
/* Sidebar navigation */
.sidebar {
position: fixed;
top: 0;
left: 0;
width: var(--sidebar-width);
height: 100vh;
background: var(--sidebar-bg);
border-right: 1px solid var(--border-color);
padding: 1.5rem 1rem;
overflow-y: auto;
z-index: 10;
}
.sidebar h2 {
font-size: 1.2rem;
color: var(--accent-color);
margin-bottom: 1.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid var(--border-color);
}
.sidebar ul {
list-style: none;
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.sidebar li {
display: block;
}
.sidebar a {
display: block;
padding: 0.4rem 0.6rem;
color: var(--text-color);
text-decoration: none;
font-size: 0.85rem;
border-radius: 4px;
transition: background 0.15s, color 0.15s;
}
.sidebar a:hover {
background: rgba(74, 144, 217, 0.15);
color: var(--accent-color);
}
/* Main content */
.content {
margin-left: var(--sidebar-width);
padding: 2rem; padding: 2rem;
} }
@@ -25,12 +80,13 @@ h1 {
color: var(--accent-color); color: var(--accent-color);
} }
h2 { .content > h2 {
font-size: 1.5rem; font-size: 1.5rem;
margin: 2rem 0 1rem; margin: 2rem 0 1rem;
color: var(--text-color); color: var(--text-color);
border-bottom: 1px solid var(--border-color); border-bottom: 1px solid var(--border-color);
padding-bottom: 0.5rem; padding-bottom: 0.5rem;
scroll-margin-top: 1rem;
} }
.diagram-container { .diagram-container {
@@ -76,20 +132,6 @@ h2 {
text-decoration: underline; text-decoration: underline;
} }
nav {
margin-bottom: 2rem;
}
nav a {
color: var(--accent-color);
text-decoration: none;
margin-right: 1.5rem;
}
nav a:hover {
text-decoration: underline;
}
.legend { .legend {
margin-top: 2rem; margin-top: 2rem;
padding: 1rem; padding: 1rem;
@@ -141,3 +183,27 @@ pre code {
background: none; background: none;
padding: 0; padding: 0;
} }
/* Responsive: collapse sidebar on small screens */
@media (max-width: 768px) {
.sidebar {
position: static;
width: 100%;
height: auto;
border-right: none;
border-bottom: 1px solid var(--border-color);
}
.sidebar ul {
flex-direction: row;
flex-wrap: wrap;
}
.content {
margin-left: 0;
}
.diagram {
min-width: 100%;
}
}

151
docs/index.html
View File

@@ -7,6 +7,21 @@
<link rel="stylesheet" href="architecture/styles.css" /> <link rel="stylesheet" href="architecture/styles.css" />
</head> </head>
<body> <body>
<nav class="sidebar">
<h2>MPR</h2>
<ul>
<li><a href="#overview">System Overview</a></li>
<li><a href="#data-model">Data Model</a></li>
<li><a href="#job-flow">Job Flow</a></li>
<li><a href="#media-storage">Media Storage</a></li>
<li><a href="#chunker-pipeline">Chunker Pipeline</a></li>
<li><a href="#api">API (GraphQL)</a></li>
<li><a href="#access-points">Access Points</a></li>
<li><a href="#quick-reference">Quick Reference</a></li>
</ul>
</nav>
<main class="content">
<h1>MPR - Media Processor</h1> <h1>MPR - Media Processor</h1>
<p> <p>
Media transcoding platform with three execution modes: local (Celery Media transcoding platform with three execution modes: local (Celery
@@ -14,13 +29,6 @@
Jobs + GCS). Storage is S3-compatible across all environments. Jobs + GCS). Storage is S3-compatible across all environments.
</p> </p>
<nav>
<a href="#overview">System Overview</a>
<a href="#data-model">Data Model</a>
<a href="#job-flow">Job Flow</a>
<a href="#media-storage">Media Storage</a>
</nav>
<h2 id="overview">System Overview</h2> <h2 id="overview">System Overview</h2>
<div class="diagram-container"> <div class="diagram-container">
<div class="diagram"> <div class="diagram">
@@ -183,6 +191,21 @@
CANCELLED - User cancelled CANCELLED - User cancelled
</li> </li>
</ul> </ul>
<h3>Execution Modes</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4e8"></span>
Local: Celery + MinIO (S3 API) + FFmpeg
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
Lambda: Step Functions + Lambda + AWS S3
</li>
<li>
<span class="color-box" style="background: #e8f0fd"></span>
GCP: Cloud Run Jobs + GCS (S3 compat)
</li>
</ul>
</div> </div>
<h2 id="media-storage">Media Storage</h2> <h2 id="media-storage">Media Storage</h2>
@@ -192,8 +215,7 @@
<strong>output</strong> paths, each independently configurable. <strong>output</strong> paths, each independently configurable.
File paths are stored File paths are stored
<strong>relative to their respective root</strong> to ensure <strong>relative to their respective root</strong> to ensure
portability between local development and cloud deployments (AWS portability between local development and cloud deployments.
S3, etc.).
</p> </p>
</div> </div>
@@ -241,25 +263,89 @@ MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/</code></pre>
</p> </p>
</div> </div>
<p>
<a href="architecture/04-media-storage.md" target="_blank"
>Full Media Storage Documentation &rarr;</a
>
</p>
<h2 id="chunker-pipeline">Chunker Pipeline</h2>
<div class="diagram-container">
<p>
The chunker pipeline splits media into time-based segments,
streaming real-time events from worker threads through Redis
and gRPC-Web to the browser UI. 7 hops from worker thread to pixel.
</p>
</div>
<div class="legend">
<h3>Event Path</h3>
<pre><code>Worker thread → Pipeline._emit() → event_bridge() → Redis RPUSH
→ [50ms poll] gRPC server LRANGE → yield protobuf
→ HTTP/2 frame → Envoy (grpc-web filter)
→ HTTP/1.1 chunk → nginx (proxy_buffering off)
→ fetch ReadableStream → protobuf-ts decode
→ setEvents([...prev, evt]) → React re-render</code></pre>
</div>
<div class="legend">
<h3>Thread Model (inside Celery worker)</h3>
<pre><code>Celery worker process
└─ run_job task thread
└─ Pipeline.run()
├─ Producer thread — enqueues chunks
├─ Monitor thread — emits progress every 500ms
├─ Worker thread 0 — pulls from queue, processes
├─ Worker thread 1 — pulls from queue, processes
├─ Worker thread 2 — pulls from queue, processes
└─ Worker thread 3 — pulls from queue, processes</code></pre>
</div>
<div class="legend">
<h3>Infrastructure</h3>
<ul>
<li><code>nginx :80</code> - Reverse proxy, static file serving</li>
<li><code>fastapi :8702</code> - GraphQL API (Strawberry)</li>
<li><code>celery</code> - Task worker (runs pipeline)</li>
<li><code>redis :6379</code> - Event bus + Celery broker</li>
<li><code>grpc :50051</code> - gRPC server (StreamChunkPipeline)</li>
<li><code>envoy :8090</code> - gRPC-Web &harr; native gRPC translation</li>
<li><code>minio :9000</code> - S3-compatible source media storage</li>
<li><code>postgres :5432</code> - Job/asset metadata</li>
</ul>
</div>
<p>
<a href="architecture/05-chunker-pipeline.md" target="_blank"
>Full Chunker Pipeline Documentation &rarr;</a
>
</p>
<h2 id="api">API (GraphQL)</h2>
<div class="legend"> <div class="legend">
<h3>API (GraphQL)</h3>
<p> <p>
All client interactions go through GraphQL at All client interactions go through GraphQL at
<code>/graphql</code>. <code>/graphql</code>.
</p> </p>
<ul> <pre><code># GraphiQL IDE
<li> http://mpr.local.ar/graphql
<code>scanMediaFolder</code> - Scan S3 bucket for media
files # Queries
</li> query { assets(status: "ready") { id filename duration } }
<li><code>createJob</code> - Create transcode/trim job</li> query { jobs(status: "processing") { id status progress } }
<li> query { presets { id name container videoCodec } }
<code>cancelJob / retryJob</code> - Job lifecycle management query { systemStatus { status version } }
</li>
<li> # Mutations
<code>updateAsset / deleteAsset</code> - Asset management mutation { scanMediaFolder { found registered skipped } }
</li> mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
</ul> mutation { cancelJob(id: "...") { id status } }
mutation { retryJob(id: "...") { id status } }
mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
mutation { deleteAsset(id: "...") { ok } }
# Lambda callback (REST)
POST /api/jobs/{id}/callback - Lambda completion webhook</code></pre>
<p><strong>Supported File Types:</strong></p> <p><strong>Supported File Types:</strong></p>
<p> <p>
Video: mp4, mkv, avi, mov, webm, flv, wmv, m4v<br /> Video: mp4, mkv, avi, mov, webm, flv, wmv, m4v<br />
@@ -267,13 +353,28 @@ MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/</code></pre>
</p> </p>
</div> </div>
<h2>Access Points</h2> <h2 id="access-points">Access Points</h2>
<pre><code># Add to /etc/hosts <pre><code># Add to /etc/hosts
127.0.0.1 mpr.local.ar 127.0.0.1 mpr.local.ar
# URLs # URLs
http://mpr.local.ar/admin - Django Admin http://mpr.local.ar/admin - Django Admin
http://mpr.local.ar/graphql - GraphiQL IDE http://mpr.local.ar/graphql - GraphiQL IDE
http://mpr.local.ar/ - Timeline UI</code></pre> http://mpr.local.ar/ - Timeline UI
http://mpr.local.ar/chunker/ - Chunker UI
http://localhost:9001 - MinIO Console
# AWS deployment
https://mpr.mcrn.ar/ - Production</code></pre>
<h2 id="quick-reference">Quick Reference</h2>
<pre><code># Render SVGs from DOT files
for f in docs/architecture/*.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
# Switch executor mode
MPR_EXECUTOR=local # Celery + MinIO
MPR_EXECUTOR=lambda # Step Functions + Lambda + S3
MPR_EXECUTOR=gcp # Cloud Run Jobs + GCS</code></pre>
</main>
</body> </body>
</html> </html>

6
modelgen/loader/schema.py
View File

@@ -101,6 +101,12 @@ class SchemaLoader:
for enum_cls in enums: for enum_cls in enums:
self.enums.append(self._parse_enum(enum_cls)) self.enums.append(self._parse_enum(enum_cls))
# Extract VIEWS (view/event projections)
if load_all or "views" in include:
views = getattr(module, "VIEWS", [])
for cls in views:
self.api_models.append(self._parse_dataclass(cls))
# Extract GRPC_MESSAGES (optional) # Extract GRPC_MESSAGES (optional)
if load_all or "grpc" in include: if load_all or "grpc" in include:
grpc_messages = getattr(module, "GRPC_MESSAGES", []) grpc_messages = getattr(module, "GRPC_MESSAGES", [])

120
ui/chunker/package-lock.json generated
View File

@@ -8,10 +8,15 @@
"name": "mpr-chunker", "name": "mpr-chunker",
"version": "0.1.0", "version": "0.1.0",
"dependencies": { "dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"react": "^18.2.0", "react": "^18.2.0",
"react-dom": "^18.2.0" "react-dom": "^18.2.0"
}, },
"devDependencies": { "devDependencies": {
"@protobuf-ts/grpcweb-transport": "^2.11.1",
"@protobuf-ts/plugin": "^2.11.1",
"@protobuf-ts/protoc": "^2.11.1",
"@types/react": "^18.2.0", "@types/react": "^18.2.0",
"@types/react-dom": "^18.2.0", "@types/react-dom": "^18.2.0",
"@vitejs/plugin-react": "^4.2.0", "@vitejs/plugin-react": "^4.2.0",
@@ -301,6 +306,39 @@
"node": ">=6.9.0" "node": ">=6.9.0"
} }
}, },
"node_modules/@bufbuild/protobuf": {
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz",
"integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==",
"dev": true,
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@bufbuild/protoplugin": {
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/@bufbuild/protoplugin/-/protoplugin-2.11.0.tgz",
"integrity": "sha512-lyZVNFUHArIOt4W0+dwYBe5GBwbKzbOy8ObaloEqsw9Mmiwv2O48TwddDoHN4itylC+BaEGqFdI1W8WQt2vWJQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@bufbuild/protobuf": "2.11.0",
"@typescript/vfs": "^1.6.2",
"typescript": "5.4.5"
}
},
"node_modules/@bufbuild/protoplugin/node_modules/typescript": {
"version": "5.4.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
"integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/@esbuild/aix-ppc64": { "node_modules/@esbuild/aix-ppc64": {
"version": "0.21.5", "version": "0.21.5",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
@@ -742,6 +780,75 @@
"@jridgewell/sourcemap-codec": "^1.4.14" "@jridgewell/sourcemap-codec": "^1.4.14"
} }
}, },
"node_modules/@protobuf-ts/grpcweb-transport": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/grpcweb-transport/-/grpcweb-transport-2.11.1.tgz",
"integrity": "sha512-1W4utDdvOB+RHMFQ0soL4JdnxjXV+ddeGIUg08DvZrA8Ms6k5NN6GBFU2oHZdTOcJVpPrDJ02RJlqtaoCMNBtw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1"
}
},
"node_modules/@protobuf-ts/plugin": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/plugin/-/plugin-2.11.1.tgz",
"integrity": "sha512-HyuprDcw0bEEJqkOWe1rnXUP0gwYLij8YhPuZyZk6cJbIgc/Q0IFgoHQxOXNIXAcXM4Sbehh6kjVnCzasElw1A==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@bufbuild/protobuf": "^2.4.0",
"@bufbuild/protoplugin": "^2.4.0",
"@protobuf-ts/protoc": "^2.11.1",
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"typescript": "^3.9"
},
"bin": {
"protoc-gen-dump": "bin/protoc-gen-dump",
"protoc-gen-ts": "bin/protoc-gen-ts"
}
},
"node_modules/@protobuf-ts/plugin/node_modules/typescript": {
"version": "3.9.10",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz",
"integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=4.2.0"
}
},
"node_modules/@protobuf-ts/protoc": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/protoc/-/protoc-2.11.1.tgz",
"integrity": "sha512-mUZJaV0daGO6HUX90o/atzQ6A7bbN2RSuHtdwo8SSF2Qoe3zHwa4IHyCN1evftTeHfLmdz+45qo47sL+5P8nyg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"protoc": "protoc.js"
}
},
"node_modules/@protobuf-ts/runtime": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz",
"integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==",
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@protobuf-ts/runtime-rpc": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime-rpc/-/runtime-rpc-2.11.1.tgz",
"integrity": "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ==",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1"
}
},
"node_modules/@rolldown/pluginutils": { "node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.27", "version": "1.0.0-beta.27",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz",
@@ -1179,6 +1286,19 @@
"@types/react": "^18.0.0" "@types/react": "^18.0.0"
} }
}, },
"node_modules/@typescript/vfs": {
"version": "1.6.4",
"resolved": "https://registry.npmjs.org/@typescript/vfs/-/vfs-1.6.4.tgz",
"integrity": "sha512-PJFXFS4ZJKiJ9Qiuix6Dz/OwEIqHD7Dme1UwZhTK11vR+5dqW2ACbdndWQexBzCx+CPuMe5WBYQWCsFyGlQLlQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"debug": "^4.4.3"
},
"peerDependencies": {
"typescript": "*"
}
},
"node_modules/@vitejs/plugin-react": { "node_modules/@vitejs/plugin-react": {
"version": "4.7.0", "version": "4.7.0",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz",

5
ui/chunker/package.json
View File

@@ -9,10 +9,15 @@
"preview": "vite preview" "preview": "vite preview"
}, },
"dependencies": { "dependencies": {
"@protobuf-ts/grpcweb-transport": "^2.11.1",
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"react": "^18.2.0", "react": "^18.2.0",
"react-dom": "^18.2.0" "react-dom": "^18.2.0"
}, },
"devDependencies": { "devDependencies": {
"@protobuf-ts/plugin": "^2.11.1",
"@protobuf-ts/protoc": "^2.11.1",
"@types/react": "^18.2.0", "@types/react": "^18.2.0",
"@types/react-dom": "^18.2.0", "@types/react-dom": "^18.2.0",
"@vitejs/plugin-react": "^4.2.0", "@vitejs/plugin-react": "^4.2.0",

475
ui/chunker/src/App.css
View File

@@ -1,16 +1,4 @@
* { @import "../../common/styles/theme.css";
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
"Fira Code", monospace, sans-serif;
background: #0f0f0f;
color: #e0e0e0;
font-size: 14px;
}
/* ---- Layout ---- */ /* ---- Layout ---- */
@@ -25,8 +13,8 @@ body {
justify-content: space-between; justify-content: space-between;
align-items: center; align-items: center;
padding: 0.75rem 1.25rem; padding: 0.75rem 1.25rem;
background: #1a1a1a; background: var(--bg-panel);
border-bottom: 1px solid #2a2a2a; border-bottom: 1px solid var(--border);
} }
.header h1 { .header h1 {
@@ -40,19 +28,19 @@ body {
align-items: center; align-items: center;
gap: 0.5rem; gap: 0.5rem;
font-size: 0.8rem; font-size: 0.8rem;
color: #666; color: var(--text-muted);
} }
.dot { .dot {
width: 8px; width: 8px;
height: 8px; height: 8px;
border-radius: 50%; border-radius: 50%;
background: #555; background: var(--text-muted);
} }
.dot.connected { .dot.connected {
background: #10b981; background: var(--success);
box-shadow: 0 0 6px #10b981; box-shadow: 0 0 6px var(--success);
} }
.error-banner { .error-banner {
@@ -70,8 +58,8 @@ body {
.sidebar { .sidebar {
width: 300px; width: 300px;
background: #141414; background: var(--bg-surface);
border-right: 1px solid #2a2a2a; border-right: 1px solid var(--border);
overflow-y: auto; overflow-y: auto;
} }
@@ -97,163 +85,20 @@ body {
gap: 1rem; gap: 1rem;
} }
/* ---- Panel shared ---- */ /* ---- Selected Asset Info ---- */
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.75rem;
}
.panel-header h2 {
font-size: 0.85rem;
text-transform: uppercase;
letter-spacing: 0.05em;
color: #888;
}
.badge-row {
display: flex;
gap: 0.25rem;
}
/* ---- Topic Badge ---- */
.topic-badge {
display: inline-flex;
align-items: center;
gap: 0.25rem;
padding: 0.15rem 0.5rem;
font-size: 0.65rem;
background: #1e293b;
border: 1px solid #334155;
border-radius: 12px;
cursor: pointer;
transition: all 0.2s;
flex-shrink: 0;
}
.topic-badge:hover {
border-color: #3b82f6;
}
.topic-badge.expanded {
flex-direction: column;
align-items: flex-start;
border-radius: 8px;
padding: 0.5rem;
position: relative;
z-index: 10;
background: #1e293b;
}
.topic-number {
color: #3b82f6;
font-weight: 700;
}
.topic-title {
color: #94a3b8;
}
.topic-detail {
margin-top: 0.25rem;
font-size: 0.7rem;
line-height: 1.4;
}
.topic-detail p {
color: #cbd5e1;
margin-bottom: 0.25rem;
}
.topic-detail code {
color: #10b981;
font-size: 0.65rem;
}
/* ---- Asset List ---- */
.scan-button {
padding: 0.25rem 0.5rem;
font-size: 0.7rem;
background: #1e293b;
color: #94a3b8;
border: 1px solid #334155;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.scan-button:hover:not(:disabled) {
background: #334155;
color: #e0e0e0;
}
.scan-button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.asset-list {
list-style: none;
max-height: 200px;
overflow-y: auto;
margin-bottom: 0.75rem;
}
.asset-item {
padding: 0.4rem 0.5rem;
cursor: pointer;
border-left: 2px solid transparent;
transition: all 0.15s;
display: flex;
flex-direction: column;
gap: 0.1rem;
}
.asset-item:hover {
background: #1a1a1a;
}
.asset-item.selected {
background: #1e293b;
border-left-color: #3b82f6;
}
.asset-filename {
font-size: 0.8rem;
color: #e0e0e0;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.asset-meta {
font-size: 0.65rem;
color: #555;
}
.asset-empty {
font-size: 0.8rem;
color: #444;
padding: 0.75rem 0.5rem;
text-align: center;
}
.selected-asset-info { .selected-asset-info {
padding: 0.5rem; padding: 0.5rem;
background: #1e293b; background: #1e293b;
border: 1px solid #334155; border: 1px solid #334155;
border-radius: 4px; border-radius: var(--radius);
margin-bottom: 0.75rem; margin-bottom: 0.75rem;
} }
.asset-detail { .asset-detail {
display: block; display: block;
font-size: 0.8rem; font-size: 0.8rem;
color: #e0e0e0; color: var(--text-primary);
font-weight: 500; font-weight: 500;
} }
@@ -277,12 +122,12 @@ body {
.config-field label { .config-field label {
display: block; display: block;
font-size: 0.75rem; font-size: 0.75rem;
color: #888; color: var(--text-secondary);
margin-bottom: 0.25rem; margin-bottom: 0.25rem;
} }
.config-field .default { .config-field .default {
color: #555; color: var(--text-muted);
font-style: italic; font-style: italic;
} }
@@ -291,26 +136,26 @@ body {
width: 100%; width: 100%;
padding: 0.4rem 0.5rem; padding: 0.4rem 0.5rem;
font-size: 0.8rem; font-size: 0.8rem;
background: #222; background: var(--bg-input);
color: #e0e0e0; color: var(--text-primary);
border: 1px solid #333; border: 1px solid var(--border);
border-radius: 4px; border-radius: var(--radius);
} }
.config-field input:focus, .config-field input:focus,
.config-field select:focus { .config-field select:focus {
outline: none; outline: none;
border-color: #3b82f6; border-color: var(--accent);
} }
.start-button { .start-button {
width: 100%; width: 100%;
padding: 0.5rem; padding: 0.5rem;
font-size: 0.85rem; font-size: 0.85rem;
background: #10b981; background: var(--success);
color: #000; color: #000;
border: none; border: none;
border-radius: 4px; border-radius: var(--radius);
cursor: pointer; cursor: pointer;
font-weight: 600; font-weight: 600;
margin-top: 0.5rem; margin-top: 0.5rem;
@@ -322,116 +167,86 @@ body {
} }
.start-button:disabled { .start-button:disabled {
background: #333; background: var(--bg-input);
color: #666; color: var(--text-muted);
cursor: not-allowed; cursor: not-allowed;
} }
/* ---- Pipeline Diagram ---- */ .stop-button {
width: 100%;
.pipeline-diagram { padding: 0.5rem;
background: #141414; font-size: 0.85rem;
border: 1px solid #2a2a2a; background: var(--error);
border-radius: 8px; color: #fff;
padding: 1rem; border: none;
} border-radius: var(--radius);
cursor: pointer;
.stage-flow {
display: flex;
align-items: center;
gap: 0;
overflow-x: auto;
}
.stage-wrapper {
display: flex;
align-items: center;
}
.stage {
padding: 0.5rem 0.75rem;
background: #1a1a1a;
border: 1px solid #333;
border-radius: 6px;
text-align: center;
min-width: 120px;
transition: all 0.3s;
}
.stage.active {
border-color: #3b82f6;
background: #1e293b;
box-shadow: 0 0 12px rgba(59, 130, 246, 0.2);
}
.stage-label {
font-size: 0.8rem;
font-weight: 600; font-weight: 600;
color: #e0e0e0; margin-top: 0.5rem;
transition: background 0.2s;
} }
.stage-sub { .stop-button:hover {
font-size: 0.65rem; background: #dc2626;
color: #666;
margin-top: 0.15rem;
} }
.stage-arrow { .reset-button {
width: 24px; width: 100%;
height: 2px; padding: 0.5rem;
background: #444; font-size: 0.85rem;
position: relative; background: #1e293b;
}
.stage-arrow::after {
content: "";
position: absolute;
right: 0;
top: -3px;
border: 4px solid transparent;
border-left: 6px solid #444;
}
.processor-hierarchy {
margin-top: 0.75rem;
padding-top: 0.75rem;
border-top: 1px solid #222;
}
.hierarchy-title {
font-size: 0.7rem;
color: #666;
margin-bottom: 0.35rem;
font-style: italic;
}
.hierarchy-children {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.hierarchy-node {
font-size: 0.7rem;
padding: 0.15rem 0.5rem;
background: #1a1a1a;
border: 1px solid #333;
border-radius: 4px;
color: #94a3b8; color: #94a3b8;
border: 1px solid #334155;
border-radius: var(--radius);
cursor: pointer;
font-weight: 600;
margin-top: 0.5rem;
transition: all 0.2s;
}
.reset-button:hover {
background: #334155;
color: var(--text-primary);
}
.range-row {
display: flex;
align-items: center;
gap: 0.5rem;
}
.range-row input {
flex: 1;
padding: 0.4rem 0.5rem;
font-size: 0.8rem;
background: var(--bg-input);
color: var(--text-primary);
border: 1px solid var(--border);
border-radius: var(--radius);
}
.range-row input:focus {
outline: none;
border-color: var(--accent);
}
.range-sep {
font-size: 0.75rem;
color: var(--text-muted);
} }
/* ---- Chunk Grid ---- */ /* ---- Chunk Grid ---- */
.chunk-grid-panel { .chunk-grid-panel {
background: #141414; background: var(--bg-surface);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 8px; border-radius: 8px;
padding: 1rem; padding: 1rem;
} }
.chunk-count { .chunk-count {
font-size: 0.7rem; font-size: 0.7rem;
color: #555; color: var(--text-muted);
font-weight: 400; font-weight: 400;
} }
@@ -466,7 +281,7 @@ body {
align-items: center; align-items: center;
gap: 0.25rem; gap: 0.25rem;
font-size: 0.65rem; font-size: 0.65rem;
color: #888; color: var(--text-secondary);
} }
.legend-dot { .legend-dot {
@@ -478,8 +293,8 @@ body {
/* ---- Worker Panel ---- */ /* ---- Worker Panel ---- */
.worker-panel { .worker-panel {
background: #141414; background: var(--bg-surface);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 8px; border-radius: 8px;
padding: 1rem; padding: 1rem;
} }
@@ -492,8 +307,8 @@ body {
.worker-card { .worker-card {
padding: 0.5rem 0.75rem; padding: 0.5rem 0.75rem;
background: #1a1a1a; background: var(--bg-panel);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 6px; border-radius: 6px;
} }
@@ -516,7 +331,7 @@ body {
.worker-chunk { .worker-chunk {
font-size: 0.7rem; font-size: 0.7rem;
color: #555; color: var(--text-muted);
margin-top: 0.15rem; margin-top: 0.15rem;
} }
@@ -524,13 +339,13 @@ body {
display: flex; display: flex;
gap: 0.75rem; gap: 0.75rem;
font-size: 0.65rem; font-size: 0.65rem;
color: #555; color: var(--text-muted);
margin-top: 0.25rem; margin-top: 0.25rem;
} }
.worker-empty { .worker-empty {
font-size: 0.8rem; font-size: 0.8rem;
color: #444; color: var(--text-muted);
text-align: center; text-align: center;
padding: 1rem; padding: 1rem;
} }
@@ -538,8 +353,8 @@ body {
/* ---- Queue Gauge ---- */ /* ---- Queue Gauge ---- */
.queue-gauge { .queue-gauge {
background: #141414; background: var(--bg-surface);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 8px; border-radius: 8px;
padding: 1rem; padding: 1rem;
} }
@@ -550,38 +365,38 @@ body {
.gauge-label { .gauge-label {
font-size: 0.75rem; font-size: 0.75rem;
color: #888; color: var(--text-secondary);
margin-bottom: 0.25rem; margin-bottom: 0.25rem;
} }
.gauge-value { .gauge-value {
color: #e0e0e0; color: var(--text-primary);
font-weight: 600; font-weight: 600;
} }
.gauge-bar { .gauge-bar {
height: 8px; height: 8px;
background: #222; background: var(--bg-input);
border-radius: 4px; border-radius: var(--radius);
overflow: hidden; overflow: hidden;
} }
.gauge-fill { .gauge-fill {
height: 100%; height: 100%;
border-radius: 4px; border-radius: var(--radius);
transition: width 0.3s, background 0.3s; transition: width 0.3s, background 0.3s;
} }
.gauge-note { .gauge-note {
font-size: 0.65rem; font-size: 0.65rem;
color: #555; color: var(--text-muted);
} }
/* ---- Stats Panel ---- */ /* ---- Stats Panel ---- */
.stats-panel { .stats-panel {
background: #141414; background: var(--bg-surface);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 8px; border-radius: 8px;
padding: 1rem; padding: 1rem;
} }
@@ -595,52 +410,29 @@ body {
.stat { .stat {
text-align: center; text-align: center;
padding: 0.5rem; padding: 0.5rem;
background: #1a1a1a; background: var(--bg-panel);
border-radius: 6px; border-radius: 6px;
} }
.stat-value { .stat-value {
font-size: 1.1rem; font-size: 1.1rem;
font-weight: 700; font-weight: 700;
color: #e0e0e0; color: var(--text-primary);
} }
.stat-label { .stat-label {
font-size: 0.6rem; font-size: 0.6rem;
color: #666; color: var(--text-muted);
text-transform: uppercase; text-transform: uppercase;
letter-spacing: 0.05em; letter-spacing: 0.05em;
margin-top: 0.15rem; margin-top: 0.15rem;
} }
.test-info {
margin-top: 0.75rem;
padding-top: 0.5rem;
border-top: 1px solid #222;
display: flex;
align-items: center;
gap: 0.5rem;
}
.test-badge {
font-size: 0.65rem;
padding: 0.15rem 0.4rem;
background: #10b981;
color: #000;
border-radius: 3px;
font-weight: 600;
}
.test-note {
font-size: 0.65rem;
color: #555;
}
/* ---- Error Log ---- */ /* ---- Error Log ---- */
.error-log { .error-log {
background: #141414; background: var(--bg-surface);
border: 1px solid #2a2a2a; border: 1px solid var(--border);
border-radius: 8px; border-radius: 8px;
padding: 1rem; padding: 1rem;
} }
@@ -654,41 +446,6 @@ body {
font-weight: 400; font-weight: 400;
} }
.exception-tree {
margin-bottom: 0.75rem;
padding: 0.5rem;
background: #1a1a1a;
border-radius: 6px;
font-size: 0.7rem;
font-family: "Fira Code", monospace;
}
.tree-node {
color: #94a3b8;
padding: 0.1rem 0;
}
.tree-node.root {
color: #f59e0b;
font-weight: 600;
}
.tree-node.leaf {
color: #64748b;
}
.tree-children {
padding-left: 1rem;
border-left: 1px solid #333;
margin-left: 0.5rem;
}
.tree-grandchildren {
padding-left: 1rem;
border-left: 1px solid #333;
margin-left: 0.5rem;
}
.error-entries { .error-entries {
max-height: 150px; max-height: 150px;
overflow-y: auto; overflow-y: auto;
@@ -696,7 +453,7 @@ body {
.error-empty { .error-empty {
font-size: 0.8rem; font-size: 0.8rem;
color: #444; color: var(--text-muted);
text-align: center; text-align: center;
padding: 0.5rem; padding: 0.5rem;
} }
@@ -706,26 +463,26 @@ body {
gap: 0.5rem; gap: 0.5rem;
align-items: center; align-items: center;
padding: 0.35rem 0; padding: 0.35rem 0;
border-bottom: 1px solid #1a1a1a; border-bottom: 1px solid var(--bg-panel);
font-size: 0.7rem; font-size: 0.7rem;
flex-wrap: wrap; flex-wrap: wrap;
} }
.error-type { .error-type {
color: #ef4444; color: var(--error);
font-weight: 500; font-weight: 500;
} }
.error-seq { .error-seq {
color: #f59e0b; color: var(--warning);
} }
.error-worker { .error-worker {
color: #3b82f6; color: var(--accent);
} }
.error-msg { .error-msg {
color: #888; color: var(--text-secondary);
flex: 1; flex: 1;
} }
@@ -733,3 +490,15 @@ body {
color: #f97316; color: #f97316;
font-size: 0.65rem; font-size: 0.65rem;
} }
/* ---- Output download link ---- */
.fm-download-link {
font-size: 0.7rem;
color: var(--accent);
text-decoration: none;
}
.fm-download-link:hover {
text-decoration: underline;
}

161
ui/chunker/src/App.tsx
View File

@@ -1,16 +1,23 @@
import { useCallback, useEffect, useMemo, useState } from "react"; import { useCallback, useEffect, useMemo, useState } from "react";
import "./App.css"; import "./App.css";
import { createChunkJob, getAssets, scanMediaFolder } from "./api"; import {
cancelChunkJob,
createChunkJob,
getAssets,
getChunkOutputFiles,
scanMediaFolder,
} from "./api";
import { ChunkGrid } from "./components/ChunkGrid"; import { ChunkGrid } from "./components/ChunkGrid";
import { ConfigPanel } from "./components/ConfigPanel"; import { ConfigPanel } from "./components/ConfigPanel";
import { ErrorLog } from "./components/ErrorLog"; import { ErrorLog } from "./components/ErrorLog";
import { PipelineDiagram } from "./components/PipelineDiagram"; import { OutputFiles } from "./components/OutputFiles";
import { QueueGauge } from "./components/QueueGauge"; import { QueueGauge } from "./components/QueueGauge";
import { StatsPanel } from "./components/StatsPanel"; import { StatsPanel } from "./components/StatsPanel";
import { WorkerPanel } from "./components/WorkerPanel"; import { WorkerPanel } from "./components/WorkerPanel";
import { useEventStream } from "./hooks/useEventStream"; import { useGrpcStream } from "./hooks/useGrpcStream";
import type { import type {
ChunkInfo, ChunkInfo,
ChunkOutputFile,
ErrorEntry, ErrorEntry,
MediaAsset, MediaAsset,
PipelineConfig, PipelineConfig,
@@ -20,6 +27,7 @@ import type {
export default function App() { export default function App() {
const [jobId, setJobId] = useState<string | null>(null); const [jobId, setJobId] = useState<string | null>(null);
const [celeryTaskId, setCeleryTaskId] = useState<string | null>(null);
const [running, setRunning] = useState(false); const [running, setRunning] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
@@ -28,15 +36,36 @@ export default function App() {
const [selectedAsset, setSelectedAsset] = useState<MediaAsset | null>(null); const [selectedAsset, setSelectedAsset] = useState<MediaAsset | null>(null);
const [scanning, setScanning] = useState(false); const [scanning, setScanning] = useState(false);
const { events, connected, done } = useEventStream(jobId); // Output files
const [outputFiles, setOutputFiles] = useState<ChunkOutputFile[]>([]);
const {
events,
connected,
done,
reset: resetStream,
} = useGrpcStream(jobId);
// Load assets on mount // Load assets on mount
useEffect(() => { useEffect(() => {
getAssets() getAssets()
.then((data) => setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename)))) .then((data) =>
.catch((e) => setError(e instanceof Error ? e.message : "Failed to load assets")); setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename))),
)
.catch((e) =>
setError(e instanceof Error ? e.message : "Failed to load assets"),
);
}, []); }, []);
// Fetch output files when job completes
useEffect(() => {
if (done && jobId) {
getChunkOutputFiles(jobId)
.then(setOutputFiles)
.catch(() => setOutputFiles([]));
}
}, [done, jobId]);
const handleScan = useCallback(async () => { const handleScan = useCallback(async () => {
setScanning(true); setScanning(true);
setError(null); setError(null);
@@ -51,8 +80,8 @@ export default function App() {
} }
}, []); }, []);
// Derive state from events // Derive state from raw events
const { chunks, workers, stats, errors, activeStage, queueSize } = const { chunks, workers, stats, errors, queueSize } =
useMemo(() => { useMemo(() => {
const chunkMap = new Map<number, ChunkInfo>(); const chunkMap = new Map<number, ChunkInfo>();
const workerMap = new Map<string, WorkerInfo>(); const workerMap = new Map<string, WorkerInfo>();
@@ -64,45 +93,54 @@ export default function App() {
let elapsed = 0; let elapsed = 0;
let throughput = 0; let throughput = 0;
let queueSize = 0; let queueSize = 0;
let stage = "pending"; let pipelineDone = false;
for (const evt of events) { for (const evt of events) {
const evtType = evt.event_type || "";
if (evt.total_chunks) totalChunks = evt.total_chunks; if (evt.total_chunks) totalChunks = evt.total_chunks;
if (evt.processed_chunks) processed = evt.processed_chunks; if (evt.processed_chunks) processed = evt.processed_chunks;
if (evt.failed_chunks) failed = evt.failed_chunks; if (evt.failed_chunks) failed = evt.failed_chunks;
if (evt.elapsed) elapsed = evt.elapsed; if (evt.elapsed) elapsed = evt.elapsed;
if (evt.throughput_mbps) throughput = evt.throughput_mbps; if (evt.throughput_mbps) throughput = evt.throughput_mbps;
if (evt.queue_size !== undefined) queueSize = evt.queue_size; if (evt.queue_size !== undefined) queueSize = evt.queue_size;
if (evt.status && evt.status !== "waiting") stage = evt.status;
// Track chunks if (evtType === "pipeline_complete" || evtType === "pipeline_error") {
pipelineDone = true;
queueSize = 0;
}
// Track chunks by raw event type
if (evt.sequence !== undefined) { if (evt.sequence !== undefined) {
const existing = chunkMap.get(evt.sequence) || { const existing = chunkMap.get(evt.sequence) || {
sequence: evt.sequence, sequence: evt.sequence,
state: "pending" as const, state: "pending" as const,
}; };
if (evt.status === "chunking" || evt.status === "pending") { if (evtType === "chunk_queued") {
existing.state = "queued"; existing.state = "queued";
} else if (evt.status === "processing") { } else if (evtType === "chunk_processing") {
existing.state = "processing"; existing.state = "processing";
if (evt.worker_id) existing.worker_id = evt.worker_id; if (evt.worker_id) existing.worker_id = evt.worker_id;
} else if (evt.status === "completed") { } else if (evtType === "chunk_done") {
existing.state = "done"; existing.state = "done";
if (evt.processing_time) if (evt.processing_time)
existing.processing_time = evt.processing_time; existing.processing_time = evt.processing_time;
if (evt.retries) existing.retries = evt.retries; if (evt.retries) existing.retries = evt.retries;
} else if (evt.status === "failed") { } else if (evtType === "chunk_error") {
existing.state = "error"; existing.state = "error";
if (evt.error) existing.error = evt.error; if (evt.error) existing.error = evt.error;
} else if (evtType === "chunk_retry") {
existing.state = "retry";
if (evt.retries) existing.retries = evt.retries;
} }
if (evt.size) existing.size = evt.size; if (evt.size) existing.size = evt.size;
chunkMap.set(evt.sequence, existing); chunkMap.set(evt.sequence, existing);
} }
// Track workers // Track workers from worker_status events
if (evt.worker_id) { if (evt.worker_id && evtType === "worker_status") {
const w = workerMap.get(evt.worker_id) || { const w = workerMap.get(evt.worker_id) || {
worker_id: evt.worker_id, worker_id: evt.worker_id,
state: "idle" as const, state: "idle" as const,
@@ -119,12 +157,38 @@ export default function App() {
w.current_chunk = undefined; w.current_chunk = undefined;
} else if (evt.state === "stopped") { } else if (evt.state === "stopped") {
w.state = "stopped"; w.state = "stopped";
w.current_chunk = undefined;
} }
if (evt.success !== undefined) { workerMap.set(evt.worker_id, w);
if (evt.success) w.processed++;
else w.errors++;
} }
// Also update workers from chunk lifecycle events
if (
evt.worker_id &&
(evtType === "chunk_processing" ||
evtType === "chunk_done" ||
evtType === "chunk_error")
) {
const w = workerMap.get(evt.worker_id) || {
worker_id: evt.worker_id,
state: "idle" as const,
processed: 0,
errors: 0,
retries: 0,
};
if (evtType === "chunk_processing") {
w.state = "processing";
w.current_chunk = evt.sequence;
} else if (evtType === "chunk_done") {
w.processed++;
w.state = "idle";
w.current_chunk = undefined;
} else if (evtType === "chunk_error") {
w.errors++;
}
if (evt.retries) { if (evt.retries) {
retries += evt.retries; retries += evt.retries;
w.retries += evt.retries; w.retries += evt.retries;
@@ -141,11 +205,19 @@ export default function App() {
worker_id: evt.worker_id, worker_id: evt.worker_id,
error: evt.error, error: evt.error,
retries: evt.retries, retries: evt.retries,
event_type: evt.status || "error", event_type: evtType,
}); });
} }
} }
// When pipeline is done, mark all workers as stopped
if (pipelineDone) {
for (const w of workerMap.values()) {
w.state = "stopped";
w.current_chunk = undefined;
}
}
const statsObj: PipelineStats = { const statsObj: PipelineStats = {
total_chunks: totalChunks, total_chunks: totalChunks,
processed, processed,
@@ -158,12 +230,11 @@ export default function App() {
return { return {
chunks: Array.from(chunkMap.values()).sort( chunks: Array.from(chunkMap.values()).sort(
(a, b) => a.sequence - b.sequence (a, b) => a.sequence - b.sequence,
), ),
workers: Array.from(workerMap.values()), workers: Array.from(workerMap.values()),
stats: statsObj, stats: statsObj,
errors: errorList, errors: errorList,
activeStage: stage,
queueSize, queueSize,
}; };
}, [events]); }, [events]);
@@ -171,15 +242,45 @@ export default function App() {
const handleStart = useCallback(async (config: PipelineConfig) => { const handleStart = useCallback(async (config: PipelineConfig) => {
setError(null); setError(null);
setRunning(true); setRunning(true);
setOutputFiles([]);
try { try {
const result = await createChunkJob(config); const result = await createChunkJob(config);
setJobId(result.id); setJobId(result.id);
setCeleryTaskId(result.celery_task_id);
} catch (e) { } catch (e) {
setError(e instanceof Error ? e.message : "Failed to start"); setError(e instanceof Error ? e.message : "Failed to start");
setRunning(false); setRunning(false);
} }
}, []); }, []);
const handleStop = useCallback(async () => {
if (!celeryTaskId) {
setError("No task ID to cancel");
return;
}
try {
const result = await cancelChunkJob(celeryTaskId);
if (result.ok) {
resetStream();
setRunning(false);
setError(null);
} else {
setError(result.message || "Failed to cancel");
}
} catch (e) {
setError(e instanceof Error ? e.message : "Failed to cancel");
}
}, [celeryTaskId, resetStream]);
const handleReset = useCallback(() => {
setJobId(null);
setCeleryTaskId(null);
setRunning(false);
setError(null);
setOutputFiles([]);
resetStream();
}, [resetStream]);
// Reset running state when done // Reset running state when done
if (done && running) { if (done && running) {
setRunning(false); setRunning(false);
@@ -211,7 +312,10 @@ export default function App() {
<aside className="sidebar"> <aside className="sidebar">
<ConfigPanel <ConfigPanel
onStart={handleStart} onStart={handleStart}
onStop={handleStop}
onReset={handleReset}
running={running} running={running}
done={done}
assets={assets} assets={assets}
selectedAsset={selectedAsset} selectedAsset={selectedAsset}
onSelectAsset={setSelectedAsset} onSelectAsset={setSelectedAsset}
@@ -221,16 +325,13 @@ export default function App() {
</aside> </aside>
<main className="main"> <main className="main">
<PipelineDiagram activeStage={activeStage} />
<div className="main-grid"> <div className="main-grid">
<div className="main-left"> <div className="main-left">
<ChunkGrid chunks={chunks} totalChunks={stats.total_chunks} /> <ChunkGrid chunks={chunks} totalChunks={stats.total_chunks} />
<QueueGauge <QueueGauge current={queueSize} max={10} buffered={0} />
current={queueSize} {done && outputFiles.length > 0 && (
max={10} <OutputFiles files={outputFiles} />
buffered={0} )}
/>
</div> </div>
<div className="main-right"> <div className="main-right">
<WorkerPanel workers={workers} /> <WorkerPanel workers={workers} />

115
ui/chunker/src/api.ts
View File

@@ -1,55 +1,13 @@
/** /**
* GraphQL API client for the chunker UI. * Chunker-specific API functions.
* Shared functions (getAssets, scanMediaFolder) come from common.
*/ */
import type { MediaAsset } from "./types"; import { gql } from "../../common/api/graphql";
import type { ChunkOutputFile } from "../../common/types/generated";
const GRAPHQL_URL = "/api/graphql"; // Re-export shared functions
export { getAssets, scanMediaFolder } from "../../common/api/media";
async function gql<T>(query: string, variables?: Record<string, unknown>): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}
/** Fetch all media assets. */
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
/** Scan media/in/ folder for new files. */
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}
/** Create a chunk job via GraphQL mutation. */ /** Create a chunk job via GraphQL mutation. */
export async function createChunkJob(config: { export async function createChunkJob(config: {
@@ -58,15 +16,70 @@ export async function createChunkJob(config: {
num_workers: number; num_workers: number;
max_retries: number; max_retries: number;
processor_type: string; processor_type: string;
}): Promise<{ id: string }> { start_time?: number | null;
const data = await gql<{ create_chunk_job: { id: string; status: string } }>(` end_time?: number | null;
}): Promise<{ id: string; celery_task_id: string | null }> {
const data = await gql<{
create_chunk_job: {
id: string;
status: string;
celery_task_id: string | null;
};
}>(
`
mutation CreateChunkJob($input: CreateChunkJobInput!) { mutation CreateChunkJob($input: CreateChunkJobInput!) {
create_chunk_job(input: $input) { create_chunk_job(input: $input) {
id id
status status
celery_task_id
} }
} }
`, { input: config }); `,
{ input: config },
);
return data.create_chunk_job; return data.create_chunk_job;
} }
/** Cancel a running chunk job. */
export async function cancelChunkJob(
celeryTaskId: string,
): Promise<{ ok: boolean; message: string | null }> {
const data = await gql<{
cancel_chunk_job: { ok: boolean; message: string | null };
}>(
`
mutation CancelChunkJob($celery_task_id: String!) {
cancel_chunk_job(celery_task_id: $celery_task_id) {
ok
message
}
}
`,
{ celery_task_id: celeryTaskId },
);
return data.cancel_chunk_job;
}
/** Fetch output chunk files for a completed job. */
export async function getChunkOutputFiles(
jobId: string,
): Promise<ChunkOutputFile[]> {
const data = await gql<{
chunk_output_files: ChunkOutputFile[];
}>(
`
query ChunkOutputFiles($job_id: String!) {
chunk_output_files(job_id: $job_id) {
key
size
url
}
}
`,
{ job_id: jobId },
);
return data.chunk_output_files;
}

19
ui/chunker/src/components/ChunkGrid.tsx
View File

@@ -1,5 +1,4 @@
import type { ChunkInfo } from "../types"; import type { ChunkInfo } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
chunks: ChunkInfo[]; chunks: ChunkInfo[];
@@ -7,19 +6,14 @@ interface Props {
} }
const STATE_COLORS: Record<string, string> = { const STATE_COLORS: Record<string, string> = {
pending: "#333", pending: "var(--bg-input)",
queued: "#f59e0b", queued: "var(--warning)",
processing: "#3b82f6", processing: "var(--processing)",
done: "#10b981", done: "var(--success)",
error: "#ef4444", error: "var(--error)",
retry: "#f97316", retry: "#f97316",
}; };
/**
* Grid of chunks colored by processing state.
* Chunks appear incrementally as the generator yields them.
* Interview Topic 3: Generators & iteration.
*/
export function ChunkGrid({ chunks, totalChunks }: Props) { export function ChunkGrid({ chunks, totalChunks }: Props) {
return ( return (
<div className="chunk-grid-panel"> <div className="chunk-grid-panel">
@@ -30,14 +24,13 @@ export function ChunkGrid({ chunks, totalChunks }: Props) {
{chunks.length} / {totalChunks || "?"} {chunks.length} / {totalChunks || "?"}
</span> </span>
</h2> </h2>
<TopicBadge topic={TOPICS.iteration} />
</div> </div>
<div className="chunk-grid"> <div className="chunk-grid">
{chunks.map((chunk) => ( {chunks.map((chunk) => (
<div <div
key={chunk.sequence} key={chunk.sequence}
className="chunk-cell" className="chunk-cell"
style={{ background: STATE_COLORS[chunk.state] || "#333" }} style={{ background: STATE_COLORS[chunk.state] || "var(--bg-input)" }}
title={`#${chunk.sequence} — ${chunk.state}${ title={`#${chunk.sequence} — ${chunk.state}${
chunk.worker_id ? ` (${chunk.worker_id})` : "" chunk.worker_id ? ` (${chunk.worker_id})` : ""
}${chunk.retries ? ` retries: ${chunk.retries}` : ""}`} }${chunk.retries ? ` retries: ${chunk.retries}` : ""}`}

156
ui/chunker/src/components/ConfigPanel.tsx
View File

@@ -1,10 +1,15 @@
import { useState } from "react"; import { useMemo, useState } from "react";
import { FileManager } from "../../../common/components/FileManager";
import type { FileEntry } from "../../../common/components/FileManager";
import { formatDuration, formatSize } from "../../../common/utils/format";
import type { MediaAsset, PipelineConfig } from "../types"; import type { MediaAsset, PipelineConfig } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
onStart: (config: PipelineConfig) => void; onStart: (config: PipelineConfig) => void;
onStop: () => void;
onReset: () => void;
running: boolean; running: boolean;
done: boolean;
assets: MediaAsset[]; assets: MediaAsset[];
selectedAsset: MediaAsset | null; selectedAsset: MediaAsset | null;
onSelectAsset: (asset: MediaAsset) => void; onSelectAsset: (asset: MediaAsset) => void;
@@ -12,13 +17,12 @@ interface Props {
scanning: boolean; scanning: boolean;
} }
/**
* Pipeline configuration form with file browser.
* Each parameter shows its default — Interview Topic 1: Function params & defaults.
*/
export function ConfigPanel({ export function ConfigPanel({
onStart, onStart,
onStop,
onReset,
running, running,
done,
assets, assets,
selectedAsset, selectedAsset,
onSelectAsset, onSelectAsset,
@@ -31,6 +35,25 @@ export function ConfigPanel({
const [processorType, setProcessorType] = useState< const [processorType, setProcessorType] = useState<
"ffmpeg" | "checksum" | "simulated_decode" | "composite" "ffmpeg" | "checksum" | "simulated_decode" | "composite"
>("ffmpeg"); >("ffmpeg");
const [startTime, setStartTime] = useState<string>("");
const [endTime, setEndTime] = useState<string>("");
// Map assets to FileEntry for FileManager
const fileEntries: FileEntry[] = useMemo(
() =>
assets.map((a) => ({
key: a.id,
name: a.filename,
size: a.file_size ?? undefined,
meta: formatDuration(a.duration),
})),
[assets],
);
const handleFileSelect = (file: FileEntry) => {
const asset = assets.find((a) => a.id === file.key);
if (asset) onSelectAsset(asset);
};
const handleSubmit = (e: React.FormEvent) => { const handleSubmit = (e: React.FormEvent) => {
e.preventDefault(); e.preventDefault();
@@ -41,61 +64,31 @@ export function ConfigPanel({
num_workers: numWorkers, num_workers: numWorkers,
max_retries: maxRetries, max_retries: maxRetries,
processor_type: processorType, processor_type: processorType,
start_time: startTime ? parseFloat(startTime) : null,
end_time: endTime ? parseFloat(endTime) : null,
}); });
}; };
const formatSize = (bytes: number | null) => {
if (!bytes) return "—";
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
};
const formatDuration = (seconds: number | null) => {
if (!seconds) return "—";
const m = Math.floor(seconds / 60);
const s = Math.floor(seconds % 60);
return `${m}:${s.toString().padStart(2, "0")}`;
};
return ( return (
<div className="config-panel"> <div className="config-panel">
{/* Asset Browser */} <FileManager
<div className="panel-header"> title="Assets"
<h2>Assets</h2> files={fileEntries}
<button selectedKey={selectedAsset?.id ?? null}
onClick={onScan} onSelect={handleFileSelect}
disabled={scanning} onScan={onScan}
className="scan-button" scanning={scanning}
> emptyMessage="No assets — click Scan Folder"
{scanning ? "Scanning..." : "Scan Folder"} disabled={running}
</button> />
</div>
<ul className="asset-list">
{assets.length === 0 ? (
<li className="asset-empty">No assets — click Scan Folder</li>
) : (
assets.map((asset) => (
<li
key={asset.id}
className={`asset-item ${selectedAsset?.id === asset.id ? "selected" : ""}`}
onClick={() => onSelectAsset(asset)}
title={asset.filename}
>
<span className="asset-filename">{asset.filename}</span>
<span className="asset-meta">
{formatSize(asset.file_size)} · {formatDuration(asset.duration)}
</span>
</li>
))
)}
</ul>
{selectedAsset && ( {selectedAsset && (
<div className="selected-asset-info"> <div className="selected-asset-info">
<span className="asset-detail">{selectedAsset.filename}</span> <span className="asset-detail">{selectedAsset.filename}</span>
<span className="asset-detail-meta"> <span className="asset-detail-meta">
{selectedAsset.video_codec} · {selectedAsset.width}x{selectedAsset.height} · {formatDuration(selectedAsset.duration)} {selectedAsset.video_codec} · {selectedAsset.width}x
{selectedAsset.height} · {formatDuration(selectedAsset.duration)} ·{" "}
{formatSize(selectedAsset.file_size)}
</span> </span>
</div> </div>
)} )}
@@ -103,9 +96,35 @@ export function ConfigPanel({
{/* Pipeline Config */} {/* Pipeline Config */}
<div className="panel-header" style={{ marginTop: "1rem" }}> <div className="panel-header" style={{ marginTop: "1rem" }}>
<h2>Pipeline Config</h2> <h2>Pipeline Config</h2>
<TopicBadge topic={TOPICS.params} />
</div> </div>
<form onSubmit={handleSubmit}> <form onSubmit={handleSubmit}>
<div className="config-field">
<label>
Time Range (seconds){" "}
<span className="default">optional — limits what gets chunked</span>
</label>
<div className="range-row">
<input
type="number"
min={0}
step={1}
placeholder="start"
value={startTime}
onChange={(e) => setStartTime(e.target.value)}
disabled={running}
/>
<span className="range-sep">to</span>
<input
type="number"
min={0}
step={1}
placeholder="end"
value={endTime}
onChange={(e) => setEndTime(e.target.value)}
disabled={running}
/>
</div>
</div>
<div className="config-field"> <div className="config-field">
<label> <label>
Chunk Duration <span className="default">default: 10s</span> Chunk Duration <span className="default">default: 10s</span>
@@ -113,6 +132,7 @@ export function ConfigPanel({
<select <select
value={chunkDuration} value={chunkDuration}
onChange={(e) => setChunkDuration(Number(e.target.value))} onChange={(e) => setChunkDuration(Number(e.target.value))}
disabled={running}
> >
<option value={5}>5 seconds</option> <option value={5}>5 seconds</option>
<option value={10}>10 seconds</option> <option value={10}>10 seconds</option>
@@ -131,6 +151,7 @@ export function ConfigPanel({
max={16} max={16}
value={numWorkers} value={numWorkers}
onChange={(e) => setNumWorkers(Number(e.target.value))} onChange={(e) => setNumWorkers(Number(e.target.value))}
disabled={running}
/> />
</div> </div>
<div className="config-field"> <div className="config-field">
@@ -143,6 +164,7 @@ export function ConfigPanel({
max={10} max={10}
value={maxRetries} value={maxRetries}
onChange={(e) => setMaxRetries(Number(e.target.value))} onChange={(e) => setMaxRetries(Number(e.target.value))}
disabled={running}
/> />
</div> </div>
<div className="config-field"> <div className="config-field">
@@ -153,9 +175,14 @@ export function ConfigPanel({
value={processorType} value={processorType}
onChange={(e) => onChange={(e) =>
setProcessorType( setProcessorType(
e.target.value as "ffmpeg" | "checksum" | "simulated_decode" | "composite" e.target.value as
| "ffmpeg"
| "checksum"
| "simulated_decode"
| "composite",
) )
} }
disabled={running}
> >
<option value="ffmpeg">FFmpegExtractProcessor</option> <option value="ffmpeg">FFmpegExtractProcessor</option>
<option value="checksum">ChecksumProcessor</option> <option value="checksum">ChecksumProcessor</option>
@@ -163,10 +190,29 @@ export function ConfigPanel({
<option value="composite">CompositeProcessor</option> <option value="composite">CompositeProcessor</option>
</select> </select>
</div> </div>
<button type="submit" className="start-button" disabled={running || !selectedAsset}>
{running ? "Running..." : "Launch Pipeline"} {!running && !done && (
<button
type="submit"
className="start-button"
disabled={!selectedAsset}
>
Launch Pipeline
</button> </button>
)}
</form> </form>
{running && (
<button type="button" className="stop-button" onClick={onStop}>
Stop Pipeline
</button>
)}
{done && (
<button type="button" className="reset-button" onClick={onReset}>
Reset
</button>
)}
</div> </div>
); );
} }

23
ui/chunker/src/components/ErrorLog.tsx
View File

@@ -1,15 +1,9 @@
import type { ErrorEntry } from "../types"; import type { ErrorEntry } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
errors: ErrorEntry[]; errors: ErrorEntry[];
} }
/**
* Error and retry event log.
* Shows exception types, retry counts, backoff delays.
* Interview Topic 7: Exception handling & resilient code.
*/
export function ErrorLog({ errors }: Props) { export function ErrorLog({ errors }: Props) {
return ( return (
<div className="error-log"> <div className="error-log">
@@ -18,23 +12,6 @@ export function ErrorLog({ errors }: Props) {
Errors & Retries{" "} Errors & Retries{" "}
<span className="error-count">{errors.length}</span> <span className="error-count">{errors.length}</span>
</h2> </h2>
<TopicBadge topic={TOPICS.exceptions} />
</div>
<div className="exception-tree">
<div className="tree-node root">PipelineError</div>
<div className="tree-children">
<div className="tree-node">ChunkError</div>
<div className="tree-grandchildren">
<div className="tree-node leaf">ChunkReadError</div>
<div className="tree-node leaf">ChunkChecksumError</div>
</div>
<div className="tree-node">ProcessingError</div>
<div className="tree-grandchildren">
<div className="tree-node leaf">ProcessorTimeoutError</div>
<div className="tree-node leaf">ProcessorFailureError</div>
</div>
<div className="tree-node">ReassemblyError</div>
</div>
</div> </div>
<div className="error-entries"> <div className="error-entries">
{errors.length === 0 && ( {errors.length === 0 && (

51
ui/chunker/src/components/OutputFiles.tsx Normal file
View File

@@ -0,0 +1,51 @@
import { useMemo } from "react";
import { FileManager } from "../../../common/components/FileManager";
import type { FileEntry } from "../../../common/components/FileManager";
import { formatSize } from "../../../common/utils/format";
import type { ChunkOutputFile } from "../types";
interface Props {
files: ChunkOutputFile[];
}
export function OutputFiles({ files }: Props) {
const fileEntries: FileEntry[] = useMemo(
() =>
files.map((f) => ({
key: f.key,
name: f.key.split("/").pop() || f.key,
size: f.size,
})),
[files],
);
const urlMap = useMemo(() => {
const map = new Map<string, string>();
for (const f of files) {
map.set(f.key, f.url);
}
return map;
}, [files]);
return (
<FileManager
title="Output Files"
files={fileEntries}
emptyMessage="No output files"
renderActions={(file) => {
const url = urlMap.get(file.key);
if (!url) return null;
return (
<a
href={url}
download
className="fm-download-link"
onClick={(e) => e.stopPropagation()}
>
{formatSize(file.size)} ↓
</a>
);
}}
/>
);
}

50
ui/chunker/src/components/PipelineDiagram.tsx
View File

@@ -1,50 +0,0 @@
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
activeStage: string;
}
const STAGES = [
{ id: "chunking", label: "Chunker", sub: "File -> Chunks (generator)" },
{ id: "queued", label: "ChunkQueue", sub: "Bounded queue (backpressure)" },
{ id: "processing", label: "WorkerPool", sub: "ThreadPoolExecutor" },
{ id: "collecting", label: "ResultCollector", sub: "heapq reassembly" },
{ id: "completed", label: "PipelineResult", sub: "Aggregate stats" },
];
/**
* Visual flow diagram of pipeline stages.
* Highlights the currently active stage.
* Interview Topic 4: OOP design — shows class hierarchy.
*/
export function PipelineDiagram({ activeStage }: Props) {
return (
<div className="pipeline-diagram">
<div className="panel-header">
<h2>Pipeline Flow</h2>
<TopicBadge topic={TOPICS.oop} />
</div>
<div className="stage-flow">
{STAGES.map((stage, i) => (
<div key={stage.id} className="stage-wrapper">
<div
className={`stage ${activeStage === stage.id ? "active" : ""}`}
>
<div className="stage-label">{stage.label}</div>
<div className="stage-sub">{stage.sub}</div>
</div>
{i < STAGES.length - 1 && <div className="stage-arrow" />}
</div>
))}
</div>
<div className="processor-hierarchy">
<div className="hierarchy-title">Processor ABC</div>
<div className="hierarchy-children">
<span className="hierarchy-node">ChecksumProcessor</span>
<span className="hierarchy-node">SimulatedDecodeProcessor</span>
<span className="hierarchy-node">CompositeProcessor</span>
</div>
</div>
</div>
);
}

9
ui/chunker/src/components/QueueGauge.tsx
View File

@@ -1,15 +1,9 @@
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
current: number; current: number;
max: number; max: number;
buffered: number; buffered: number;
} }
/**
* Queue fill level gauge + collector heap buffer.
* Interview Topic 5: Data structures — queue.Queue, heapq, deque.
*/
export function QueueGauge({ current, max, buffered }: Props) { export function QueueGauge({ current, max, buffered }: Props) {
const fillPct = max > 0 ? Math.min((current / max) * 100, 100) : 0; const fillPct = max > 0 ? Math.min((current / max) * 100, 100) : 0;
@@ -17,7 +11,6 @@ export function QueueGauge({ current, max, buffered }: Props) {
<div className="queue-gauge"> <div className="queue-gauge">
<div className="panel-header"> <div className="panel-header">
<h2>Queue & Buffer</h2> <h2>Queue & Buffer</h2>
<TopicBadge topic={TOPICS.datastructures} />
</div> </div>
<div className="gauge-row"> <div className="gauge-row">
<div className="gauge-label"> <div className="gauge-label">
@@ -28,7 +21,7 @@ export function QueueGauge({ current, max, buffered }: Props) {
className="gauge-fill" className="gauge-fill"
style={{ style={{
width: `${fillPct}%`, width: `${fillPct}%`,
background: fillPct > 80 ? "#ef4444" : "#3b82f6", background: fillPct > 80 ? "var(--error)" : "var(--processing)",
}} }}
/> />
</div> </div>

16
ui/chunker/src/components/StatsPanel.tsx
View File

@@ -1,24 +1,14 @@
import type { PipelineStats } from "../types"; import type { PipelineStats } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
stats: PipelineStats; stats: PipelineStats;
} }
/**
* Throughput, timing, and error stats.
* Interview Topic 6: Algorithms — throughput calculation over sliding window.
* Interview Topic 8: TDD — test count and coverage.
*/
export function StatsPanel({ stats }: Props) { export function StatsPanel({ stats }: Props) {
return ( return (
<div className="stats-panel"> <div className="stats-panel">
<div className="panel-header"> <div className="panel-header">
<h2>Stats</h2> <h2>Stats</h2>
<div className="badge-row">
<TopicBadge topic={TOPICS.algorithms} />
<TopicBadge topic={TOPICS.testing} />
</div>
</div> </div>
<div className="stats-grid"> <div className="stats-grid">
<div className="stat"> <div className="stat">
@@ -48,12 +38,6 @@ export function StatsPanel({ stats }: Props) {
<div className="stat-label">Elapsed</div> <div className="stat-label">Elapsed</div>
</div> </div>
</div> </div>
<div className="test-info">
<span className="test-badge">64 tests</span>
<span className="test-note">
7 test files &middot; pytest &middot; parametrized
</span>
</div>
</div> </div>
); );
} }

86
ui/chunker/src/components/TopicBadge.tsx
View File

@@ -1,86 +0,0 @@
import { useState } from "react";
import type { InterviewTopic } from "../types";
/**
* Expandable pill badge annotating an interview topic.
* Click to expand and see description + code reference.
*/
export function TopicBadge({ topic }: { topic: InterviewTopic }) {
const [expanded, setExpanded] = useState(false);
return (
<div
className={`topic-badge ${expanded ? "expanded" : ""}`}
onClick={() => setExpanded(!expanded)}
>
<span className="topic-number">#{topic.number}</span>
<span className="topic-title">{topic.title}</span>
{expanded && (
<div className="topic-detail">
<p>{topic.description}</p>
<code>{topic.code_ref}</code>
</div>
)}
</div>
);
}
/** Pre-defined topics mapped to pipeline components. */
export const TOPICS: Record<string, InterviewTopic> = {
params: {
number: 1,
title: "Function Params & Defaults",
description:
"Each pipeline parameter has a sensible default (chunk_duration=10s, num_workers=4, max_retries=3). Tweaking them changes pipeline behavior.",
code_ref: "core/chunker/pipeline.py — Pipeline.__init__()",
},
concurrency: {
number: 2,
title: "Concurrency (Threading)",
description:
"Workers run in a ThreadPoolExecutor. The queue coordinates work between producer and consumer threads.",
code_ref: "core/chunker/pool.py — WorkerPool, ThreadPoolExecutor",
},
iteration: {
number: 3,
title: "Generators & Iteration",
description:
"Chunks are yielded lazily via a generator — the file is never fully loaded into memory.",
code_ref: "core/chunker/chunker.py — Chunker.chunks() generator",
},
oop: {
number: 4,
title: "OOP Design (ABC)",
description:
"Processor is an abstract base class. ChecksumProcessor, SimulatedDecodeProcessor, and CompositeProcessor inherit from it.",
code_ref: "core/chunker/processor.py — Processor ABC hierarchy",
},
datastructures: {
number: 5,
title: "Data Structures",
description:
"Bounded queue.Queue for backpressure, heapq min-heap for ordered reassembly, deque for sliding-window throughput.",
code_ref: "core/chunker/queue.py, collector.py, models.py",
},
algorithms: {
number: 6,
title: "Algorithms & Sorting",
description:
"ResultCollector uses a min-heap to reassemble chunks in sequence order, even when they arrive out of order.",
code_ref: "core/chunker/collector.py — heapq-based reassembly",
},
exceptions: {
number: 7,
title: "Exception Handling",
description:
"PipelineError hierarchy with typed exceptions. Workers retry with exponential backoff before giving up.",
code_ref: "core/chunker/exceptions.py, worker.py — retry logic",
},
testing: {
number: 8,
title: "TDD & Unit Testing",
description:
"64 tests covering every module. Parametrized tests, fixtures, edge cases, concurrency tests.",
code_ref: "tests/chunker/ — 7 test files, pytest",
},
};

15
ui/chunker/src/components/WorkerPanel.tsx
View File

@@ -1,28 +1,21 @@
import type { WorkerInfo } from "../types"; import type { WorkerInfo } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props { interface Props {
workers: WorkerInfo[]; workers: WorkerInfo[];
} }
const STATE_COLORS: Record<string, string> = { const STATE_COLORS: Record<string, string> = {
idle: "#6b7280", idle: "var(--text-muted)",
processing: "#3b82f6", processing: "var(--processing)",
retry: "#f97316", retry: "#f97316",
stopped: "#ef4444", stopped: "var(--error)",
}; };
/**
* Worker thread status cards.
* Shows each worker's real-time state and which chunk it's processing.
* Interview Topic 2: Concurrency (threading).
*/
export function WorkerPanel({ workers }: Props) { export function WorkerPanel({ workers }: Props) {
return ( return (
<div className="worker-panel"> <div className="worker-panel">
<div className="panel-header"> <div className="panel-header">
<h2>Workers</h2> <h2>Workers</h2>
<TopicBadge topic={TOPICS.concurrency} />
</div> </div>
<div className="worker-cards"> <div className="worker-cards">
{workers.map((w) => ( {workers.map((w) => (
@@ -31,7 +24,7 @@ export function WorkerPanel({ workers }: Props) {
<span className="worker-name">{w.worker_id}</span> <span className="worker-name">{w.worker_id}</span>
<span <span
className="worker-state" className="worker-state"
style={{ color: STATE_COLORS[w.state] || "#888" }} style={{ color: STATE_COLORS[w.state] || "var(--text-secondary)" }}
> >
{w.state} {w.state}
</span> </span>

33
ui/chunker/src/hooks/useEventStream.ts
View File

@@ -3,8 +3,6 @@ import type { PipelineEvent } from "../types";
/** /**
* SSE hook — connects to /api/chunker/stream/{jobId} via native EventSource. * SSE hook — connects to /api/chunker/stream/{jobId} via native EventSource.
*
* Demonstrates: real-time event streaming from backend to UI.
*/ */
export function useEventStream(jobId: string | null) { export function useEventStream(jobId: string | null) {
const [events, setEvents] = useState<PipelineEvent[]>([]); const [events, setEvents] = useState<PipelineEvent[]>([]);
@@ -20,6 +18,12 @@ export function useEventStream(jobId: string | null) {
} }
}, []); }, []);
const reset = useCallback(() => {
close();
setEvents([]);
setDone(false);
}, [close]);
useEffect(() => { useEffect(() => {
if (!jobId) return; if (!jobId) return;
@@ -35,21 +39,28 @@ export function useEventStream(jobId: string | null) {
const handleEvent = (eventType: string) => (e: MessageEvent) => { const handleEvent = (eventType: string) => (e: MessageEvent) => {
try { try {
const data = JSON.parse(e.data) as PipelineEvent; const data = JSON.parse(e.data) as PipelineEvent;
setEvents((prev) => [...prev, { ...data, status: eventType }]); setEvents((prev) => [...prev, { ...data, event_type: eventType }]);
} catch { } catch {
// ignore parse errors // ignore parse errors
} }
}; };
// Listen to all chunker event types // Listen to all raw pipeline event types
const eventTypes = [ const eventTypes = [
"waiting", "waiting",
"pending", "pipeline_start",
"chunking", "pipeline_info",
"processing", "chunk_queued",
"collecting", "chunk_processing",
"completed", "chunk_done",
"failed", "chunk_retry",
"chunk_error",
"chunk_collected",
"worker_status",
"pipeline_progress",
"pipeline_complete",
"pipeline_error",
"producer_error",
"cancelled", "cancelled",
"done", "done",
"timeout", "timeout",
@@ -77,5 +88,5 @@ export function useEventStream(jobId: string | null) {
}; };
}, [jobId]); }, [jobId]);
return { events, connected, done, close }; return { events, connected, done, close, reset };
} }

103
ui/chunker/src/hooks/useGrpcStream.ts Normal file
View File

@@ -0,0 +1,103 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { GrpcWebFetchTransport } from "@protobuf-ts/grpcweb-transport";
import { WorkerServiceClient } from "../../../common/api/grpc/worker.client";
import type { ChunkPipelineEvent } from "../../../common/api/grpc/worker";
import type { PipelineEvent } from "../types";
const GRPC_WEB_URL = "/grpc-web";
function toEvent(msg: ChunkPipelineEvent): PipelineEvent {
return {
event_type: msg.eventType,
job_id: msg.jobId,
sequence: msg.sequence || undefined,
worker_id: msg.workerId || undefined,
state: msg.state || undefined,
queue_size: msg.queueSize || undefined,
elapsed: msg.elapsed || undefined,
throughput_mbps: msg.throughputMbps || undefined,
total_chunks: msg.totalChunks || undefined,
processed_chunks: msg.processedChunks || undefined,
failed_chunks: msg.failedChunks || undefined,
error: msg.error || undefined,
processing_time: msg.processingTime || undefined,
retries: msg.retries || undefined,
};
}
/**
* gRPC-Web streaming hook — connects to WorkerService.StreamChunkPipeline
* via Envoy proxy. Replaces useEventStream (SSE+Redis).
*/
export function useGrpcStream(jobId: string | null) {
const [events, setEvents] = useState<PipelineEvent[]>([]);
const [connected, setConnected] = useState(false);
const [done, setDone] = useState(false);
const abortRef = useRef<AbortController | null>(null);
const close = useCallback(() => {
if (abortRef.current) {
abortRef.current.abort();
abortRef.current = null;
setConnected(false);
}
}, []);
const reset = useCallback(() => {
close();
setEvents([]);
setDone(false);
}, [close]);
useEffect(() => {
if (!jobId) return;
setEvents([]);
setDone(false);
const abort = new AbortController();
abortRef.current = abort;
const transport = new GrpcWebFetchTransport({
baseUrl: GRPC_WEB_URL,
abort: abort.signal,
});
const client = new WorkerServiceClient(transport);
const stream = client.streamChunkPipeline({ jobId });
setConnected(true);
(async () => {
try {
for await (const msg of stream.responses) {
const evt = toEvent(msg);
setEvents((prev) => [...prev, evt]);
if (
evt.event_type === "pipeline_complete" ||
evt.event_type === "pipeline_error"
) {
setDone(true);
setConnected(false);
break;
}
}
} catch (err) {
if (!abort.signal.aborted) {
setConnected(false);
}
} finally {
setConnected(false);
}
})();
return () => {
abort.abort();
abortRef.current = null;
};
}, [jobId]);
return { events, connected, done, close, reset };
}

76
ui/chunker/src/types.ts
View File

@@ -1,3 +1,19 @@
/**
* Chunker UI types.
*
* Domain types (MediaAsset, ChunkEvent, etc.) come from generated schema.
* This file holds UI-only types: state enums, SSE envelope, derived views.
*/
// Re-export generated types used by this app
export type {
MediaAsset,
ChunkEvent,
WorkerEvent,
PipelineStats,
ChunkOutputFile,
} from "../../common/types/generated";
/** Pipeline configuration sent to the backend. */ /** Pipeline configuration sent to the backend. */
export interface PipelineConfig { export interface PipelineConfig {
source_asset_id: string; source_asset_id: string;
@@ -5,31 +21,11 @@ export interface PipelineConfig {
num_workers: number; num_workers: number;
max_retries: number; max_retries: number;
processor_type: "ffmpeg" | "checksum" | "simulated_decode" | "composite"; processor_type: "ffmpeg" | "checksum" | "simulated_decode" | "composite";
start_time?: number | null;
end_time?: number | null;
} }
/** Media asset from the backend. */ /** UI state of an individual chunk in the grid. */
export interface MediaAsset {
id: string;
filename: string;
file_path: string;
status: string;
error_message: string | null;
file_size: number | null;
duration: number | null;
video_codec: string | null;
audio_codec: string | null;
width: number | null;
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
/** State of an individual chunk. */
export type ChunkState = export type ChunkState =
| "pending" | "pending"
| "queued" | "queued"
@@ -38,7 +34,7 @@ export type ChunkState =
| "error" | "error"
| "retry"; | "retry";
/** Tracked chunk in the UI grid. */ /** Tracked chunk in the UI grid (derived from events). */
export interface ChunkInfo { export interface ChunkInfo {
sequence: number; sequence: number;
state: ChunkState; state: ChunkState;
@@ -49,7 +45,7 @@ export interface ChunkInfo {
error?: string; error?: string;
} }
/** Worker thread status. */ /** Worker thread status (derived from events). */
export interface WorkerInfo { export interface WorkerInfo {
worker_id: string; worker_id: string;
state: "idle" | "processing" | "retry" | "stopped"; state: "idle" | "processing" | "retry" | "stopped";
@@ -59,9 +55,14 @@ export interface WorkerInfo {
retries: number; retries: number;
} }
/** SSE event from the backend. */ /**
* Raw SSE event envelope from the backend.
* The event_type field is set by useEventStream from the SSE event name.
* All other fields are optional — presence depends on event_type.
*/
export interface PipelineEvent { export interface PipelineEvent {
job_id: string; job_id?: string;
event_type?: string;
status?: string; status?: string;
progress?: number; progress?: number;
total_chunks?: number; total_chunks?: number;
@@ -84,18 +85,7 @@ export interface PipelineEvent {
backoff?: number; backoff?: number;
} }
/** Aggregate pipeline stats. */ /** Error log entry (derived from events). */
export interface PipelineStats {
total_chunks: number;
processed: number;
failed: number;
retries: number;
elapsed: number;
throughput_mbps: number;
queue_size: number;
}
/** Error log entry. */
export interface ErrorEntry { export interface ErrorEntry {
timestamp: number; timestamp: number;
sequence?: number; sequence?: number;
@@ -104,11 +94,3 @@ export interface ErrorEntry {
retries?: number; retries?: number;
event_type: string; event_type: string;
} }
/** Interview topic for annotation badges. */
export interface InterviewTopic {
number: number;
title: string;
description: string;
code_ref: string;
}

9
ui/chunker/tsconfig.json
View File

@@ -14,8 +14,13 @@
"strict": true, "strict": true,
"noUnusedLocals": true, "noUnusedLocals": true,
"noUnusedParameters": true, "noUnusedParameters": true,
"noFallthroughCasesInSwitch": true "noFallthroughCasesInSwitch": true,
"rootDir": "..",
"typeRoots": ["./node_modules/@types"],
"paths": {
"@protobuf-ts/*": ["./node_modules/@protobuf-ts/*"]
}
}, },
"include": ["src/**/*.ts", "src/**/*.tsx"], "include": ["src/**/*.ts", "src/**/*.tsx", "../common/**/*.ts", "../common/**/*.tsx"],
"references": [{ "path": "./tsconfig.node.json" }] "references": [{ "path": "./tsconfig.node.json" }]
} }

25
ui/chunker/vite.config.ts
View File

@@ -1,9 +1,26 @@
import path from "path";
import { defineConfig } from "vite"; import { defineConfig } from "vite";
import react from "@vitejs/plugin-react"; import react from "@vitejs/plugin-react";
export default defineConfig({ export default defineConfig({
base: "/chunker/", base: "/chunker/",
plugins: [react()], plugins: [react()],
resolve: {
alias: {
"@protobuf-ts/runtime": path.resolve(
__dirname,
"node_modules/@protobuf-ts/runtime",
),
"@protobuf-ts/runtime-rpc": path.resolve(
__dirname,
"node_modules/@protobuf-ts/runtime-rpc",
),
"@protobuf-ts/grpcweb-transport": path.resolve(
__dirname,
"node_modules/@protobuf-ts/grpcweb-transport",
),
},
},
server: { server: {
host: "0.0.0.0", host: "0.0.0.0",
port: 5174, port: 5174,
@@ -11,6 +28,9 @@ export default defineConfig({
hmr: { hmr: {
path: "/chunker/@vite/client", path: "/chunker/@vite/client",
}, },
fs: {
allow: [".."],
},
proxy: { proxy: {
"/api": { "/api": {
target: "http://fastapi:8702", target: "http://fastapi:8702",
@@ -20,6 +40,11 @@ export default defineConfig({
target: "http://fastapi:8702", target: "http://fastapi:8702",
changeOrigin: true, changeOrigin: true,
}, },
"/grpc-web": {
target: "http://envoy:8090",
changeOrigin: true,
rewrite: (p) => p.replace(/^\/grpc-web/, ""),
},
}, },
}, },
}); });

24
ui/common/api/graphql.ts Normal file
View File

@@ -0,0 +1,24 @@
/**
* Shared GraphQL client for all MPR UI apps.
*/
const GRAPHQL_URL = "/api/graphql";
export async function gql<T>(
query: string,
variables?: Record<string, unknown>,
): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}

95
ui/common/api/grpc/worker.client.ts Normal file
View File

@@ -0,0 +1,95 @@
// @generated by protobuf-ts 2.11.1
// @generated from protobuf file "worker.proto" (package "mpr.worker", syntax proto3)
// tslint:disable
//
// Protocol Buffer Definitions - GENERATED FILE
//
// Do not edit directly. Regenerate using modelgen.
//
import type { RpcTransport } from "@protobuf-ts/runtime-rpc";
import type { ServiceInfo } from "@protobuf-ts/runtime-rpc";
import { WorkerService } from "./worker";
import type { ChunkPipelineEvent } from "./worker";
import type { ChunkStreamRequest } from "./worker";
import type { WorkerStatus } from "./worker";
import type { Empty } from "./worker";
import type { CancelResponse } from "./worker";
import type { CancelRequest } from "./worker";
import type { ProgressUpdate } from "./worker";
import type { ProgressRequest } from "./worker";
import type { ServerStreamingCall } from "@protobuf-ts/runtime-rpc";
import { stackIntercept } from "@protobuf-ts/runtime-rpc";
import type { JobResponse } from "./worker";
import type { JobRequest } from "./worker";
import type { UnaryCall } from "@protobuf-ts/runtime-rpc";
import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
/**
* @generated from protobuf service mpr.worker.WorkerService
*/
export interface IWorkerServiceClient {
/**
* @generated from protobuf rpc: SubmitJob
*/
submitJob(input: JobRequest, options?: RpcOptions): UnaryCall<JobRequest, JobResponse>;
/**
* @generated from protobuf rpc: StreamProgress
*/
streamProgress(input: ProgressRequest, options?: RpcOptions): ServerStreamingCall<ProgressRequest, ProgressUpdate>;
/**
* @generated from protobuf rpc: CancelJob
*/
cancelJob(input: CancelRequest, options?: RpcOptions): UnaryCall<CancelRequest, CancelResponse>;
/**
* @generated from protobuf rpc: GetWorkerStatus
*/
getWorkerStatus(input: Empty, options?: RpcOptions): UnaryCall<Empty, WorkerStatus>;
/**
* @generated from protobuf rpc: StreamChunkPipeline
*/
streamChunkPipeline(input: ChunkStreamRequest, options?: RpcOptions): ServerStreamingCall<ChunkStreamRequest, ChunkPipelineEvent>;
}
/**
* @generated from protobuf service mpr.worker.WorkerService
*/
export class WorkerServiceClient implements IWorkerServiceClient, ServiceInfo {
typeName = WorkerService.typeName;
methods = WorkerService.methods;
options = WorkerService.options;
constructor(private readonly _transport: RpcTransport) {
}
/**
* @generated from protobuf rpc: SubmitJob
*/
submitJob(input: JobRequest, options?: RpcOptions): UnaryCall<JobRequest, JobResponse> {
const method = this.methods[0], opt = this._transport.mergeOptions(options);
return stackIntercept<JobRequest, JobResponse>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: StreamProgress
*/
streamProgress(input: ProgressRequest, options?: RpcOptions): ServerStreamingCall<ProgressRequest, ProgressUpdate> {
const method = this.methods[1], opt = this._transport.mergeOptions(options);
return stackIntercept<ProgressRequest, ProgressUpdate>("serverStreaming", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: CancelJob
*/
cancelJob(input: CancelRequest, options?: RpcOptions): UnaryCall<CancelRequest, CancelResponse> {
const method = this.methods[2], opt = this._transport.mergeOptions(options);
return stackIntercept<CancelRequest, CancelResponse>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: GetWorkerStatus
*/
getWorkerStatus(input: Empty, options?: RpcOptions): UnaryCall<Empty, WorkerStatus> {
const method = this.methods[3], opt = this._transport.mergeOptions(options);
return stackIntercept<Empty, WorkerStatus>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: StreamChunkPipeline
*/
streamChunkPipeline(input: ChunkStreamRequest, options?: RpcOptions): ServerStreamingCall<ChunkStreamRequest, ChunkPipelineEvent> {
const method = this.methods[4], opt = this._transport.mergeOptions(options);
return stackIntercept<ChunkStreamRequest, ChunkPipelineEvent>("serverStreaming", this._transport, method, opt, input);
}
}

946
ui/common/api/grpc/worker.ts Normal file
View File

@@ -0,0 +1,946 @@
// @generated by protobuf-ts 2.11.1
// @generated from protobuf file "worker.proto" (package "mpr.worker", syntax proto3)
// tslint:disable
//
// Protocol Buffer Definitions - GENERATED FILE
//
// Do not edit directly. Regenerate using modelgen.
//
import { ServiceType } from "@protobuf-ts/runtime-rpc";
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
import type { IBinaryWriter } from "@protobuf-ts/runtime";
import { WireType } from "@protobuf-ts/runtime";
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
import type { IBinaryReader } from "@protobuf-ts/runtime";
import { UnknownFieldHandler } from "@protobuf-ts/runtime";
import type { PartialMessage } from "@protobuf-ts/runtime";
import { reflectionMergePartial } from "@protobuf-ts/runtime";
import { MessageType } from "@protobuf-ts/runtime";
/**
* @generated from protobuf message mpr.worker.JobRequest
*/
export interface JobRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: string source_path = 2
*/
sourcePath: string;
/**
* @generated from protobuf field: string output_path = 3
*/
outputPath: string;
/**
* @generated from protobuf field: string preset_json = 4
*/
presetJson: string;
/**
* @generated from protobuf field: optional float trim_start = 5
*/
trimStart?: number;
/**
* @generated from protobuf field: optional float trim_end = 6
*/
trimEnd?: number;
}
/**
* @generated from protobuf message mpr.worker.JobResponse
*/
export interface JobResponse {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: bool accepted = 2
*/
accepted: boolean;
/**
* @generated from protobuf field: string message = 3
*/
message: string;
}
/**
* @generated from protobuf message mpr.worker.ProgressRequest
*/
export interface ProgressRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.ProgressUpdate
*/
export interface ProgressUpdate {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: int32 progress = 2
*/
progress: number;
/**
* @generated from protobuf field: int32 current_frame = 3
*/
currentFrame: number;
/**
* @generated from protobuf field: float current_time = 4
*/
currentTime: number;
/**
* @generated from protobuf field: float speed = 5
*/
speed: number;
/**
* @generated from protobuf field: string status = 6
*/
status: string;
/**
* @generated from protobuf field: optional string error = 7
*/
error?: string;
}
/**
* @generated from protobuf message mpr.worker.CancelRequest
*/
export interface CancelRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.CancelResponse
*/
export interface CancelResponse {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: bool cancelled = 2
*/
cancelled: boolean;
/**
* @generated from protobuf field: string message = 3
*/
message: string;
}
/**
* @generated from protobuf message mpr.worker.WorkerStatus
*/
export interface WorkerStatus {
/**
* @generated from protobuf field: bool available = 1
*/
available: boolean;
/**
* @generated from protobuf field: int32 active_jobs = 2
*/
activeJobs: number;
/**
* @generated from protobuf field: repeated string supported_codecs = 3
*/
supportedCodecs: string[];
/**
* @generated from protobuf field: bool gpu_available = 4
*/
gpuAvailable: boolean;
}
/**
* Empty
*
* @generated from protobuf message mpr.worker.Empty
*/
export interface Empty {
}
/**
* @generated from protobuf message mpr.worker.ChunkStreamRequest
*/
export interface ChunkStreamRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.ChunkPipelineEvent
*/
export interface ChunkPipelineEvent {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: string event_type = 2
*/
eventType: string;
/**
* @generated from protobuf field: int32 sequence = 3
*/
sequence: number;
/**
* @generated from protobuf field: string worker_id = 4
*/
workerId: string;
/**
* @generated from protobuf field: string state = 5
*/
state: string;
/**
* @generated from protobuf field: int32 queue_size = 6
*/
queueSize: number;
/**
* @generated from protobuf field: float elapsed = 7
*/
elapsed: number;
/**
* @generated from protobuf field: float throughput_mbps = 8
*/
throughputMbps: number;
/**
* @generated from protobuf field: int32 total_chunks = 9
*/
totalChunks: number;
/**
* @generated from protobuf field: int32 processed_chunks = 10
*/
processedChunks: number;
/**
* @generated from protobuf field: int32 failed_chunks = 11
*/
failedChunks: number;
/**
* @generated from protobuf field: string error = 12
*/
error: string;
/**
* @generated from protobuf field: float processing_time = 13
*/
processingTime: number;
/**
* @generated from protobuf field: int32 retries = 14
*/
retries: number;
}
// @generated message type with reflection information, may provide speed optimized methods
class JobRequest$Type extends MessageType<JobRequest> {
constructor() {
super("mpr.worker.JobRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "source_path", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 3, name: "output_path", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 4, name: "preset_json", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 5, name: "trim_start", kind: "scalar", opt: true, T: 2 /*ScalarType.FLOAT*/ },
{ no: 6, name: "trim_end", kind: "scalar", opt: true, T: 2 /*ScalarType.FLOAT*/ }
]);
}
create(value?: PartialMessage<JobRequest>): JobRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.sourcePath = "";
message.outputPath = "";
message.presetJson = "";
if (value !== undefined)
reflectionMergePartial<JobRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: JobRequest): JobRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* string source_path */ 2:
message.sourcePath = reader.string();
break;
case /* string output_path */ 3:
message.outputPath = reader.string();
break;
case /* string preset_json */ 4:
message.presetJson = reader.string();
break;
case /* optional float trim_start */ 5:
message.trimStart = reader.float();
break;
case /* optional float trim_end */ 6:
message.trimEnd = reader.float();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: JobRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* string source_path = 2; */
if (message.sourcePath !== "")
writer.tag(2, WireType.LengthDelimited).string(message.sourcePath);
/* string output_path = 3; */
if (message.outputPath !== "")
writer.tag(3, WireType.LengthDelimited).string(message.outputPath);
/* string preset_json = 4; */
if (message.presetJson !== "")
writer.tag(4, WireType.LengthDelimited).string(message.presetJson);
/* optional float trim_start = 5; */
if (message.trimStart !== undefined)
writer.tag(5, WireType.Bit32).float(message.trimStart);
/* optional float trim_end = 6; */
if (message.trimEnd !== undefined)
writer.tag(6, WireType.Bit32).float(message.trimEnd);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.JobRequest
*/
export const JobRequest = new JobRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class JobResponse$Type extends MessageType<JobResponse> {
constructor() {
super("mpr.worker.JobResponse", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 3, name: "message", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<JobResponse>): JobResponse {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.accepted = false;
message.message = "";
if (value !== undefined)
reflectionMergePartial<JobResponse>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: JobResponse): JobResponse {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* bool accepted */ 2:
message.accepted = reader.bool();
break;
case /* string message */ 3:
message.message = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: JobResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* bool accepted = 2; */
if (message.accepted !== false)
writer.tag(2, WireType.Varint).bool(message.accepted);
/* string message = 3; */
if (message.message !== "")
writer.tag(3, WireType.LengthDelimited).string(message.message);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.JobResponse
*/
export const JobResponse = new JobResponse$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ProgressRequest$Type extends MessageType<ProgressRequest> {
constructor() {
super("mpr.worker.ProgressRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ProgressRequest>): ProgressRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<ProgressRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ProgressRequest): ProgressRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ProgressRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ProgressRequest
*/
export const ProgressRequest = new ProgressRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ProgressUpdate$Type extends MessageType<ProgressUpdate> {
constructor() {
super("mpr.worker.ProgressUpdate", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "progress", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 3, name: "current_frame", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 4, name: "current_time", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 5, name: "speed", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 6, name: "status", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 7, name: "error", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ProgressUpdate>): ProgressUpdate {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.progress = 0;
message.currentFrame = 0;
message.currentTime = 0;
message.speed = 0;
message.status = "";
if (value !== undefined)
reflectionMergePartial<ProgressUpdate>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ProgressUpdate): ProgressUpdate {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* int32 progress */ 2:
message.progress = reader.int32();
break;
case /* int32 current_frame */ 3:
message.currentFrame = reader.int32();
break;
case /* float current_time */ 4:
message.currentTime = reader.float();
break;
case /* float speed */ 5:
message.speed = reader.float();
break;
case /* string status */ 6:
message.status = reader.string();
break;
case /* optional string error */ 7:
message.error = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ProgressUpdate, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* int32 progress = 2; */
if (message.progress !== 0)
writer.tag(2, WireType.Varint).int32(message.progress);
/* int32 current_frame = 3; */
if (message.currentFrame !== 0)
writer.tag(3, WireType.Varint).int32(message.currentFrame);
/* float current_time = 4; */
if (message.currentTime !== 0)
writer.tag(4, WireType.Bit32).float(message.currentTime);
/* float speed = 5; */
if (message.speed !== 0)
writer.tag(5, WireType.Bit32).float(message.speed);
/* string status = 6; */
if (message.status !== "")
writer.tag(6, WireType.LengthDelimited).string(message.status);
/* optional string error = 7; */
if (message.error !== undefined)
writer.tag(7, WireType.LengthDelimited).string(message.error);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ProgressUpdate
*/
export const ProgressUpdate = new ProgressUpdate$Type();
// @generated message type with reflection information, may provide speed optimized methods
class CancelRequest$Type extends MessageType<CancelRequest> {
constructor() {
super("mpr.worker.CancelRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<CancelRequest>): CancelRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<CancelRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: CancelRequest): CancelRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: CancelRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.CancelRequest
*/
export const CancelRequest = new CancelRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class CancelResponse$Type extends MessageType<CancelResponse> {
constructor() {
super("mpr.worker.CancelResponse", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "cancelled", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 3, name: "message", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<CancelResponse>): CancelResponse {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.cancelled = false;
message.message = "";
if (value !== undefined)
reflectionMergePartial<CancelResponse>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: CancelResponse): CancelResponse {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* bool cancelled */ 2:
message.cancelled = reader.bool();
break;
case /* string message */ 3:
message.message = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: CancelResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* bool cancelled = 2; */
if (message.cancelled !== false)
writer.tag(2, WireType.Varint).bool(message.cancelled);
/* string message = 3; */
if (message.message !== "")
writer.tag(3, WireType.LengthDelimited).string(message.message);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.CancelResponse
*/
export const CancelResponse = new CancelResponse$Type();
// @generated message type with reflection information, may provide speed optimized methods
class WorkerStatus$Type extends MessageType<WorkerStatus> {
constructor() {
super("mpr.worker.WorkerStatus", [
{ no: 1, name: "available", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 2, name: "active_jobs", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 3, name: "supported_codecs", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ },
{ no: 4, name: "gpu_available", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
]);
}
create(value?: PartialMessage<WorkerStatus>): WorkerStatus {
const message = globalThis.Object.create((this.messagePrototype!));
message.available = false;
message.activeJobs = 0;
message.supportedCodecs = [];
message.gpuAvailable = false;
if (value !== undefined)
reflectionMergePartial<WorkerStatus>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: WorkerStatus): WorkerStatus {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* bool available */ 1:
message.available = reader.bool();
break;
case /* int32 active_jobs */ 2:
message.activeJobs = reader.int32();
break;
case /* repeated string supported_codecs */ 3:
message.supportedCodecs.push(reader.string());
break;
case /* bool gpu_available */ 4:
message.gpuAvailable = reader.bool();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: WorkerStatus, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* bool available = 1; */
if (message.available !== false)
writer.tag(1, WireType.Varint).bool(message.available);
/* int32 active_jobs = 2; */
if (message.activeJobs !== 0)
writer.tag(2, WireType.Varint).int32(message.activeJobs);
/* repeated string supported_codecs = 3; */
for (let i = 0; i < message.supportedCodecs.length; i++)
writer.tag(3, WireType.LengthDelimited).string(message.supportedCodecs[i]);
/* bool gpu_available = 4; */
if (message.gpuAvailable !== false)
writer.tag(4, WireType.Varint).bool(message.gpuAvailable);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.WorkerStatus
*/
export const WorkerStatus = new WorkerStatus$Type();
// @generated message type with reflection information, may provide speed optimized methods
class Empty$Type extends MessageType<Empty> {
constructor() {
super("mpr.worker.Empty", []);
}
create(value?: PartialMessage<Empty>): Empty {
const message = globalThis.Object.create((this.messagePrototype!));
if (value !== undefined)
reflectionMergePartial<Empty>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: Empty): Empty {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: Empty, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.Empty
*/
export const Empty = new Empty$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChunkStreamRequest$Type extends MessageType<ChunkStreamRequest> {
constructor() {
super("mpr.worker.ChunkStreamRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ChunkStreamRequest>): ChunkStreamRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<ChunkStreamRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChunkStreamRequest): ChunkStreamRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChunkStreamRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ChunkStreamRequest
*/
export const ChunkStreamRequest = new ChunkStreamRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChunkPipelineEvent$Type extends MessageType<ChunkPipelineEvent> {
constructor() {
super("mpr.worker.ChunkPipelineEvent", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "event_type", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 3, name: "sequence", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 4, name: "worker_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 5, name: "state", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 6, name: "queue_size", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 7, name: "elapsed", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 8, name: "throughput_mbps", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 9, name: "total_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 10, name: "processed_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 11, name: "failed_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 12, name: "error", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 13, name: "processing_time", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 14, name: "retries", kind: "scalar", T: 5 /*ScalarType.INT32*/ }
]);
}
create(value?: PartialMessage<ChunkPipelineEvent>): ChunkPipelineEvent {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.eventType = "";
message.sequence = 0;
message.workerId = "";
message.state = "";
message.queueSize = 0;
message.elapsed = 0;
message.throughputMbps = 0;
message.totalChunks = 0;
message.processedChunks = 0;
message.failedChunks = 0;
message.error = "";
message.processingTime = 0;
message.retries = 0;
if (value !== undefined)
reflectionMergePartial<ChunkPipelineEvent>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChunkPipelineEvent): ChunkPipelineEvent {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* string event_type */ 2:
message.eventType = reader.string();
break;
case /* int32 sequence */ 3:
message.sequence = reader.int32();
break;
case /* string worker_id */ 4:
message.workerId = reader.string();
break;
case /* string state */ 5:
message.state = reader.string();
break;
case /* int32 queue_size */ 6:
message.queueSize = reader.int32();
break;
case /* float elapsed */ 7:
message.elapsed = reader.float();
break;
case /* float throughput_mbps */ 8:
message.throughputMbps = reader.float();
break;
case /* int32 total_chunks */ 9:
message.totalChunks = reader.int32();
break;
case /* int32 processed_chunks */ 10:
message.processedChunks = reader.int32();
break;
case /* int32 failed_chunks */ 11:
message.failedChunks = reader.int32();
break;
case /* string error */ 12:
message.error = reader.string();
break;
case /* float processing_time */ 13:
message.processingTime = reader.float();
break;
case /* int32 retries */ 14:
message.retries = reader.int32();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChunkPipelineEvent, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* string event_type = 2; */
if (message.eventType !== "")
writer.tag(2, WireType.LengthDelimited).string(message.eventType);
/* int32 sequence = 3; */
if (message.sequence !== 0)
writer.tag(3, WireType.Varint).int32(message.sequence);
/* string worker_id = 4; */
if (message.workerId !== "")
writer.tag(4, WireType.LengthDelimited).string(message.workerId);
/* string state = 5; */
if (message.state !== "")
writer.tag(5, WireType.LengthDelimited).string(message.state);
/* int32 queue_size = 6; */
if (message.queueSize !== 0)
writer.tag(6, WireType.Varint).int32(message.queueSize);
/* float elapsed = 7; */
if (message.elapsed !== 0)
writer.tag(7, WireType.Bit32).float(message.elapsed);
/* float throughput_mbps = 8; */
if (message.throughputMbps !== 0)
writer.tag(8, WireType.Bit32).float(message.throughputMbps);
/* int32 total_chunks = 9; */
if (message.totalChunks !== 0)
writer.tag(9, WireType.Varint).int32(message.totalChunks);
/* int32 processed_chunks = 10; */
if (message.processedChunks !== 0)
writer.tag(10, WireType.Varint).int32(message.processedChunks);
/* int32 failed_chunks = 11; */
if (message.failedChunks !== 0)
writer.tag(11, WireType.Varint).int32(message.failedChunks);
/* string error = 12; */
if (message.error !== "")
writer.tag(12, WireType.LengthDelimited).string(message.error);
/* float processing_time = 13; */
if (message.processingTime !== 0)
writer.tag(13, WireType.Bit32).float(message.processingTime);
/* int32 retries = 14; */
if (message.retries !== 0)
writer.tag(14, WireType.Varint).int32(message.retries);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ChunkPipelineEvent
*/
export const ChunkPipelineEvent = new ChunkPipelineEvent$Type();
/**
* @generated ServiceType for protobuf service mpr.worker.WorkerService
*/
export const WorkerService = new ServiceType("mpr.worker.WorkerService", [
{ name: "SubmitJob", options: {}, I: JobRequest, O: JobResponse },
{ name: "StreamProgress", serverStreaming: true, options: {}, I: ProgressRequest, O: ProgressUpdate },
{ name: "CancelJob", options: {}, I: CancelRequest, O: CancelResponse },
{ name: "GetWorkerStatus", options: {}, I: Empty, O: WorkerStatus },
{ name: "StreamChunkPipeline", serverStreaming: true, options: {}, I: ChunkStreamRequest, O: ChunkPipelineEvent }
]);

42
ui/common/api/media.ts Normal file
View File

@@ -0,0 +1,42 @@
/**
* Shared media API functions — identical across all MPR UI apps.
*/
import type { MediaAsset } from "../types/generated";
import { gql } from "./graphql";
/** Fetch all media assets. */
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
/** Scan media/in/ folder for new files. */
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{
scan_media_folder: {
found: number;
registered: number;
skipped: number;
files: string[];
};
}>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}

97
ui/common/components/FileManager.css Normal file
View File

@@ -0,0 +1,97 @@
.file-manager {
margin-bottom: 1rem;
}
.fm-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.fm-header h2 {
font-size: 0.85rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--text-secondary);
}
.fm-scan-btn {
padding: 0.25rem 0.6rem;
background: var(--bg-input);
color: var(--text-secondary);
font-size: var(--font-size-xs);
}
.fm-scan-btn:hover:not(:disabled) {
color: var(--text-primary);
background: var(--border-light);
}
.fm-list {
list-style: none;
max-height: 200px;
overflow-y: auto;
border: 1px solid var(--border);
border-radius: var(--radius-sm);
background: var(--bg-primary);
}
.fm-empty {
padding: 1rem;
text-align: center;
color: var(--text-muted);
font-size: var(--font-size-sm);
}
.fm-item {
display: flex;
justify-content: space-between;
align-items: center;
padding: 0.4rem 0.6rem;
border-bottom: 1px solid var(--border);
transition: background 0.1s;
}
.fm-item:last-child {
border-bottom: none;
}
.fm-clickable {
cursor: pointer;
}
.fm-clickable:hover {
background: var(--bg-input);
}
.fm-selected {
background: var(--accent) !important;
color: #fff;
}
.fm-selected .fm-meta {
color: rgba(255, 255, 255, 0.7);
}
.fm-item-info {
display: flex;
flex-direction: column;
gap: 0.15rem;
overflow: hidden;
min-width: 0;
}
.fm-filename {
font-size: var(--font-size-sm);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.fm-meta {
font-size: var(--font-size-xs);
color: var(--text-muted);
}
.fm-actions {
flex-shrink: 0;
margin-left: 0.5rem;
}

84
ui/common/components/FileManager.tsx Normal file
View File

@@ -0,0 +1,84 @@
/**
* FileManager — pluggable file browser for S3/MinIO files.
*
* Handles both input file selection and output file listing.
* Used by timeline (assets + output), chunker (assets + chunk output),
* and future tools.
*/
import type { ReactNode } from "react";
import { formatSize } from "../utils/format";
import "./FileManager.css";
export interface FileEntry {
key: string;
name: string;
size?: number;
meta?: string;
}
interface FileManagerProps {
title: string;
files: FileEntry[];
selectedKey?: string | null;
onSelect?: (file: FileEntry) => void;
onScan?: () => void;
scanning?: boolean;
emptyMessage?: string;
renderActions?: (file: FileEntry) => ReactNode;
disabled?: boolean;
}
export function FileManager({
title,
files,
selectedKey,
onSelect,
onScan,
scanning = false,
emptyMessage = "No files",
renderActions,
disabled = false,
}: FileManagerProps) {
return (
<div className="file-manager">
<div className="fm-header">
<h2>{title}</h2>
{onScan && (
<button
className="fm-scan-btn"
onClick={onScan}
disabled={scanning || disabled}
>
{scanning ? "Scanning..." : "Scan Folder"}
</button>
)}
</div>
<ul className="fm-list">
{files.length === 0 ? (
<li className="fm-empty">{emptyMessage}</li>
) : (
files.map((file) => (
<li
key={file.key}
className={`fm-item ${selectedKey === file.key ? "fm-selected" : ""} ${onSelect && !disabled ? "fm-clickable" : ""}`}
onClick={() => onSelect && !disabled && onSelect(file)}
title={file.name}
>
<div className="fm-item-info">
<span className="fm-filename">{file.name}</span>
<span className="fm-meta">
{file.size != null && formatSize(file.size)}
{file.meta && (file.size != null ? ` · ${file.meta}` : file.meta)}
</span>
</div>
{renderActions && (
<div className="fm-actions">{renderActions(file)}</div>
)}
</li>
))
)}
</ul>
</div>
);
}

33
ui/common/components/StatusDot.tsx Normal file
View File

@@ -0,0 +1,33 @@
/**
* StatusDot — small colored indicator for connection/state.
*/
const STATE_COLORS: Record<string, string> = {
connected: "var(--success)",
idle: "var(--text-muted)",
processing: "var(--processing)",
stopped: "var(--text-muted)",
error: "var(--error)",
done: "var(--success)",
};
interface StatusDotProps {
state: string;
glow?: boolean;
}
export function StatusDot({ state, glow = false }: StatusDotProps) {
const color = STATE_COLORS[state] || "var(--text-muted)";
return (
<span
style={{
display: "inline-block",
width: 8,
height: 8,
borderRadius: "50%",
background: color,
boxShadow: glow ? `0 0 6px ${color}` : undefined,
}}
/>
);
}

109
ui/common/styles/theme.css Normal file
View File

@@ -0,0 +1,109 @@
/**
* MPR Shared Theme — CSS custom properties + base styles.
* Import from any UI app: @import "../../common/styles/theme.css";
*/
:root {
--bg-primary: #0f0f0f;
--bg-panel: #1a1a1a;
--bg-surface: #141414;
--bg-input: #2a2a2a;
--border: #2a2a2a;
--border-light: #333;
--text-primary: #e0e0e0;
--text-secondary: #999;
--text-muted: #666;
--accent: #3b82f6;
--success: #10b981;
--warning: #f59e0b;
--error: #ef4444;
--processing: #3b82f6;
--radius: 8px;
--radius-sm: 4px;
--font-mono: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
"Fira Code", monospace, sans-serif;
--font-size: 14px;
--font-size-sm: 0.8rem;
--font-size-xs: 0.75rem;
}
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: var(--font-mono);
background: var(--bg-primary);
color: var(--text-primary);
font-size: var(--font-size);
}
/* Scrollbar */
::-webkit-scrollbar {
width: 6px;
}
::-webkit-scrollbar-track {
background: transparent;
}
::-webkit-scrollbar-thumb {
background: var(--border-light);
border-radius: 3px;
}
/* Shared button base */
button {
cursor: pointer;
border: none;
border-radius: var(--radius-sm);
font-family: var(--font-mono);
font-size: var(--font-size-sm);
transition: opacity 0.15s;
}
button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
/* Shared input base */
input,
select {
font-family: var(--font-mono);
font-size: var(--font-size-sm);
background: var(--bg-input);
color: var(--text-primary);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
padding: 0.4rem 0.5rem;
}
input:focus,
select:focus {
outline: none;
border-color: var(--accent);
}
/* Panel base */
.panel {
background: var(--bg-panel);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 1rem;
}
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.75rem;
position: relative;
}
.panel-header h2 {
font-size: 0.85rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--text-secondary);
}

170
ui/common/types/generated.ts Normal file
View File

@@ -0,0 +1,170 @@
/**
* TypeScript Types - GENERATED FILE
*
* Do not edit directly. Regenerate using modelgen.
*/
export type AssetStatus = "pending" | "ready" | "error";
export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled";
export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled";
export interface MediaAsset {
id: string;
filename: string;
file_path: string;
status: AssetStatus;
error_message: string | null;
file_size: number | null;
duration: number | null;
video_codec: string | null;
audio_codec: string | null;
width: number | null;
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodePreset {
id: string;
name: string;
description: string;
is_builtin: boolean;
container: string;
video_codec: string;
video_bitrate: string | null;
video_crf: number | null;
video_preset: string | null;
resolution: string | null;
framerate: number | null;
audio_codec: string;
audio_bitrate: string | null;
audio_channels: number | null;
audio_samplerate: number | null;
extra_args: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodeJob {
id: string;
source_asset_id: string;
preset_id: string | null;
preset_snapshot: Record<string, unknown>;
trim_start: number | null;
trim_end: number | null;
output_filename: string;
output_path: string | null;
output_asset_id: string | null;
status: JobStatus;
progress: number;
current_frame: number | null;
current_time: number | null;
speed: string | null;
error_message: string | null;
celery_task_id: string | null;
execution_arn: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface ChunkJob {
id: string;
source_asset_id: string;
chunk_duration: number;
num_workers: number;
max_retries: number;
processor_type: string;
status: ChunkJobStatus;
progress: number;
total_chunks: number;
processed_chunks: number;
failed_chunks: number;
retry_count: number;
error_message: string | null;
throughput_mbps: number | null;
elapsed_seconds: number | null;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;
trim_start: number | null;
trim_end: number | null;
output_filename: string | null;
priority: number;
}
export interface UpdateAssetRequest {
comments: string | null;
tags: string[] | null;
}
export interface SystemStatus {
status: string;
version: string;
}
export interface ScanResult {
found: number;
registered: number;
skipped: number;
files: string[];
}
export interface DeleteResult {
ok: boolean;
}
export interface WorkerStatus {
available: boolean;
active_jobs: number;
supported_codecs: string[];
gpu_available: boolean;
}
export interface ChunkEvent {
sequence: number;
status: string;
size: number | null;
worker_id: string | null;
processing_time: number | null;
error: string | null;
retries: number;
}
export interface WorkerEvent {
worker_id: string;
state: string;
current_chunk: number | null;
processed: number;
errors: number;
retries: number;
}
export interface PipelineStats {
total_chunks: number;
processed: number;
failed: number;
retries: number;
elapsed: number;
throughput_mbps: number;
queue_size: number;
}
export interface ChunkOutputFile {
key: string;
size: number;
url: string;
}

21
ui/common/utils/format.ts Normal file
View File

@@ -0,0 +1,21 @@
/**
* Shared formatting utilities.
*/
export function formatSize(bytes: number | null | undefined): string {
if (!bytes) return "—";
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
if (bytes < 1024 * 1024 * 1024)
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
}
export function formatDuration(seconds: number | null | undefined): string {
if (!seconds) return "—";
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
const s = Math.floor(seconds % 60);
if (h > 0)
return `${h}:${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
return `${m}:${s.toString().padStart(2, "0")}`;
}

1736
ui/timeline/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

13
ui/timeline/src/App.css
View File

@@ -1,15 +1,4 @@
* { @import "../../common/styles/theme.css";
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family:
-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
background: #1a1a1a;
color: #e0e0e0;
}
.app { .app {
display: flex; display: flex;

1
ui/timeline/src/App.tsx
View File

@@ -115,7 +115,6 @@ function App() {
setJobs(data); setJobs(data);
}; };
const assetJobs = jobs.filter((j) => j.source_asset_id === selectedAsset?.id);
const completedJobs = jobs.filter((j) => j.status === "completed"); const completedJobs = jobs.filter((j) => j.status === "completed");
if (loading) return <div className="loading">Loading...</div>; if (loading) return <div className="loading">Loading...</div>;

2
ui/timeline/src/JobPanel.tsx
View File

@@ -42,6 +42,8 @@ export default function JobPanel({
preset_id: selectedPresetId || null, preset_id: selectedPresetId || null,
trim_start: hasTrim ? trimStart : null, trim_start: hasTrim ? trimStart : null,
trim_end: hasTrim ? trimEnd : null, trim_end: hasTrim ? trimEnd : null,
output_filename: null,
priority: 0,
}); });
onJobCreated(); onJobCreated();
} catch (e) { } catch (e) {

50
ui/timeline/src/api.ts
View File

@@ -2,45 +2,17 @@
* GraphQL API client * GraphQL API client
*/ */
import { gql } from "../../common/api/graphql";
import { getAssets, scanMediaFolder } from "../../common/api/media";
import type { import type {
MediaAsset,
TranscodePreset, TranscodePreset,
TranscodeJob, TranscodeJob,
CreateJobRequest, CreateJobRequest,
SystemStatus, SystemStatus,
MediaAsset,
} from "./types"; } from "./types";
const GRAPHQL_URL = "/api/graphql"; export { getAssets, scanMediaFolder };
async function gql<T>(query: string, variables?: Record<string, unknown>): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}
// Assets
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
export async function getAsset(id: string): Promise<MediaAsset> { export async function getAsset(id: string): Promise<MediaAsset> {
const data = await gql<{ asset: MediaAsset }>(` const data = await gql<{ asset: MediaAsset }>(`
@@ -55,20 +27,6 @@ export async function getAsset(id: string): Promise<MediaAsset> {
return data.asset; return data.asset;
} }
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}
// Presets // Presets
export async function getPresets(): Promise<TranscodePreset[]> { export async function getPresets(): Promise<TranscodePreset[]> {
const data = await gql<{ presets: TranscodePreset[] }>(` const data = await gql<{ presets: TranscodePreset[] }>(`

146
ui/timeline/src/types.ts
View File

@@ -1,135 +1,21 @@
/** /**
* TypeScript Types - GENERATED FILE * TypeScript Types — re-exported from common generated types.
* *
* Do not edit directly. Regenerate using modelgen. * Do not edit directly. Regenerate using modelgen.
*/ */
export type AssetStatus = "pending" | "ready" | "error"; export type {
export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled"; AssetStatus,
export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled"; JobStatus,
ChunkJobStatus,
export interface MediaAsset { MediaAsset,
id: string; TranscodePreset,
filename: string; TranscodeJob,
file_path: string; ChunkJob,
status: AssetStatus; CreateJobRequest,
error_message: string | null; UpdateAssetRequest,
file_size: number | null; SystemStatus,
duration: number | null; ScanResult,
video_codec: string | null; DeleteResult,
audio_codec: string | null; WorkerStatus,
width: number | null; } from "../../common/types/generated";
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodePreset {
id: string;
name: string;
description: string;
is_builtin: boolean;
container: string;
video_codec: string;
video_bitrate: string | null;
video_crf: number | null;
video_preset: string | null;
resolution: string | null;
framerate: number | null;
audio_codec: string;
audio_bitrate: string | null;
audio_channels: number | null;
audio_samplerate: number | null;
extra_args: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodeJob {
id: string;
source_asset_id: string;
preset_id: string | null;
preset_snapshot: Record<string, unknown>;
trim_start: number | null;
trim_end: number | null;
output_filename: string;
output_path: string | null;
output_asset_id: string | null;
status: JobStatus;
progress: number;
current_frame: number | null;
current_time: number | null;
speed: string | null;
error_message: string | null;
celery_task_id: string | null;
execution_arn: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface ChunkJob {
id: string;
source_asset_id: string;
chunk_duration: number;
num_workers: number;
max_retries: number;
processor_type: string;
status: ChunkJobStatus;
progress: number;
total_chunks: number;
processed_chunks: number;
failed_chunks: number;
retry_count: number;
error_message: string | null;
throughput_mbps: number | null;
elapsed_seconds: number | null;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;
trim_start: number | null;
trim_end: number | null;
output_filename: string | null;
priority: number;
}
export interface UpdateAssetRequest {
comments: string | null;
tags: string[] | null;
}
export interface SystemStatus {
status: string;
version: string;
}
export interface ScanResult {
found: number;
registered: number;
skipped: number;
files: string[];
}
export interface DeleteResult {
ok: boolean;
}
export interface WorkerStatus {
available: boolean;
active_jobs: number;
supported_codecs: string[];
gpu_available: boolean;
}

5
ui/timeline/tsconfig.json
View File

@@ -15,7 +15,10 @@
"noUnusedLocals": true, "noUnusedLocals": true,
"noUnusedParameters": true, "noUnusedParameters": true,
"noFallthroughCasesInSwitch": true, "noFallthroughCasesInSwitch": true,
"rootDir": "..",
"typeRoots": ["./node_modules/@types"],
}, },
"include": ["src/**/*.ts", "src/**/*.tsx"], "include": ["src/**/*.ts", "src/**/*.tsx", "../common/**/*.ts", "../common/**/*.tsx"],
"exclude": ["../common/api/grpc/**"],
"references": [{ "path": "./tsconfig.node.json" }], "references": [{ "path": "./tsconfig.node.json" }],
} }

3
ui/timeline/vite.config.ts
View File

@@ -11,6 +11,9 @@ export default defineConfig({
hmr: { hmr: {
path: "/timeline/@vite/client", path: "/timeline/@vite/client",
}, },
fs: {
allow: [".."],
},
proxy: { proxy: {
"/api": { "/api": {
target: "http://fastapi:8702", target: "http://fastapi:8702",