chunker ui redo

This commit is contained in:
2026-03-15 16:03:53 -03:00
parent d5a3372d6b
commit b40bd68411
62 changed files with 5460 additions and 1493 deletions

View File

@@ -1,8 +1,10 @@
"""
SSE endpoint for chunker pipeline events.
Bridges gRPC StreamProgress to browser-native EventSource.
GET /api/chunker/stream/{job_id} → text/event-stream
Uses Redis as the event bus between Celery workers and the SSE stream.
Celery worker pushes events via core.events, SSE endpoint polls them.
GET /chunker/stream/{job_id} → text/event-stream
"""
import asyncio
@@ -14,46 +16,39 @@ from typing import AsyncGenerator
from fastapi import APIRouter
from starlette.responses import StreamingResponse
from core.events import poll_events
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/chunker", tags=["chunker"])
router = APIRouter(prefix="/chunker", tags=["chunker"])
async def _event_generator(job_id: str) -> AsyncGenerator[str, None]:
"""
Generate SSE events by polling gRPC job state.
Yields server-sent events in the format:
event: <event_type>
data: <json_payload>
Generate SSE events by polling Redis for chunk job events.
"""
from core.rpc.server import _active_jobs
last_state = None
cursor = 0
timeout = time.monotonic() + 600 # 10 min max
while time.monotonic() < timeout:
job_state = _active_jobs.get(job_id)
events, cursor = poll_events(job_id, cursor)
if job_state is None:
# Job not found yet — may not have started
if not events:
yield f"event: waiting\ndata: {json.dumps({'job_id': job_id})}\n\n"
await asyncio.sleep(0.5)
await asyncio.sleep(0.1)
continue
# Only send if state changed
if job_state != last_state:
last_state = dict(job_state)
event_type = job_state.get("status", "update")
for data in events:
event_type = data.pop("event", "update")
payload = {**data, "job_id": job_id}
yield f"event: {event_type}\ndata: {json.dumps({**job_state, 'job_id': job_id})}\n\n"
yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n"
# End stream when job is terminal
if event_type in ("completed", "failed", "cancelled"):
if event_type in ("pipeline_complete", "pipeline_error", "cancelled"):
yield f"event: done\ndata: {json.dumps({'job_id': job_id})}\n\n"
break
return
await asyncio.sleep(0.2)
await asyncio.sleep(0.05)
yield f"event: timeout\ndata: {json.dumps({'job_id': job_id})}\n\n"

View File

@@ -15,7 +15,9 @@ from strawberry.schema.config import StrawberryConfig
from strawberry.types import Info
from core.api.schema.graphql import (
CancelResultType,
ChunkJobType,
ChunkOutputFileType,
CreateChunkJobInput,
CreateJobInput,
DeleteResultType,
@@ -26,7 +28,7 @@ from core.api.schema.graphql import (
TranscodePresetType,
UpdateAssetInput,
)
from core.storage import BUCKET_IN, list_objects
from core.storage import BUCKET_IN, list_objects, upload_file
VIDEO_EXTS = {".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v"}
AUDIO_EXTS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a"}
@@ -90,6 +92,25 @@ class Query:
def system_status(self, info: Info) -> SystemStatusType:
return SystemStatusType(status="ok", version="0.1.0")
@strawberry.field
def chunk_output_files(self, info: Info, job_id: str) -> List[ChunkOutputFileType]:
"""List output chunk files for a completed job from media/out/."""
from pathlib import Path
media_out = os.environ.get("MEDIA_OUT_DIR", "/app/media/out")
output_dir = Path(media_out) / "chunks" / job_id
if not output_dir.is_dir():
return []
return [
ChunkOutputFileType(
key=f.name,
size=f.stat().st_size,
url=f"/media/out/chunks/{job_id}/{f.name}",
)
for f in sorted(output_dir.iterdir())
if f.is_file()
]
# ---------------------------------------------------------------------------
# Mutations
@@ -100,8 +121,26 @@ class Query:
class Mutation:
@strawberry.mutation
def scan_media_folder(self, info: Info) -> ScanResultType:
import logging
from pathlib import Path
from core.db import create_asset, get_asset_filenames
logger = logging.getLogger(__name__)
# Sync local media/in/ files to MinIO (handles fresh installs / pruned volumes)
local_media = Path("/app/media/in")
if local_media.is_dir():
existing_keys = {o["key"] for o in list_objects(BUCKET_IN)}
for f in local_media.iterdir():
if f.is_file() and f.suffix.lower() in MEDIA_EXTS:
if f.name not in existing_keys:
try:
upload_file(str(f), BUCKET_IN, f.name)
logger.info("Uploaded %s to MinIO", f.name)
except Exception as e:
logger.warning("Failed to upload %s: %s", f.name, e)
objects = list_objects(BUCKET_IN, extensions=MEDIA_EXTS)
existing = get_asset_filenames()
@@ -284,6 +323,8 @@ class Mutation:
"num_workers": input.num_workers,
"max_retries": input.max_retries,
"processor_type": input.processor_type,
"start_time": input.start_time,
"end_time": input.end_time,
}
executor_mode = os.environ.get("MPR_EXECUTOR", "local")
@@ -320,6 +361,17 @@ class Mutation:
celery_task_id=celery_task_id,
)
@strawberry.mutation
def cancel_chunk_job(self, info: Info, celery_task_id: str) -> CancelResultType:
"""Cancel a running chunk job by revoking its Celery task."""
try:
from admin.mpr.celery import app as celery_app
celery_app.control.revoke(celery_task_id, terminate=True, signal="SIGTERM")
return CancelResultType(ok=True, message="Task revoked")
except Exception as e:
return CancelResultType(ok=False, message=str(e))
# ---------------------------------------------------------------------------
# Schema

View File

@@ -37,7 +37,7 @@ class MediaAssetType:
file_path: Optional[str] = None
status: Optional[str] = None
error_message: Optional[str] = None
file_size: Optional[int] = None
file_size: Optional[float] = None
duration: Optional[float] = None
video_codec: Optional[str] = None
audio_codec: Optional[str] = None
@@ -205,3 +205,22 @@ class CreateChunkJobInput:
max_retries: int = 3
processor_type: str = "ffmpeg"
priority: int = 0
start_time: Optional[float] = None
end_time: Optional[float] = None
@strawberry.type
class CancelResultType:
"""Result of cancelling a chunk job."""
ok: bool = False
message: Optional[str] = None
@strawberry.type
class ChunkOutputFileType:
"""A chunk output file in S3/MinIO with presigned download URL."""
key: str
size: int = 0
url: str = ""

View File

@@ -28,7 +28,13 @@ class Chunker:
chunk_duration: Duration of each chunk in seconds (default: 10.0)
"""
def __init__(self, file_path: str, chunk_duration: float = 10.0):
def __init__(
self,
file_path: str,
chunk_duration: float = 10.0,
start_time: float | None = None,
end_time: float | None = None,
):
if not os.path.isfile(file_path):
raise ChunkReadError(f"File not found: {file_path}")
if chunk_duration <= 0:
@@ -37,7 +43,16 @@ class Chunker:
self.file_path = file_path
self.chunk_duration = chunk_duration
self.file_size = os.path.getsize(file_path)
self.source_duration = self._probe_duration()
full_duration = self._probe_duration()
# Apply time range
self.range_start = max(start_time or 0.0, 0.0)
self.range_end = min(end_time or full_duration, full_duration)
if self.range_start >= self.range_end:
raise ValueError(
f"Invalid range: start={self.range_start} >= end={self.range_end}"
)
self.source_duration = self.range_end - self.range_start
def _probe_duration(self) -> float:
"""Get source file duration via FFmpeg probe."""
@@ -71,9 +86,9 @@ class Chunker:
"""
total = self.expected_chunks
for sequence in range(total):
start_time = sequence * self.chunk_duration
start_time = self.range_start + sequence * self.chunk_duration
end_time = min(
start_time + self.chunk_duration, self.source_duration
start_time + self.chunk_duration, self.range_end
)
duration = end_time - start_time

View File

@@ -57,6 +57,8 @@ class Pipeline:
queue_size: int = 10,
event_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None,
output_dir: Optional[str] = None,
start_time: Optional[float] = None,
end_time: Optional[float] = None,
):
self.source = source
self.chunk_duration = chunk_duration
@@ -66,6 +68,8 @@ class Pipeline:
self.queue_size = queue_size
self.event_callback = event_callback
self.output_dir = output_dir
self.start_time = start_time
self.end_time = end_time
def _emit(self, event_type: str, data: Dict[str, Any]) -> None:
"""Emit an event if callback is registered."""
@@ -92,6 +96,19 @@ class Pipeline:
finally:
chunk_queue.close()
def _monitor_progress(
self, start_time: float, file_size: int, stop_event: threading.Event
) -> None:
"""Monitor thread: emit pipeline_progress every 500ms."""
while not stop_event.is_set():
elapsed = time.monotonic() - start_time
mb = file_size / (1024 * 1024)
self._emit("pipeline_progress", {
"elapsed": round(elapsed, 2),
"throughput_mbps": round(mb / elapsed, 2) if elapsed > 0 else 0,
})
stop_event.wait(0.5)
def _write_manifest(
self, result: PipelineResult, source_duration: float
) -> None:
@@ -146,7 +163,12 @@ class Pipeline:
try:
# Stage 1: Set up chunker (probes file for duration)
chunker = Chunker(self.source, self.chunk_duration)
chunker = Chunker(
self.source,
self.chunk_duration,
start_time=self.start_time,
end_time=self.end_time,
)
total_chunks = chunker.expected_chunks
if total_chunks == 0:
@@ -170,9 +192,18 @@ class Pipeline:
output_dir=self.output_dir,
)
# Stage 3: Start workers, then produce chunks
# Stage 3: Start workers, monitor, then produce chunks
pool.start()
monitor_stop = threading.Event()
monitor = threading.Thread(
target=self._monitor_progress,
args=(start_time, chunker.file_size, monitor_stop),
name="progress-monitor",
daemon=True,
)
monitor.start()
producer = threading.Thread(
target=self._produce_chunks,
args=(chunker, chunk_queue),
@@ -185,6 +216,10 @@ class Pipeline:
all_results = pool.wait()
producer.join(timeout=5.0)
# Stop monitor
monitor_stop.set()
monitor.join(timeout=2.0)
# Stage 5: Collect results in order
collector = ResultCollector(total_chunks)
for r in all_results:

View File

@@ -124,6 +124,7 @@ class Worker:
self._emit("chunk_processing", {
"sequence": chunk.sequence,
"state": "processing",
"queue_size": self.chunk_queue.qsize(),
})
result = self._process_with_retry(chunk)
@@ -135,6 +136,7 @@ class Worker:
"success": result.success,
"processing_time": result.processing_time,
"retries": result.retries,
"queue_size": self.chunk_queue.qsize(),
})
self._emit("worker_status", {"state": "stopped"})

40
core/events.py Normal file
View File

@@ -0,0 +1,40 @@
"""
Redis-based event bus for pipeline job progress.
Celery workers push events, SSE endpoints poll them.
Only depends on redis — safe to import from any context.
"""
import json
import os
import redis
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
def _get_redis():
return redis.from_url(REDIS_URL, decode_responses=True)
def push_event(job_id: str, event_type: str, data: dict) -> None:
"""Push an event to the Redis list for a job."""
r = _get_redis()
key = f"chunk_events:{job_id}"
event = json.dumps({"event": event_type, **data})
r.rpush(key, event)
r.expire(key, 3600)
def poll_events(job_id: str, cursor: int = 0) -> tuple[list[dict], int]:
"""Poll new events from Redis. Returns (events, new_cursor)."""
r = _get_redis()
key = f"chunk_events:{job_id}"
raw_events = r.lrange(key, cursor, -1)
parsed = []
for raw in raw_events:
try:
parsed.append(json.loads(raw))
except (json.JSONDecodeError, TypeError):
pass
return parsed, cursor + len(raw_events)

View File

@@ -2,22 +2,24 @@
ChunkHandler — job handler that wraps the chunker Pipeline.
Downloads source from S3/MinIO, runs FFmpeg chunking pipeline,
uploads mp4 segments + manifest back to S3/MinIO.
writes mp4 segments + manifest to media/out/chunks/{job_id}/.
Pushes real-time events to Redis for SSE consumption.
"""
import logging
import os
import shutil
import tempfile
from typing import Any, Callable, Dict, Optional
from core.events import push_event as push_chunk_event
from core.chunker import Pipeline
from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file
from core.storage import BUCKET_IN, download_to_temp
from .base import Handler
logger = logging.getLogger(__name__)
MEDIA_OUT_DIR = os.environ.get("MEDIA_OUT_DIR", "/app/media/out")
class ChunkHandler(Handler):
"""
@@ -44,14 +46,19 @@ class ChunkHandler(Handler):
logger.info(f"ChunkHandler starting job {job_id}: {source_key}")
# Download source from S3/MinIO
push_chunk_event(job_id, "pipeline_start", {"status": "downloading", "source_key": source_key})
tmp_source = download_to_temp(BUCKET_IN, source_key)
# Create temp output directory for chunks
tmp_output_dir = tempfile.mkdtemp(prefix=f"chunks-{job_id}-")
# Output directory: media/out/chunks/{job_id}/
output_dir = os.path.join(MEDIA_OUT_DIR, "chunks", job_id)
if processor_type == "ffmpeg":
os.makedirs(output_dir, exist_ok=True)
try:
def event_bridge(event_type: str, data: Dict[str, Any]) -> None:
"""Bridge pipeline events to the job progress callback."""
"""Bridge pipeline events to Redis + optional progress callback."""
push_chunk_event(job_id, event_type, data)
if progress_callback and event_type == "pipeline_complete":
progress_callback(100, data)
elif progress_callback and event_type == "chunk_done":
@@ -68,29 +75,28 @@ class ChunkHandler(Handler):
processor_type=processor_type,
queue_size=payload.get("queue_size", 10),
event_callback=event_bridge,
output_dir=tmp_output_dir if processor_type == "ffmpeg" else None,
output_dir=output_dir if processor_type == "ffmpeg" else None,
start_time=payload.get("start_time"),
end_time=payload.get("end_time"),
)
result = pipeline.run()
# Upload chunks + manifest to S3/MinIO
# Files are already in media/out/chunks/{job_id}/
output_prefix = f"chunks/{job_id}"
uploaded_files = []
output_files = [
f"{output_prefix}/{os.path.basename(f)}"
for f in result.chunk_files
]
for chunk_file in result.chunk_files:
filename = os.path.basename(chunk_file)
output_key = f"{output_prefix}/{filename}"
upload_file(chunk_file, BUCKET_OUT, output_key)
uploaded_files.append(output_key)
logger.info(f"Uploaded {output_key}")
# Upload manifest
manifest_path = os.path.join(tmp_output_dir, "manifest.json")
if os.path.exists(manifest_path):
manifest_key = f"{output_prefix}/manifest.json"
upload_file(manifest_path, BUCKET_OUT, manifest_key)
uploaded_files.append(manifest_key)
logger.info(f"Uploaded {manifest_key}")
push_chunk_event(job_id, "pipeline_complete", {
"status": "completed",
"total_chunks": result.total_chunks,
"processed": result.processed,
"failed": result.failed,
"elapsed": result.elapsed_time,
"throughput_mbps": result.throughput_mbps,
})
return {
"status": "completed" if result.failed == 0 else "completed_with_errors",
@@ -104,16 +110,16 @@ class ChunkHandler(Handler):
"errors": result.errors,
"chunks_in_order": result.chunks_in_order,
"output_prefix": output_prefix,
"uploaded_files": uploaded_files,
"output_files": output_files,
}
except Exception as e:
push_chunk_event(job_id, "pipeline_error", {"status": "failed", "error": str(e)})
raise
finally:
# Cleanup temp files
# Cleanup temp source file only (output dir is persistent)
try:
os.unlink(tmp_source)
except OSError:
pass
try:
shutil.rmtree(tmp_output_dir, ignore_errors=True)
except OSError:
pass

View File

@@ -11,6 +11,7 @@ service WorkerService {
rpc StreamProgress(ProgressRequest) returns (stream ProgressUpdate);
rpc CancelJob(CancelRequest) returns (CancelResponse);
rpc GetWorkerStatus(Empty) returns (WorkerStatus);
rpc StreamChunkPipeline(ChunkStreamRequest) returns (stream ChunkPipelineEvent);
}
message JobRequest {
@@ -62,3 +63,24 @@ message WorkerStatus {
message Empty {
// Empty
}
message ChunkStreamRequest {
string job_id = 1;
}
message ChunkPipelineEvent {
string job_id = 1;
string event_type = 2;
int32 sequence = 3;
string worker_id = 4;
string state = 5;
int32 queue_size = 6;
float elapsed = 7;
float throughput_mbps = 8;
int32 total_chunks = 9;
int32 processed_chunks = 10;
int32 failed_chunks = 11;
string error = 12;
float processing_time = 13;
int32 retries = 14;
}

View File

@@ -173,6 +173,43 @@ class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer):
message="Job not found",
)
def StreamChunkPipeline(self, request, context) -> Iterator[worker_pb2.ChunkPipelineEvent]:
"""Stream chunk pipeline events for a job."""
from core.events import poll_events
job_id = request.job_id
logger.info(f"StreamChunkPipeline: {job_id}")
cursor = 0
timeout = time.monotonic() + 600 # 10 min max
while context.is_active() and time.monotonic() < timeout:
events, cursor = poll_events(job_id, cursor)
for data in events:
event_type = data.pop("event", "")
yield worker_pb2.ChunkPipelineEvent(
job_id=job_id,
event_type=event_type,
sequence=data.get("sequence", 0),
worker_id=data.get("worker_id", ""),
state=data.get("state", ""),
queue_size=data.get("queue_size", 0),
elapsed=data.get("elapsed", 0.0),
throughput_mbps=data.get("throughput_mbps", 0.0),
total_chunks=data.get("total_chunks", 0),
processed_chunks=data.get("processed_chunks", 0),
failed_chunks=data.get("failed_chunks", 0),
error=data.get("error", ""),
processing_time=data.get("processing_time", 0.0),
retries=data.get("retries", 0),
)
if event_type in ("pipeline_complete", "pipeline_error"):
return
time.sleep(0.05)
def GetWorkerStatus(self, request, context):
"""Get worker health and capabilities."""
try:

View File

@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty2\x9e\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatusb\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cworker.proto\x12\nmpr.worker\"\xa7\x01\n\nJobRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x13\n\x0bsource_path\x18\x02 \x01(\t\x12\x13\n\x0boutput_path\x18\x03 \x01(\t\x12\x13\n\x0bpreset_json\x18\x04 \x01(\t\x12\x17\n\ntrim_start\x18\x05 \x01(\x02H\x00\x88\x01\x01\x12\x15\n\x08trim_end\x18\x06 \x01(\x02H\x01\x88\x01\x01\x42\r\n\x0b_trim_startB\x0b\n\t_trim_end\"@\n\x0bJobResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"!\n\x0fProgressRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9c\x01\n\x0eProgressUpdate\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\x05\x12\x15\n\rcurrent_frame\x18\x03 \x01(\x05\x12\x14\n\x0c\x63urrent_time\x18\x04 \x01(\x02\x12\r\n\x05speed\x18\x05 \x01(\x02\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x12\n\x05\x65rror\x18\x07 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_error\"\x1f\n\rCancelRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"D\n\x0e\x43\x61ncelResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tcancelled\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x0cWorkerStatus\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x13\n\x0b\x61\x63tive_jobs\x18\x02 \x01(\x05\x12\x18\n\x10supported_codecs\x18\x03 \x03(\t\x12\x15\n\rgpu_available\x18\x04 \x01(\x08\"\x07\n\x05\x45mpty\"$\n\x12\x43hunkStreamRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\xaa\x02\n\x12\x43hunkPipelineEvent\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x12\n\nevent_type\x18\x02 \x01(\t\x12\x10\n\x08sequence\x18\x03 \x01(\x05\x12\x11\n\tworker_id\x18\x04 \x01(\t\x12\r\n\x05state\x18\x05 \x01(\t\x12\x12\n\nqueue_size\x18\x06 \x01(\x05\x12\x0f\n\x07\x65lapsed\x18\x07 \x01(\x02\x12\x17\n\x0fthroughput_mbps\x18\x08 \x01(\x02\x12\x14\n\x0ctotal_chunks\x18\t \x01(\x05\x12\x18\n\x10processed_chunks\x18\n \x01(\x05\x12\x15\n\rfailed_chunks\x18\x0b \x01(\x05\x12\r\n\x05\x65rror\x18\x0c \x01(\t\x12\x17\n\x0fprocessing_time\x18\r \x01(\x02\x12\x0f\n\x07retries\x18\x0e \x01(\x05\x32\xf7\x02\n\rWorkerService\x12<\n\tSubmitJob\x12\x16.mpr.worker.JobRequest\x1a\x17.mpr.worker.JobResponse\x12K\n\x0eStreamProgress\x12\x1b.mpr.worker.ProgressRequest\x1a\x1a.mpr.worker.ProgressUpdate0\x01\x12\x42\n\tCancelJob\x12\x19.mpr.worker.CancelRequest\x1a\x1a.mpr.worker.CancelResponse\x12>\n\x0fGetWorkerStatus\x12\x11.mpr.worker.Empty\x1a\x18.mpr.worker.WorkerStatus\x12W\n\x13StreamChunkPipeline\x12\x1e.mpr.worker.ChunkStreamRequest\x1a\x1e.mpr.worker.ChunkPipelineEvent0\x01\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -47,6 +47,10 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals['_WORKERSTATUS']._serialized_end=664
_globals['_EMPTY']._serialized_start=666
_globals['_EMPTY']._serialized_end=673
_globals['_WORKERSERVICE']._serialized_start=676
_globals['_WORKERSERVICE']._serialized_end=962
_globals['_CHUNKSTREAMREQUEST']._serialized_start=675
_globals['_CHUNKSTREAMREQUEST']._serialized_end=711
_globals['_CHUNKPIPELINEEVENT']._serialized_start=714
_globals['_CHUNKPIPELINEEVENT']._serialized_end=1012
_globals['_WORKERSERVICE']._serialized_start=1015
_globals['_WORKERSERVICE']._serialized_end=1390
# @@protoc_insertion_point(module_scope)

View File

@@ -5,7 +5,7 @@ import warnings
from . import worker_pb2 as worker__pb2
GRPC_GENERATED_VERSION = '1.76.0'
GRPC_GENERATED_VERSION = '1.78.0'
GRPC_VERSION = grpc.__version__
_version_not_supported = False
@@ -54,6 +54,11 @@ class WorkerServiceStub(object):
request_serializer=worker__pb2.Empty.SerializeToString,
response_deserializer=worker__pb2.WorkerStatus.FromString,
_registered_method=True)
self.StreamChunkPipeline = channel.unary_stream(
'/mpr.worker.WorkerService/StreamChunkPipeline',
request_serializer=worker__pb2.ChunkStreamRequest.SerializeToString,
response_deserializer=worker__pb2.ChunkPipelineEvent.FromString,
_registered_method=True)
class WorkerServiceServicer(object):
@@ -83,6 +88,12 @@ class WorkerServiceServicer(object):
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def StreamChunkPipeline(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_WorkerServiceServicer_to_server(servicer, server):
rpc_method_handlers = {
@@ -106,6 +117,11 @@ def add_WorkerServiceServicer_to_server(servicer, server):
request_deserializer=worker__pb2.Empty.FromString,
response_serializer=worker__pb2.WorkerStatus.SerializeToString,
),
'StreamChunkPipeline': grpc.unary_stream_rpc_method_handler(
servicer.StreamChunkPipeline,
request_deserializer=worker__pb2.ChunkStreamRequest.FromString,
response_serializer=worker__pb2.ChunkPipelineEvent.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'mpr.worker.WorkerService', rpc_method_handlers)
@@ -224,3 +240,30 @@ class WorkerService(object):
timeout,
metadata,
_registered_method=True)
@staticmethod
def StreamChunkPipeline(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_stream(
request,
target,
'/mpr.worker.WorkerService/StreamChunkPipeline',
worker__pb2.ChunkStreamRequest.SerializeToString,
worker__pb2.ChunkPipelineEvent.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)

View File

@@ -13,8 +13,8 @@
},
{
"target": "typescript",
"output": "ui/timeline/src/types.ts",
"include": ["dataclasses", "enums", "api"]
"output": "ui/common/types/generated.ts",
"include": ["dataclasses", "enums", "api", "views"]
},
{
"target": "protobuf",

View File

@@ -16,6 +16,8 @@ from .grpc import (
GRPC_SERVICE,
CancelRequest,
CancelResponse,
ChunkPipelineEvent,
ChunkStreamRequest,
Empty,
JobRequest,
JobResponse,
@@ -26,6 +28,7 @@ from .grpc import (
from .jobs import ChunkJob, ChunkJobStatus, JobStatus, TranscodeJob
from .media import AssetStatus, MediaAsset
from .presets import BUILTIN_PRESETS, TranscodePreset
from .views import ChunkEvent, ChunkOutputFile, PipelineStats, WorkerEvent
# Core domain models - generates Django, Pydantic, TypeScript
DATACLASSES = [MediaAsset, TranscodePreset, TranscodeJob, ChunkJob]
@@ -44,6 +47,9 @@ API_MODELS = [
# Status enums - included in generated code
ENUMS = [AssetStatus, JobStatus, ChunkJobStatus]
# View/event models - generates TypeScript for UI consumption
VIEWS = [ChunkEvent, WorkerEvent, PipelineStats, ChunkOutputFile]
# gRPC messages - generates Proto
GRPC_MESSAGES = [
JobRequest,
@@ -54,6 +60,8 @@ GRPC_MESSAGES = [
CancelResponse,
WorkerStatus,
Empty,
ChunkStreamRequest,
ChunkPipelineEvent,
]
__all__ = [
@@ -82,10 +90,18 @@ __all__ = [
"CancelResponse",
"WorkerStatus",
"Empty",
"ChunkStreamRequest",
"ChunkPipelineEvent",
# Views
"ChunkEvent",
"WorkerEvent",
"PipelineStats",
"ChunkOutputFile",
# For generator
"DATACLASSES",
"API_MODELS",
"ENUMS",
"VIEWS",
"GRPC_MESSAGES",
"BUILTIN_PRESETS",
]

View File

@@ -41,6 +41,13 @@ class CancelRequest:
job_id: str
@dataclass
class ChunkStreamRequest:
"""Request to stream chunk pipeline events."""
job_id: str
@dataclass
class Empty:
"""Empty message for requests with no parameters."""
@@ -94,6 +101,26 @@ class WorkerStatus:
gpu_available: bool
@dataclass
class ChunkPipelineEvent:
"""Streaming chunk pipeline event."""
job_id: str
event_type: str # pipeline_start, chunk_queued, chunk_done, etc.
sequence: int = 0
worker_id: str = ""
state: str = ""
queue_size: int = 0
elapsed: float = 0.0
throughput_mbps: float = 0.0
total_chunks: int = 0
processed_chunks: int = 0
failed_chunks: int = 0
error: str = ""
processing_time: float = 0.0
retries: int = 0
# -----------------------------------------------------------------------------
# Service Definition (for documentation, generator uses this)
# -----------------------------------------------------------------------------
@@ -126,5 +153,11 @@ GRPC_SERVICE = {
"response": WorkerStatus,
"stream_response": False,
},
{
"name": "StreamChunkPipeline",
"request": ChunkStreamRequest,
"response": ChunkPipelineEvent,
"stream_response": True, # Server streaming
},
],
}

View File

@@ -0,0 +1,57 @@
"""
View/Event Schema Definitions
Projections of domain models for UI consumption via SSE events.
These reference existing schema types (e.g., ChunkJobStatus) to maintain
type-level dependencies — if the domain model changes, views update too.
"""
from dataclasses import dataclass
from typing import Optional
@dataclass
class ChunkEvent:
"""SSE event for a single chunk's lifecycle."""
sequence: int
status: str
size: Optional[int] = None
worker_id: Optional[str] = None
processing_time: Optional[float] = None
error: Optional[str] = None
retries: int = 0
@dataclass
class WorkerEvent:
"""SSE event for worker state changes."""
worker_id: str
state: str
current_chunk: Optional[int] = None
processed: int = 0
errors: int = 0
retries: int = 0
@dataclass
class PipelineStats:
"""Aggregate pipeline statistics, updated via SSE."""
total_chunks: int = 0
processed: int = 0
failed: int = 0
retries: int = 0
elapsed: float = 0.0
throughput_mbps: float = 0.0
queue_size: int = 0
@dataclass
class ChunkOutputFile:
"""A chunk output file in S3/MinIO with presigned download URL."""
key: str
size: int = 0
url: str = ""

View File

@@ -89,6 +89,15 @@ services:
mc anonymous set download local/mpr-media-in
mc anonymous set download local/mpr-media-out
envoy:
image: envoyproxy/envoy:v1.28-latest
ports:
- "8090:8090"
volumes:
- ./envoy.yaml:/etc/envoy/envoy.yaml:ro
depends_on:
- grpc
nginx:
image: nginx:alpine
ports:
@@ -96,12 +105,14 @@ services:
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
- ./landing.html:/etc/nginx/landing.html:ro
- ../media/out:/app/media/out:ro
depends_on:
- django
- fastapi
- timeline
- chunker
- minio
- envoy
# =============================================================================
# Application Services
@@ -139,7 +150,7 @@ services:
build:
context: ..
dockerfile: ctrl/Dockerfile.worker
command: celery -A admin.mpr worker -l info -Q transcode -c 2
command: celery -A admin.mpr worker -l info -Q celery,transcode -c 2
environment:
<<: *common-env
MPR_EXECUTOR: local
@@ -163,6 +174,8 @@ services:
VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-}
volumes:
- ../ui/timeline/src:/app/src
- ../ui/timeline/vite.config.ts:/app/vite.config.ts
- ../ui/common:/common
chunker:
build:
@@ -174,6 +187,8 @@ services:
VITE_ALLOWED_HOSTS: ${VITE_ALLOWED_HOSTS:-}
volumes:
- ../ui/chunker/src:/app/src
- ../ui/chunker/vite.config.ts:/app/vite.config.ts
- ../ui/common:/common
volumes:
postgres-data:

64
ctrl/envoy.yaml Normal file
View File

@@ -0,0 +1,64 @@
admin:
address:
socket_address: { address: 0.0.0.0, port_value: 9901 }
static_resources:
listeners:
- name: listener_0
address:
socket_address: { address: 0.0.0.0, port_value: 8090 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
codec_type: auto
stat_prefix: ingress_http
route_config:
name: local_route
virtual_hosts:
- name: local_service
domains: ["*"]
routes:
- match: { prefix: "/" }
route:
cluster: grpc_service
timeout: 600s
max_stream_duration:
grpc_timeout_header_max: 600s
cors:
allow_origin_string_match:
- prefix: "*"
allow_methods: GET, PUT, DELETE, POST, OPTIONS
allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout
expose_headers: grpc-status,grpc-message
max_age: "1728000"
http_filters:
- name: envoy.filters.http.grpc_web
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
- name: envoy.filters.http.cors
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
clusters:
- name: grpc_service
connect_timeout: 5s
type: logical_dns
lb_policy: round_robin
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options: {}
load_assignment:
cluster_name: grpc_service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: grpc
port_value: 50051

View File

@@ -19,4 +19,13 @@ python -m grpc_tools.protoc \
# Fix relative import in generated grpc stub
sed -i 's/^import worker_pb2/from . import worker_pb2/' core/rpc/worker_pb2_grpc.py
# Generate TypeScript gRPC-Web client from proto
echo "Generating TypeScript gRPC-Web client..."
cd ui/chunker
npx protoc \
--ts_out ../common/api/grpc \
--proto_path ../../core/rpc/protos \
worker.proto
cd ../..
echo "Done!"

View File

@@ -29,6 +29,10 @@ http {
server minio:9000;
}
upstream envoy {
server envoy:8090;
}
server {
listen 80;
server_name mpr.local.ar;
@@ -106,8 +110,24 @@ http {
}
location /media/out/ {
proxy_pass http://minio/mpr-media-out/;
proxy_set_header Host $http_host;
alias /app/media/out/;
autoindex on;
}
# gRPC-Web proxy via Envoy
location /grpc-web/ {
proxy_pass http://envoy/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 600s;
# Critical for streaming: disable nginx response buffering
proxy_buffering off;
proxy_cache off;
chunked_transfer_encoding on;
}
}
}

View File

@@ -0,0 +1,290 @@
# Chunker Pipeline — Execution Path
## Overview
The chunker pipeline splits a media file into time-based segments using FFmpeg stream-copy. Events flow from worker threads through Redis and gRPC-Web streaming to the browser UI in real time.
**7 hops from worker thread to pixel:**
```
Worker thread → Pipeline._emit() → event_bridge() → Redis RPUSH
→ [50ms poll] gRPC server LRANGE → yield protobuf
→ HTTP/2 frame → Envoy (grpc-web filter)
→ HTTP/1.1 chunk → nginx (proxy_buffering off)
→ fetch ReadableStream → protobuf-ts decode
→ setEvents([...prev, evt]) → React re-render
```
---
## Step 1: Job Creation (Browser → GraphQL → Celery)
```
User clicks "Start"
→ App.tsx: handleStart(config)
→ api.ts: createChunkJob(config)
→ POST /graphql (nginx :80 → fastapi:8702)
→ graphql.py: Mutation.create_chunk_job()
→ core.db: creates ChunkJob row in Postgres
→ Celery: run_job.delay(job_type="chunk", job_id=..., payload=...)
→ Returns { id, celery_task_id } to browser
→ App.tsx: setJobId(id) — triggers gRPC stream subscription
```
**Files:** `ui/chunker/src/api.ts`, `core/api/graphql.py`, `core/jobs/task.py`
---
## Step 2: gRPC-Web Stream (Browser → nginx → Envoy → gRPC Server)
Once `jobId` is set, `useGrpcStream(jobId)` opens a server-streaming RPC:
```
useGrpcStream(jobId) fires useEffect
→ GrpcWebFetchTransport({ baseUrl: "/grpc-web" })
→ WorkerServiceClient.streamChunkPipeline({ jobId })
→ fetch() POST to /grpc-web/worker.WorkerService/StreamChunkPipeline
→ nginx :80 /grpc-web/ (proxy_pass → envoy:8090, proxy_buffering off)
→ Envoy :8090 (grpc_web filter: HTTP/1.1 grpc-web → HTTP/2 native gRPC)
→ gRPC server :50051 WorkerServicer.StreamChunkPipeline()
→ Enters Redis polling loop (Step 5)
```
**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ctrl/nginx.conf`, `ctrl/envoy.yaml`, `core/rpc/server.py`
**Key nginx config:** `proxy_buffering off` is critical — without it, nginx collects the entire upstream response before forwarding, defeating streaming entirely.
---
## Step 3: Celery Worker → ChunkHandler
```
Celery picks up run_job task
→ task.py: run_job(job_type="chunk", job_id, payload)
→ registry.get_handler("chunk") → ChunkHandler
→ chunk.py: ChunkHandler.process(job_id, payload)
→ download_to_temp(BUCKET_IN, source_key) — pulls source from MinIO/S3
→ Creates output_dir: /app/media/out/chunks/{job_id}/
→ Constructs event_bridge callback (bridges Pipeline events → Redis)
→ pipeline = Pipeline(source, ..., event_callback=event_bridge, output_dir=...)
→ pipeline.run()
```
**Files:** `core/jobs/task.py`, `core/jobs/handlers/chunk.py`
The `event_bridge` closure wraps every `Pipeline._emit()` call, forwarding to `push_event(job_id, event_type, data)` which writes to Redis.
---
## Step 4: Pipeline Orchestration (inside Celery worker process)
`Pipeline.run()` spawns multiple threads:
```
pipeline.run():
├─ Chunker(source, chunk_duration)
│ → ffprobe source file → gets duration, file_size
│ → calculates total_chunks = ceil(duration / chunk_duration)
├─ _emit("pipeline_start", {...}) → event_bridge → Redis
├─ _emit("pipeline_info", {file_size, duration, total_chunks}) → Redis
├─ Creates ChunkQueue(maxsize=10)
├─ Creates WorkerPool(num_workers=N, chunk_queue, processor, event_callback)
├─ pool.start() — spawns N worker threads
├─ MONITOR THREAD starts (_monitor_progress)
│ → Every 500ms: _emit("pipeline_progress", {elapsed, throughput_mbps}) → Redis
├─ PRODUCER THREAD starts (_produce_chunks)
│ → Iterates chunker.chunks() → yields Chunk(sequence, start_time, end_time)
│ → For each: chunk_queue.put(chunk)
│ → _emit("chunk_queued", {sequence, start_time, end_time, queue_size}) → Redis
│ → chunk_queue.close() when done (sends N sentinel Nones)
├─ WORKER THREADS (N concurrent, each runs worker.py:Worker.run())
│ │ Each worker loops:
│ │
│ ├─ chunk = chunk_queue.get(timeout=1.0)
│ ├─ _emit("chunk_processing", {sequence, state:"processing", queue_size}) → Redis
│ │
│ ├─ processor.process(chunk)
│ │ ├─ ffmpeg: runs `ffmpeg -ss start -to end -c copy chunk_NNNN.mp4`
│ │ ├─ simulated_decode: sleep(random) + checksum
│ │ └─ checksum: reads bytes, computes hash
│ │
│ ├─ On success: _emit("chunk_done", {sequence, processing_time, retries, queue_size}) → Redis
│ ├─ On failure: retries with exponential backoff (0.1s, 0.2s, 0.4s...)
│ │ └─ _emit("chunk_retry", {sequence, attempt, backoff}) → Redis
│ │ └─ _emit("chunk_error", {sequence, error, retries}) → Redis (after exhaustion)
│ │
│ └─ On sentinel (None): _emit("worker_status", {state:"stopped"}) → Redis
├─ pool.wait() — joins all worker threads, collects results
├─ monitor_stop.set() — stops progress monitor
├─ ResultCollector — reassembles results in sequence order
│ └─ _emit("chunk_collected", {sequence, buffered, emitted}) → Redis
├─ Writes manifest.json to output_dir
└─ _emit("pipeline_complete", {total_chunks, processed, failed, elapsed, throughput}) → Redis
```
**Files:** `core/chunker/pipeline.py`, `core/chunker/worker.py`, `core/chunker/pool.py`, `core/chunker/chunker.py`, `core/chunker/collector.py`
---
## Step 5: Redis — the Event Bus
```
WRITE side (Celery worker, all threads):
push_event(job_id, event_type, data)
→ json.dumps({"event": event_type, ...data})
→ Redis RPUSH to key "chunk_events:{job_id}"
→ Redis EXPIRE 3600 (1 hour TTL)
READ side (gRPC server, StreamChunkPipeline):
poll_events(job_id, cursor)
→ Redis LRANGE "chunk_events:{job_id}" cursor -1
→ Returns (parsed_events, new_cursor)
→ Called every 50ms (time.sleep(0.05) in server loop)
```
Redis acts as a decoupling layer between the Celery worker process (which runs the pipeline) and the gRPC server process (which streams to browsers). Events are appended with RPUSH and read with cursor-based LRANGE polling.
**Files:** `core/events.py`
---
## Step 6: gRPC Server → Envoy → nginx → Browser
```
server.py: StreamChunkPipeline polling loop:
while context.is_active():
events, cursor = poll_events(job_id, cursor) ← Redis LRANGE
for data in events:
yield worker_pb2.ChunkPipelineEvent( ← serialized protobuf message
job_id, event_type, sequence, worker_id,
state, queue_size, elapsed, throughput_mbps,
total_chunks, processed_chunks, failed_chunks,
error, processing_time, retries
)
if event_type in ("pipeline_complete", "pipeline_error"):
return ← ends the stream
time.sleep(0.05) ← 50ms poll interval
Each yield sends:
→ gRPC HTTP/2 DATA frame to Envoy
→ Envoy grpc_web filter: HTTP/2 → base64-encoded grpc-web-text
→ nginx proxy_pass (proxy_buffering off) → chunked HTTP/1.1 to browser
→ fetch() ReadableStream in GrpcWebFetchTransport
→ @protobuf-ts decodes protobuf → ChunkPipelineEvent TypeScript object
```
**Files:** `core/rpc/server.py`, `ctrl/envoy.yaml`, `ctrl/nginx.conf`, `ui/common/api/grpc/worker.ts`, `ui/common/api/grpc/worker.client.ts`
---
## Step 7: React State Derivation and Rendering
```
useGrpcStream.ts:
for await (const msg of stream.responses):
const evt = toEvent(msg) ← maps protobuf camelCase → snake_case PipelineEvent
setEvents(prev => [...prev, evt]) ← appends to events array
if pipeline_complete/error → setDone(true), break
App.tsx useMemo(events):
Iterates ALL events on every update, derives:
├─ chunkMap: Map<sequence, ChunkInfo> — state machine per chunk
│ pending → queued → processing → done/error/retry
├─ workerMap: Map<worker_id, WorkerInfo> — state per worker
│ idle → processing → idle → ... → stopped
├─ stats: PipelineStats
│ total_chunks, processed, failed, retries, elapsed, throughput_mbps, queue_size
├─ errors: ErrorEntry[] — every event containing an error field
└─ queueSize: number — last seen queue_size value
Renders:
├─ ChunkGrid — colored cells per chunk (pending/queued/processing/done/error)
├─ QueueGauge — current queue depth / max
├─ WorkerPanel — per-worker state + current chunk assignment
├─ StatsPanel — elapsed time, throughput, processed/failed counts
├─ ErrorLog — scrollable error list
└─ OutputFiles — download links (when done)
```
**Files:** `ui/chunker/src/hooks/useGrpcStream.ts`, `ui/chunker/src/App.tsx`
---
## Step 8: Output File Access (after pipeline completes)
```
App.tsx useEffect([done, jobId]):
→ api.ts: getChunkOutputFiles(jobId)
→ POST /graphql → graphql.py: chunk_output_files(job_id)
→ Reads /app/media/out/chunks/{job_id}/ directory listing from disk
→ Returns [{key, size, url: "/media/out/chunks/{job_id}/chunk_0001.mp4"}]
→ Browser renders download links
→ Click link → nginx /media/out/ → alias /app/media/out/ → serves file from disk
```
Chunks are written directly to `media/out/chunks/{job_id}/` by the ffmpeg processor — no MinIO upload needed for output. Nginx serves them with `autoindex on`.
**Files:** `core/api/graphql.py`, `core/jobs/handlers/chunk.py`, `ctrl/nginx.conf`
---
## Event Types Reference
| Event | Source | Key Fields |
|-------|--------|------------|
| `pipeline_start` | Pipeline.run() | source, chunk_duration, num_workers, processor_type |
| `pipeline_info` | Pipeline.run() | file_size, source_duration, total_chunks |
| `pipeline_progress` | Monitor thread (500ms) | elapsed, throughput_mbps |
| `chunk_queued` | Producer thread | sequence, start_time, end_time, duration, queue_size |
| `chunk_processing` | Worker thread | sequence, worker_id, state, queue_size |
| `chunk_done` | Worker thread | sequence, processing_time, retries, queue_size |
| `chunk_retry` | Worker thread | sequence, attempt, backoff |
| `chunk_error` | Worker thread | sequence, error, retries |
| `chunk_collected` | ResultCollector | sequence, buffered, emitted |
| `worker_status` | Worker thread | worker_id, state (idle/processing/stopped) |
| `pipeline_complete` | Pipeline.run() | total_chunks, processed, failed, elapsed, throughput_mbps |
| `pipeline_error` | Pipeline.run() | error |
---
## Thread Model (inside Celery worker)
```
Celery worker process
└─ run_job task thread
└─ Pipeline.run()
├─ Producer thread — enqueues chunks
├─ Monitor thread — emits progress every 500ms
├─ Worker thread 0 — pulls from queue, processes
├─ Worker thread 1 — pulls from queue, processes
├─ Worker thread 2 — pulls from queue, processes
└─ Worker thread 3 — pulls from queue, processes
```
All threads share the same `event_callback``event_bridge``push_event()`, which creates a new Redis connection per call. Thread-safe via Redis atomic RPUSH.
---
## Infrastructure
| Service | Port | Role |
|---------|------|------|
| nginx | 80 | Reverse proxy, static file serving |
| fastapi | 8702 | GraphQL API (Strawberry) |
| celery | — | Task worker (runs pipeline) |
| redis | 6379 | Event bus + Celery broker |
| grpc | 50051 | gRPC server (StreamChunkPipeline) |
| envoy | 8090 | gRPC-Web ↔ native gRPC translation |
| minio | 9000 | S3-compatible source media storage |
| postgres | 5432 | Job/asset metadata |

View File

@@ -1,212 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>MPR - Architecture</title>
<link rel="stylesheet" href="styles.css" />
</head>
<body>
<h1>MPR - Media Processor</h1>
<p>
Media transcoding platform with dual execution modes: local (Celery
+ MinIO) and cloud (AWS Step Functions + Lambda + S3).
</p>
<nav>
<a href="#overview">System Overview</a>
<a href="#data-model">Data Model</a>
<a href="#job-flow">Job Flow</a>
<a href="#media-storage">Media Storage</a>
</nav>
<h2 id="overview">System Overview</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Local Architecture (Development)</h3>
<object type="image/svg+xml" data="01a-local-architecture.svg">
<img
src="01a-local-architecture.svg"
alt="Local Architecture"
/>
</object>
<a href="01a-local-architecture.svg" target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>AWS Architecture (Production)</h3>
<object type="image/svg+xml" data="01b-aws-architecture.svg">
<img
src="01b-aws-architecture.svg"
alt="AWS Architecture"
/>
</object>
<a href="01b-aws-architecture.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Components</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4f8"></span>
Reverse Proxy (nginx)
</li>
<li>
<span class="color-box" style="background: #f0f8e8"></span>
Application Layer (Django Admin, GraphQL API, Timeline UI)
</li>
<li>
<span class="color-box" style="background: #fff8e8"></span>
Worker Layer (Celery local mode)
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
AWS (Step Functions, Lambda - cloud mode)
</li>
<li>
<span class="color-box" style="background: #f8e8f0"></span>
Data Layer (PostgreSQL, Redis)
</li>
<li>
<span class="color-box" style="background: #f0f0f0"></span>
S3 Storage (MinIO local / AWS S3 cloud)
</li>
</ul>
</div>
<h2 id="data-model">Data Model</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Entity Relationships</h3>
<object type="image/svg+xml" data="02-data-model.svg">
<img src="02-data-model.svg" alt="Data Model" />
</object>
<a href="02-data-model.svg" target="_blank">Open full size</a>
</div>
</div>
<div class="legend">
<h3>Entities</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
MediaAsset - Video/audio files (S3 keys as paths)
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
TranscodePreset - Encoding configurations
</li>
<li>
<span class="color-box" style="background: #d9534f"></span>
TranscodeJob - Processing queue (celery_task_id or
execution_arn)
</li>
</ul>
</div>
<h2 id="job-flow">Job Flow</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Job Lifecycle</h3>
<object type="image/svg+xml" data="03-job-flow.svg">
<img src="03-job-flow.svg" alt="Job Flow" />
</object>
<a href="03-job-flow.svg" target="_blank">Open full size</a>
</div>
</div>
<div class="legend">
<h3>Job States</h3>
<ul>
<li>
<span class="color-box" style="background: #ffc107"></span>
PENDING - Waiting in queue
</li>
<li>
<span class="color-box" style="background: #17a2b8"></span>
PROCESSING - Worker executing
</li>
<li>
<span class="color-box" style="background: #28a745"></span>
COMPLETED - Success
</li>
<li>
<span class="color-box" style="background: #dc3545"></span>
FAILED - Error occurred
</li>
<li>
<span class="color-box" style="background: #6c757d"></span>
CANCELLED - User cancelled
</li>
</ul>
<h3>Execution Modes</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4e8"></span>
Local: Celery + MinIO (S3 API) + FFmpeg
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
Lambda: Step Functions + Lambda + AWS S3
</li>
</ul>
</div>
<h2 id="media-storage">Media Storage</h2>
<div class="diagram-container">
<p>
MPR separates media into input and output paths for flexible
storage configuration.
</p>
<p>
<a href="04-media-storage.md" target="_blank"
>View Media Storage Documentation →</a
>
</p>
</div>
<h2>API (GraphQL)</h2>
<pre><code># GraphiQL IDE
http://mpr.local.ar/graphql
# Queries
query { assets(status: "ready") { id filename duration } }
query { jobs(status: "processing") { id status progress } }
query { presets { id name container videoCodec } }
query { systemStatus { status version } }
# Mutations
mutation { scanMediaFolder { found registered skipped } }
mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
mutation { cancelJob(id: "...") { id status } }
mutation { retryJob(id: "...") { id status } }
mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
mutation { deleteAsset(id: "...") { ok } }
# Lambda callback (REST)
POST /api/jobs/{id}/callback - Lambda completion webhook</code></pre>
<h2>Access Points</h2>
<pre><code># Local development
127.0.0.1 mpr.local.ar
http://mpr.local.ar/admin - Django Admin
http://mpr.local.ar/graphql - GraphiQL
http://mpr.local.ar/ - Timeline UI
http://localhost:9001 - MinIO Console
# AWS deployment
https://mpr.mcrn.ar/ - Production</code></pre>
<h2>Quick Reference</h2>
<pre><code># Render SVGs from DOT files
for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
# Switch executor mode
MPR_EXECUTOR=local # Celery + MinIO
MPR_EXECUTOR=lambda # Step Functions + Lambda + S3</code></pre>
</body>
</html>

View File

@@ -3,6 +3,8 @@
--text-color: #e8e8e8;
--accent-color: #4a90d9;
--border-color: #333;
--sidebar-width: 220px;
--sidebar-bg: #151528;
}
* {
@@ -16,6 +18,59 @@ body {
background-color: var(--bg-color);
color: var(--text-color);
line-height: 1.6;
}
/* Sidebar navigation */
.sidebar {
position: fixed;
top: 0;
left: 0;
width: var(--sidebar-width);
height: 100vh;
background: var(--sidebar-bg);
border-right: 1px solid var(--border-color);
padding: 1.5rem 1rem;
overflow-y: auto;
z-index: 10;
}
.sidebar h2 {
font-size: 1.2rem;
color: var(--accent-color);
margin-bottom: 1.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid var(--border-color);
}
.sidebar ul {
list-style: none;
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.sidebar li {
display: block;
}
.sidebar a {
display: block;
padding: 0.4rem 0.6rem;
color: var(--text-color);
text-decoration: none;
font-size: 0.85rem;
border-radius: 4px;
transition: background 0.15s, color 0.15s;
}
.sidebar a:hover {
background: rgba(74, 144, 217, 0.15);
color: var(--accent-color);
}
/* Main content */
.content {
margin-left: var(--sidebar-width);
padding: 2rem;
}
@@ -25,12 +80,13 @@ h1 {
color: var(--accent-color);
}
h2 {
.content > h2 {
font-size: 1.5rem;
margin: 2rem 0 1rem;
color: var(--text-color);
border-bottom: 1px solid var(--border-color);
padding-bottom: 0.5rem;
scroll-margin-top: 1rem;
}
.diagram-container {
@@ -76,20 +132,6 @@ h2 {
text-decoration: underline;
}
nav {
margin-bottom: 2rem;
}
nav a {
color: var(--accent-color);
text-decoration: none;
margin-right: 1.5rem;
}
nav a:hover {
text-decoration: underline;
}
.legend {
margin-top: 2rem;
padding: 1rem;
@@ -141,3 +183,27 @@ pre code {
background: none;
padding: 0;
}
/* Responsive: collapse sidebar on small screens */
@media (max-width: 768px) {
.sidebar {
position: static;
width: 100%;
height: auto;
border-right: none;
border-bottom: 1px solid var(--border-color);
}
.sidebar ul {
flex-direction: row;
flex-wrap: wrap;
}
.content {
margin-left: 0;
}
.diagram {
min-width: 100%;
}
}

View File

@@ -7,219 +7,241 @@
<link rel="stylesheet" href="architecture/styles.css" />
</head>
<body>
<h1>MPR - Media Processor</h1>
<p>
Media transcoding platform with three execution modes: local (Celery
+ MinIO), AWS (Step Functions + Lambda + S3), and GCP (Cloud Run
Jobs + GCS). Storage is S3-compatible across all environments.
</p>
<nav>
<a href="#overview">System Overview</a>
<a href="#data-model">Data Model</a>
<a href="#job-flow">Job Flow</a>
<a href="#media-storage">Media Storage</a>
<nav class="sidebar">
<h2>MPR</h2>
<ul>
<li><a href="#overview">System Overview</a></li>
<li><a href="#data-model">Data Model</a></li>
<li><a href="#job-flow">Job Flow</a></li>
<li><a href="#media-storage">Media Storage</a></li>
<li><a href="#chunker-pipeline">Chunker Pipeline</a></li>
<li><a href="#api">API (GraphQL)</a></li>
<li><a href="#access-points">Access Points</a></li>
<li><a href="#quick-reference">Quick Reference</a></li>
</ul>
</nav>
<h2 id="overview">System Overview</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Local Architecture (Development)</h3>
<object
type="image/svg+xml"
data="architecture/01a-local-architecture.svg"
>
<img
src="architecture/01a-local-architecture.svg"
alt="Local Architecture"
/>
</object>
<a
href="architecture/01a-local-architecture.svg"
target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>AWS Architecture (Production)</h3>
<object
type="image/svg+xml"
data="architecture/01b-aws-architecture.svg"
>
<img
src="architecture/01b-aws-architecture.svg"
alt="AWS Architecture"
/>
</object>
<a href="architecture/01b-aws-architecture.svg" target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>GCP Architecture (Production)</h3>
<object
type="image/svg+xml"
data="architecture/01c-gcp-architecture.svg"
>
<img
src="architecture/01c-gcp-architecture.svg"
alt="GCP Architecture"
/>
</object>
<a href="architecture/01c-gcp-architecture.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Components</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4f8"></span>
Reverse Proxy (nginx)
</li>
<li>
<span class="color-box" style="background: #f0f8e8"></span>
Application Layer (Django Admin, GraphQL API, Timeline UI)
</li>
<li>
<span class="color-box" style="background: #fff8e8"></span>
Worker Layer (Celery local mode)
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
AWS (Step Functions, Lambda)
</li>
<li>
<span class="color-box" style="background: #e8f0fd"></span>
GCP (Cloud Run Jobs + GCS)
</li>
<li>
<span class="color-box" style="background: #f8e8f0"></span>
Data Layer (PostgreSQL, Redis)
</li>
<li>
<span class="color-box" style="background: #f0f0f0"></span>
S3-compatible Storage (MinIO / AWS S3 / GCS)
</li>
</ul>
</div>
<h2 id="data-model">Data Model</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Entity Relationships</h3>
<object
type="image/svg+xml"
data="architecture/02-data-model.svg"
>
<img
src="architecture/02-data-model.svg"
alt="Data Model"
/>
</object>
<a href="architecture/02-data-model.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Entities</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
MediaAsset - Video/audio files with metadata
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
TranscodePreset - Encoding configurations
</li>
<li>
<span class="color-box" style="background: #d9534f"></span>
TranscodeJob - Processing queue items
</li>
</ul>
</div>
<h2 id="job-flow">Job Flow</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Job Lifecycle</h3>
<object
type="image/svg+xml"
data="architecture/03-job-flow.svg"
>
<img src="architecture/03-job-flow.svg" alt="Job Flow" />
</object>
<a href="architecture/03-job-flow.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Job States</h3>
<ul>
<li>
<span class="color-box" style="background: #ffc107"></span>
PENDING - Waiting in queue
</li>
<li>
<span class="color-box" style="background: #17a2b8"></span>
PROCESSING - Worker executing
</li>
<li>
<span class="color-box" style="background: #28a745"></span>
COMPLETED - Success
</li>
<li>
<span class="color-box" style="background: #dc3545"></span>
FAILED - Error occurred
</li>
<li>
<span class="color-box" style="background: #6c757d"></span>
CANCELLED - User cancelled
</li>
</ul>
</div>
<h2 id="media-storage">Media Storage</h2>
<div class="diagram-container">
<main class="content">
<h1>MPR - Media Processor</h1>
<p>
MPR separates media into <strong>input</strong> and
<strong>output</strong> paths, each independently configurable.
File paths are stored
<strong>relative to their respective root</strong> to ensure
portability between local development and cloud deployments (AWS
S3, etc.).
Media transcoding platform with three execution modes: local (Celery
+ MinIO), AWS (Step Functions + Lambda + S3), and GCP (Cloud Run
Jobs + GCS). Storage is S3-compatible across all environments.
</p>
</div>
<div class="legend">
<h3>Input / Output Separation</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
<code>MEDIA_IN</code> - Source media files to process
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
<code>MEDIA_OUT</code> - Transcoded/trimmed output files
</li>
</ul>
<p><strong>Why Relative Paths?</strong></p>
<ul>
<li>Portability: Same database works locally and in cloud</li>
<li>Flexibility: Easy to switch between storage backends</li>
<li>Simplicity: No need to update paths when migrating</li>
</ul>
</div>
<h2 id="overview">System Overview</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Local Architecture (Development)</h3>
<object
type="image/svg+xml"
data="architecture/01a-local-architecture.svg"
>
<img
src="architecture/01a-local-architecture.svg"
alt="Local Architecture"
/>
</object>
<a
href="architecture/01a-local-architecture.svg"
target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>AWS Architecture (Production)</h3>
<object
type="image/svg+xml"
data="architecture/01b-aws-architecture.svg"
>
<img
src="architecture/01b-aws-architecture.svg"
alt="AWS Architecture"
/>
</object>
<a href="architecture/01b-aws-architecture.svg" target="_blank"
>Open full size</a
>
</div>
<div class="diagram">
<h3>GCP Architecture (Production)</h3>
<object
type="image/svg+xml"
data="architecture/01c-gcp-architecture.svg"
>
<img
src="architecture/01c-gcp-architecture.svg"
alt="GCP Architecture"
/>
</object>
<a href="architecture/01c-gcp-architecture.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Local Development</h3>
<pre><code>MEDIA_IN=/app/media/in
<div class="legend">
<h3>Components</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4f8"></span>
Reverse Proxy (nginx)
</li>
<li>
<span class="color-box" style="background: #f0f8e8"></span>
Application Layer (Django Admin, GraphQL API, Timeline UI)
</li>
<li>
<span class="color-box" style="background: #fff8e8"></span>
Worker Layer (Celery local mode)
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
AWS (Step Functions, Lambda)
</li>
<li>
<span class="color-box" style="background: #e8f0fd"></span>
GCP (Cloud Run Jobs + GCS)
</li>
<li>
<span class="color-box" style="background: #f8e8f0"></span>
Data Layer (PostgreSQL, Redis)
</li>
<li>
<span class="color-box" style="background: #f0f0f0"></span>
S3-compatible Storage (MinIO / AWS S3 / GCS)
</li>
</ul>
</div>
<h2 id="data-model">Data Model</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Entity Relationships</h3>
<object
type="image/svg+xml"
data="architecture/02-data-model.svg"
>
<img
src="architecture/02-data-model.svg"
alt="Data Model"
/>
</object>
<a href="architecture/02-data-model.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Entities</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
MediaAsset - Video/audio files with metadata
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
TranscodePreset - Encoding configurations
</li>
<li>
<span class="color-box" style="background: #d9534f"></span>
TranscodeJob - Processing queue items
</li>
</ul>
</div>
<h2 id="job-flow">Job Flow</h2>
<div class="diagram-container">
<div class="diagram">
<h3>Job Lifecycle</h3>
<object
type="image/svg+xml"
data="architecture/03-job-flow.svg"
>
<img src="architecture/03-job-flow.svg" alt="Job Flow" />
</object>
<a href="architecture/03-job-flow.svg" target="_blank"
>Open full size</a
>
</div>
</div>
<div class="legend">
<h3>Job States</h3>
<ul>
<li>
<span class="color-box" style="background: #ffc107"></span>
PENDING - Waiting in queue
</li>
<li>
<span class="color-box" style="background: #17a2b8"></span>
PROCESSING - Worker executing
</li>
<li>
<span class="color-box" style="background: #28a745"></span>
COMPLETED - Success
</li>
<li>
<span class="color-box" style="background: #dc3545"></span>
FAILED - Error occurred
</li>
<li>
<span class="color-box" style="background: #6c757d"></span>
CANCELLED - User cancelled
</li>
</ul>
<h3>Execution Modes</h3>
<ul>
<li>
<span class="color-box" style="background: #e8f4e8"></span>
Local: Celery + MinIO (S3 API) + FFmpeg
</li>
<li>
<span class="color-box" style="background: #fde8d0"></span>
Lambda: Step Functions + Lambda + AWS S3
</li>
<li>
<span class="color-box" style="background: #e8f0fd"></span>
GCP: Cloud Run Jobs + GCS (S3 compat)
</li>
</ul>
</div>
<h2 id="media-storage">Media Storage</h2>
<div class="diagram-container">
<p>
MPR separates media into <strong>input</strong> and
<strong>output</strong> paths, each independently configurable.
File paths are stored
<strong>relative to their respective root</strong> to ensure
portability between local development and cloud deployments.
</p>
</div>
<div class="legend">
<h3>Input / Output Separation</h3>
<ul>
<li>
<span class="color-box" style="background: #4a90d9"></span>
<code>MEDIA_IN</code> - Source media files to process
</li>
<li>
<span class="color-box" style="background: #50b050"></span>
<code>MEDIA_OUT</code> - Transcoded/trimmed output files
</li>
</ul>
<p><strong>Why Relative Paths?</strong></p>
<ul>
<li>Portability: Same database works locally and in cloud</li>
<li>Flexibility: Easy to switch between storage backends</li>
<li>Simplicity: No need to update paths when migrating</li>
</ul>
</div>
<div class="legend">
<h3>Local Development</h3>
<pre><code>MEDIA_IN=/app/media/in
MEDIA_OUT=/app/media/out
/app/media/
@@ -228,52 +250,131 @@ MEDIA_OUT=/app/media/out
│ └── subfolder/video3.mp4
└── out/ # Transcoded output
└── video1_h264.mp4</code></pre>
</div>
</div>
<div class="legend">
<h3>AWS/Cloud Deployment</h3>
<pre><code>MEDIA_IN=s3://source-bucket/media/
<div class="legend">
<h3>AWS/Cloud Deployment</h3>
<pre><code>MEDIA_IN=s3://source-bucket/media/
MEDIA_OUT=s3://output-bucket/transcoded/
MEDIA_BASE_URL=https://source-bucket.s3.amazonaws.com/media/</code></pre>
<p>
Database paths remain unchanged (already relative). Just upload
files to S3 and update environment variables.
</p>
</div>
<p>
Database paths remain unchanged (already relative). Just upload
files to S3 and update environment variables.
</p>
</div>
<div class="legend">
<h3>API (GraphQL)</h3>
<p>
All client interactions go through GraphQL at
<code>/graphql</code>.
<a href="architecture/04-media-storage.md" target="_blank"
>Full Media Storage Documentation &rarr;</a
>
</p>
<ul>
<li>
<code>scanMediaFolder</code> - Scan S3 bucket for media
files
</li>
<li><code>createJob</code> - Create transcode/trim job</li>
<li>
<code>cancelJob / retryJob</code> - Job lifecycle management
</li>
<li>
<code>updateAsset / deleteAsset</code> - Asset management
</li>
</ul>
<p><strong>Supported File Types:</strong></p>
<p>
Video: mp4, mkv, avi, mov, webm, flv, wmv, m4v<br />
Audio: mp3, wav, flac, aac, ogg, m4a
</p>
</div>
<h2>Access Points</h2>
<pre><code># Add to /etc/hosts
<h2 id="chunker-pipeline">Chunker Pipeline</h2>
<div class="diagram-container">
<p>
The chunker pipeline splits media into time-based segments,
streaming real-time events from worker threads through Redis
and gRPC-Web to the browser UI. 7 hops from worker thread to pixel.
</p>
</div>
<div class="legend">
<h3>Event Path</h3>
<pre><code>Worker thread → Pipeline._emit() → event_bridge() → Redis RPUSH
→ [50ms poll] gRPC server LRANGE → yield protobuf
→ HTTP/2 frame → Envoy (grpc-web filter)
→ HTTP/1.1 chunk → nginx (proxy_buffering off)
→ fetch ReadableStream → protobuf-ts decode
→ setEvents([...prev, evt]) → React re-render</code></pre>
</div>
<div class="legend">
<h3>Thread Model (inside Celery worker)</h3>
<pre><code>Celery worker process
└─ run_job task thread
└─ Pipeline.run()
├─ Producer thread — enqueues chunks
├─ Monitor thread — emits progress every 500ms
├─ Worker thread 0 — pulls from queue, processes
├─ Worker thread 1 — pulls from queue, processes
├─ Worker thread 2 — pulls from queue, processes
└─ Worker thread 3 — pulls from queue, processes</code></pre>
</div>
<div class="legend">
<h3>Infrastructure</h3>
<ul>
<li><code>nginx :80</code> - Reverse proxy, static file serving</li>
<li><code>fastapi :8702</code> - GraphQL API (Strawberry)</li>
<li><code>celery</code> - Task worker (runs pipeline)</li>
<li><code>redis :6379</code> - Event bus + Celery broker</li>
<li><code>grpc :50051</code> - gRPC server (StreamChunkPipeline)</li>
<li><code>envoy :8090</code> - gRPC-Web &harr; native gRPC translation</li>
<li><code>minio :9000</code> - S3-compatible source media storage</li>
<li><code>postgres :5432</code> - Job/asset metadata</li>
</ul>
</div>
<p>
<a href="architecture/05-chunker-pipeline.md" target="_blank"
>Full Chunker Pipeline Documentation &rarr;</a
>
</p>
<h2 id="api">API (GraphQL)</h2>
<div class="legend">
<p>
All client interactions go through GraphQL at
<code>/graphql</code>.
</p>
<pre><code># GraphiQL IDE
http://mpr.local.ar/graphql
# Queries
query { assets(status: "ready") { id filename duration } }
query { jobs(status: "processing") { id status progress } }
query { presets { id name container videoCodec } }
query { systemStatus { status version } }
# Mutations
mutation { scanMediaFolder { found registered skipped } }
mutation { createJob(input: { sourceAssetId: "...", presetId: "..." }) { id status } }
mutation { cancelJob(id: "...") { id status } }
mutation { retryJob(id: "...") { id status } }
mutation { updateAsset(id: "...", input: { comments: "..." }) { id comments } }
mutation { deleteAsset(id: "...") { ok } }
# Lambda callback (REST)
POST /api/jobs/{id}/callback - Lambda completion webhook</code></pre>
<p><strong>Supported File Types:</strong></p>
<p>
Video: mp4, mkv, avi, mov, webm, flv, wmv, m4v<br />
Audio: mp3, wav, flac, aac, ogg, m4a
</p>
</div>
<h2 id="access-points">Access Points</h2>
<pre><code># Add to /etc/hosts
127.0.0.1 mpr.local.ar
# URLs
http://mpr.local.ar/admin - Django Admin
http://mpr.local.ar/graphql - GraphiQL IDE
http://mpr.local.ar/ - Timeline UI</code></pre>
http://mpr.local.ar/admin - Django Admin
http://mpr.local.ar/graphql - GraphiQL IDE
http://mpr.local.ar/ - Timeline UI
http://mpr.local.ar/chunker/ - Chunker UI
http://localhost:9001 - MinIO Console
# AWS deployment
https://mpr.mcrn.ar/ - Production</code></pre>
<h2 id="quick-reference">Quick Reference</h2>
<pre><code># Render SVGs from DOT files
for f in docs/architecture/*.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; done
# Switch executor mode
MPR_EXECUTOR=local # Celery + MinIO
MPR_EXECUTOR=lambda # Step Functions + Lambda + S3
MPR_EXECUTOR=gcp # Cloud Run Jobs + GCS</code></pre>
</main>
</body>
</html>

View File

@@ -101,6 +101,12 @@ class SchemaLoader:
for enum_cls in enums:
self.enums.append(self._parse_enum(enum_cls))
# Extract VIEWS (view/event projections)
if load_all or "views" in include:
views = getattr(module, "VIEWS", [])
for cls in views:
self.api_models.append(self._parse_dataclass(cls))
# Extract GRPC_MESSAGES (optional)
if load_all or "grpc" in include:
grpc_messages = getattr(module, "GRPC_MESSAGES", [])

View File

@@ -8,10 +8,15 @@
"name": "mpr-chunker",
"version": "0.1.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
"devDependencies": {
"@protobuf-ts/grpcweb-transport": "^2.11.1",
"@protobuf-ts/plugin": "^2.11.1",
"@protobuf-ts/protoc": "^2.11.1",
"@types/react": "^18.2.0",
"@types/react-dom": "^18.2.0",
"@vitejs/plugin-react": "^4.2.0",
@@ -301,6 +306,39 @@
"node": ">=6.9.0"
}
},
"node_modules/@bufbuild/protobuf": {
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz",
"integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==",
"dev": true,
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@bufbuild/protoplugin": {
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/@bufbuild/protoplugin/-/protoplugin-2.11.0.tgz",
"integrity": "sha512-lyZVNFUHArIOt4W0+dwYBe5GBwbKzbOy8ObaloEqsw9Mmiwv2O48TwddDoHN4itylC+BaEGqFdI1W8WQt2vWJQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@bufbuild/protobuf": "2.11.0",
"@typescript/vfs": "^1.6.2",
"typescript": "5.4.5"
}
},
"node_modules/@bufbuild/protoplugin/node_modules/typescript": {
"version": "5.4.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
"integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.21.5",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
@@ -742,6 +780,75 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@protobuf-ts/grpcweb-transport": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/grpcweb-transport/-/grpcweb-transport-2.11.1.tgz",
"integrity": "sha512-1W4utDdvOB+RHMFQ0soL4JdnxjXV+ddeGIUg08DvZrA8Ms6k5NN6GBFU2oHZdTOcJVpPrDJ02RJlqtaoCMNBtw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1"
}
},
"node_modules/@protobuf-ts/plugin": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/plugin/-/plugin-2.11.1.tgz",
"integrity": "sha512-HyuprDcw0bEEJqkOWe1rnXUP0gwYLij8YhPuZyZk6cJbIgc/Q0IFgoHQxOXNIXAcXM4Sbehh6kjVnCzasElw1A==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@bufbuild/protobuf": "^2.4.0",
"@bufbuild/protoplugin": "^2.4.0",
"@protobuf-ts/protoc": "^2.11.1",
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"typescript": "^3.9"
},
"bin": {
"protoc-gen-dump": "bin/protoc-gen-dump",
"protoc-gen-ts": "bin/protoc-gen-ts"
}
},
"node_modules/@protobuf-ts/plugin/node_modules/typescript": {
"version": "3.9.10",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz",
"integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=4.2.0"
}
},
"node_modules/@protobuf-ts/protoc": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/protoc/-/protoc-2.11.1.tgz",
"integrity": "sha512-mUZJaV0daGO6HUX90o/atzQ6A7bbN2RSuHtdwo8SSF2Qoe3zHwa4IHyCN1evftTeHfLmdz+45qo47sL+5P8nyg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"protoc": "protoc.js"
}
},
"node_modules/@protobuf-ts/runtime": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz",
"integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==",
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@protobuf-ts/runtime-rpc": {
"version": "2.11.1",
"resolved": "https://registry.npmjs.org/@protobuf-ts/runtime-rpc/-/runtime-rpc-2.11.1.tgz",
"integrity": "sha512-4CqqUmNA+/uMz00+d3CYKgElXO9VrEbucjnBFEjqI4GuDrEQ32MaI3q+9qPBvIGOlL4PmHXrzM32vBPWRhQKWQ==",
"license": "Apache-2.0",
"dependencies": {
"@protobuf-ts/runtime": "^2.11.1"
}
},
"node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.27",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz",
@@ -1179,6 +1286,19 @@
"@types/react": "^18.0.0"
}
},
"node_modules/@typescript/vfs": {
"version": "1.6.4",
"resolved": "https://registry.npmjs.org/@typescript/vfs/-/vfs-1.6.4.tgz",
"integrity": "sha512-PJFXFS4ZJKiJ9Qiuix6Dz/OwEIqHD7Dme1UwZhTK11vR+5dqW2ACbdndWQexBzCx+CPuMe5WBYQWCsFyGlQLlQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"debug": "^4.4.3"
},
"peerDependencies": {
"typescript": "*"
}
},
"node_modules/@vitejs/plugin-react": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz",

View File

@@ -9,10 +9,15 @@
"preview": "vite preview"
},
"dependencies": {
"@protobuf-ts/grpcweb-transport": "^2.11.1",
"@protobuf-ts/runtime": "^2.11.1",
"@protobuf-ts/runtime-rpc": "^2.11.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
"devDependencies": {
"@protobuf-ts/plugin": "^2.11.1",
"@protobuf-ts/protoc": "^2.11.1",
"@types/react": "^18.2.0",
"@types/react-dom": "^18.2.0",
"@vitejs/plugin-react": "^4.2.0",

View File

@@ -1,16 +1,4 @@
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
"Fira Code", monospace, sans-serif;
background: #0f0f0f;
color: #e0e0e0;
font-size: 14px;
}
@import "../../common/styles/theme.css";
/* ---- Layout ---- */
@@ -25,8 +13,8 @@ body {
justify-content: space-between;
align-items: center;
padding: 0.75rem 1.25rem;
background: #1a1a1a;
border-bottom: 1px solid #2a2a2a;
background: var(--bg-panel);
border-bottom: 1px solid var(--border);
}
.header h1 {
@@ -40,19 +28,19 @@ body {
align-items: center;
gap: 0.5rem;
font-size: 0.8rem;
color: #666;
color: var(--text-muted);
}
.dot {
width: 8px;
height: 8px;
border-radius: 50%;
background: #555;
background: var(--text-muted);
}
.dot.connected {
background: #10b981;
box-shadow: 0 0 6px #10b981;
background: var(--success);
box-shadow: 0 0 6px var(--success);
}
.error-banner {
@@ -70,8 +58,8 @@ body {
.sidebar {
width: 300px;
background: #141414;
border-right: 1px solid #2a2a2a;
background: var(--bg-surface);
border-right: 1px solid var(--border);
overflow-y: auto;
}
@@ -97,163 +85,20 @@ body {
gap: 1rem;
}
/* ---- Panel shared ---- */
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.75rem;
}
.panel-header h2 {
font-size: 0.85rem;
text-transform: uppercase;
letter-spacing: 0.05em;
color: #888;
}
.badge-row {
display: flex;
gap: 0.25rem;
}
/* ---- Topic Badge ---- */
.topic-badge {
display: inline-flex;
align-items: center;
gap: 0.25rem;
padding: 0.15rem 0.5rem;
font-size: 0.65rem;
background: #1e293b;
border: 1px solid #334155;
border-radius: 12px;
cursor: pointer;
transition: all 0.2s;
flex-shrink: 0;
}
.topic-badge:hover {
border-color: #3b82f6;
}
.topic-badge.expanded {
flex-direction: column;
align-items: flex-start;
border-radius: 8px;
padding: 0.5rem;
position: relative;
z-index: 10;
background: #1e293b;
}
.topic-number {
color: #3b82f6;
font-weight: 700;
}
.topic-title {
color: #94a3b8;
}
.topic-detail {
margin-top: 0.25rem;
font-size: 0.7rem;
line-height: 1.4;
}
.topic-detail p {
color: #cbd5e1;
margin-bottom: 0.25rem;
}
.topic-detail code {
color: #10b981;
font-size: 0.65rem;
}
/* ---- Asset List ---- */
.scan-button {
padding: 0.25rem 0.5rem;
font-size: 0.7rem;
background: #1e293b;
color: #94a3b8;
border: 1px solid #334155;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.scan-button:hover:not(:disabled) {
background: #334155;
color: #e0e0e0;
}
.scan-button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.asset-list {
list-style: none;
max-height: 200px;
overflow-y: auto;
margin-bottom: 0.75rem;
}
.asset-item {
padding: 0.4rem 0.5rem;
cursor: pointer;
border-left: 2px solid transparent;
transition: all 0.15s;
display: flex;
flex-direction: column;
gap: 0.1rem;
}
.asset-item:hover {
background: #1a1a1a;
}
.asset-item.selected {
background: #1e293b;
border-left-color: #3b82f6;
}
.asset-filename {
font-size: 0.8rem;
color: #e0e0e0;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.asset-meta {
font-size: 0.65rem;
color: #555;
}
.asset-empty {
font-size: 0.8rem;
color: #444;
padding: 0.75rem 0.5rem;
text-align: center;
}
/* ---- Selected Asset Info ---- */
.selected-asset-info {
padding: 0.5rem;
background: #1e293b;
border: 1px solid #334155;
border-radius: 4px;
border-radius: var(--radius);
margin-bottom: 0.75rem;
}
.asset-detail {
display: block;
font-size: 0.8rem;
color: #e0e0e0;
color: var(--text-primary);
font-weight: 500;
}
@@ -277,12 +122,12 @@ body {
.config-field label {
display: block;
font-size: 0.75rem;
color: #888;
color: var(--text-secondary);
margin-bottom: 0.25rem;
}
.config-field .default {
color: #555;
color: var(--text-muted);
font-style: italic;
}
@@ -291,26 +136,26 @@ body {
width: 100%;
padding: 0.4rem 0.5rem;
font-size: 0.8rem;
background: #222;
color: #e0e0e0;
border: 1px solid #333;
border-radius: 4px;
background: var(--bg-input);
color: var(--text-primary);
border: 1px solid var(--border);
border-radius: var(--radius);
}
.config-field input:focus,
.config-field select:focus {
outline: none;
border-color: #3b82f6;
border-color: var(--accent);
}
.start-button {
width: 100%;
padding: 0.5rem;
font-size: 0.85rem;
background: #10b981;
background: var(--success);
color: #000;
border: none;
border-radius: 4px;
border-radius: var(--radius);
cursor: pointer;
font-weight: 600;
margin-top: 0.5rem;
@@ -322,116 +167,86 @@ body {
}
.start-button:disabled {
background: #333;
color: #666;
background: var(--bg-input);
color: var(--text-muted);
cursor: not-allowed;
}
/* ---- Pipeline Diagram ---- */
.pipeline-diagram {
background: #141414;
border: 1px solid #2a2a2a;
border-radius: 8px;
padding: 1rem;
}
.stage-flow {
display: flex;
align-items: center;
gap: 0;
overflow-x: auto;
}
.stage-wrapper {
display: flex;
align-items: center;
}
.stage {
padding: 0.5rem 0.75rem;
background: #1a1a1a;
border: 1px solid #333;
border-radius: 6px;
text-align: center;
min-width: 120px;
transition: all 0.3s;
}
.stage.active {
border-color: #3b82f6;
background: #1e293b;
box-shadow: 0 0 12px rgba(59, 130, 246, 0.2);
}
.stage-label {
font-size: 0.8rem;
.stop-button {
width: 100%;
padding: 0.5rem;
font-size: 0.85rem;
background: var(--error);
color: #fff;
border: none;
border-radius: var(--radius);
cursor: pointer;
font-weight: 600;
color: #e0e0e0;
margin-top: 0.5rem;
transition: background 0.2s;
}
.stage-sub {
font-size: 0.65rem;
color: #666;
margin-top: 0.15rem;
.stop-button:hover {
background: #dc2626;
}
.stage-arrow {
width: 24px;
height: 2px;
background: #444;
position: relative;
}
.stage-arrow::after {
content: "";
position: absolute;
right: 0;
top: -3px;
border: 4px solid transparent;
border-left: 6px solid #444;
}
.processor-hierarchy {
margin-top: 0.75rem;
padding-top: 0.75rem;
border-top: 1px solid #222;
}
.hierarchy-title {
font-size: 0.7rem;
color: #666;
margin-bottom: 0.35rem;
font-style: italic;
}
.hierarchy-children {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.hierarchy-node {
font-size: 0.7rem;
padding: 0.15rem 0.5rem;
background: #1a1a1a;
border: 1px solid #333;
border-radius: 4px;
.reset-button {
width: 100%;
padding: 0.5rem;
font-size: 0.85rem;
background: #1e293b;
color: #94a3b8;
border: 1px solid #334155;
border-radius: var(--radius);
cursor: pointer;
font-weight: 600;
margin-top: 0.5rem;
transition: all 0.2s;
}
.reset-button:hover {
background: #334155;
color: var(--text-primary);
}
.range-row {
display: flex;
align-items: center;
gap: 0.5rem;
}
.range-row input {
flex: 1;
padding: 0.4rem 0.5rem;
font-size: 0.8rem;
background: var(--bg-input);
color: var(--text-primary);
border: 1px solid var(--border);
border-radius: var(--radius);
}
.range-row input:focus {
outline: none;
border-color: var(--accent);
}
.range-sep {
font-size: 0.75rem;
color: var(--text-muted);
}
/* ---- Chunk Grid ---- */
.chunk-grid-panel {
background: #141414;
border: 1px solid #2a2a2a;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
.chunk-count {
font-size: 0.7rem;
color: #555;
color: var(--text-muted);
font-weight: 400;
}
@@ -466,7 +281,7 @@ body {
align-items: center;
gap: 0.25rem;
font-size: 0.65rem;
color: #888;
color: var(--text-secondary);
}
.legend-dot {
@@ -478,8 +293,8 @@ body {
/* ---- Worker Panel ---- */
.worker-panel {
background: #141414;
border: 1px solid #2a2a2a;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
@@ -492,8 +307,8 @@ body {
.worker-card {
padding: 0.5rem 0.75rem;
background: #1a1a1a;
border: 1px solid #2a2a2a;
background: var(--bg-panel);
border: 1px solid var(--border);
border-radius: 6px;
}
@@ -516,7 +331,7 @@ body {
.worker-chunk {
font-size: 0.7rem;
color: #555;
color: var(--text-muted);
margin-top: 0.15rem;
}
@@ -524,13 +339,13 @@ body {
display: flex;
gap: 0.75rem;
font-size: 0.65rem;
color: #555;
color: var(--text-muted);
margin-top: 0.25rem;
}
.worker-empty {
font-size: 0.8rem;
color: #444;
color: var(--text-muted);
text-align: center;
padding: 1rem;
}
@@ -538,8 +353,8 @@ body {
/* ---- Queue Gauge ---- */
.queue-gauge {
background: #141414;
border: 1px solid #2a2a2a;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
@@ -550,38 +365,38 @@ body {
.gauge-label {
font-size: 0.75rem;
color: #888;
color: var(--text-secondary);
margin-bottom: 0.25rem;
}
.gauge-value {
color: #e0e0e0;
color: var(--text-primary);
font-weight: 600;
}
.gauge-bar {
height: 8px;
background: #222;
border-radius: 4px;
background: var(--bg-input);
border-radius: var(--radius);
overflow: hidden;
}
.gauge-fill {
height: 100%;
border-radius: 4px;
border-radius: var(--radius);
transition: width 0.3s, background 0.3s;
}
.gauge-note {
font-size: 0.65rem;
color: #555;
color: var(--text-muted);
}
/* ---- Stats Panel ---- */
.stats-panel {
background: #141414;
border: 1px solid #2a2a2a;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
@@ -595,52 +410,29 @@ body {
.stat {
text-align: center;
padding: 0.5rem;
background: #1a1a1a;
background: var(--bg-panel);
border-radius: 6px;
}
.stat-value {
font-size: 1.1rem;
font-weight: 700;
color: #e0e0e0;
color: var(--text-primary);
}
.stat-label {
font-size: 0.6rem;
color: #666;
color: var(--text-muted);
text-transform: uppercase;
letter-spacing: 0.05em;
margin-top: 0.15rem;
}
.test-info {
margin-top: 0.75rem;
padding-top: 0.5rem;
border-top: 1px solid #222;
display: flex;
align-items: center;
gap: 0.5rem;
}
.test-badge {
font-size: 0.65rem;
padding: 0.15rem 0.4rem;
background: #10b981;
color: #000;
border-radius: 3px;
font-weight: 600;
}
.test-note {
font-size: 0.65rem;
color: #555;
}
/* ---- Error Log ---- */
.error-log {
background: #141414;
border: 1px solid #2a2a2a;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
@@ -654,41 +446,6 @@ body {
font-weight: 400;
}
.exception-tree {
margin-bottom: 0.75rem;
padding: 0.5rem;
background: #1a1a1a;
border-radius: 6px;
font-size: 0.7rem;
font-family: "Fira Code", monospace;
}
.tree-node {
color: #94a3b8;
padding: 0.1rem 0;
}
.tree-node.root {
color: #f59e0b;
font-weight: 600;
}
.tree-node.leaf {
color: #64748b;
}
.tree-children {
padding-left: 1rem;
border-left: 1px solid #333;
margin-left: 0.5rem;
}
.tree-grandchildren {
padding-left: 1rem;
border-left: 1px solid #333;
margin-left: 0.5rem;
}
.error-entries {
max-height: 150px;
overflow-y: auto;
@@ -696,7 +453,7 @@ body {
.error-empty {
font-size: 0.8rem;
color: #444;
color: var(--text-muted);
text-align: center;
padding: 0.5rem;
}
@@ -706,26 +463,26 @@ body {
gap: 0.5rem;
align-items: center;
padding: 0.35rem 0;
border-bottom: 1px solid #1a1a1a;
border-bottom: 1px solid var(--bg-panel);
font-size: 0.7rem;
flex-wrap: wrap;
}
.error-type {
color: #ef4444;
color: var(--error);
font-weight: 500;
}
.error-seq {
color: #f59e0b;
color: var(--warning);
}
.error-worker {
color: #3b82f6;
color: var(--accent);
}
.error-msg {
color: #888;
color: var(--text-secondary);
flex: 1;
}
@@ -733,3 +490,15 @@ body {
color: #f97316;
font-size: 0.65rem;
}
/* ---- Output download link ---- */
.fm-download-link {
font-size: 0.7rem;
color: var(--accent);
text-decoration: none;
}
.fm-download-link:hover {
text-decoration: underline;
}

View File

@@ -1,16 +1,23 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import "./App.css";
import { createChunkJob, getAssets, scanMediaFolder } from "./api";
import {
cancelChunkJob,
createChunkJob,
getAssets,
getChunkOutputFiles,
scanMediaFolder,
} from "./api";
import { ChunkGrid } from "./components/ChunkGrid";
import { ConfigPanel } from "./components/ConfigPanel";
import { ErrorLog } from "./components/ErrorLog";
import { PipelineDiagram } from "./components/PipelineDiagram";
import { OutputFiles } from "./components/OutputFiles";
import { QueueGauge } from "./components/QueueGauge";
import { StatsPanel } from "./components/StatsPanel";
import { WorkerPanel } from "./components/WorkerPanel";
import { useEventStream } from "./hooks/useEventStream";
import { useGrpcStream } from "./hooks/useGrpcStream";
import type {
ChunkInfo,
ChunkOutputFile,
ErrorEntry,
MediaAsset,
PipelineConfig,
@@ -20,6 +27,7 @@ import type {
export default function App() {
const [jobId, setJobId] = useState<string | null>(null);
const [celeryTaskId, setCeleryTaskId] = useState<string | null>(null);
const [running, setRunning] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -28,15 +36,36 @@ export default function App() {
const [selectedAsset, setSelectedAsset] = useState<MediaAsset | null>(null);
const [scanning, setScanning] = useState(false);
const { events, connected, done } = useEventStream(jobId);
// Output files
const [outputFiles, setOutputFiles] = useState<ChunkOutputFile[]>([]);
const {
events,
connected,
done,
reset: resetStream,
} = useGrpcStream(jobId);
// Load assets on mount
useEffect(() => {
getAssets()
.then((data) => setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename))))
.catch((e) => setError(e instanceof Error ? e.message : "Failed to load assets"));
.then((data) =>
setAssets(data.sort((a, b) => a.filename.localeCompare(b.filename))),
)
.catch((e) =>
setError(e instanceof Error ? e.message : "Failed to load assets"),
);
}, []);
// Fetch output files when job completes
useEffect(() => {
if (done && jobId) {
getChunkOutputFiles(jobId)
.then(setOutputFiles)
.catch(() => setOutputFiles([]));
}
}, [done, jobId]);
const handleScan = useCallback(async () => {
setScanning(true);
setError(null);
@@ -51,8 +80,8 @@ export default function App() {
}
}, []);
// Derive state from events
const { chunks, workers, stats, errors, activeStage, queueSize } =
// Derive state from raw events
const { chunks, workers, stats, errors, queueSize } =
useMemo(() => {
const chunkMap = new Map<number, ChunkInfo>();
const workerMap = new Map<string, WorkerInfo>();
@@ -64,45 +93,54 @@ export default function App() {
let elapsed = 0;
let throughput = 0;
let queueSize = 0;
let stage = "pending";
let pipelineDone = false;
for (const evt of events) {
const evtType = evt.event_type || "";
if (evt.total_chunks) totalChunks = evt.total_chunks;
if (evt.processed_chunks) processed = evt.processed_chunks;
if (evt.failed_chunks) failed = evt.failed_chunks;
if (evt.elapsed) elapsed = evt.elapsed;
if (evt.throughput_mbps) throughput = evt.throughput_mbps;
if (evt.queue_size !== undefined) queueSize = evt.queue_size;
if (evt.status && evt.status !== "waiting") stage = evt.status;
// Track chunks
if (evtType === "pipeline_complete" || evtType === "pipeline_error") {
pipelineDone = true;
queueSize = 0;
}
// Track chunks by raw event type
if (evt.sequence !== undefined) {
const existing = chunkMap.get(evt.sequence) || {
sequence: evt.sequence,
state: "pending" as const,
};
if (evt.status === "chunking" || evt.status === "pending") {
if (evtType === "chunk_queued") {
existing.state = "queued";
} else if (evt.status === "processing") {
} else if (evtType === "chunk_processing") {
existing.state = "processing";
if (evt.worker_id) existing.worker_id = evt.worker_id;
} else if (evt.status === "completed") {
} else if (evtType === "chunk_done") {
existing.state = "done";
if (evt.processing_time)
existing.processing_time = evt.processing_time;
if (evt.retries) existing.retries = evt.retries;
} else if (evt.status === "failed") {
} else if (evtType === "chunk_error") {
existing.state = "error";
if (evt.error) existing.error = evt.error;
} else if (evtType === "chunk_retry") {
existing.state = "retry";
if (evt.retries) existing.retries = evt.retries;
}
if (evt.size) existing.size = evt.size;
chunkMap.set(evt.sequence, existing);
}
// Track workers
if (evt.worker_id) {
// Track workers from worker_status events
if (evt.worker_id && evtType === "worker_status") {
const w = workerMap.get(evt.worker_id) || {
worker_id: evt.worker_id,
state: "idle" as const,
@@ -119,12 +157,38 @@ export default function App() {
w.current_chunk = undefined;
} else if (evt.state === "stopped") {
w.state = "stopped";
w.current_chunk = undefined;
}
if (evt.success !== undefined) {
if (evt.success) w.processed++;
else w.errors++;
workerMap.set(evt.worker_id, w);
}
// Also update workers from chunk lifecycle events
if (
evt.worker_id &&
(evtType === "chunk_processing" ||
evtType === "chunk_done" ||
evtType === "chunk_error")
) {
const w = workerMap.get(evt.worker_id) || {
worker_id: evt.worker_id,
state: "idle" as const,
processed: 0,
errors: 0,
retries: 0,
};
if (evtType === "chunk_processing") {
w.state = "processing";
w.current_chunk = evt.sequence;
} else if (evtType === "chunk_done") {
w.processed++;
w.state = "idle";
w.current_chunk = undefined;
} else if (evtType === "chunk_error") {
w.errors++;
}
if (evt.retries) {
retries += evt.retries;
w.retries += evt.retries;
@@ -141,11 +205,19 @@ export default function App() {
worker_id: evt.worker_id,
error: evt.error,
retries: evt.retries,
event_type: evt.status || "error",
event_type: evtType,
});
}
}
// When pipeline is done, mark all workers as stopped
if (pipelineDone) {
for (const w of workerMap.values()) {
w.state = "stopped";
w.current_chunk = undefined;
}
}
const statsObj: PipelineStats = {
total_chunks: totalChunks,
processed,
@@ -158,12 +230,11 @@ export default function App() {
return {
chunks: Array.from(chunkMap.values()).sort(
(a, b) => a.sequence - b.sequence
(a, b) => a.sequence - b.sequence,
),
workers: Array.from(workerMap.values()),
stats: statsObj,
errors: errorList,
activeStage: stage,
queueSize,
};
}, [events]);
@@ -171,15 +242,45 @@ export default function App() {
const handleStart = useCallback(async (config: PipelineConfig) => {
setError(null);
setRunning(true);
setOutputFiles([]);
try {
const result = await createChunkJob(config);
setJobId(result.id);
setCeleryTaskId(result.celery_task_id);
} catch (e) {
setError(e instanceof Error ? e.message : "Failed to start");
setRunning(false);
}
}, []);
const handleStop = useCallback(async () => {
if (!celeryTaskId) {
setError("No task ID to cancel");
return;
}
try {
const result = await cancelChunkJob(celeryTaskId);
if (result.ok) {
resetStream();
setRunning(false);
setError(null);
} else {
setError(result.message || "Failed to cancel");
}
} catch (e) {
setError(e instanceof Error ? e.message : "Failed to cancel");
}
}, [celeryTaskId, resetStream]);
const handleReset = useCallback(() => {
setJobId(null);
setCeleryTaskId(null);
setRunning(false);
setError(null);
setOutputFiles([]);
resetStream();
}, [resetStream]);
// Reset running state when done
if (done && running) {
setRunning(false);
@@ -197,10 +298,10 @@ export default function App() {
{!jobId
? "Configure and launch"
: connected
? "Streaming"
: done
? "Complete"
: "Connecting..."}
? "Streaming"
: done
? "Complete"
: "Connecting..."}
</span>
</div>
</header>
@@ -211,7 +312,10 @@ export default function App() {
<aside className="sidebar">
<ConfigPanel
onStart={handleStart}
onStop={handleStop}
onReset={handleReset}
running={running}
done={done}
assets={assets}
selectedAsset={selectedAsset}
onSelectAsset={setSelectedAsset}
@@ -221,16 +325,13 @@ export default function App() {
</aside>
<main className="main">
<PipelineDiagram activeStage={activeStage} />
<div className="main-grid">
<div className="main-left">
<ChunkGrid chunks={chunks} totalChunks={stats.total_chunks} />
<QueueGauge
current={queueSize}
max={10}
buffered={0}
/>
<QueueGauge current={queueSize} max={10} buffered={0} />
{done && outputFiles.length > 0 && (
<OutputFiles files={outputFiles} />
)}
</div>
<div className="main-right">
<WorkerPanel workers={workers} />

View File

@@ -1,55 +1,13 @@
/**
* GraphQL API client for the chunker UI.
* Chunker-specific API functions.
* Shared functions (getAssets, scanMediaFolder) come from common.
*/
import type { MediaAsset } from "./types";
import { gql } from "../../common/api/graphql";
import type { ChunkOutputFile } from "../../common/types/generated";
const GRAPHQL_URL = "/api/graphql";
async function gql<T>(query: string, variables?: Record<string, unknown>): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}
/** Fetch all media assets. */
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
/** Scan media/in/ folder for new files. */
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}
// Re-export shared functions
export { getAssets, scanMediaFolder } from "../../common/api/media";
/** Create a chunk job via GraphQL mutation. */
export async function createChunkJob(config: {
@@ -58,15 +16,70 @@ export async function createChunkJob(config: {
num_workers: number;
max_retries: number;
processor_type: string;
}): Promise<{ id: string }> {
const data = await gql<{ create_chunk_job: { id: string; status: string } }>(`
start_time?: number | null;
end_time?: number | null;
}): Promise<{ id: string; celery_task_id: string | null }> {
const data = await gql<{
create_chunk_job: {
id: string;
status: string;
celery_task_id: string | null;
};
}>(
`
mutation CreateChunkJob($input: CreateChunkJobInput!) {
create_chunk_job(input: $input) {
id
status
celery_task_id
}
}
`, { input: config });
`,
{ input: config },
);
return data.create_chunk_job;
}
/** Cancel a running chunk job. */
export async function cancelChunkJob(
celeryTaskId: string,
): Promise<{ ok: boolean; message: string | null }> {
const data = await gql<{
cancel_chunk_job: { ok: boolean; message: string | null };
}>(
`
mutation CancelChunkJob($celery_task_id: String!) {
cancel_chunk_job(celery_task_id: $celery_task_id) {
ok
message
}
}
`,
{ celery_task_id: celeryTaskId },
);
return data.cancel_chunk_job;
}
/** Fetch output chunk files for a completed job. */
export async function getChunkOutputFiles(
jobId: string,
): Promise<ChunkOutputFile[]> {
const data = await gql<{
chunk_output_files: ChunkOutputFile[];
}>(
`
query ChunkOutputFiles($job_id: String!) {
chunk_output_files(job_id: $job_id) {
key
size
url
}
}
`,
{ job_id: jobId },
);
return data.chunk_output_files;
}

View File

@@ -1,5 +1,4 @@
import type { ChunkInfo } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
chunks: ChunkInfo[];
@@ -7,19 +6,14 @@ interface Props {
}
const STATE_COLORS: Record<string, string> = {
pending: "#333",
queued: "#f59e0b",
processing: "#3b82f6",
done: "#10b981",
error: "#ef4444",
pending: "var(--bg-input)",
queued: "var(--warning)",
processing: "var(--processing)",
done: "var(--success)",
error: "var(--error)",
retry: "#f97316",
};
/**
* Grid of chunks colored by processing state.
* Chunks appear incrementally as the generator yields them.
* Interview Topic 3: Generators & iteration.
*/
export function ChunkGrid({ chunks, totalChunks }: Props) {
return (
<div className="chunk-grid-panel">
@@ -30,14 +24,13 @@ export function ChunkGrid({ chunks, totalChunks }: Props) {
{chunks.length} / {totalChunks || "?"}
</span>
</h2>
<TopicBadge topic={TOPICS.iteration} />
</div>
<div className="chunk-grid">
{chunks.map((chunk) => (
<div
key={chunk.sequence}
className="chunk-cell"
style={{ background: STATE_COLORS[chunk.state] || "#333" }}
style={{ background: STATE_COLORS[chunk.state] || "var(--bg-input)" }}
title={`#${chunk.sequence}${chunk.state}${
chunk.worker_id ? ` (${chunk.worker_id})` : ""
}${chunk.retries ? ` retries: ${chunk.retries}` : ""}`}

View File

@@ -1,10 +1,15 @@
import { useState } from "react";
import { useMemo, useState } from "react";
import { FileManager } from "../../../common/components/FileManager";
import type { FileEntry } from "../../../common/components/FileManager";
import { formatDuration, formatSize } from "../../../common/utils/format";
import type { MediaAsset, PipelineConfig } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
onStart: (config: PipelineConfig) => void;
onStop: () => void;
onReset: () => void;
running: boolean;
done: boolean;
assets: MediaAsset[];
selectedAsset: MediaAsset | null;
onSelectAsset: (asset: MediaAsset) => void;
@@ -12,13 +17,12 @@ interface Props {
scanning: boolean;
}
/**
* Pipeline configuration form with file browser.
* Each parameter shows its default — Interview Topic 1: Function params & defaults.
*/
export function ConfigPanel({
onStart,
onStop,
onReset,
running,
done,
assets,
selectedAsset,
onSelectAsset,
@@ -31,6 +35,25 @@ export function ConfigPanel({
const [processorType, setProcessorType] = useState<
"ffmpeg" | "checksum" | "simulated_decode" | "composite"
>("ffmpeg");
const [startTime, setStartTime] = useState<string>("");
const [endTime, setEndTime] = useState<string>("");
// Map assets to FileEntry for FileManager
const fileEntries: FileEntry[] = useMemo(
() =>
assets.map((a) => ({
key: a.id,
name: a.filename,
size: a.file_size ?? undefined,
meta: formatDuration(a.duration),
})),
[assets],
);
const handleFileSelect = (file: FileEntry) => {
const asset = assets.find((a) => a.id === file.key);
if (asset) onSelectAsset(asset);
};
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
@@ -41,61 +64,31 @@ export function ConfigPanel({
num_workers: numWorkers,
max_retries: maxRetries,
processor_type: processorType,
start_time: startTime ? parseFloat(startTime) : null,
end_time: endTime ? parseFloat(endTime) : null,
});
};
const formatSize = (bytes: number | null) => {
if (!bytes) return "—";
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
};
const formatDuration = (seconds: number | null) => {
if (!seconds) return "—";
const m = Math.floor(seconds / 60);
const s = Math.floor(seconds % 60);
return `${m}:${s.toString().padStart(2, "0")}`;
};
return (
<div className="config-panel">
{/* Asset Browser */}
<div className="panel-header">
<h2>Assets</h2>
<button
onClick={onScan}
disabled={scanning}
className="scan-button"
>
{scanning ? "Scanning..." : "Scan Folder"}
</button>
</div>
<ul className="asset-list">
{assets.length === 0 ? (
<li className="asset-empty">No assets click Scan Folder</li>
) : (
assets.map((asset) => (
<li
key={asset.id}
className={`asset-item ${selectedAsset?.id === asset.id ? "selected" : ""}`}
onClick={() => onSelectAsset(asset)}
title={asset.filename}
>
<span className="asset-filename">{asset.filename}</span>
<span className="asset-meta">
{formatSize(asset.file_size)} · {formatDuration(asset.duration)}
</span>
</li>
))
)}
</ul>
<FileManager
title="Assets"
files={fileEntries}
selectedKey={selectedAsset?.id ?? null}
onSelect={handleFileSelect}
onScan={onScan}
scanning={scanning}
emptyMessage="No assets — click Scan Folder"
disabled={running}
/>
{selectedAsset && (
<div className="selected-asset-info">
<span className="asset-detail">{selectedAsset.filename}</span>
<span className="asset-detail-meta">
{selectedAsset.video_codec} · {selectedAsset.width}x{selectedAsset.height} · {formatDuration(selectedAsset.duration)}
{selectedAsset.video_codec} · {selectedAsset.width}x
{selectedAsset.height} · {formatDuration(selectedAsset.duration)} ·{" "}
{formatSize(selectedAsset.file_size)}
</span>
</div>
)}
@@ -103,9 +96,35 @@ export function ConfigPanel({
{/* Pipeline Config */}
<div className="panel-header" style={{ marginTop: "1rem" }}>
<h2>Pipeline Config</h2>
<TopicBadge topic={TOPICS.params} />
</div>
<form onSubmit={handleSubmit}>
<div className="config-field">
<label>
Time Range (seconds){" "}
<span className="default">optional limits what gets chunked</span>
</label>
<div className="range-row">
<input
type="number"
min={0}
step={1}
placeholder="start"
value={startTime}
onChange={(e) => setStartTime(e.target.value)}
disabled={running}
/>
<span className="range-sep">to</span>
<input
type="number"
min={0}
step={1}
placeholder="end"
value={endTime}
onChange={(e) => setEndTime(e.target.value)}
disabled={running}
/>
</div>
</div>
<div className="config-field">
<label>
Chunk Duration <span className="default">default: 10s</span>
@@ -113,6 +132,7 @@ export function ConfigPanel({
<select
value={chunkDuration}
onChange={(e) => setChunkDuration(Number(e.target.value))}
disabled={running}
>
<option value={5}>5 seconds</option>
<option value={10}>10 seconds</option>
@@ -131,6 +151,7 @@ export function ConfigPanel({
max={16}
value={numWorkers}
onChange={(e) => setNumWorkers(Number(e.target.value))}
disabled={running}
/>
</div>
<div className="config-field">
@@ -143,6 +164,7 @@ export function ConfigPanel({
max={10}
value={maxRetries}
onChange={(e) => setMaxRetries(Number(e.target.value))}
disabled={running}
/>
</div>
<div className="config-field">
@@ -153,9 +175,14 @@ export function ConfigPanel({
value={processorType}
onChange={(e) =>
setProcessorType(
e.target.value as "ffmpeg" | "checksum" | "simulated_decode" | "composite"
e.target.value as
| "ffmpeg"
| "checksum"
| "simulated_decode"
| "composite",
)
}
disabled={running}
>
<option value="ffmpeg">FFmpegExtractProcessor</option>
<option value="checksum">ChecksumProcessor</option>
@@ -163,10 +190,29 @@ export function ConfigPanel({
<option value="composite">CompositeProcessor</option>
</select>
</div>
<button type="submit" className="start-button" disabled={running || !selectedAsset}>
{running ? "Running..." : "Launch Pipeline"}
</button>
{!running && !done && (
<button
type="submit"
className="start-button"
disabled={!selectedAsset}
>
Launch Pipeline
</button>
)}
</form>
{running && (
<button type="button" className="stop-button" onClick={onStop}>
Stop Pipeline
</button>
)}
{done && (
<button type="button" className="reset-button" onClick={onReset}>
Reset
</button>
)}
</div>
);
}

View File

@@ -1,15 +1,9 @@
import type { ErrorEntry } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
errors: ErrorEntry[];
}
/**
* Error and retry event log.
* Shows exception types, retry counts, backoff delays.
* Interview Topic 7: Exception handling & resilient code.
*/
export function ErrorLog({ errors }: Props) {
return (
<div className="error-log">
@@ -18,23 +12,6 @@ export function ErrorLog({ errors }: Props) {
Errors & Retries{" "}
<span className="error-count">{errors.length}</span>
</h2>
<TopicBadge topic={TOPICS.exceptions} />
</div>
<div className="exception-tree">
<div className="tree-node root">PipelineError</div>
<div className="tree-children">
<div className="tree-node">ChunkError</div>
<div className="tree-grandchildren">
<div className="tree-node leaf">ChunkReadError</div>
<div className="tree-node leaf">ChunkChecksumError</div>
</div>
<div className="tree-node">ProcessingError</div>
<div className="tree-grandchildren">
<div className="tree-node leaf">ProcessorTimeoutError</div>
<div className="tree-node leaf">ProcessorFailureError</div>
</div>
<div className="tree-node">ReassemblyError</div>
</div>
</div>
<div className="error-entries">
{errors.length === 0 && (

View File

@@ -0,0 +1,51 @@
import { useMemo } from "react";
import { FileManager } from "../../../common/components/FileManager";
import type { FileEntry } from "../../../common/components/FileManager";
import { formatSize } from "../../../common/utils/format";
import type { ChunkOutputFile } from "../types";
interface Props {
files: ChunkOutputFile[];
}
export function OutputFiles({ files }: Props) {
const fileEntries: FileEntry[] = useMemo(
() =>
files.map((f) => ({
key: f.key,
name: f.key.split("/").pop() || f.key,
size: f.size,
})),
[files],
);
const urlMap = useMemo(() => {
const map = new Map<string, string>();
for (const f of files) {
map.set(f.key, f.url);
}
return map;
}, [files]);
return (
<FileManager
title="Output Files"
files={fileEntries}
emptyMessage="No output files"
renderActions={(file) => {
const url = urlMap.get(file.key);
if (!url) return null;
return (
<a
href={url}
download
className="fm-download-link"
onClick={(e) => e.stopPropagation()}
>
{formatSize(file.size)}
</a>
);
}}
/>
);
}

View File

@@ -1,50 +0,0 @@
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
activeStage: string;
}
const STAGES = [
{ id: "chunking", label: "Chunker", sub: "File -> Chunks (generator)" },
{ id: "queued", label: "ChunkQueue", sub: "Bounded queue (backpressure)" },
{ id: "processing", label: "WorkerPool", sub: "ThreadPoolExecutor" },
{ id: "collecting", label: "ResultCollector", sub: "heapq reassembly" },
{ id: "completed", label: "PipelineResult", sub: "Aggregate stats" },
];
/**
* Visual flow diagram of pipeline stages.
* Highlights the currently active stage.
* Interview Topic 4: OOP design — shows class hierarchy.
*/
export function PipelineDiagram({ activeStage }: Props) {
return (
<div className="pipeline-diagram">
<div className="panel-header">
<h2>Pipeline Flow</h2>
<TopicBadge topic={TOPICS.oop} />
</div>
<div className="stage-flow">
{STAGES.map((stage, i) => (
<div key={stage.id} className="stage-wrapper">
<div
className={`stage ${activeStage === stage.id ? "active" : ""}`}
>
<div className="stage-label">{stage.label}</div>
<div className="stage-sub">{stage.sub}</div>
</div>
{i < STAGES.length - 1 && <div className="stage-arrow" />}
</div>
))}
</div>
<div className="processor-hierarchy">
<div className="hierarchy-title">Processor ABC</div>
<div className="hierarchy-children">
<span className="hierarchy-node">ChecksumProcessor</span>
<span className="hierarchy-node">SimulatedDecodeProcessor</span>
<span className="hierarchy-node">CompositeProcessor</span>
</div>
</div>
</div>
);
}

View File

@@ -1,15 +1,9 @@
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
current: number;
max: number;
buffered: number;
}
/**
* Queue fill level gauge + collector heap buffer.
* Interview Topic 5: Data structures — queue.Queue, heapq, deque.
*/
export function QueueGauge({ current, max, buffered }: Props) {
const fillPct = max > 0 ? Math.min((current / max) * 100, 100) : 0;
@@ -17,7 +11,6 @@ export function QueueGauge({ current, max, buffered }: Props) {
<div className="queue-gauge">
<div className="panel-header">
<h2>Queue & Buffer</h2>
<TopicBadge topic={TOPICS.datastructures} />
</div>
<div className="gauge-row">
<div className="gauge-label">
@@ -28,7 +21,7 @@ export function QueueGauge({ current, max, buffered }: Props) {
className="gauge-fill"
style={{
width: `${fillPct}%`,
background: fillPct > 80 ? "#ef4444" : "#3b82f6",
background: fillPct > 80 ? "var(--error)" : "var(--processing)",
}}
/>
</div>

View File

@@ -1,24 +1,14 @@
import type { PipelineStats } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
stats: PipelineStats;
}
/**
* Throughput, timing, and error stats.
* Interview Topic 6: Algorithms — throughput calculation over sliding window.
* Interview Topic 8: TDD — test count and coverage.
*/
export function StatsPanel({ stats }: Props) {
return (
<div className="stats-panel">
<div className="panel-header">
<h2>Stats</h2>
<div className="badge-row">
<TopicBadge topic={TOPICS.algorithms} />
<TopicBadge topic={TOPICS.testing} />
</div>
</div>
<div className="stats-grid">
<div className="stat">
@@ -48,12 +38,6 @@ export function StatsPanel({ stats }: Props) {
<div className="stat-label">Elapsed</div>
</div>
</div>
<div className="test-info">
<span className="test-badge">64 tests</span>
<span className="test-note">
7 test files &middot; pytest &middot; parametrized
</span>
</div>
</div>
);
}

View File

@@ -1,86 +0,0 @@
import { useState } from "react";
import type { InterviewTopic } from "../types";
/**
* Expandable pill badge annotating an interview topic.
* Click to expand and see description + code reference.
*/
export function TopicBadge({ topic }: { topic: InterviewTopic }) {
const [expanded, setExpanded] = useState(false);
return (
<div
className={`topic-badge ${expanded ? "expanded" : ""}`}
onClick={() => setExpanded(!expanded)}
>
<span className="topic-number">#{topic.number}</span>
<span className="topic-title">{topic.title}</span>
{expanded && (
<div className="topic-detail">
<p>{topic.description}</p>
<code>{topic.code_ref}</code>
</div>
)}
</div>
);
}
/** Pre-defined topics mapped to pipeline components. */
export const TOPICS: Record<string, InterviewTopic> = {
params: {
number: 1,
title: "Function Params & Defaults",
description:
"Each pipeline parameter has a sensible default (chunk_duration=10s, num_workers=4, max_retries=3). Tweaking them changes pipeline behavior.",
code_ref: "core/chunker/pipeline.py — Pipeline.__init__()",
},
concurrency: {
number: 2,
title: "Concurrency (Threading)",
description:
"Workers run in a ThreadPoolExecutor. The queue coordinates work between producer and consumer threads.",
code_ref: "core/chunker/pool.py — WorkerPool, ThreadPoolExecutor",
},
iteration: {
number: 3,
title: "Generators & Iteration",
description:
"Chunks are yielded lazily via a generator — the file is never fully loaded into memory.",
code_ref: "core/chunker/chunker.py — Chunker.chunks() generator",
},
oop: {
number: 4,
title: "OOP Design (ABC)",
description:
"Processor is an abstract base class. ChecksumProcessor, SimulatedDecodeProcessor, and CompositeProcessor inherit from it.",
code_ref: "core/chunker/processor.py — Processor ABC hierarchy",
},
datastructures: {
number: 5,
title: "Data Structures",
description:
"Bounded queue.Queue for backpressure, heapq min-heap for ordered reassembly, deque for sliding-window throughput.",
code_ref: "core/chunker/queue.py, collector.py, models.py",
},
algorithms: {
number: 6,
title: "Algorithms & Sorting",
description:
"ResultCollector uses a min-heap to reassemble chunks in sequence order, even when they arrive out of order.",
code_ref: "core/chunker/collector.py — heapq-based reassembly",
},
exceptions: {
number: 7,
title: "Exception Handling",
description:
"PipelineError hierarchy with typed exceptions. Workers retry with exponential backoff before giving up.",
code_ref: "core/chunker/exceptions.py, worker.py — retry logic",
},
testing: {
number: 8,
title: "TDD & Unit Testing",
description:
"64 tests covering every module. Parametrized tests, fixtures, edge cases, concurrency tests.",
code_ref: "tests/chunker/ — 7 test files, pytest",
},
};

View File

@@ -1,28 +1,21 @@
import type { WorkerInfo } from "../types";
import { TopicBadge, TOPICS } from "./TopicBadge";
interface Props {
workers: WorkerInfo[];
}
const STATE_COLORS: Record<string, string> = {
idle: "#6b7280",
processing: "#3b82f6",
idle: "var(--text-muted)",
processing: "var(--processing)",
retry: "#f97316",
stopped: "#ef4444",
stopped: "var(--error)",
};
/**
* Worker thread status cards.
* Shows each worker's real-time state and which chunk it's processing.
* Interview Topic 2: Concurrency (threading).
*/
export function WorkerPanel({ workers }: Props) {
return (
<div className="worker-panel">
<div className="panel-header">
<h2>Workers</h2>
<TopicBadge topic={TOPICS.concurrency} />
</div>
<div className="worker-cards">
{workers.map((w) => (
@@ -31,7 +24,7 @@ export function WorkerPanel({ workers }: Props) {
<span className="worker-name">{w.worker_id}</span>
<span
className="worker-state"
style={{ color: STATE_COLORS[w.state] || "#888" }}
style={{ color: STATE_COLORS[w.state] || "var(--text-secondary)" }}
>
{w.state}
</span>

View File

@@ -3,8 +3,6 @@ import type { PipelineEvent } from "../types";
/**
* SSE hook — connects to /api/chunker/stream/{jobId} via native EventSource.
*
* Demonstrates: real-time event streaming from backend to UI.
*/
export function useEventStream(jobId: string | null) {
const [events, setEvents] = useState<PipelineEvent[]>([]);
@@ -20,6 +18,12 @@ export function useEventStream(jobId: string | null) {
}
}, []);
const reset = useCallback(() => {
close();
setEvents([]);
setDone(false);
}, [close]);
useEffect(() => {
if (!jobId) return;
@@ -35,21 +39,28 @@ export function useEventStream(jobId: string | null) {
const handleEvent = (eventType: string) => (e: MessageEvent) => {
try {
const data = JSON.parse(e.data) as PipelineEvent;
setEvents((prev) => [...prev, { ...data, status: eventType }]);
setEvents((prev) => [...prev, { ...data, event_type: eventType }]);
} catch {
// ignore parse errors
}
};
// Listen to all chunker event types
// Listen to all raw pipeline event types
const eventTypes = [
"waiting",
"pending",
"chunking",
"processing",
"collecting",
"completed",
"failed",
"pipeline_start",
"pipeline_info",
"chunk_queued",
"chunk_processing",
"chunk_done",
"chunk_retry",
"chunk_error",
"chunk_collected",
"worker_status",
"pipeline_progress",
"pipeline_complete",
"pipeline_error",
"producer_error",
"cancelled",
"done",
"timeout",
@@ -77,5 +88,5 @@ export function useEventStream(jobId: string | null) {
};
}, [jobId]);
return { events, connected, done, close };
return { events, connected, done, close, reset };
}

View File

@@ -0,0 +1,103 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { GrpcWebFetchTransport } from "@protobuf-ts/grpcweb-transport";
import { WorkerServiceClient } from "../../../common/api/grpc/worker.client";
import type { ChunkPipelineEvent } from "../../../common/api/grpc/worker";
import type { PipelineEvent } from "../types";
const GRPC_WEB_URL = "/grpc-web";
function toEvent(msg: ChunkPipelineEvent): PipelineEvent {
return {
event_type: msg.eventType,
job_id: msg.jobId,
sequence: msg.sequence || undefined,
worker_id: msg.workerId || undefined,
state: msg.state || undefined,
queue_size: msg.queueSize || undefined,
elapsed: msg.elapsed || undefined,
throughput_mbps: msg.throughputMbps || undefined,
total_chunks: msg.totalChunks || undefined,
processed_chunks: msg.processedChunks || undefined,
failed_chunks: msg.failedChunks || undefined,
error: msg.error || undefined,
processing_time: msg.processingTime || undefined,
retries: msg.retries || undefined,
};
}
/**
* gRPC-Web streaming hook — connects to WorkerService.StreamChunkPipeline
* via Envoy proxy. Replaces useEventStream (SSE+Redis).
*/
export function useGrpcStream(jobId: string | null) {
const [events, setEvents] = useState<PipelineEvent[]>([]);
const [connected, setConnected] = useState(false);
const [done, setDone] = useState(false);
const abortRef = useRef<AbortController | null>(null);
const close = useCallback(() => {
if (abortRef.current) {
abortRef.current.abort();
abortRef.current = null;
setConnected(false);
}
}, []);
const reset = useCallback(() => {
close();
setEvents([]);
setDone(false);
}, [close]);
useEffect(() => {
if (!jobId) return;
setEvents([]);
setDone(false);
const abort = new AbortController();
abortRef.current = abort;
const transport = new GrpcWebFetchTransport({
baseUrl: GRPC_WEB_URL,
abort: abort.signal,
});
const client = new WorkerServiceClient(transport);
const stream = client.streamChunkPipeline({ jobId });
setConnected(true);
(async () => {
try {
for await (const msg of stream.responses) {
const evt = toEvent(msg);
setEvents((prev) => [...prev, evt]);
if (
evt.event_type === "pipeline_complete" ||
evt.event_type === "pipeline_error"
) {
setDone(true);
setConnected(false);
break;
}
}
} catch (err) {
if (!abort.signal.aborted) {
setConnected(false);
}
} finally {
setConnected(false);
}
})();
return () => {
abort.abort();
abortRef.current = null;
};
}, [jobId]);
return { events, connected, done, close, reset };
}

View File

@@ -1,3 +1,19 @@
/**
* Chunker UI types.
*
* Domain types (MediaAsset, ChunkEvent, etc.) come from generated schema.
* This file holds UI-only types: state enums, SSE envelope, derived views.
*/
// Re-export generated types used by this app
export type {
MediaAsset,
ChunkEvent,
WorkerEvent,
PipelineStats,
ChunkOutputFile,
} from "../../common/types/generated";
/** Pipeline configuration sent to the backend. */
export interface PipelineConfig {
source_asset_id: string;
@@ -5,31 +21,11 @@ export interface PipelineConfig {
num_workers: number;
max_retries: number;
processor_type: "ffmpeg" | "checksum" | "simulated_decode" | "composite";
start_time?: number | null;
end_time?: number | null;
}
/** Media asset from the backend. */
export interface MediaAsset {
id: string;
filename: string;
file_path: string;
status: string;
error_message: string | null;
file_size: number | null;
duration: number | null;
video_codec: string | null;
audio_codec: string | null;
width: number | null;
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
/** State of an individual chunk. */
/** UI state of an individual chunk in the grid. */
export type ChunkState =
| "pending"
| "queued"
@@ -38,7 +34,7 @@ export type ChunkState =
| "error"
| "retry";
/** Tracked chunk in the UI grid. */
/** Tracked chunk in the UI grid (derived from events). */
export interface ChunkInfo {
sequence: number;
state: ChunkState;
@@ -49,7 +45,7 @@ export interface ChunkInfo {
error?: string;
}
/** Worker thread status. */
/** Worker thread status (derived from events). */
export interface WorkerInfo {
worker_id: string;
state: "idle" | "processing" | "retry" | "stopped";
@@ -59,9 +55,14 @@ export interface WorkerInfo {
retries: number;
}
/** SSE event from the backend. */
/**
* Raw SSE event envelope from the backend.
* The event_type field is set by useEventStream from the SSE event name.
* All other fields are optional — presence depends on event_type.
*/
export interface PipelineEvent {
job_id: string;
job_id?: string;
event_type?: string;
status?: string;
progress?: number;
total_chunks?: number;
@@ -84,18 +85,7 @@ export interface PipelineEvent {
backoff?: number;
}
/** Aggregate pipeline stats. */
export interface PipelineStats {
total_chunks: number;
processed: number;
failed: number;
retries: number;
elapsed: number;
throughput_mbps: number;
queue_size: number;
}
/** Error log entry. */
/** Error log entry (derived from events). */
export interface ErrorEntry {
timestamp: number;
sequence?: number;
@@ -104,11 +94,3 @@ export interface ErrorEntry {
retries?: number;
event_type: string;
}
/** Interview topic for annotation badges. */
export interface InterviewTopic {
number: number;
title: string;
description: string;
code_ref: string;
}

View File

@@ -14,8 +14,13 @@
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true
"noFallthroughCasesInSwitch": true,
"rootDir": "..",
"typeRoots": ["./node_modules/@types"],
"paths": {
"@protobuf-ts/*": ["./node_modules/@protobuf-ts/*"]
}
},
"include": ["src/**/*.ts", "src/**/*.tsx"],
"include": ["src/**/*.ts", "src/**/*.tsx", "../common/**/*.ts", "../common/**/*.tsx"],
"references": [{ "path": "./tsconfig.node.json" }]
}

View File

@@ -1,9 +1,26 @@
import path from "path";
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
export default defineConfig({
base: "/chunker/",
plugins: [react()],
resolve: {
alias: {
"@protobuf-ts/runtime": path.resolve(
__dirname,
"node_modules/@protobuf-ts/runtime",
),
"@protobuf-ts/runtime-rpc": path.resolve(
__dirname,
"node_modules/@protobuf-ts/runtime-rpc",
),
"@protobuf-ts/grpcweb-transport": path.resolve(
__dirname,
"node_modules/@protobuf-ts/grpcweb-transport",
),
},
},
server: {
host: "0.0.0.0",
port: 5174,
@@ -11,6 +28,9 @@ export default defineConfig({
hmr: {
path: "/chunker/@vite/client",
},
fs: {
allow: [".."],
},
proxy: {
"/api": {
target: "http://fastapi:8702",
@@ -20,6 +40,11 @@ export default defineConfig({
target: "http://fastapi:8702",
changeOrigin: true,
},
"/grpc-web": {
target: "http://envoy:8090",
changeOrigin: true,
rewrite: (p) => p.replace(/^\/grpc-web/, ""),
},
},
},
});

24
ui/common/api/graphql.ts Normal file
View File

@@ -0,0 +1,24 @@
/**
* Shared GraphQL client for all MPR UI apps.
*/
const GRAPHQL_URL = "/api/graphql";
export async function gql<T>(
query: string,
variables?: Record<string, unknown>,
): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}

View File

@@ -0,0 +1,95 @@
// @generated by protobuf-ts 2.11.1
// @generated from protobuf file "worker.proto" (package "mpr.worker", syntax proto3)
// tslint:disable
//
// Protocol Buffer Definitions - GENERATED FILE
//
// Do not edit directly. Regenerate using modelgen.
//
import type { RpcTransport } from "@protobuf-ts/runtime-rpc";
import type { ServiceInfo } from "@protobuf-ts/runtime-rpc";
import { WorkerService } from "./worker";
import type { ChunkPipelineEvent } from "./worker";
import type { ChunkStreamRequest } from "./worker";
import type { WorkerStatus } from "./worker";
import type { Empty } from "./worker";
import type { CancelResponse } from "./worker";
import type { CancelRequest } from "./worker";
import type { ProgressUpdate } from "./worker";
import type { ProgressRequest } from "./worker";
import type { ServerStreamingCall } from "@protobuf-ts/runtime-rpc";
import { stackIntercept } from "@protobuf-ts/runtime-rpc";
import type { JobResponse } from "./worker";
import type { JobRequest } from "./worker";
import type { UnaryCall } from "@protobuf-ts/runtime-rpc";
import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
/**
* @generated from protobuf service mpr.worker.WorkerService
*/
export interface IWorkerServiceClient {
/**
* @generated from protobuf rpc: SubmitJob
*/
submitJob(input: JobRequest, options?: RpcOptions): UnaryCall<JobRequest, JobResponse>;
/**
* @generated from protobuf rpc: StreamProgress
*/
streamProgress(input: ProgressRequest, options?: RpcOptions): ServerStreamingCall<ProgressRequest, ProgressUpdate>;
/**
* @generated from protobuf rpc: CancelJob
*/
cancelJob(input: CancelRequest, options?: RpcOptions): UnaryCall<CancelRequest, CancelResponse>;
/**
* @generated from protobuf rpc: GetWorkerStatus
*/
getWorkerStatus(input: Empty, options?: RpcOptions): UnaryCall<Empty, WorkerStatus>;
/**
* @generated from protobuf rpc: StreamChunkPipeline
*/
streamChunkPipeline(input: ChunkStreamRequest, options?: RpcOptions): ServerStreamingCall<ChunkStreamRequest, ChunkPipelineEvent>;
}
/**
* @generated from protobuf service mpr.worker.WorkerService
*/
export class WorkerServiceClient implements IWorkerServiceClient, ServiceInfo {
typeName = WorkerService.typeName;
methods = WorkerService.methods;
options = WorkerService.options;
constructor(private readonly _transport: RpcTransport) {
}
/**
* @generated from protobuf rpc: SubmitJob
*/
submitJob(input: JobRequest, options?: RpcOptions): UnaryCall<JobRequest, JobResponse> {
const method = this.methods[0], opt = this._transport.mergeOptions(options);
return stackIntercept<JobRequest, JobResponse>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: StreamProgress
*/
streamProgress(input: ProgressRequest, options?: RpcOptions): ServerStreamingCall<ProgressRequest, ProgressUpdate> {
const method = this.methods[1], opt = this._transport.mergeOptions(options);
return stackIntercept<ProgressRequest, ProgressUpdate>("serverStreaming", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: CancelJob
*/
cancelJob(input: CancelRequest, options?: RpcOptions): UnaryCall<CancelRequest, CancelResponse> {
const method = this.methods[2], opt = this._transport.mergeOptions(options);
return stackIntercept<CancelRequest, CancelResponse>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: GetWorkerStatus
*/
getWorkerStatus(input: Empty, options?: RpcOptions): UnaryCall<Empty, WorkerStatus> {
const method = this.methods[3], opt = this._transport.mergeOptions(options);
return stackIntercept<Empty, WorkerStatus>("unary", this._transport, method, opt, input);
}
/**
* @generated from protobuf rpc: StreamChunkPipeline
*/
streamChunkPipeline(input: ChunkStreamRequest, options?: RpcOptions): ServerStreamingCall<ChunkStreamRequest, ChunkPipelineEvent> {
const method = this.methods[4], opt = this._transport.mergeOptions(options);
return stackIntercept<ChunkStreamRequest, ChunkPipelineEvent>("serverStreaming", this._transport, method, opt, input);
}
}

View File

@@ -0,0 +1,946 @@
// @generated by protobuf-ts 2.11.1
// @generated from protobuf file "worker.proto" (package "mpr.worker", syntax proto3)
// tslint:disable
//
// Protocol Buffer Definitions - GENERATED FILE
//
// Do not edit directly. Regenerate using modelgen.
//
import { ServiceType } from "@protobuf-ts/runtime-rpc";
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
import type { IBinaryWriter } from "@protobuf-ts/runtime";
import { WireType } from "@protobuf-ts/runtime";
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
import type { IBinaryReader } from "@protobuf-ts/runtime";
import { UnknownFieldHandler } from "@protobuf-ts/runtime";
import type { PartialMessage } from "@protobuf-ts/runtime";
import { reflectionMergePartial } from "@protobuf-ts/runtime";
import { MessageType } from "@protobuf-ts/runtime";
/**
* @generated from protobuf message mpr.worker.JobRequest
*/
export interface JobRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: string source_path = 2
*/
sourcePath: string;
/**
* @generated from protobuf field: string output_path = 3
*/
outputPath: string;
/**
* @generated from protobuf field: string preset_json = 4
*/
presetJson: string;
/**
* @generated from protobuf field: optional float trim_start = 5
*/
trimStart?: number;
/**
* @generated from protobuf field: optional float trim_end = 6
*/
trimEnd?: number;
}
/**
* @generated from protobuf message mpr.worker.JobResponse
*/
export interface JobResponse {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: bool accepted = 2
*/
accepted: boolean;
/**
* @generated from protobuf field: string message = 3
*/
message: string;
}
/**
* @generated from protobuf message mpr.worker.ProgressRequest
*/
export interface ProgressRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.ProgressUpdate
*/
export interface ProgressUpdate {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: int32 progress = 2
*/
progress: number;
/**
* @generated from protobuf field: int32 current_frame = 3
*/
currentFrame: number;
/**
* @generated from protobuf field: float current_time = 4
*/
currentTime: number;
/**
* @generated from protobuf field: float speed = 5
*/
speed: number;
/**
* @generated from protobuf field: string status = 6
*/
status: string;
/**
* @generated from protobuf field: optional string error = 7
*/
error?: string;
}
/**
* @generated from protobuf message mpr.worker.CancelRequest
*/
export interface CancelRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.CancelResponse
*/
export interface CancelResponse {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: bool cancelled = 2
*/
cancelled: boolean;
/**
* @generated from protobuf field: string message = 3
*/
message: string;
}
/**
* @generated from protobuf message mpr.worker.WorkerStatus
*/
export interface WorkerStatus {
/**
* @generated from protobuf field: bool available = 1
*/
available: boolean;
/**
* @generated from protobuf field: int32 active_jobs = 2
*/
activeJobs: number;
/**
* @generated from protobuf field: repeated string supported_codecs = 3
*/
supportedCodecs: string[];
/**
* @generated from protobuf field: bool gpu_available = 4
*/
gpuAvailable: boolean;
}
/**
* Empty
*
* @generated from protobuf message mpr.worker.Empty
*/
export interface Empty {
}
/**
* @generated from protobuf message mpr.worker.ChunkStreamRequest
*/
export interface ChunkStreamRequest {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
}
/**
* @generated from protobuf message mpr.worker.ChunkPipelineEvent
*/
export interface ChunkPipelineEvent {
/**
* @generated from protobuf field: string job_id = 1
*/
jobId: string;
/**
* @generated from protobuf field: string event_type = 2
*/
eventType: string;
/**
* @generated from protobuf field: int32 sequence = 3
*/
sequence: number;
/**
* @generated from protobuf field: string worker_id = 4
*/
workerId: string;
/**
* @generated from protobuf field: string state = 5
*/
state: string;
/**
* @generated from protobuf field: int32 queue_size = 6
*/
queueSize: number;
/**
* @generated from protobuf field: float elapsed = 7
*/
elapsed: number;
/**
* @generated from protobuf field: float throughput_mbps = 8
*/
throughputMbps: number;
/**
* @generated from protobuf field: int32 total_chunks = 9
*/
totalChunks: number;
/**
* @generated from protobuf field: int32 processed_chunks = 10
*/
processedChunks: number;
/**
* @generated from protobuf field: int32 failed_chunks = 11
*/
failedChunks: number;
/**
* @generated from protobuf field: string error = 12
*/
error: string;
/**
* @generated from protobuf field: float processing_time = 13
*/
processingTime: number;
/**
* @generated from protobuf field: int32 retries = 14
*/
retries: number;
}
// @generated message type with reflection information, may provide speed optimized methods
class JobRequest$Type extends MessageType<JobRequest> {
constructor() {
super("mpr.worker.JobRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "source_path", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 3, name: "output_path", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 4, name: "preset_json", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 5, name: "trim_start", kind: "scalar", opt: true, T: 2 /*ScalarType.FLOAT*/ },
{ no: 6, name: "trim_end", kind: "scalar", opt: true, T: 2 /*ScalarType.FLOAT*/ }
]);
}
create(value?: PartialMessage<JobRequest>): JobRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.sourcePath = "";
message.outputPath = "";
message.presetJson = "";
if (value !== undefined)
reflectionMergePartial<JobRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: JobRequest): JobRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* string source_path */ 2:
message.sourcePath = reader.string();
break;
case /* string output_path */ 3:
message.outputPath = reader.string();
break;
case /* string preset_json */ 4:
message.presetJson = reader.string();
break;
case /* optional float trim_start */ 5:
message.trimStart = reader.float();
break;
case /* optional float trim_end */ 6:
message.trimEnd = reader.float();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: JobRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* string source_path = 2; */
if (message.sourcePath !== "")
writer.tag(2, WireType.LengthDelimited).string(message.sourcePath);
/* string output_path = 3; */
if (message.outputPath !== "")
writer.tag(3, WireType.LengthDelimited).string(message.outputPath);
/* string preset_json = 4; */
if (message.presetJson !== "")
writer.tag(4, WireType.LengthDelimited).string(message.presetJson);
/* optional float trim_start = 5; */
if (message.trimStart !== undefined)
writer.tag(5, WireType.Bit32).float(message.trimStart);
/* optional float trim_end = 6; */
if (message.trimEnd !== undefined)
writer.tag(6, WireType.Bit32).float(message.trimEnd);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.JobRequest
*/
export const JobRequest = new JobRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class JobResponse$Type extends MessageType<JobResponse> {
constructor() {
super("mpr.worker.JobResponse", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "accepted", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 3, name: "message", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<JobResponse>): JobResponse {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.accepted = false;
message.message = "";
if (value !== undefined)
reflectionMergePartial<JobResponse>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: JobResponse): JobResponse {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* bool accepted */ 2:
message.accepted = reader.bool();
break;
case /* string message */ 3:
message.message = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: JobResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* bool accepted = 2; */
if (message.accepted !== false)
writer.tag(2, WireType.Varint).bool(message.accepted);
/* string message = 3; */
if (message.message !== "")
writer.tag(3, WireType.LengthDelimited).string(message.message);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.JobResponse
*/
export const JobResponse = new JobResponse$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ProgressRequest$Type extends MessageType<ProgressRequest> {
constructor() {
super("mpr.worker.ProgressRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ProgressRequest>): ProgressRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<ProgressRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ProgressRequest): ProgressRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ProgressRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ProgressRequest
*/
export const ProgressRequest = new ProgressRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ProgressUpdate$Type extends MessageType<ProgressUpdate> {
constructor() {
super("mpr.worker.ProgressUpdate", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "progress", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 3, name: "current_frame", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 4, name: "current_time", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 5, name: "speed", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 6, name: "status", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 7, name: "error", kind: "scalar", opt: true, T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ProgressUpdate>): ProgressUpdate {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.progress = 0;
message.currentFrame = 0;
message.currentTime = 0;
message.speed = 0;
message.status = "";
if (value !== undefined)
reflectionMergePartial<ProgressUpdate>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ProgressUpdate): ProgressUpdate {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* int32 progress */ 2:
message.progress = reader.int32();
break;
case /* int32 current_frame */ 3:
message.currentFrame = reader.int32();
break;
case /* float current_time */ 4:
message.currentTime = reader.float();
break;
case /* float speed */ 5:
message.speed = reader.float();
break;
case /* string status */ 6:
message.status = reader.string();
break;
case /* optional string error */ 7:
message.error = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ProgressUpdate, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* int32 progress = 2; */
if (message.progress !== 0)
writer.tag(2, WireType.Varint).int32(message.progress);
/* int32 current_frame = 3; */
if (message.currentFrame !== 0)
writer.tag(3, WireType.Varint).int32(message.currentFrame);
/* float current_time = 4; */
if (message.currentTime !== 0)
writer.tag(4, WireType.Bit32).float(message.currentTime);
/* float speed = 5; */
if (message.speed !== 0)
writer.tag(5, WireType.Bit32).float(message.speed);
/* string status = 6; */
if (message.status !== "")
writer.tag(6, WireType.LengthDelimited).string(message.status);
/* optional string error = 7; */
if (message.error !== undefined)
writer.tag(7, WireType.LengthDelimited).string(message.error);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ProgressUpdate
*/
export const ProgressUpdate = new ProgressUpdate$Type();
// @generated message type with reflection information, may provide speed optimized methods
class CancelRequest$Type extends MessageType<CancelRequest> {
constructor() {
super("mpr.worker.CancelRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<CancelRequest>): CancelRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<CancelRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: CancelRequest): CancelRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: CancelRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.CancelRequest
*/
export const CancelRequest = new CancelRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class CancelResponse$Type extends MessageType<CancelResponse> {
constructor() {
super("mpr.worker.CancelResponse", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "cancelled", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 3, name: "message", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<CancelResponse>): CancelResponse {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.cancelled = false;
message.message = "";
if (value !== undefined)
reflectionMergePartial<CancelResponse>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: CancelResponse): CancelResponse {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* bool cancelled */ 2:
message.cancelled = reader.bool();
break;
case /* string message */ 3:
message.message = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: CancelResponse, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* bool cancelled = 2; */
if (message.cancelled !== false)
writer.tag(2, WireType.Varint).bool(message.cancelled);
/* string message = 3; */
if (message.message !== "")
writer.tag(3, WireType.LengthDelimited).string(message.message);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.CancelResponse
*/
export const CancelResponse = new CancelResponse$Type();
// @generated message type with reflection information, may provide speed optimized methods
class WorkerStatus$Type extends MessageType<WorkerStatus> {
constructor() {
super("mpr.worker.WorkerStatus", [
{ no: 1, name: "available", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
{ no: 2, name: "active_jobs", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 3, name: "supported_codecs", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 9 /*ScalarType.STRING*/ },
{ no: 4, name: "gpu_available", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
]);
}
create(value?: PartialMessage<WorkerStatus>): WorkerStatus {
const message = globalThis.Object.create((this.messagePrototype!));
message.available = false;
message.activeJobs = 0;
message.supportedCodecs = [];
message.gpuAvailable = false;
if (value !== undefined)
reflectionMergePartial<WorkerStatus>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: WorkerStatus): WorkerStatus {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* bool available */ 1:
message.available = reader.bool();
break;
case /* int32 active_jobs */ 2:
message.activeJobs = reader.int32();
break;
case /* repeated string supported_codecs */ 3:
message.supportedCodecs.push(reader.string());
break;
case /* bool gpu_available */ 4:
message.gpuAvailable = reader.bool();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: WorkerStatus, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* bool available = 1; */
if (message.available !== false)
writer.tag(1, WireType.Varint).bool(message.available);
/* int32 active_jobs = 2; */
if (message.activeJobs !== 0)
writer.tag(2, WireType.Varint).int32(message.activeJobs);
/* repeated string supported_codecs = 3; */
for (let i = 0; i < message.supportedCodecs.length; i++)
writer.tag(3, WireType.LengthDelimited).string(message.supportedCodecs[i]);
/* bool gpu_available = 4; */
if (message.gpuAvailable !== false)
writer.tag(4, WireType.Varint).bool(message.gpuAvailable);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.WorkerStatus
*/
export const WorkerStatus = new WorkerStatus$Type();
// @generated message type with reflection information, may provide speed optimized methods
class Empty$Type extends MessageType<Empty> {
constructor() {
super("mpr.worker.Empty", []);
}
create(value?: PartialMessage<Empty>): Empty {
const message = globalThis.Object.create((this.messagePrototype!));
if (value !== undefined)
reflectionMergePartial<Empty>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: Empty): Empty {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: Empty, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.Empty
*/
export const Empty = new Empty$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChunkStreamRequest$Type extends MessageType<ChunkStreamRequest> {
constructor() {
super("mpr.worker.ChunkStreamRequest", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ChunkStreamRequest>): ChunkStreamRequest {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
if (value !== undefined)
reflectionMergePartial<ChunkStreamRequest>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChunkStreamRequest): ChunkStreamRequest {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChunkStreamRequest, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ChunkStreamRequest
*/
export const ChunkStreamRequest = new ChunkStreamRequest$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChunkPipelineEvent$Type extends MessageType<ChunkPipelineEvent> {
constructor() {
super("mpr.worker.ChunkPipelineEvent", [
{ no: 1, name: "job_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 2, name: "event_type", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 3, name: "sequence", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 4, name: "worker_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 5, name: "state", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 6, name: "queue_size", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 7, name: "elapsed", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 8, name: "throughput_mbps", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 9, name: "total_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 10, name: "processed_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 11, name: "failed_chunks", kind: "scalar", T: 5 /*ScalarType.INT32*/ },
{ no: 12, name: "error", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 13, name: "processing_time", kind: "scalar", T: 2 /*ScalarType.FLOAT*/ },
{ no: 14, name: "retries", kind: "scalar", T: 5 /*ScalarType.INT32*/ }
]);
}
create(value?: PartialMessage<ChunkPipelineEvent>): ChunkPipelineEvent {
const message = globalThis.Object.create((this.messagePrototype!));
message.jobId = "";
message.eventType = "";
message.sequence = 0;
message.workerId = "";
message.state = "";
message.queueSize = 0;
message.elapsed = 0;
message.throughputMbps = 0;
message.totalChunks = 0;
message.processedChunks = 0;
message.failedChunks = 0;
message.error = "";
message.processingTime = 0;
message.retries = 0;
if (value !== undefined)
reflectionMergePartial<ChunkPipelineEvent>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChunkPipelineEvent): ChunkPipelineEvent {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string job_id */ 1:
message.jobId = reader.string();
break;
case /* string event_type */ 2:
message.eventType = reader.string();
break;
case /* int32 sequence */ 3:
message.sequence = reader.int32();
break;
case /* string worker_id */ 4:
message.workerId = reader.string();
break;
case /* string state */ 5:
message.state = reader.string();
break;
case /* int32 queue_size */ 6:
message.queueSize = reader.int32();
break;
case /* float elapsed */ 7:
message.elapsed = reader.float();
break;
case /* float throughput_mbps */ 8:
message.throughputMbps = reader.float();
break;
case /* int32 total_chunks */ 9:
message.totalChunks = reader.int32();
break;
case /* int32 processed_chunks */ 10:
message.processedChunks = reader.int32();
break;
case /* int32 failed_chunks */ 11:
message.failedChunks = reader.int32();
break;
case /* string error */ 12:
message.error = reader.string();
break;
case /* float processing_time */ 13:
message.processingTime = reader.float();
break;
case /* int32 retries */ 14:
message.retries = reader.int32();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChunkPipelineEvent, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string job_id = 1; */
if (message.jobId !== "")
writer.tag(1, WireType.LengthDelimited).string(message.jobId);
/* string event_type = 2; */
if (message.eventType !== "")
writer.tag(2, WireType.LengthDelimited).string(message.eventType);
/* int32 sequence = 3; */
if (message.sequence !== 0)
writer.tag(3, WireType.Varint).int32(message.sequence);
/* string worker_id = 4; */
if (message.workerId !== "")
writer.tag(4, WireType.LengthDelimited).string(message.workerId);
/* string state = 5; */
if (message.state !== "")
writer.tag(5, WireType.LengthDelimited).string(message.state);
/* int32 queue_size = 6; */
if (message.queueSize !== 0)
writer.tag(6, WireType.Varint).int32(message.queueSize);
/* float elapsed = 7; */
if (message.elapsed !== 0)
writer.tag(7, WireType.Bit32).float(message.elapsed);
/* float throughput_mbps = 8; */
if (message.throughputMbps !== 0)
writer.tag(8, WireType.Bit32).float(message.throughputMbps);
/* int32 total_chunks = 9; */
if (message.totalChunks !== 0)
writer.tag(9, WireType.Varint).int32(message.totalChunks);
/* int32 processed_chunks = 10; */
if (message.processedChunks !== 0)
writer.tag(10, WireType.Varint).int32(message.processedChunks);
/* int32 failed_chunks = 11; */
if (message.failedChunks !== 0)
writer.tag(11, WireType.Varint).int32(message.failedChunks);
/* string error = 12; */
if (message.error !== "")
writer.tag(12, WireType.LengthDelimited).string(message.error);
/* float processing_time = 13; */
if (message.processingTime !== 0)
writer.tag(13, WireType.Bit32).float(message.processingTime);
/* int32 retries = 14; */
if (message.retries !== 0)
writer.tag(14, WireType.Varint).int32(message.retries);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message mpr.worker.ChunkPipelineEvent
*/
export const ChunkPipelineEvent = new ChunkPipelineEvent$Type();
/**
* @generated ServiceType for protobuf service mpr.worker.WorkerService
*/
export const WorkerService = new ServiceType("mpr.worker.WorkerService", [
{ name: "SubmitJob", options: {}, I: JobRequest, O: JobResponse },
{ name: "StreamProgress", serverStreaming: true, options: {}, I: ProgressRequest, O: ProgressUpdate },
{ name: "CancelJob", options: {}, I: CancelRequest, O: CancelResponse },
{ name: "GetWorkerStatus", options: {}, I: Empty, O: WorkerStatus },
{ name: "StreamChunkPipeline", serverStreaming: true, options: {}, I: ChunkStreamRequest, O: ChunkPipelineEvent }
]);

42
ui/common/api/media.ts Normal file
View File

@@ -0,0 +1,42 @@
/**
* Shared media API functions — identical across all MPR UI apps.
*/
import type { MediaAsset } from "../types/generated";
import { gql } from "./graphql";
/** Fetch all media assets. */
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
/** Scan media/in/ folder for new files. */
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{
scan_media_folder: {
found: number;
registered: number;
skipped: number;
files: string[];
};
}>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}

View File

@@ -0,0 +1,97 @@
.file-manager {
margin-bottom: 1rem;
}
.fm-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.fm-header h2 {
font-size: 0.85rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--text-secondary);
}
.fm-scan-btn {
padding: 0.25rem 0.6rem;
background: var(--bg-input);
color: var(--text-secondary);
font-size: var(--font-size-xs);
}
.fm-scan-btn:hover:not(:disabled) {
color: var(--text-primary);
background: var(--border-light);
}
.fm-list {
list-style: none;
max-height: 200px;
overflow-y: auto;
border: 1px solid var(--border);
border-radius: var(--radius-sm);
background: var(--bg-primary);
}
.fm-empty {
padding: 1rem;
text-align: center;
color: var(--text-muted);
font-size: var(--font-size-sm);
}
.fm-item {
display: flex;
justify-content: space-between;
align-items: center;
padding: 0.4rem 0.6rem;
border-bottom: 1px solid var(--border);
transition: background 0.1s;
}
.fm-item:last-child {
border-bottom: none;
}
.fm-clickable {
cursor: pointer;
}
.fm-clickable:hover {
background: var(--bg-input);
}
.fm-selected {
background: var(--accent) !important;
color: #fff;
}
.fm-selected .fm-meta {
color: rgba(255, 255, 255, 0.7);
}
.fm-item-info {
display: flex;
flex-direction: column;
gap: 0.15rem;
overflow: hidden;
min-width: 0;
}
.fm-filename {
font-size: var(--font-size-sm);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.fm-meta {
font-size: var(--font-size-xs);
color: var(--text-muted);
}
.fm-actions {
flex-shrink: 0;
margin-left: 0.5rem;
}

View File

@@ -0,0 +1,84 @@
/**
* FileManager — pluggable file browser for S3/MinIO files.
*
* Handles both input file selection and output file listing.
* Used by timeline (assets + output), chunker (assets + chunk output),
* and future tools.
*/
import type { ReactNode } from "react";
import { formatSize } from "../utils/format";
import "./FileManager.css";
export interface FileEntry {
key: string;
name: string;
size?: number;
meta?: string;
}
interface FileManagerProps {
title: string;
files: FileEntry[];
selectedKey?: string | null;
onSelect?: (file: FileEntry) => void;
onScan?: () => void;
scanning?: boolean;
emptyMessage?: string;
renderActions?: (file: FileEntry) => ReactNode;
disabled?: boolean;
}
export function FileManager({
title,
files,
selectedKey,
onSelect,
onScan,
scanning = false,
emptyMessage = "No files",
renderActions,
disabled = false,
}: FileManagerProps) {
return (
<div className="file-manager">
<div className="fm-header">
<h2>{title}</h2>
{onScan && (
<button
className="fm-scan-btn"
onClick={onScan}
disabled={scanning || disabled}
>
{scanning ? "Scanning..." : "Scan Folder"}
</button>
)}
</div>
<ul className="fm-list">
{files.length === 0 ? (
<li className="fm-empty">{emptyMessage}</li>
) : (
files.map((file) => (
<li
key={file.key}
className={`fm-item ${selectedKey === file.key ? "fm-selected" : ""} ${onSelect && !disabled ? "fm-clickable" : ""}`}
onClick={() => onSelect && !disabled && onSelect(file)}
title={file.name}
>
<div className="fm-item-info">
<span className="fm-filename">{file.name}</span>
<span className="fm-meta">
{file.size != null && formatSize(file.size)}
{file.meta && (file.size != null ? ` · ${file.meta}` : file.meta)}
</span>
</div>
{renderActions && (
<div className="fm-actions">{renderActions(file)}</div>
)}
</li>
))
)}
</ul>
</div>
);
}

View File

@@ -0,0 +1,33 @@
/**
* StatusDot — small colored indicator for connection/state.
*/
const STATE_COLORS: Record<string, string> = {
connected: "var(--success)",
idle: "var(--text-muted)",
processing: "var(--processing)",
stopped: "var(--text-muted)",
error: "var(--error)",
done: "var(--success)",
};
interface StatusDotProps {
state: string;
glow?: boolean;
}
export function StatusDot({ state, glow = false }: StatusDotProps) {
const color = STATE_COLORS[state] || "var(--text-muted)";
return (
<span
style={{
display: "inline-block",
width: 8,
height: 8,
borderRadius: "50%",
background: color,
boxShadow: glow ? `0 0 6px ${color}` : undefined,
}}
/>
);
}

109
ui/common/styles/theme.css Normal file
View File

@@ -0,0 +1,109 @@
/**
* MPR Shared Theme — CSS custom properties + base styles.
* Import from any UI app: @import "../../common/styles/theme.css";
*/
:root {
--bg-primary: #0f0f0f;
--bg-panel: #1a1a1a;
--bg-surface: #141414;
--bg-input: #2a2a2a;
--border: #2a2a2a;
--border-light: #333;
--text-primary: #e0e0e0;
--text-secondary: #999;
--text-muted: #666;
--accent: #3b82f6;
--success: #10b981;
--warning: #f59e0b;
--error: #ef4444;
--processing: #3b82f6;
--radius: 8px;
--radius-sm: 4px;
--font-mono: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
"Fira Code", monospace, sans-serif;
--font-size: 14px;
--font-size-sm: 0.8rem;
--font-size-xs: 0.75rem;
}
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: var(--font-mono);
background: var(--bg-primary);
color: var(--text-primary);
font-size: var(--font-size);
}
/* Scrollbar */
::-webkit-scrollbar {
width: 6px;
}
::-webkit-scrollbar-track {
background: transparent;
}
::-webkit-scrollbar-thumb {
background: var(--border-light);
border-radius: 3px;
}
/* Shared button base */
button {
cursor: pointer;
border: none;
border-radius: var(--radius-sm);
font-family: var(--font-mono);
font-size: var(--font-size-sm);
transition: opacity 0.15s;
}
button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
/* Shared input base */
input,
select {
font-family: var(--font-mono);
font-size: var(--font-size-sm);
background: var(--bg-input);
color: var(--text-primary);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
padding: 0.4rem 0.5rem;
}
input:focus,
select:focus {
outline: none;
border-color: var(--accent);
}
/* Panel base */
.panel {
background: var(--bg-panel);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 1rem;
}
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.75rem;
position: relative;
}
.panel-header h2 {
font-size: 0.85rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--text-secondary);
}

View File

@@ -0,0 +1,170 @@
/**
* TypeScript Types - GENERATED FILE
*
* Do not edit directly. Regenerate using modelgen.
*/
export type AssetStatus = "pending" | "ready" | "error";
export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled";
export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled";
export interface MediaAsset {
id: string;
filename: string;
file_path: string;
status: AssetStatus;
error_message: string | null;
file_size: number | null;
duration: number | null;
video_codec: string | null;
audio_codec: string | null;
width: number | null;
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodePreset {
id: string;
name: string;
description: string;
is_builtin: boolean;
container: string;
video_codec: string;
video_bitrate: string | null;
video_crf: number | null;
video_preset: string | null;
resolution: string | null;
framerate: number | null;
audio_codec: string;
audio_bitrate: string | null;
audio_channels: number | null;
audio_samplerate: number | null;
extra_args: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodeJob {
id: string;
source_asset_id: string;
preset_id: string | null;
preset_snapshot: Record<string, unknown>;
trim_start: number | null;
trim_end: number | null;
output_filename: string;
output_path: string | null;
output_asset_id: string | null;
status: JobStatus;
progress: number;
current_frame: number | null;
current_time: number | null;
speed: string | null;
error_message: string | null;
celery_task_id: string | null;
execution_arn: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface ChunkJob {
id: string;
source_asset_id: string;
chunk_duration: number;
num_workers: number;
max_retries: number;
processor_type: string;
status: ChunkJobStatus;
progress: number;
total_chunks: number;
processed_chunks: number;
failed_chunks: number;
retry_count: number;
error_message: string | null;
throughput_mbps: number | null;
elapsed_seconds: number | null;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;
trim_start: number | null;
trim_end: number | null;
output_filename: string | null;
priority: number;
}
export interface UpdateAssetRequest {
comments: string | null;
tags: string[] | null;
}
export interface SystemStatus {
status: string;
version: string;
}
export interface ScanResult {
found: number;
registered: number;
skipped: number;
files: string[];
}
export interface DeleteResult {
ok: boolean;
}
export interface WorkerStatus {
available: boolean;
active_jobs: number;
supported_codecs: string[];
gpu_available: boolean;
}
export interface ChunkEvent {
sequence: number;
status: string;
size: number | null;
worker_id: string | null;
processing_time: number | null;
error: string | null;
retries: number;
}
export interface WorkerEvent {
worker_id: string;
state: string;
current_chunk: number | null;
processed: number;
errors: number;
retries: number;
}
export interface PipelineStats {
total_chunks: number;
processed: number;
failed: number;
retries: number;
elapsed: number;
throughput_mbps: number;
queue_size: number;
}
export interface ChunkOutputFile {
key: string;
size: number;
url: string;
}

21
ui/common/utils/format.ts Normal file
View File

@@ -0,0 +1,21 @@
/**
* Shared formatting utilities.
*/
export function formatSize(bytes: number | null | undefined): string {
if (!bytes) return "—";
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`;
if (bytes < 1024 * 1024 * 1024)
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
}
export function formatDuration(seconds: number | null | undefined): string {
if (!seconds) return "—";
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
const s = Math.floor(seconds % 60);
if (h > 0)
return `${h}:${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
return `${m}:${s.toString().padStart(2, "0")}`;
}

1736
ui/timeline/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,15 +1,4 @@
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family:
-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
background: #1a1a1a;
color: #e0e0e0;
}
@import "../../common/styles/theme.css";
.app {
display: flex;

View File

@@ -115,7 +115,6 @@ function App() {
setJobs(data);
};
const assetJobs = jobs.filter((j) => j.source_asset_id === selectedAsset?.id);
const completedJobs = jobs.filter((j) => j.status === "completed");
if (loading) return <div className="loading">Loading...</div>;

View File

@@ -42,6 +42,8 @@ export default function JobPanel({
preset_id: selectedPresetId || null,
trim_start: hasTrim ? trimStart : null,
trim_end: hasTrim ? trimEnd : null,
output_filename: null,
priority: 0,
});
onJobCreated();
} catch (e) {

View File

@@ -2,45 +2,17 @@
* GraphQL API client
*/
import { gql } from "../../common/api/graphql";
import { getAssets, scanMediaFolder } from "../../common/api/media";
import type {
MediaAsset,
TranscodePreset,
TranscodeJob,
CreateJobRequest,
SystemStatus,
MediaAsset,
} from "./types";
const GRAPHQL_URL = "/api/graphql";
async function gql<T>(query: string, variables?: Record<string, unknown>): Promise<T> {
const response = await fetch(GRAPHQL_URL, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ query, variables }),
});
const json = await response.json();
if (json.errors?.length) {
throw new Error(json.errors[0].message);
}
return json.data as T;
}
// Assets
export async function getAssets(): Promise<MediaAsset[]> {
const data = await gql<{ assets: MediaAsset[] }>(`
query {
assets {
id filename file_path status error_message file_size duration
video_codec audio_codec width height framerate bitrate
properties comments tags created_at updated_at
}
}
`);
return data.assets;
}
export { getAssets, scanMediaFolder };
export async function getAsset(id: string): Promise<MediaAsset> {
const data = await gql<{ asset: MediaAsset }>(`
@@ -55,20 +27,6 @@ export async function getAsset(id: string): Promise<MediaAsset> {
return data.asset;
}
export async function scanMediaFolder(): Promise<{
found: number;
registered: number;
skipped: number;
files: string[];
}> {
const data = await gql<{ scan_media_folder: { found: number; registered: number; skipped: number; files: string[] } }>(`
mutation {
scan_media_folder { found registered skipped files }
}
`);
return data.scan_media_folder;
}
// Presets
export async function getPresets(): Promise<TranscodePreset[]> {
const data = await gql<{ presets: TranscodePreset[] }>(`

View File

@@ -1,135 +1,21 @@
/**
* TypeScript Types - GENERATED FILE
* TypeScript Types — re-exported from common generated types.
*
* Do not edit directly. Regenerate using modelgen.
*/
export type AssetStatus = "pending" | "ready" | "error";
export type JobStatus = "pending" | "processing" | "completed" | "failed" | "cancelled";
export type ChunkJobStatus = "pending" | "chunking" | "processing" | "collecting" | "completed" | "failed" | "cancelled";
export interface MediaAsset {
id: string;
filename: string;
file_path: string;
status: AssetStatus;
error_message: string | null;
file_size: number | null;
duration: number | null;
video_codec: string | null;
audio_codec: string | null;
width: number | null;
height: number | null;
framerate: number | null;
bitrate: number | null;
properties: Record<string, unknown>;
comments: string;
tags: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodePreset {
id: string;
name: string;
description: string;
is_builtin: boolean;
container: string;
video_codec: string;
video_bitrate: string | null;
video_crf: number | null;
video_preset: string | null;
resolution: string | null;
framerate: number | null;
audio_codec: string;
audio_bitrate: string | null;
audio_channels: number | null;
audio_samplerate: number | null;
extra_args: string[];
created_at: string | null;
updated_at: string | null;
}
export interface TranscodeJob {
id: string;
source_asset_id: string;
preset_id: string | null;
preset_snapshot: Record<string, unknown>;
trim_start: number | null;
trim_end: number | null;
output_filename: string;
output_path: string | null;
output_asset_id: string | null;
status: JobStatus;
progress: number;
current_frame: number | null;
current_time: number | null;
speed: string | null;
error_message: string | null;
celery_task_id: string | null;
execution_arn: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface ChunkJob {
id: string;
source_asset_id: string;
chunk_duration: number;
num_workers: number;
max_retries: number;
processor_type: string;
status: ChunkJobStatus;
progress: number;
total_chunks: number;
processed_chunks: number;
failed_chunks: number;
retry_count: number;
error_message: string | null;
throughput_mbps: number | null;
elapsed_seconds: number | null;
celery_task_id: string | null;
priority: number;
created_at: string | null;
started_at: string | null;
completed_at: string | null;
}
export interface CreateJobRequest {
source_asset_id: string;
preset_id: string | null;
trim_start: number | null;
trim_end: number | null;
output_filename: string | null;
priority: number;
}
export interface UpdateAssetRequest {
comments: string | null;
tags: string[] | null;
}
export interface SystemStatus {
status: string;
version: string;
}
export interface ScanResult {
found: number;
registered: number;
skipped: number;
files: string[];
}
export interface DeleteResult {
ok: boolean;
}
export interface WorkerStatus {
available: boolean;
active_jobs: number;
supported_codecs: string[];
gpu_available: boolean;
}
export type {
AssetStatus,
JobStatus,
ChunkJobStatus,
MediaAsset,
TranscodePreset,
TranscodeJob,
ChunkJob,
CreateJobRequest,
UpdateAssetRequest,
SystemStatus,
ScanResult,
DeleteResult,
WorkerStatus,
} from "../../common/types/generated";

View File

@@ -15,7 +15,10 @@
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"rootDir": "..",
"typeRoots": ["./node_modules/@types"],
},
"include": ["src/**/*.ts", "src/**/*.tsx"],
"include": ["src/**/*.ts", "src/**/*.tsx", "../common/**/*.ts", "../common/**/*.tsx"],
"exclude": ["../common/api/grpc/**"],
"references": [{ "path": "./tsconfig.node.json" }],
}

View File

@@ -11,6 +11,9 @@ export default defineConfig({
hmr: {
path: "/timeline/@vite/client",
},
fs: {
allow: [".."],
},
proxy: {
"/api": {
target: "http://fastapi:8702",