phase 1
This commit is contained in:
@@ -1,15 +1,13 @@
|
||||
"""
|
||||
MPR Jobs Module
|
||||
|
||||
Provides executor abstraction and task dispatch for job processing.
|
||||
Provides executor abstraction for job dispatch (local, Lambda, GCP).
|
||||
"""
|
||||
|
||||
from .executor import Executor, LocalExecutor, get_executor
|
||||
from .task import run_job
|
||||
|
||||
__all__ = [
|
||||
"Executor",
|
||||
"LocalExecutor",
|
||||
"get_executor",
|
||||
"run_job",
|
||||
]
|
||||
|
||||
@@ -42,7 +42,7 @@ class Executor(ABC):
|
||||
|
||||
|
||||
class LocalExecutor(Executor):
|
||||
"""Execute jobs locally using registered handlers."""
|
||||
"""Execute jobs locally by calling the stage function directly."""
|
||||
|
||||
def run(
|
||||
self,
|
||||
@@ -51,16 +51,10 @@ class LocalExecutor(Executor):
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||
) -> bool:
|
||||
"""Execute job using the appropriate local handler."""
|
||||
from .registry import get_handler
|
||||
|
||||
handler = get_handler(job_type)
|
||||
result = handler.process(
|
||||
job_id=job_id,
|
||||
payload=payload,
|
||||
progress_callback=progress_callback,
|
||||
"""Execute job locally. Socket for PipelineRunner integration."""
|
||||
raise NotImplementedError(
|
||||
"LocalExecutor.run() — will be wired to PipelineRunner in Phase 3"
|
||||
)
|
||||
return result.get("status") == "completed"
|
||||
|
||||
|
||||
class LambdaExecutor(Executor):
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Job handlers — type-specific execution logic."""
|
||||
|
||||
from .base import Handler
|
||||
|
||||
__all__ = ["Handler"]
|
||||
@@ -1,33 +0,0 @@
|
||||
"""
|
||||
Base Handler ABC — defines the interface for job-type-specific execution logic.
|
||||
|
||||
A Handler knows HOW to execute a specific kind of job (transcode, chunk, etc.).
|
||||
The Executor decides WHERE to run it (local, Lambda, GCP).
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
|
||||
class Handler(ABC):
|
||||
"""Abstract base class for job handlers."""
|
||||
|
||||
@abstractmethod
|
||||
def process(
|
||||
self,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute job-specific logic.
|
||||
|
||||
Args:
|
||||
job_id: Unique job identifier
|
||||
payload: Job-type-specific configuration
|
||||
progress_callback: Called with (percent, details_dict)
|
||||
|
||||
Returns:
|
||||
Result dict with at least {"status": "completed"} or raises
|
||||
"""
|
||||
pass
|
||||
@@ -1,125 +0,0 @@
|
||||
"""
|
||||
ChunkHandler — job handler that wraps the chunker Pipeline.
|
||||
|
||||
Downloads source from S3/MinIO, runs FFmpeg chunking pipeline,
|
||||
writes mp4 segments + manifest to media/out/chunks/{job_id}/.
|
||||
Pushes real-time events to Redis for SSE consumption.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.events import push_event as push_chunk_event
|
||||
from core.chunker import Pipeline
|
||||
from core.storage import BUCKET_IN, download_to_temp
|
||||
|
||||
from .base import Handler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MEDIA_OUT_DIR = os.environ.get("MEDIA_OUT_DIR", "/app/media/out")
|
||||
|
||||
|
||||
class ChunkHandler(Handler):
|
||||
"""
|
||||
Handles chunk processing jobs by delegating to the chunker Pipeline.
|
||||
|
||||
Expected payload keys:
|
||||
source_key: str — S3 key of the source file in BUCKET_IN
|
||||
chunk_duration: float — seconds per chunk (default: 10.0)
|
||||
num_workers: int — concurrent workers (default: 4)
|
||||
max_retries: int — retries per chunk (default: 3)
|
||||
processor_type: str — "ffmpeg", "checksum", "simulated_decode", "composite"
|
||||
queue_size: int — max queue depth (default: 10)
|
||||
"""
|
||||
|
||||
def process(
|
||||
self,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
source_key = payload["source_key"]
|
||||
processor_type = payload.get("processor_type", "ffmpeg")
|
||||
|
||||
logger.info(f"ChunkHandler starting job {job_id}: {source_key}")
|
||||
|
||||
# Download source from S3/MinIO
|
||||
push_chunk_event(job_id, "pipeline_start", {"status": "downloading", "source_key": source_key})
|
||||
tmp_source = download_to_temp(BUCKET_IN, source_key)
|
||||
|
||||
# Output directory: media/out/chunks/{job_id}/
|
||||
output_dir = os.path.join(MEDIA_OUT_DIR, "chunks", job_id)
|
||||
if processor_type == "ffmpeg":
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
try:
|
||||
def event_bridge(event_type: str, data: Dict[str, Any]) -> None:
|
||||
"""Bridge pipeline events to Redis + optional progress callback."""
|
||||
push_chunk_event(job_id, event_type, data)
|
||||
|
||||
if progress_callback and event_type == "pipeline_complete":
|
||||
progress_callback(100, data)
|
||||
elif progress_callback and event_type == "chunk_done":
|
||||
total = data.get("total_chunks", 1)
|
||||
if total > 0:
|
||||
pct = min(int((data.get("sequence", 0) + 1) / total * 100), 99)
|
||||
progress_callback(pct, data)
|
||||
|
||||
pipeline = Pipeline(
|
||||
source=tmp_source,
|
||||
chunk_duration=payload.get("chunk_duration", 10.0),
|
||||
num_workers=payload.get("num_workers", 4),
|
||||
max_retries=payload.get("max_retries", 3),
|
||||
processor_type=processor_type,
|
||||
queue_size=payload.get("queue_size", 10),
|
||||
event_callback=event_bridge,
|
||||
output_dir=output_dir if processor_type == "ffmpeg" else None,
|
||||
start_time=payload.get("start_time"),
|
||||
end_time=payload.get("end_time"),
|
||||
)
|
||||
|
||||
result = pipeline.run()
|
||||
|
||||
# Files are already in media/out/chunks/{job_id}/
|
||||
output_prefix = f"chunks/{job_id}"
|
||||
output_files = [
|
||||
f"{output_prefix}/{os.path.basename(f)}"
|
||||
for f in result.chunk_files
|
||||
]
|
||||
|
||||
push_chunk_event(job_id, "pipeline_complete", {
|
||||
"status": "completed",
|
||||
"total_chunks": result.total_chunks,
|
||||
"processed": result.processed,
|
||||
"failed": result.failed,
|
||||
"elapsed": result.elapsed_time,
|
||||
"throughput_mbps": result.throughput_mbps,
|
||||
})
|
||||
|
||||
return {
|
||||
"status": "completed" if result.failed == 0 else "completed_with_errors",
|
||||
"total_chunks": result.total_chunks,
|
||||
"processed": result.processed,
|
||||
"failed": result.failed,
|
||||
"retries": result.retries,
|
||||
"elapsed_time": result.elapsed_time,
|
||||
"throughput_mbps": result.throughput_mbps,
|
||||
"worker_stats": result.worker_stats,
|
||||
"errors": result.errors,
|
||||
"chunks_in_order": result.chunks_in_order,
|
||||
"output_prefix": output_prefix,
|
||||
"output_files": output_files,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
push_chunk_event(job_id, "pipeline_error", {"status": "failed", "error": str(e)})
|
||||
raise
|
||||
|
||||
finally:
|
||||
# Cleanup temp source file only (output dir is persistent)
|
||||
try:
|
||||
os.unlink(tmp_source)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -1,130 +0,0 @@
|
||||
"""
|
||||
DetectHandler — runs the detection pipeline as a Celery job.
|
||||
|
||||
Supports three modes via payload:
|
||||
- Initial run: {"video_path": "...", "profile_name": "..."}
|
||||
- Replay: {"replay_from": "run_ocr", "source_job_id": "...", "config_overrides": {...}}
|
||||
- Retry: {"retry_from": "escalate_vlm", "source_job_id": "...", "config_overrides": {...}}
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from .base import Handler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DetectHandler(Handler):
|
||||
|
||||
def process(
|
||||
self,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
replay_from = payload.get("replay_from")
|
||||
source_job_id = payload.get("source_job_id")
|
||||
|
||||
if replay_from and source_job_id:
|
||||
return self._run_replay(job_id, source_job_id, replay_from, payload, progress_callback)
|
||||
|
||||
return self._run_initial(job_id, payload, progress_callback)
|
||||
|
||||
def _run_initial(
|
||||
self,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable],
|
||||
) -> Dict[str, Any]:
|
||||
from detect import emit
|
||||
from detect.graph import get_pipeline
|
||||
from detect.state import DetectState
|
||||
|
||||
video_path = payload["video_path"]
|
||||
profile_name = payload.get("profile_name", "soccer_broadcast")
|
||||
source_asset_id = payload.get("source_asset_id", "")
|
||||
checkpoint_enabled = payload.get("checkpoint", os.environ.get("MPR_CHECKPOINT") == "1")
|
||||
|
||||
emit.set_run_context(
|
||||
run_id=job_id,
|
||||
parent_job_id=payload.get("parent_job_id", job_id),
|
||||
run_type="initial",
|
||||
)
|
||||
|
||||
logger.info("DetectHandler: initial run job=%s video=%s profile=%s checkpoint=%s",
|
||||
job_id, video_path, profile_name, checkpoint_enabled)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(0, {"stage": "starting"})
|
||||
|
||||
pipeline = get_pipeline(checkpoint=checkpoint_enabled)
|
||||
|
||||
initial_state = DetectState(
|
||||
video_path=video_path,
|
||||
job_id=job_id,
|
||||
profile_name=profile_name,
|
||||
source_asset_id=source_asset_id,
|
||||
)
|
||||
|
||||
try:
|
||||
result = pipeline.invoke(initial_state)
|
||||
finally:
|
||||
emit.clear_run_context()
|
||||
|
||||
detections = result.get("detections", [])
|
||||
report = result.get("report")
|
||||
brands_found = len(report.brands) if report else 0
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(100, {"stage": "completed"})
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"job_id": job_id,
|
||||
"detections": len(detections),
|
||||
"brands_found": brands_found,
|
||||
}
|
||||
|
||||
def _run_replay(
|
||||
self,
|
||||
job_id: str,
|
||||
source_job_id: str,
|
||||
start_stage: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable],
|
||||
) -> Dict[str, Any]:
|
||||
from detect.checkpoint import replay_from
|
||||
|
||||
config_overrides = payload.get("config_overrides", {})
|
||||
|
||||
logger.info("DetectHandler: replay job=%s from=%s source=%s overrides=%s",
|
||||
job_id, start_stage, source_job_id, config_overrides)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(0, {"stage": f"replaying from {start_stage}"})
|
||||
|
||||
result = replay_from(
|
||||
job_id=source_job_id,
|
||||
start_stage=start_stage,
|
||||
config_overrides=config_overrides,
|
||||
)
|
||||
|
||||
detections = result.get("detections", [])
|
||||
report = result.get("report")
|
||||
brands_found = len(report.brands) if report else 0
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(100, {"stage": "completed"})
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"job_id": job_id,
|
||||
"source_job_id": source_job_id,
|
||||
"replay_from": start_stage,
|
||||
"detections": len(detections),
|
||||
"brands_found": brands_found,
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
"""
|
||||
TranscodeHandler — executes transcode/trim jobs using FFmpeg.
|
||||
|
||||
Extracted from the old tasks.py Celery task logic.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.ffmpeg.transcode import TranscodeConfig, transcode
|
||||
from core.storage import BUCKET_IN, BUCKET_OUT, download_to_temp, upload_file
|
||||
|
||||
from .base import Handler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TranscodeHandler(Handler):
|
||||
"""Handle transcode and trim jobs via FFmpeg."""
|
||||
|
||||
def process(
|
||||
self,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
progress_callback: Optional[Callable[[int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
source_key = payload["source_key"]
|
||||
output_key = payload["output_key"]
|
||||
preset = payload.get("preset")
|
||||
trim_start = payload.get("trim_start")
|
||||
trim_end = payload.get("trim_end")
|
||||
duration = payload.get("duration")
|
||||
|
||||
logger.info(f"TranscodeHandler: {source_key} -> {output_key}")
|
||||
|
||||
# Download source
|
||||
tmp_source = download_to_temp(BUCKET_IN, source_key)
|
||||
|
||||
ext = Path(output_key).suffix or ".mp4"
|
||||
fd, tmp_output = tempfile.mkstemp(suffix=ext)
|
||||
os.close(fd)
|
||||
|
||||
try:
|
||||
if preset:
|
||||
config = TranscodeConfig(
|
||||
input_path=tmp_source,
|
||||
output_path=tmp_output,
|
||||
video_codec=preset.get("video_codec", "libx264"),
|
||||
video_bitrate=preset.get("video_bitrate"),
|
||||
video_crf=preset.get("video_crf"),
|
||||
video_preset=preset.get("video_preset"),
|
||||
resolution=preset.get("resolution"),
|
||||
framerate=preset.get("framerate"),
|
||||
audio_codec=preset.get("audio_codec", "aac"),
|
||||
audio_bitrate=preset.get("audio_bitrate"),
|
||||
audio_channels=preset.get("audio_channels"),
|
||||
audio_samplerate=preset.get("audio_samplerate"),
|
||||
container=preset.get("container", "mp4"),
|
||||
extra_args=preset.get("extra_args", []),
|
||||
trim_start=trim_start,
|
||||
trim_end=trim_end,
|
||||
)
|
||||
else:
|
||||
config = TranscodeConfig(
|
||||
input_path=tmp_source,
|
||||
output_path=tmp_output,
|
||||
video_codec="copy",
|
||||
audio_codec="copy",
|
||||
trim_start=trim_start,
|
||||
trim_end=trim_end,
|
||||
)
|
||||
|
||||
def wrapped_callback(percent: float, details: Dict[str, Any]) -> None:
|
||||
if progress_callback:
|
||||
progress_callback(int(percent), details)
|
||||
|
||||
success = transcode(
|
||||
config,
|
||||
duration=duration,
|
||||
progress_callback=wrapped_callback if progress_callback else None,
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise RuntimeError("Transcode returned False")
|
||||
|
||||
# Upload result
|
||||
logger.info(f"Uploading {output_key} to {BUCKET_OUT}")
|
||||
upload_file(tmp_output, BUCKET_OUT, output_key)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"job_id": job_id,
|
||||
"output_key": output_key,
|
||||
}
|
||||
|
||||
finally:
|
||||
for f in [tmp_source, tmp_output]:
|
||||
try:
|
||||
os.unlink(f)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -1,35 +0,0 @@
|
||||
"""
|
||||
Handler registry — maps job_type strings to Handler classes.
|
||||
"""
|
||||
|
||||
from typing import Dict, Type
|
||||
|
||||
from .handlers.base import Handler
|
||||
|
||||
_handlers: Dict[str, Type[Handler]] = {}
|
||||
|
||||
|
||||
def register_handler(job_type: str, handler_class: Type[Handler]) -> None:
|
||||
"""Register a handler class for a job type."""
|
||||
_handlers[job_type] = handler_class
|
||||
|
||||
|
||||
def get_handler(job_type: str) -> Handler:
|
||||
"""Get an instantiated handler for a job type."""
|
||||
if job_type not in _handlers:
|
||||
raise ValueError(f"Unknown job type: {job_type}")
|
||||
return _handlers[job_type]()
|
||||
|
||||
|
||||
def _register_defaults() -> None:
|
||||
"""Register built-in handlers."""
|
||||
from .handlers.chunk import ChunkHandler
|
||||
from .handlers.transcode import TranscodeHandler
|
||||
from .handlers.detect import DetectHandler
|
||||
|
||||
register_handler("transcode", TranscodeHandler)
|
||||
register_handler("chunk", ChunkHandler)
|
||||
register_handler("detect", DetectHandler)
|
||||
|
||||
|
||||
_register_defaults()
|
||||
@@ -1,64 +0,0 @@
|
||||
"""
|
||||
Celery task for job processing.
|
||||
|
||||
Generic dispatcher — routes to the appropriate handler based on job_type.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
from core.rpc.server import update_job_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(bind=True, max_retries=3, default_retry_delay=60)
|
||||
def run_job(
|
||||
self,
|
||||
job_type: str,
|
||||
job_id: str,
|
||||
payload: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Generic Celery task — dispatches to the registered handler for job_type.
|
||||
"""
|
||||
logger.info(f"Starting {job_type} job {job_id}")
|
||||
|
||||
update_job_progress(job_id, progress=0, status="processing")
|
||||
|
||||
def progress_callback(percent: int, details: Dict[str, Any]) -> None:
|
||||
update_job_progress(
|
||||
job_id,
|
||||
progress=percent,
|
||||
current_time=details.get("time", 0.0),
|
||||
status="processing",
|
||||
)
|
||||
|
||||
try:
|
||||
from .registry import get_handler
|
||||
|
||||
handler = get_handler(job_type)
|
||||
result = handler.process(
|
||||
job_id=job_id,
|
||||
payload=payload,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
logger.info(f"Job {job_id} completed successfully")
|
||||
update_job_progress(job_id, progress=100, status="completed")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Job {job_id} failed: {e}")
|
||||
update_job_progress(job_id, progress=0, status="failed", error=str(e))
|
||||
|
||||
if self.request.retries < self.max_retries:
|
||||
raise self.retry(exc=e)
|
||||
|
||||
return {
|
||||
"status": "failed",
|
||||
"job_id": job_id,
|
||||
"error": str(e),
|
||||
}
|
||||
Reference in New Issue
Block a user